@lde/search-typesense 0.27.4 → 0.29.0

package/dist/import.js

@@ -1,60 +0,0 @@
-/** Documents imported per Typesense request unless a writer is told otherwise. */
-export const DEFAULT_BATCH_SIZE = 1000;
-/**
- * Accumulates documents across writes and upserts them into a collection in
- * fixed-size batches, so many small writes (one dataset at a time) still land
- * as few Typesense requests. Call {@link flush} to import the remainder.
- */
-export class BatchImporter {
-    client;
-    collection;
-    batchSize;
-    buffer = [];
-    constructor(client, collection, batchSize) {
-        this.client = client;
-        this.collection = collection;
-        this.batchSize = batchSize;
-    }
-    /** Add a stream of documents, importing a batch whenever one fills up. */
-    async add(documents) {
-        for await (const document of documents) {
-            this.buffer.push(document);
-            if (this.buffer.length >= this.batchSize) {
-                await this.importBuffer();
-            }
-        }
-    }
-    /** Import any remaining documents. */
-    async flush() {
-        if (this.buffer.length > 0) {
-            await this.importBuffer();
-        }
-    }
-    async importBuffer() {
-        const batch = this.buffer;
-        this.buffer = [];
-        await importBatch(this.client, this.collection, batch);
-    }
-}
-/**
- * Create-or-replace one batch of whole documents, keyed on `id`, throwing if any
- * individual document fails (Typesense’s bulk import otherwise reports
- * per-document failures without rejecting).
- */
-export async function importBatch(client, collection, batch) {
-    const results = (await client
-        .collections(collection)
-        .documents()
-        .import(batch, {
-        action: 'upsert',
-        // Collect per-document outcomes instead of throwing the client’s opaque
-        // ImportError, so we can report which documents failed and why.
-        throwOnFail: false,
-    }));
-    const failures = results.filter((result) => !result.success);
-    if (failures.length > 0) {
-        throw new Error(`Typesense upsert into “${collection}” failed for ${failures.length}/${results.length} documents: ${failures
-            .map((failure) => failure.error)
-            .join('; ')}`);
-    }
-}

package/dist/sweep.d.ts

@@ -1,70 +0,0 @@
-import type { SearchType } from '@lde/search';
-/** The private field carrying the dataset IRI a document came from, for a type
- *  that declares none of its own – see {@link provenanceField}. */
-export declare const SOURCE_FIELD = "source";
-/** The document field carrying the id of the run that last wrote a document. */
-export declare const LAST_SEEN_FIELD = "last_seen";
-/**
- * The field a collection carries its documents’ dataset IRI in – the one the
- * membership sweep enumerates, filters and deletes by.
- *
- * A type that declares a field over the dataset
- * ({@link datasetField `from: 'dataset'`}) *is* its provenance: the writer keeps
- * its bookkeeping on that field rather than stamping a private one beside it,
- * so the facet, the label resolution, any `derive` and the sweep all read one
- * column that cannot drift. A type declaring nothing – or declaring the dataset
- * only as an *internal* field, which the projection prunes before the writer
- * ever sees it – falls back to the private {@link SOURCE_FIELD}, and behaves
- * exactly as it always has.
- */
-export declare function provenanceField(searchType: SearchType): string;
-/**
- * Sources whose documents must leave the index: indexed, but no longer part
- * of the run’s selection. Selection is membership, not processing – a dataset
- * skipped as unchanged is still selected, so its documents survive.
- *
- * @param indexedSources Source IRIs present in the collection
- * @param selectedSources Source IRIs the run’s selector produced
- */
-export declare function departedSources(indexedSources: Iterable<string>, selectedSources: Iterable<string>): string[];
-/**
- * Typesense filter matching a source’s documents that this run did not touch:
- * everything the source no longer contains, ready for a per-source sweep.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run; documents it wrote carry it as `last_seen`
- */
-export declare function staleDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
-/**
- * Typesense filter matching all of a source’s documents, ready to drop a whole
- * source (a departed source’s membership sweep, or a Blue/green writer rolling
- * a failed dataset out of its not-yet-live collection).
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- */
-export declare function sourceDocumentsFilter(sourceField: string, sourceIri: string): string;
-/**
- * Typesense filter matching the documents **this run** wrote for a source: the
- * inverse of {@link staleDocumentsFilter}. An In-place writer deletes these to
- * discard a source’s in-progress writes (the dump-fallback reset) without
- * touching its prior-run documents, which the success sweep or a failed run
- * still owns.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run, stamped on its documents as `last_seen`
- */
-export declare function thisRunDocumentsFilter(sourceField: string, sourceIri: string, runId: string): string;
-/**
- * Typesense filters deleting every departed source’s documents, combined into
- * as few filters as fit: deletes travel in the URL query string, so each
- * filter stays under a conservative length budget rather than listing every
- * source in one string.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param departed The departed source IRIs ({@link departedSources})
- */
-export declare function membershipSweepFilters(sourceField: string, departed: readonly string[]): string[];
-//# sourceMappingURL=sweep.d.ts.map

package/dist/sweep.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"sweep.d.ts","sourceRoot":"","sources":["../src/sweep.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C;mEACmE;AACnE,eAAO,MAAM,YAAY,WAAW,CAAC;AAErC,gFAAgF;AAChF,eAAO,MAAM,eAAe,cAAc,CAAC;AAE3C;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAK9D;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,GAChC,MAAM,EAAE,CAGV;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,MAAM,CAER;AAED;;;;;;;;GAQG;AACH,wBAAgB,sBAAsB,CACpC,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,SAAS,MAAM,EAAE,GAC1B,MAAM,EAAE,CAqBV"}

package/dist/sweep.js

@@ -1,113 +0,0 @@
-// Pure deletion planning shared by the rebuild writers: which documents leave
-// a collection, expressed as source sets and Typesense filter strings. Kept
-// free of the Typesense client so the logic is unit-testable. This module owns
-// the private bookkeeping field names and resolves which field a collection
-// carries its provenance in, so stamping and every filter that reads it can
-// never disagree.
-import { datasetField, isInternalField } from '@lde/search/adapter';
-import { escapeFilterValue } from './query-compiler.js';
-/** The private field carrying the dataset IRI a document came from, for a type
- *  that declares none of its own – see {@link provenanceField}. */
-export const SOURCE_FIELD = 'source';
-/** The document field carrying the id of the run that last wrote a document. */
-export const LAST_SEEN_FIELD = 'last_seen';
-/**
- * The field a collection carries its documents’ dataset IRI in – the one the
- * membership sweep enumerates, filters and deletes by.
- *
- * A type that declares a field over the dataset
- * ({@link datasetField `from: 'dataset'`}) *is* its provenance: the writer keeps
- * its bookkeeping on that field rather than stamping a private one beside it,
- * so the facet, the label resolution, any `derive` and the sweep all read one
- * column that cannot drift. A type declaring nothing – or declaring the dataset
- * only as an *internal* field, which the projection prunes before the writer
- * ever sees it – falls back to the private {@link SOURCE_FIELD}, and behaves
- * exactly as it always has.
- */
-export function provenanceField(searchType) {
-    const declared = datasetField(searchType);
-    return declared === undefined || isInternalField(declared)
-        ? SOURCE_FIELD
-        : declared.name;
-}
-/**
- * Sources whose documents must leave the index: indexed, but no longer part
- * of the run’s selection. Selection is membership, not processing – a dataset
- * skipped as unchanged is still selected, so its documents survive.
- *
- * @param indexedSources Source IRIs present in the collection
- * @param selectedSources Source IRIs the run’s selector produced
- */
-export function departedSources(indexedSources, selectedSources) {
-    const selected = new Set(selectedSources);
-    return [...indexedSources].filter((source) => !selected.has(source));
-}
-/**
- * Typesense filter matching a source’s documents that this run did not touch:
- * everything the source no longer contains, ready for a per-source sweep.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run; documents it wrote carry it as `last_seen`
- */
-export function staleDocumentsFilter(sourceField, sourceIri, runId) {
-    return `${sourceDocumentsFilter(sourceField, sourceIri)} && ${LAST_SEEN_FIELD}:!=${escapeFilterValue(runId)}`;
-}
-/**
- * Typesense filter matching all of a source’s documents, ready to drop a whole
- * source (a departed source’s membership sweep, or a Blue/green writer rolling
- * a failed dataset out of its not-yet-live collection).
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- */
-export function sourceDocumentsFilter(sourceField, sourceIri) {
-    return `${sourceField}:=${escapeFilterValue(sourceIri)}`;
-}
-/**
- * Typesense filter matching the documents **this run** wrote for a source: the
- * inverse of {@link staleDocumentsFilter}. An In-place writer deletes these to
- * discard a source’s in-progress writes (the dump-fallback reset) without
- * touching its prior-run documents, which the success sweep or a failed run
- * still owns.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param sourceIri The dataset IRI the documents carry in that field
- * @param runId The current run, stamped on its documents as `last_seen`
- */
-export function thisRunDocumentsFilter(sourceField, sourceIri, runId) {
-    return `${sourceDocumentsFilter(sourceField, sourceIri)} && ${LAST_SEEN_FIELD}:=${escapeFilterValue(runId)}`;
-}
-/**
- * Typesense filters deleting every departed source’s documents, combined into
- * as few filters as fit: deletes travel in the URL query string, so each
- * filter stays under a conservative length budget rather than listing every
- * source in one string.
- *
- * @param sourceField The collection’s {@link provenanceField}
- * @param departed The departed source IRIs ({@link departedSources})
- */
-export function membershipSweepFilters(sourceField, departed) {
-    const filters = [];
-    let chunk = [];
-    let chunkLength = 0;
-    const flush = () => {
-        if (chunk.length > 0) {
-            filters.push(`${sourceField}:=[${chunk.join(',')}]`);
-            chunk = [];
-            chunkLength = 0;
-        }
-    };
-    for (const source of departed) {
-        const escaped = escapeFilterValue(source);
-        if (chunkLength + escaped.length > MAX_FILTER_VALUES_LENGTH) {
-            flush();
-        }
-        chunk.push(escaped);
-        chunkLength += escaped.length + 1;
-    }
-    flush();
-    return filters;
-}
-/** Stay well under Typesense’s ~4000-char URL query-string limit per delete. */
-const MAX_FILTER_VALUES_LENGTH = 3000;