mindoodb 0.0.23 → 0.0.32

package/dist/core/indexing/fulltext/DocumentFullTextIndex.d.ts

@@ -0,0 +1,180 @@
+import type { MindooDB, ProcessChangesCursor } from "../../types";
+import type { LocalCacheStore } from "../../cache/LocalCacheStore";
+import type { CacheManager, ICacheable } from "../../cache/CacheManager";
+import type { VirtualViewUpdateOptions } from "../virtualviews/IVirtualViewDataProvider";
+import type { AttachmentTextExtractor, FulltextConfig, FulltextCoverage, FulltextSearchOptions, FulltextSearchResult, ResolvedFulltextConfig } from "./types";
+/**
+ * The document full-text index: a changefeed-maintained, encrypted-at-rest
+ * search index over extracted document text (see docs/fulltext-search.md).
+ *
+ * Sits next to the {@link DocumentSummaryStore} as a second derived index:
+ * same changefeed pipeline (cursor, resumable backfill, config
+ * fingerprint), same encrypted persistence through the tenant's
+ * {@link LocalCacheStore}, same `dbsetup` reconciliation (field
+ * `fulltextSetup`). Unlike the summary it is OPT-IN (`enabled: true`);
+ * text is extracted from the materialized documents the changefeed yields
+ * anyway — including long-text and rich-text fields that must stay out of
+ * the RAM-resident summary buffer.
+ *
+ * E2EE note: the index is built client-side from decrypted content, so
+ * devices whose KeyBag lacks certain decryption keys index (and find)
+ * fewer documents. Purge paths remove entries immediately so plaintext
+ * tokens never outlive key revocation.
+ */
+export declare class DocumentFullTextIndex implements ICacheable {
+    private readonly db;
+    private config;
+    private configFingerprint;
+    /**
+     * `true` when the configuration came from code (constructor arg or
+     * `setConfig`). Without an explicit configuration the index follows the
+     * `fulltextSetup` field of the synced {@link DB_SETUP_DOC_ID} document.
+     */
+    private hasExplicitConfig;
+    /** Whether the setup document was already consulted (once per instance). */
+    private setupDocSeeded;
+    private engine;
+    private cursor;
+    /**
+     * Attachment text extractors, provided by the host environment (see
+     * `MindooDB.registerAttachmentTextExtractor`). Read lazily so
+     * registrations after index creation are honored for subsequently
+     * indexed documents.
+     */
+    private readonly getAttachmentExtractors;
+    private cacheManager;
+    private cachePrefix;
+    private engineDirty;
+    private metaDirty;
+    private restorePromise;
+    private restored;
+    private needsBackfill;
+    private backfillCursor;
+    private updatePromise;
+    constructor(db: MindooDB, config?: FulltextConfig, options?: {
+        getAttachmentExtractors?: () => AttachmentTextExtractor[];
+    });
+    private createEngine;
+    getConfig(): ResolvedFulltextConfig;
+    getConfigFingerprint(): string;
+    isEnabled(): boolean;
+    /**
+     * Replace the full-text configuration. When the fingerprint changes,
+     * the engine is recreated (tokenizer/language and field extraction may
+     * differ) and a resumable backfill re-indexes all documents on the next
+     * `update()` run. Until the backfill finishes the index reports
+     * `"rebuilding"` and serves the (growing) partial result.
+     *
+     * Calling this marks the configuration as explicit: the index stops
+     * following the {@link DB_SETUP_DOC_ID} document.
+     */
+    setConfig(config?: FulltextConfig): void;
+    /** Shared config-swap logic for explicit and setup-document paths. */
+    private applyConfig;
+    private scheduleBackfill;
+    /**
+     * Adopt the configuration from the synced {@link DB_SETUP_DOC_ID}
+     * document (its `fulltextSetup` field) unless an explicit configuration
+     * was provided in code. Runs once per instance, BEFORE the persisted
+     * state is restored, so the fingerprint comparison on restore already
+     * sees the setup-document configuration.
+     */
+    private seedConfigFromSetupDoc;
+    /** Fresh read of the setup document (no-op with an explicit config). */
+    private reloadConfigFromSetupDoc;
+    /**
+     * Make sure the effective configuration is settled: consults the
+     * {@link DB_SETUP_DOC_ID} document once (unless the config was provided
+     * in code) and returns the resolved configuration.
+     */
+    ensureConfigLoaded(): Promise<ResolvedFulltextConfig>;
+    /**
+     * `"full"` when the index reflects the current configuration,
+     * `"rebuilding"` while a configuration-change backfill is in progress.
+     */
+    getCoverage(): FulltextCoverage;
+    getSize(): number;
+    getCursor(): ProcessChangesCursor | null;
+    /** Index field names currently known (document paths + `_attachments`). */
+    getFieldNames(): string[];
+    /**
+     * Full-text search. Brings the index up to date first (same catch-up
+     * semantics as queries against the summary buffer), then returns
+     * matching documents best-score-first.
+     *
+     * @throws Error when full-text indexing is not enabled for this
+     *   database (neither via `fulltextSetup` nor an explicit config).
+     */
+    search(query: string, options?: FulltextSearchOptions & VirtualViewUpdateOptions): Promise<FulltextSearchResult>;
+    /**
+     * Search without a catch-up run — evaluates against the current index
+     * state. Used by the query engine, which coordinates update passes
+     * itself.
+     */
+    searchSync(query: string, options?: FulltextSearchOptions): FulltextSearchResult;
+    /**
+     * Bring the index up to date with the changefeed. Restores persisted
+     * state on first use, then consumes `iterateChangesSince` from the
+     * saved cursor and finally works down a pending configuration backfill.
+     * While the configuration has `enabled: false` this is a cheap no-op
+     * (one setup-document read to detect a later enable).
+     *
+     * Accepts the same batching/progress/cancellation options as summary
+     * updates; an interrupted run resumes from the saved cursors. Calls are
+     * single-flight: concurrent callers share one run.
+     */
+    update(options?: VirtualViewUpdateOptions): Promise<void>;
+    private runUpdate;
+    private applyDocument;
+    /**
+     * Collect the searchable text of a document's attachments into the
+     * synthetic {@link FULLTEXT_ATTACHMENT_FIELD} (capped at
+     * `maxFieldBytes` like every other index field). Two sources feed it:
+     *
+     * 1. **Persisted extraction results** (`extractedText` on the
+     *    attachment entry, written via
+     *    `MindooDoc.setAttachmentExtractedText()`, e.g. by Haven's OCR
+     *    service). These are always indexed — the text is already there,
+     *    costs nothing, and is obviously meant to be searchable.
+     * 2. **Registered extractors** (cheap formats: plain text, PDF text
+     *    layer, Office) — only when `config.attachments` is enabled, and
+     *    only for attachments without a persisted result (a persisted
+     *    "failed"/"skipped" marker also suppresses the extractor run).
+     *
+     * Extraction failures skip the attachment — a broken PDF must never
+     * stall the changefeed pipeline.
+     */
+    private collectAttachmentText;
+    /**
+     * Drop a document's index entry immediately (called from the DB's purge
+     * paths so extracted plaintext tokens never outlive key
+     * revocation/purges; the changefeed tombstone would remove it anyway,
+     * but only on the next update run).
+     */
+    removeDocument(docId: string): void;
+    /** Drop all indexed content and cursors (e.g. after the underlying store was cleared). */
+    reset(): void;
+    /**
+     * Attach cache persistence. `cachePrefix` scopes all records of this
+     * index (recommended: `<dbCachePrefix>/fulltext`). Registers with the
+     * CacheManager for periodic flushing; the persisted state is restored
+     * lazily on the first `update()` call.
+     */
+    attachCache(cacheManager: CacheManager, cachePrefix: string): void;
+    getCachePrefix(): string;
+    hasDirtyState(): boolean;
+    clearDirty(): void;
+    private markEngineDirty;
+    flushToCache(store: LocalCacheStore, _options?: {
+        force?: boolean;
+    }): Promise<number>;
+    private ensureRestored;
+    /**
+     * Restore persisted state. Unlike the summary, index content is only
+     * usable when the config fingerprint matches exactly (tokenization is
+     * config-dependent); a mismatch or unreadable blob starts empty and
+     * schedules a full backfill instead.
+     */
+    private restoreFromCache;
+}
+//# sourceMappingURL=DocumentFullTextIndex.d.ts.map

package/dist/core/indexing/fulltext/DocumentFullTextIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DocumentFullTextIndex.d.ts","sourceRoot":"","sources":["../../../../src/core/indexing/fulltext/DocumentFullTextIndex.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAa,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AAEzF,OAAO,KAAK,EACV,uBAAuB,EACvB,cAAc,EACd,gBAAgB,EAEhB,qBAAqB,EACrB,oBAAoB,EACpB,sBAAsB,EACvB,MAAM,SAAS,CAAC;AAkBjB;;;;;;;;;;;;;;;;;GAiBG;AACH,qBAAa,qBAAsB,YAAW,UAAU;IACtD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAW;IAC9B,OAAO,CAAC,MAAM,CAAyB;IACvC,OAAO,CAAC,iBAAiB,CAAS;IAClC;;;;OAIG;IACH,OAAO,CAAC,iBAAiB,CAAU;IACnC,4EAA4E;IAC5E,OAAO,CAAC,cAAc,CAAkB;IAExC,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,MAAM,CAAqC;IAEnD;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAkC;IAG1E,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,WAAW,CAAuB;IAC1C,OAAO,CAAC,WAAW,CAAkB;IACrC,OAAO,CAAC,SAAS,CAAkB;IACnC,OAAO,CAAC,cAAc,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAkB;IAGlC,OAAO,CAAC,aAAa,CAAkB;IACvC,OAAO,CAAC,cAAc,CAAqC;IAG3D,OAAO,CAAC,aAAa,CAA8B;gBAGjD,EAAE,EAAE,QAAQ,EACZ,MAAM,CAAC,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE;QAAE,uBAAuB,CAAC,EAAE,MAAM,uBAAuB,EAAE,CAAA;KAAE;IAUzE,OAAO,CAAC,YAAY;IAQpB,SAAS,IAAI,sBAAsB;IAInC,oBAAoB,IAAI,MAAM;IAI9B,SAAS,IAAI,OAAO;IAIpB;;;;;;;;;OASG;IACH,SAAS,CAAC,MAAM,CAAC,EAAE,cAAc,GAAG,IAAI;IAKxC,sEAAsE;IACtE,OAAO,CAAC,WAAW;IAoBnB,OAAO,CAAC,gBAAgB;IAKxB;;;;;;OAMG;YACW,sBAAsB;IASpC,wEAAwE;YAC1D,wBAAwB;IAatC;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC,sBAAsB,CAAC;IAK3D;;;OAGG;IACH,WAAW,IAAI,gBAAgB;IAQ/B,OAAO,IAAI,MAAM;IAIjB,SAAS,IAAI,oBAAoB,GAAG,IAAI;IAIxC,2EAA2E;IAC3E,aAAa,IAAI,MAAM,EAAE;IAIzB;;;;;;;OAOG;IACG,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,qBAAqB,GAAG,wBAAwB,GACzD,OAAO,CAAC,oBAAoB,CAAC;IAchC;;;;OAIG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,qBAAqB,GAAG,oBAAoB;IAWhF;;;;;;;;;;OAUG;IACG,MAAM,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC;YAUjD,SAAS;YA6FT,aAAa;IAiD3B;;;;;;;;;;;;;;;;;OAiBG;YACW,qBAAqB;IAiEnC;;;;;OAKG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQnC,0FAA0F;IAC1F,KAAK,IAAI,IAAI;IAcb;;;;;OAKG;IACH,WAAW,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,IAAI;IAMlE,cAAc,IAAI,MAAM;IAIxB,aAAa,IAAI,OAAO;IAIxB,UAAU,IAAI,IAAI;IAKlB,OAAO,CAAC,eAAe;IAKjB,YAAY,CAAC,KAAK,EAAE,eAAe,EAAE,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC;YAqB7E,cAAc;IAa5B;;;;;OAKG;YACW,gBAAgB;CAgD/B"}