@songsid/agend 2.3.0-beta.8 → 2.3.0

package/dist/search/search-index.js

@@ -0,0 +1,440 @@
+/**
+ * #2293: the fleet's side of global search. The fleet loop only builds small docs (search-docs.ts) and posts them; the
+ * worker (search-worker.ts) owns search.db — every write, query, prune and the SQLite part of the backfill run there.
+ *
+ * - Writes are batched: posted every BATCH_MS or at BATCH_MAX docs. While the worker is starting (or restarting) they
+ *   wait in a bounded queue (QUEUE_MAX; the oldest go first — the backfill finds what is still kept).
+ * - A query has QUERY_BUDGET_MS; past it the page is told so and the worker carries on (a query is bounded: LIMIT, or
+ *   the newest SHORT_SCAN_ROWS rows). At most PENDING_MAX at a time.
+ * - A new or rebuilt index (`ready` with built=false: a first start, a schema change, a corrupt file moved aside) is
+ *   backfilled: the chat histories the fleet holds, in chunks between turns of the loop, then the worker's read of the
+ *   outbox and the Task Board; then marked built. Search answers meanwhile and says it is still building.
+ * - A worker that exits is started again after a backoff (1 s doubling to a minute); one that cannot open the file at
+ *   all ("failed") leaves search unavailable until the next fleet start. The fleet runs on either way.
+ * - Retention: docs older than retention_days, and the oldest past ROW_CAP, are pruned after the backfill and daily.
+ */
+import { Worker } from "node:worker_threads";
+export const BATCH_MS = 250;
+export const BATCH_MAX = 200;
+export const QUEUE_MAX = 5_000;
+export const QUERY_BUDGET_MS = 3_000;
+export const PENDING_MAX = 16;
+/** A query the worker still holds this long after it was asked: the worker is stuck, and is restarted. */
+export const STALL_MS = 30_000;
+/** How long close() waits for the worker to write what it was sent and exit. */
+export const CLOSE_WAIT_MS = 3_000;
+export const PRUNE_EVERY_MS = 24 * 3600_000;
+export const BACKFILL_CHUNK = 200;
+/** web.search.retention_days when unset. */
+export const SEARCH_RETENTION_DAYS = 90;
+const RESTART_MIN_MS = 1_000, RESTART_MAX_MS = 60_000;
+function createIsolate(path) {
+    return import.meta.url.endsWith(".ts")
+        ? new Worker(`const { workerData } = require("node:worker_threads"); import("tsx/esm/api").then(({ tsImport }) => tsImport(workerData.entry, workerData.entry));`, { eval: true, execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 256 }, workerData: { path, entry: new URL("./search-worker.ts", import.meta.url).href } })
+        : new Worker(new URL("./search-worker.js", import.meta.url), { execArgv: [], resourceLimits: { maxOldGenerationSizeMb: 256 }, workerData: { path } });
+}
+export class SearchIndex {
+    opts;
+    worker = null;
+    state = "starting";
+    queue = [];
+    flushTimer = null;
+    restartTimer = null;
+    pruneTimer = null;
+    restartDelay = RESTART_MIN_MS;
+    nextId = 0;
+    /**
+     * Queries the worker holds: PENDING_MAX counts these. Past QUERY_BUDGET_MS the caller is answered ("timeout"), but the
+     * entry stays until the worker replies or exits — the worker still runs or queues it. One still held at STALL_MS: the
+     * worker is retired (terminated) and started again.
+     */
+    queries = new Map();
+    /** The index's own requests (backfill, prune): no budget; settled on the reply or when the worker goes. */
+    requests = new Map();
+    /**
+     * Batches posted and not yet committed (the worker acks each after its write). A worker that goes, or reopens a fresh
+     * file, gets them again (an upsert by key: twice is harmless). Flushing waits while they hold QUEUE_MAX docs.
+     */
+    inflight = new Map();
+    inflightDocs = 0;
+    nextBatch = 0;
+    /**
+     * Instance removals not yet known to be done, with the last batch posted before each: sent again before anything else
+     * to a worker that (re)starts. Done when the worker acks it, or acks a later batch (it works in order).
+     */
+    removals = new Map();
+    removalGen = 0;
+    /** A backfill waiting for room under QUEUE_MAX uncommitted docs: woken by a commit, the worker going, or close. */
+    roomWaiters = [];
+    /** Instances removed while a backfill runs: the rest of it leaves their chat out (it read the history before). */
+    removedDuringBackfill = null;
+    /** Bumped on every (re)start and on close: a backfill of an older worker stops. */
+    epoch = 0;
+    closed = false;
+    factory;
+    now;
+    constructor(opts) {
+        this.opts = opts;
+        this.factory = opts.factory ?? createIsolate;
+        this.now = opts.now ?? Date.now;
+    }
+    start() {
+        if (this.closed || this.worker)
+            return;
+        const epoch = ++this.epoch;
+        let worker;
+        try {
+            worker = this.factory(this.opts.path);
+        }
+        catch (err) {
+            this.opts.log?.("warn", `search: cannot start the index worker: ${String(err?.message ?? err)}`);
+            this.scheduleRestart();
+            return;
+        }
+        this.worker = worker;
+        this.state = "starting";
+        worker.unref();
+        worker.on("message", (m) => {
+            if (this.worker !== worker)
+                return; // an old worker's late reply or ack changes nothing
+            if (m.op === "reply" && m.id !== undefined) {
+                this.replied(m.id, m.ok ? { ok: true, value: m.value } : { ok: false, reason: "unavailable" });
+                return;
+            }
+            if (m.op === "added" && m.batch !== undefined) {
+                this.committed(m.batch);
+                return;
+            }
+            // A removal's ack confirms that removal only: a later one of the same name (re-created, removed again) stays.
+            if (m.op === "removed" && typeof m.instance === "string") {
+                if (this.removals.get(m.instance)?.gen === m.gen)
+                    this.removals.delete(m.instance);
+                return;
+            }
+            if (m.op === "failed") {
+                this.opts.log?.("warn", `search: the index cannot be opened (${m.reason ?? "unknown"}); search is unavailable`);
+                this.state = "unavailable";
+                this.closed = true;
+                return;
+            }
+            if (m.op !== "ready")
+                return;
+            this.restartDelay = RESTART_MIN_MS;
+            // Whatever this worker's file may be missing: removals first, then the batches no worker committed.
+            for (const [instance, { gen }] of this.removals)
+                this.post({ op: "removeInstance", instance, gen });
+            for (const [batch, docs] of this.inflight)
+                this.post({ op: "add", batch, docs });
+            if (m.built) {
+                this.state = "ready";
+                this.flush();
+                this.startPruning();
+            }
+            else
+                void this.backfill(epoch);
+        });
+        worker.on("error", (err) => { if (this.worker === worker)
+            this.opts.log?.("warn", `search: index worker error: ${String(err?.message ?? err)}`); });
+        worker.on("exit", () => {
+            if (this.worker !== worker)
+                return;
+            this.worker = null;
+            this.dropWorkerWork();
+            this.wakeRoom();
+            if (this.closed) {
+                if (this.state !== "unavailable")
+                    this.state = "unavailable";
+                return;
+            }
+            this.state = "starting";
+            this.scheduleRestart();
+        });
+    }
+    /** What a page shows: whether search answers, and whether the index is still being built. */
+    status() { return this.state; }
+    /** A doc to index (null: nothing to index). */
+    record(doc) {
+        if (!doc || this.closed)
+            return;
+        this.queue.push(doc);
+        if (this.queue.length > QUEUE_MAX)
+            this.queue.splice(0, this.queue.length - QUEUE_MAX);
+        if (this.state !== "ready" && this.state !== "building")
+            return;
+        if (this.queue.length >= BATCH_MAX)
+            this.flush();
+        else
+            this.flushTimer ??= setTimeout(() => this.flush(), BATCH_MS);
+    }
+    /**
+     * An instance removed from the fleet: its chat leaves the index — queued and uncommitted docs, what the worker has, and
+     * the rest of a backfill running now (see SearchStore.removeInstance). Its later chat, under the same name, is a new
+     * instance's and is indexed as usual.
+     */
+    removeInstance(instance) {
+        const gone = (d) => d.source === "chat" && d.instance === instance;
+        this.queue = this.queue.filter(d => !gone(d));
+        for (const [batch, docs] of this.inflight) {
+            const kept = docs.filter(d => !gone(d));
+            this.inflightDocs -= docs.length - kept.length;
+            this.inflight.set(batch, kept);
+        }
+        this.removedDuringBackfill?.add(instance);
+        const gen = ++this.removalGen;
+        this.removals.set(instance, { gen, before: this.nextBatch });
+        this.post({ op: "removeInstance", instance, gen });
+    }
+    query(filters) { return this.ask("query", { filters }); }
+    context(key) { return this.ask("context", { key }); }
+    /**
+     * Stop: everything queued goes to the worker first (bounded: the queue is at most QUEUE_MAX), then "close" — the worker
+     * writes in order and then closes the file. Resolves when the worker has exited, or after CLOSE_WAIT_MS.
+     */
+    close() {
+        if (this.closed && !this.worker)
+            return Promise.resolve();
+        this.closed = true;
+        this.epoch++;
+        for (const t of [this.flushTimer, this.restartTimer])
+            if (t)
+                clearTimeout(t);
+        if (this.pruneTimer)
+            clearInterval(this.pruneTimer);
+        this.flushTimer = this.restartTimer = this.pruneTimer = null;
+        while (this.queue.length && this.worker)
+            if (!this.postBatch(this.queue.splice(0, BATCH_MAX)))
+                break;
+        this.dropWorkerWork();
+        this.wakeRoom();
+        const worker = this.worker;
+        this.post({ op: "close" });
+        this.state = "unavailable";
+        if (!worker)
+            return Promise.resolve();
+        return new Promise(resolve => {
+            const timer = setTimeout(resolve, CLOSE_WAIT_MS);
+            timer.unref?.();
+            worker.on("exit", () => { clearTimeout(timer); resolve(); });
+        });
+    }
+    ask(op, body) {
+        if (!this.worker || (this.state !== "ready" && this.state !== "building"))
+            return Promise.resolve({ ok: false, reason: "unavailable" });
+        if (this.queries.size >= PENDING_MAX)
+            return Promise.resolve({ ok: false, reason: "busy" });
+        const id = ++this.nextId, building = this.state === "building", worker = this.worker;
+        return new Promise(resolve => {
+            const entry = {
+                answered: false,
+                resolve: (r) => resolve(r.ok ? { ok: true, value: r.value, building } : { ok: false, reason: r.reason ?? "unavailable" }),
+                timer: setTimeout(() => {
+                    this.answer(id, { ok: false, reason: "timeout" });
+                    // Still held long after: the worker is stuck — retire it (its exit ends the hold, and it is started again).
+                    const held = this.queries.get(id);
+                    if (held)
+                        held.stall = setTimeout(() => { if (this.queries.has(id) && this.worker === worker)
+                            this.retire(); }, STALL_MS - QUERY_BUDGET_MS);
+                }, QUERY_BUDGET_MS),
+            };
+            this.queries.set(id, entry);
+            if (!this.post({ op, id, ...body }))
+                this.replied(id, { ok: false, reason: "unavailable" });
+        });
+    }
+    /** The caller's answer (once); the worker may still hold the query. */
+    answer(id, r) {
+        const q = this.queries.get(id);
+        if (!q || q.answered)
+            return;
+        q.answered = true;
+        clearTimeout(q.timer);
+        q.resolve(r);
+    }
+    /** The worker is done with `id`: a query's hold ends (answered now if it was not), or a request settles. */
+    replied(id, r) {
+        if (this.queries.has(id)) {
+            this.answer(id, r);
+            const q = this.queries.get(id);
+            if (q.stall)
+                clearTimeout(q.stall);
+            this.queries.delete(id);
+            return;
+        }
+        const request = this.requests.get(id);
+        if (request) {
+            this.requests.delete(id);
+            request({ ok: r.ok, value: r.value });
+        }
+    }
+    /** The worker went (or the index closes): nothing it held will be answered. Uncommitted batches are kept. */
+    dropWorkerWork() {
+        for (const id of [...this.queries.keys()])
+            this.replied(id, { ok: false, reason: "unavailable" });
+        for (const id of [...this.requests.keys()])
+            this.replied(id, { ok: false });
+    }
+    retire() {
+        const w = this.worker;
+        if (!w)
+            return;
+        this.opts.log?.("warn", "search: a query held the index worker too long; restarting it");
+        try {
+            void w.terminate().catch(() => { });
+        }
+        catch { /* its exit (or none) decides */ }
+    }
+    committed(batch) {
+        const docs = this.inflight.get(batch);
+        if (docs) {
+            this.inflight.delete(batch);
+            this.inflightDocs -= docs.length;
+        }
+        // The worker works in order: a removal posted before this batch is done too.
+        for (const [instance, { before }] of this.removals)
+            if (before < batch)
+                this.removals.delete(instance);
+        this.wakeRoom();
+        if (this.queue.length && (this.state === "ready" || this.state === "building"))
+            this.flushTimer ??= setTimeout(() => this.flush(), BATCH_MS);
+    }
+    wakeRoom() { const waiters = this.roomWaiters; this.roomWaiters = []; for (const w of waiters)
+        w(); }
+    /**
+     * A backfill's chunk, admitted under the same QUEUE_MAX as the queue's batches: while that many wait for a commit, it
+     * waits for one. False when this backfill is over (the worker went — the next one backfills again — or close).
+     */
+    async admitChunk(docs, epoch, kept) {
+        while (this.inflightDocs > 0 && this.inflightDocs + docs.length > QUEUE_MAX) {
+            if (epoch !== this.epoch || !this.worker || this.closed)
+                return null;
+            await new Promise(resolve => this.roomWaiters.push(resolve));
+        }
+        if (epoch !== this.epoch || !this.worker || this.closed)
+            return null;
+        const admitted = docs.filter(kept); // an instance removed while it waited, too
+        return this.postBatch(admitted) ? admitted.length : null;
+    }
+    post(message) {
+        if (!this.worker)
+            return false;
+        try {
+            this.worker.postMessage(message);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** A batch to the worker, kept until it is committed. False when there is no worker to take it. */
+    postBatch(docs) {
+        if (!docs.length)
+            return true;
+        const batch = ++this.nextBatch;
+        this.inflight.set(batch, docs);
+        this.inflightDocs += docs.length;
+        if (this.post({ op: "add", batch, docs }))
+            return true;
+        this.inflight.delete(batch);
+        this.inflightDocs -= docs.length;
+        return false;
+    }
+    flush(force = false) {
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = null;
+        }
+        if (!this.queue.length || (!force && this.state !== "ready" && this.state !== "building"))
+            return;
+        // Bounded: while QUEUE_MAX docs wait for their commit, new ones wait in the (capped) queue.
+        if (!force && this.inflightDocs >= QUEUE_MAX)
+            return;
+        const docs = this.queue.splice(0, BATCH_MAX);
+        if (!this.postBatch(docs)) {
+            this.queue = docs.concat(this.queue).slice(-QUEUE_MAX);
+            return;
+        }
+        if (this.queue.length)
+            this.flushTimer = setTimeout(() => this.flush(), 0);
+    }
+    scheduleRestart() {
+        if (this.closed || this.restartTimer)
+            return;
+        const delay = this.restartDelay;
+        this.restartDelay = Math.min(RESTART_MAX_MS, this.restartDelay * 2);
+        this.restartTimer = setTimeout(() => { this.restartTimer = null; this.start(); }, delay);
+        this.restartTimer.unref?.();
+    }
+    cutoff() { return this.now() - this.opts.retentionDays * 24 * 3600_000; }
+    async backfill(epoch) {
+        this.state = "building";
+        this.flush();
+        const started = this.now();
+        const removed = this.removedDuringBackfill = new Set();
+        const kept = (d) => !(d.source === "chat" && removed.has(d.instance));
+        let chats = 0;
+        try {
+            const cutoff = this.cutoff();
+            let chunk = [];
+            for (const doc of this.opts.chatDocs()) {
+                if (doc.ts < cutoff || !kept(doc))
+                    continue;
+                chunk.push(doc);
+                if (chunk.length < BACKFILL_CHUNK)
+                    continue;
+                const sent = await this.admitChunk(chunk, epoch, kept);
+                if (sent === null)
+                    return;
+                chats += sent;
+                chunk = [];
+                await new Promise(resolve => setImmediate(resolve));
+            }
+            if (epoch !== this.epoch)
+                return;
+            const sent = await this.admitChunk(chunk, epoch, kept);
+            if (sent === null)
+                return;
+            chats += sent;
+            const sql = await this.request("backfill", { outboxPath: this.opts.outboxPath, schedulerPath: this.opts.schedulerPath, cutoff });
+            if (epoch !== this.epoch || !sql.ok)
+                return;
+            this.post({ op: "markBuilt", now: this.now() });
+            const { xmsg = 0, tasks = 0 } = (sql.value ?? {});
+            this.opts.log?.("info", `search: index built (${chats} chat, ${xmsg} cross-instance, ${tasks} task docs in ${this.now() - started} ms)`);
+            this.state = "ready";
+            this.flush();
+            this.startPruning();
+        }
+        catch (err) {
+            this.opts.log?.("warn", `search: backfill failed: ${String(err?.message ?? err)}`);
+        }
+        finally {
+            if (this.removedDuringBackfill === removed)
+                this.removedDuringBackfill = null;
+        }
+    }
+    /** A worker request without the query budget (the backfill and prune run as long as they need). */
+    request(op, body) {
+        const id = ++this.nextId;
+        return new Promise(resolve => {
+            this.requests.set(id, resolve);
+            if (!this.post({ op, id, ...body }))
+                this.replied(id, { ok: false });
+        });
+    }
+    startPruning() {
+        void this.prune();
+        if (this.pruneTimer || this.closed)
+            return;
+        this.pruneTimer = setInterval(() => void this.prune(), PRUNE_EVERY_MS);
+        this.pruneTimer.unref?.();
+    }
+    async prune() {
+        for (let rounds = 0; rounds < 1_000; rounds++) {
+            const r = await this.request("prune", { cutoff: this.cutoff() });
+            if (!r.ok || !r.value?.more)
+                return;
+            await new Promise(resolve => setImmediate(resolve));
+        }
+    }
+}
+//# sourceMappingURL=search-index.js.map

package/dist/search/search-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-index.js","sourceRoot":"","sources":["../../src/search/search-index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAG,CAAC;AAC5B,MAAM,CAAC,MAAM,SAAS,GAAG,GAAG,CAAC;AAC7B,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;AAC/B,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,CAAC;AACrC,MAAM,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC9B,0GAA0G;AAC1G,MAAM,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC;AAC/B,gFAAgF;AAChF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,CAAC;AACnC,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,GAAG,QAAQ,CAAC;AAC5C,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,4CAA4C;AAC5C,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACxC,MAAM,cAAc,GAAG,KAAK,EAAE,cAAc,GAAG,MAAM,CAAC;AAUtD,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC;QACpC,CAAC,CAAC,IAAI,MAAM,CAAC,oJAAoJ,EAC/J,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,sBAAsB,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAClK,CAAC,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,cAAc,EAAE,EAAE,sBAAsB,EAAE,GAAG,EAAE,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;AAC1J,CAAC;AAiBD,MAAM,OAAO,WAAW;IAwCO,IAAI;IAvCzB,MAAM,GAAmB,IAAI,CAAC;IAC9B,KAAK,GAAgB,UAAU,CAAC;IAChC,KAAK,GAAgB,EAAE,CAAC;IACxB,UAAU,GAAyC,IAAI,CAAC;IACxD,YAAY,GAAyC,IAAI,CAAC;IAC1D,UAAU,GAA0C,IAAI,CAAC;IACzD,YAAY,GAAG,cAAc,CAAC;IAC9B,MAAM,GAAG,CAAC,CAAC;IACnB;;;;OAIG;IACK,OAAO,GAAG,IAAI,GAAG,EAAmL,CAAC;IAC7M,2GAA2G;IACnG,QAAQ,GAAG,IAAI,GAAG,EAAyD,CAAC;IACpF;;;OAGG;IACK,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC1C,YAAY,GAAG,CAAC,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACtB;;;OAGG;IACK,QAAQ,GAAG,IAAI,GAAG,EAA2C,CAAC;IAC9D,UAAU,GAAG,CAAC,CAAC;IACvB,mHAAmH;IAC3G,WAAW,GAAsB,EAAE,CAAC;IAC5C,kHAAkH;IAC1G,qBAAqB,GAAuB,IAAI,CAAC;IACzD,mFAAmF;IAC3E,KAAK,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,KAAK,CAAC;IACN,OAAO,CAA4B;IACnC,GAAG,CAAe;IAEnC,YAA6B,IAAwB;oBAAxB,IAAI;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC;QAC7C,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IAClC,CAAC;IAED,KAAK;QACH,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACvC,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;QAC3B,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAC,CAAC;QAC9C,OAAO,GAAG,EAAE,CAAC;YAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,0CAA0C,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;YAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAAC,OAAO;QAAC,CAAC;QAC5J,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAgK,EAAE,EAAE;YACxL,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,CAAU,oDAAoD;YACjG,IAAI,CAAC,CAAC,EAAE,KAAK,OAAO,IAAI,CAAC,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;gBAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACvJ,IAAI,CAAC,CAAC,EAAE,KAAK,OAAO,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnF,8GAA8G;YAC9G,IAAI,CAAC,CAAC,EAAE,KAAK,SAAS,IAAI,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;gBAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC,GAAG;oBAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBAAC,OAAO;YAAC,CAAC;YACzJ,IAAI,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,uCAAuC,CAAC,CAAC,MAAM,IAAI,SAAS,0BAA0B,CAAC,CAAC;gBAAC,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC;gBAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;gBAAC,OAAO;YAAC,CAAC;YACnM,IAAI,CAAC,CAAC,EAAE,KAAK,OAAO;gBAAE,OAAO;YAC7B,IAAI,CAAC,YAAY,GAAG,cAAc,CAAC;YACnC,oGAAoG;YACpG,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,gBAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACpG,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;gBAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YAAC,CAAC;;gBACpE,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAY,EAAE,EAAE,GAAG,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,+BAA+B,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxK,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACrB,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO;YACnC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAAC,IAAI,IAAI,CAAC,KAAK,KAAK,aAAa;oBAAE,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC;gBAAC,OAAO;YAAC,CAAC;YAC1F,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;YACxB,IAAI,CAAC,eAAe,EAAE,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6FAA6F;IAC7F,MAAM,KAAkB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAE5C,+CAA+C;IAC/C,MAAM,CAAC,GAAqB;QAC1B,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAChC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QACvF,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU;YAAE,OAAO;QAChE,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,SAAS;YAAE,IAAI,CAAC,KAAK,EAAE,CAAC;;YAC5C,IAAI,CAAC,UAAU,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAC;IACpE,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,QAAgB;QAC7B,MAAM,IAAI,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;QAC9E,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,qBAAqB,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,GAAG,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC;QAC9B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,gBAAgB,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,OAAsB,IAAyC,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,CAAwC,CAAC,CAAC,CAAC;IACpJ,OAAO,CAAC,GAAW,IAAiD,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAgD,CAAC,CAAC,CAAC;IAEzJ;;;OAGG;IACH,KAAK;QACH,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,YAAY,CAAC;YAAE,IAAI,CAAC;gBAAE,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,IAAI,CAAC,UAAU;YAAE,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC7D,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;gBAAE,MAAM;QACrG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,KAAK,GAAG,aAAa,CAAC;QAC3B,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QACtC,OAAO,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE;YACjC,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YACjD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,GAAG,CAAC,EAAuB,EAAE,IAA6B;QAChE,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QACxI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,WAAW;YAAE,OAAO,OAAO,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAC5F,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,IAAI,CAAC,KAAK,KAAK,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACrF,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,MAAM,KAAK,GAAG;gBACZ,QAAQ,EAAE,KAAK;gBACf,OAAO,EAAE,CAAC,CAAoD,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAG,CAAC,CAAC,MAA6C,IAAI,aAAa,EAAE,CAAC;gBACpN,KAAK,EAAE,UAAU,CAAC,GAAG,EAAE;oBACrB,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;oBAClD,4GAA4G;oBAC5G,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;oBAClC,IAAI,IAAI;wBAAE,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;4BAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,GAAG,eAAe,CAAC,CAAC;gBAC9I,CAAC,EAAE,eAAe,CAAC;aACuJ,CAAC;YAC7K,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC;gBAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAC9F,CAAC,CAAC,CAAC;IACL,CAAC;IACD,uEAAuE;IAC/D,MAAM,CAAC,EAAU,EAAE,CAAoD;QAC7E,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ;YAAE,OAAO;QAC7B,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC;QAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzD,CAAC;IACD,4GAA4G;IACpG,OAAO,CAAC,EAAU,EAAE,CAAoD;QAC9E,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACnB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;YAChC,IAAI,CAAC,CAAC,KAAK;gBAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;QAAC,CAAC;IACnF,CAAC;IACD,6GAA6G;IACrG,cAAc;QACpB,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;QAClG,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9E,CAAC;IACO,MAAM;QACZ,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QACtB,IAAI,CAAC,CAAC;YAAE,OAAO;QACf,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,+DAA+D,CAAC,CAAC;QACzF,IAAI,CAAC;YAAC,KAAK,CAAC,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,gCAAgC,CAAC,CAAC;IACxF,CAAC;IACO,SAAS,CAAC,KAAa;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,IAAI,EAAE,CAAC;YAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC;QAAC,CAAC;QAC5E,6EAA6E;QAC7E,KAAK,MAAM,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,IAAI,MAAM,GAAG,KAAK;gBAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAChB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC;YAAE,IAAI,CAAC,UAAU,KAAK,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,QAAQ,CAAC,CAAC;IAC/I,CAAC;IACO,QAAQ,KAAW,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,OAAO;QAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACnH;;;OAGG;IACK,KAAK,CAAC,UAAU,CAAC,IAAiB,EAAE,KAAa,EAAE,IAA+B;QACxF,OAAO,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC5E,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YACrE,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QACrE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAoB,2CAA2C;QAClG,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3D,CAAC;IACO,IAAI,CAAC,OAAgC;QAC3C,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC/B,IAAI,CAAC;YAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAAC,OAAO,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAChF,CAAC;IACD,mGAAmG;IAC3F,SAAS,CAAC,IAAiB;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAC9B,MAAM,KAAK,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC;QACjE,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;YAAE,OAAO,IAAI,CAAC;QACvD,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,MAAM,CAAC;QAC9D,OAAO,KAAK,CAAC;IACf,CAAC;IACO,KAAK,CAAC,KAAK,GAAG,KAAK;QACzB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAAC,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAAC,CAAC;QAC/E,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,KAAK,UAAU,CAAC;YAAE,OAAO;QAClG,4FAA4F;QAC5F,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,YAAY,IAAI,SAAS;YAAE,OAAO;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;YAAC,OAAO;QAAC,CAAC;QAC9F,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM;YAAE,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;IAC7E,CAAC;IACO,eAAe;QACrB,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY;YAAE,OAAO;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACzF,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,CAAC;IAC9B,CAAC;IACO,MAAM,KAAa,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;IACjF,KAAK,CAAC,QAAQ,CAAC,KAAa;QAClC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC;QACxB,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/D,MAAM,IAAI,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjF,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,GAAgB,EAAE,CAAC;YAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACvC,IAAI,GAAG,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAC5C,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChB,IAAI,KAAK,CAAC,MAAM,GAAG,cAAc;oBAAE,SAAS;gBAC5C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;gBACvD,IAAI,IAAI,KAAK,IAAI;oBAAE,OAAO;gBAC1B,KAAK,IAAI,IAAI,CAAC;gBAAC,KAAK,GAAG,EAAE,CAAC;gBAC1B,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5D,CAAC;YACD,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK;gBAAE,OAAO;YACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACvD,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO;YAC1B,KAAK,IAAI,IAAI,CAAC;YACd,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,EAAE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;YACjI,IAAI,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO;YAC5C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAChD,MAAM,EAAE,IAAI,GAAG,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAsC,CAAC;YACvF,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,wBAAwB,KAAK,UAAU,IAAI,oBAAoB,KAAK,iBAAiB,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,MAAM,CAAC,CAAC;YACzI,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC;YACrB,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,EAAE,4BAA4B,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,qBAAqB,KAAK,OAAO;gBAAE,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAChF,CAAC;IACH,CAAC;IACD,mGAAmG;IAC3F,OAAO,CAAC,EAAU,EAAE,IAA6B;QACvD,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE;YAC3B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,IAAI,EAAE,CAAC;gBAAE,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACvE,CAAC,CAAC,CAAC;IACL,CAAC;IACO,YAAY;QAClB,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAC3C,IAAI,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,EAAE,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,CAAC;IAC5B,CAAC;IACO,KAAK,CAAC,KAAK;QACjB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACjE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAE,CAAC,CAAC,KAAwC,EAAE,IAAI;gBAAE,OAAO;YACxE,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5D,CAAC;IACH,CAAC;CACF"}

package/dist/search/search-store.d.ts

@@ -0,0 +1,122 @@
+export type SearchSource = "chat" | "xmsg" | "task";
+export declare const SEARCH_SOURCES: readonly SearchSource[];
+/**
+ * One searchable thing. `key` is unique per thing (a later doc with the same key replaces it: a task updated).
+ * - chat: `instance` its chat, `role` user|agent, `mine` the account that wrote it (`<connection>:<user>`, or
+ *   `web:<name>` for a web message of an owner-less session), `ref` { boot, id } — the chat row it opens.
+ * - xmsg: `instance` the sender, `peer` the target, `role` the request kind, `ref` { correlationId?, kind }.
+ * - task: `instance` the assignee (else who created it, else "tasks"), `role` its status, `ref` { taskId }.
+ */
+export interface SearchDoc {
+    key: string;
+    source: SearchSource;
+    instance: string;
+    peer?: string;
+    sender: string;
+    role?: string;
+    mine?: string;
+    /** Epoch ms. */
+    ts: number;
+    ref: Record<string, unknown>;
+    text: string;
+}
+export interface SearchFilters {
+    /** At most QUERY_MAX_TERMS terms, split on whitespace (case-insensitive). */
+    q: string;
+    /** Only these instances (a doc matches by its instance or, for a cross-instance message, its peer). */
+    instances?: readonly string[];
+    sources?: readonly SearchSource[];
+    /** Epoch ms bounds (inclusive since, exclusive until). */
+    since?: number;
+    until?: number;
+    /** Only docs whose `mine` is one of these. */
+    mine?: readonly string[];
+    /**
+     * Instances whose cross_instance_visibility is "hidden" now: a cross-instance message to or from one is not found
+     * (fleet-traffic's rule). Judged at query time, so changing the setting applies to what was indexed before it.
+     */
+    hidden?: readonly string[];
+    /**
+     * When the fleet's default is "hidden": a cross-instance message is found only when both its sides are among these
+     * (the instances set to show theirs); every other name — another instance, an external session — follows the default.
+     */
+    xmsgAmong?: readonly string[];
+    limit?: number;
+}
+export interface SearchHit {
+    key: string;
+    source: SearchSource;
+    instance: string;
+    peer: string | null;
+    sender: string;
+    role: string | null;
+    ts: number;
+    ref: Record<string, unknown>;
+    /** The matching sentence (around the first hit) and where the hits are in it: [start, end) offsets. */
+    snippet: string;
+    marks: Array<[number, number]>;
+}
+export interface SearchResult {
+    hits: SearchHit[];
+    limited: boolean;
+}
+export declare const QUERY_MAX_CHARS = 200;
+export declare const QUERY_MAX_TERMS = 8;
+export declare const RESULT_MAX = 200;
+export declare const SHORT_SCAN_ROWS = 5000;
+export declare const ROW_CAP = 300000;
+export declare const TEXT_MAX = 16000;
+/**
+ * Masking reads this much past TEXT_MAX before the text is cut (#2295 review): a secret cut at the edge loses what makes
+ * it recognisable (a closing quote), so the cut comes after the mask. One starting in the kept text and shorter than
+ * this is masked whole — a private key block is a few KB. A text cut before it got here (the web chat's history keeps
+ * 16000 characters) can end inside a quoted secret: its open tail is masked too (maskOpenSecretTail).
+ */
+export declare const MASK_MARGIN = 16000;
+/** The terms of a query: whitespace-split, lower-cased, deduplicated; long (3+ characters) and short ones apart. */
+export declare function parseQuery(q: string): {
+    long: string[];
+    short: string[];
+};
+/** The sentence around the first hit of any term, and every hit's place in it. */
+export declare function snippetOf(text: string, terms: readonly string[]): {
+    snippet: string;
+    marks: Array<[number, number]>;
+};
+export declare class SearchStore {
+    private db;
+    constructor(path: string);
+    /** Whether this index was built (backfilled) under the current schema; false on a new or older one. */
+    built(): boolean;
+    /** Marks a finished backfill (a rebuild starts over: everything is replaced or added by key). */
+    markBuilt(now: number): void;
+    /** Add or replace docs (by key), in one transaction. Text is masked, one-spaced at its edges and bounded. */
+    add(docs: readonly SearchDoc[]): number;
+    /** An instance's docs go (the instance was deleted). */
+    /**
+     * An instance removed from the fleet: its chat goes, as the fleet forgets it. Its cross-instance messages and tasks
+     * stay — the outbox and the Task Board still hold them, and the other side's view of them is not this instance's.
+     */
+    removeInstance(instance: string): number;
+    /** Docs older than `cutoff` go, then the oldest beyond `cap`; in chunks (the caller yields between calls). */
+    prune(cutoff: number, cap?: number, chunk?: number): {
+        removed: number;
+        more: boolean;
+    };
+    count(): number;
+    /** The newest matches, newest first. */
+    search(f: SearchFilters): SearchResult;
+    /** One doc by key, with (for a chat message) up to `around` messages of its chat before and after it. */
+    context(key: string, around?: number): {
+        doc: SearchHit & {
+            text: string;
+        };
+        before: Array<SearchHit & {
+            text: string;
+        }>;
+        after: Array<SearchHit & {
+            text: string;
+        }>;
+    } | null;
+    close(): void;
+}