memtrails 0.0.0-stage → 0.2.0

package/dist/core/qdrant-binary.js

@@ -0,0 +1,108 @@
+/**
+ * The Qdrant binary memory runs when you give it no Qdrant server of your own: one pinned release, fetched on first use
+ * from Qdrant's GitHub release, verified against the size and SHA-256 pinned below, unpacked next to it (cache dir,
+ * paths.ts), and trusted after that through a `.verified` marker.
+ *
+ * Qdrant publishes no checksums, so the hashes are pinned here from its release. Upgrading means a new version and new
+ * hashes; Qdrant reads storage only from the minor version before it, so after a version change the index is written
+ * again from the logs (rebuild.ts) rather than migrated: it is derived data.
+ */
+import { createHash } from 'node:crypto';
+import { chmodSync, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { spawn } from 'node:child_process';
+import { cacheDir } from './paths.js';
+export const QDRANT_VERSION = '1.20.0';
+/** Per `${platform}-${arch}`. Linux takes the static musl build: it runs on any distribution. */
+export const QDRANT_ASSETS = {
+    'darwin-arm64': { asset: 'qdrant-aarch64-apple-darwin.tar.gz', size: 29_070_255, sha256: '531904e822050790bb8c1da6705f801ee48b91eb4c229d0d605ff06f6331b20a', exe: 'qdrant' },
+    'darwin-x64': { asset: 'qdrant-x86_64-apple-darwin.tar.gz', size: 31_917_921, sha256: '94a2e6f6b8ef3a3b5756a11172fd66554ac753e0a71c1d8d8b5349356c084f4d', exe: 'qdrant' },
+    'linux-arm64': { asset: 'qdrant-aarch64-unknown-linux-musl.tar.gz', size: 31_475_234, sha256: 'bd8a458b8ef8006ce2c603c5bd6d1d973e39d787f794008dd959112e747c3bb7', exe: 'qdrant' },
+    'linux-x64': { asset: 'qdrant-x86_64-unknown-linux-musl.tar.gz', size: 33_773_335, sha256: 'b03a64616a1f41ddab78b480a318466541765f19835318113b1821cbea8df52c', exe: 'qdrant' },
+    'win32-x64': { asset: 'qdrant-x86_64-pc-windows-msvc.zip', size: 30_801_664, sha256: '6d13553584fd0411f00637519e293331ff0aa1fb79bd9e30db45839bb75de2be', exe: 'qdrant.exe' },
+};
+export function qdrantAsset(platform = process.platform, arch = process.arch) {
+    return QDRANT_ASSETS[`${platform}-${arch}`] ?? null;
+}
+export function qdrantDir() {
+    return join(cacheDir(), `qdrant-${QDRANT_VERSION}`);
+}
+export function qdrantBinaryPath(a = qdrantAsset()) {
+    return a ? join(qdrantDir(), a.exe) : null;
+}
+export const QDRANT_RELEASE_BASE = `https://github.com/qdrant/qdrant/releases/download/v${QDRANT_VERSION}`;
+const assetUrl = (a) => `${QDRANT_RELEASE_BASE}/${a.asset}`;
+/** The binary is in place and its archive was verified when it was unpacked. */
+export function qdrantReady(a = qdrantAsset()) {
+    const bin = qdrantBinaryPath(a);
+    if (!a || !bin || !existsSync(bin))
+        return false;
+    try {
+        return readFileSync(join(qdrantDir(), '.verified'), 'utf8').trim() === `${a.size}:${a.sha256}`;
+    }
+    catch {
+        return false;
+    }
+}
+/** Unpack one archive with the system's tar (bsdtar on macOS and Windows reads zip too). */
+function untar(archive, into) {
+    return new Promise((resolve, reject) => {
+        const child = spawn('tar', ['-xf', archive, '-C', into], { stdio: 'ignore' });
+        child.once('error', reject);
+        child.once('exit', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited ${code}`))));
+    });
+}
+/**
+ * Fetch, verify and unpack the binary unless it is already there. Throws on a network failure, a size or hash
+ * mismatch (the download is deleted), or a platform Qdrant does not ship for; the caller retries with backoff.
+ */
+export async function ensureQdrant(opts = {}) {
+    const a = qdrantAsset();
+    if (!a)
+        throw new Error(`no Qdrant build for ${process.platform}-${process.arch}`);
+    const bin = qdrantBinaryPath(a);
+    if (qdrantReady(a))
+        return bin;
+    const dir = qdrantDir();
+    mkdirSync(dir, { recursive: true });
+    const archive = join(dir, a.asset);
+    const part = `${archive}.part`;
+    rmSync(part, { force: true });
+    opts.log?.(`[memtrails] fetching Qdrant ${QDRANT_VERSION} (${Math.round(a.size / 1e6)} MB)`);
+    const res = await (opts.fetchImpl ?? fetch)(assetUrl(a), { signal: opts.signal, redirect: 'follow' });
+    if (!res.ok || !res.body)
+        throw new Error(`HTTP ${res.status} for ${a.asset}`);
+    const hash = createHash('sha256');
+    let size = 0;
+    try {
+        await pipeline(Readable.fromWeb(res.body), async function* (src) {
+            for await (const chunk of src) {
+                const b = chunk;
+                hash.update(b);
+                size += b.length;
+                yield b;
+            }
+        }, createWriteStream(part));
+    }
+    catch (err) {
+        rmSync(part, { force: true });
+        throw err;
+    }
+    const digest = hash.digest('hex');
+    if (size !== a.size || digest !== a.sha256) {
+        rmSync(part, { force: true });
+        throw new Error(`${a.asset}: got ${size} bytes sha256 ${digest}, expected ${a.size} ${a.sha256}`);
+    }
+    renameSync(part, archive);
+    await untar(archive, dir);
+    rmSync(archive, { force: true });
+    if (!existsSync(bin) || statSync(bin).size === 0)
+        throw new Error(`${a.asset} did not contain ${a.exe}`);
+    if (process.platform !== 'win32')
+        chmodSync(bin, 0o755);
+    writeFileSync(join(dir, '.verified'), `${a.size}:${a.sha256}\n`);
+    opts.log?.(`[memtrails] Qdrant ${QDRANT_VERSION} ready`);
+    return bin;
+}

package/dist/core/qdrant-client.d.ts

@@ -0,0 +1,115 @@
+import type { QdrantEndpoint } from './qdrant-process.js';
+export declare const COLLECTION = "memory";
+/** The payload field a tenant client keys every point by, and the original point id it keeps beside the scoped one. */
+export declare const TENANT_FIELD = "user";
+export declare const DIMS: 768;
+/** What the index is made of; a change empties it and it is written in again from the logs (rebuild.ts).
+ *  2: binary quantisation with rescoring (½ the RAM of int8, recall 0.98–1.00 measured at 1–3M points), keyword terms
+ *  folded and segmented for every language (text.ts); 3: on-disk binary copy, HNSW and payload indexes;
+ *  4: remote EmbeddingGemma GPU vectors (768 dimensions, explicit runtime/pooling tag in embedder.ts); 5: nothing cut —
+ *  every chunk of a body (no 12-chunk limit), short-term entries chunked too, cues numbered below the key, full payloads;
+ *  6: every captured request indexed for good — however old, consolidated or not (the raw record is the truth). */
+export declare const INDEX_LAYOUT = 6;
+export interface SparseVector {
+    indices: number[];
+    values: number[];
+}
+export interface Filter {
+    must?: unknown[];
+    must_not?: unknown[];
+    should?: unknown[];
+}
+export interface PointStruct {
+    id: string;
+    vector: Record<string, number[] | SparseVector>;
+    payload: Record<string, unknown>;
+}
+/** A point's id from its item and its place (0 = the key, n = the n-th child): the same item writes the same ids. */
+export declare function pointId(item: string, n: number): string;
+export declare class QdrantError extends Error {
+    readonly status: number;
+    constructor(message: string, status: number);
+}
+export interface QdrantClientOptions {
+    /** The collection (COLLECTION when not given). */
+    collection?: string;
+    /** Many users in one collection: the user every call is for — read on each call from the caller's context; null or ''
+     *  refuses the call. */
+    tenant?: () => string | null | undefined;
+}
+export declare class QdrantClient {
+    private readonly endpoint;
+    private readonly timeoutMs;
+    private readonly opts;
+    private readonly collection;
+    /** `endpoint` may start Qdrant (the service runs it only while memory has work): every call waits for it. */
+    constructor(endpoint: () => QdrantEndpoint | null | Promise<QdrantEndpoint | null>, timeoutMs?: number, opts?: QdrantClientOptions);
+    /** The user this call is for (a tenant client), or null (one person's own index). Throws when a tenant client has none. */
+    private user;
+    /** A filter with the user added — unchanged for one person's own index. */
+    private scoped;
+    /** A point's id in a shared collection: the user's and the item's (pointId of both), so users never share a point. */
+    private scopedId;
+    /** What a read hands back: the point's own id (as the writer named it) and only this user's points. */
+    private own;
+    private call;
+    /** Drop the whole collection: the index is derived, written again from the logs (rebuild.ts). */
+    dropCollection(): Promise<void>;
+    /** The collection and its payload indexes, made if missing — an index added since a collection was made is added to
+     *  it in place. All of it on disk, memory-mapped — the vectors, their 1-bit copy, the graph, the payloads and their
+     *  indexes: RAM holds what queries touch, and the system may take it back. Measured at 30k / 100k items (0.33M /
+     *  1.1M points): 205 → 68 MB and 591 → 144 MB that cannot be given back,
+     *  for 12–16 ms more of Qdrant's CPU a turn (26 → 38, 81 → 97). */
+    ensureCollection(): Promise<void>;
+    upsert(points: PointStruct[], wait?: boolean): Promise<void>;
+    /** Every point of these items — before an item is written again (its children may be fewer now). */
+    deleteItems(items: string[]): Promise<void>;
+    /** One kind of point of these items (`cue`), before that part is written again. */
+    deleteParts(items: string[], part: string): Promise<void>;
+    /** Change some payload fields of these items' points (status, a merged key's display fields). */
+    setPayload(items: string[], payload: Record<string, unknown>, keyOnly?: boolean): Promise<void>;
+    /** The nearest key points to one vector (consolidation's "do I know this already?"), with their cosine. */
+    neighbors(vector: number[], filter: Filter, limit?: number): Promise<Array<{
+        item: string;
+        score: number;
+        payload: Record<string, unknown>;
+    }>>;
+    /** Each of these items' best cosine to one vector among its points of one part (key, child, cue) — the ranker's
+     *  per-family similarities. An item with no such point is not in the map. */
+    bestPerItem(vector: number[], items: string[], part: string, timeoutMs?: number): Promise<Map<string, number>>;
+    /** `nearest` for several vectors in one call. */
+    /** `fields`: only these payload fields (and the item and its user), for a caller that needs no more — a walk over
+     *  many neighbours reads ids, not the bodies of long requests. */
+    nearestMany(vectors: number[][], filter: Filter, limit?: number, timeoutMs?: number, fields?: string[]): Promise<Array<Array<{
+        item: string;
+        score: number;
+        payload: Record<string, unknown>;
+    }>>>;
+    /** Points within a filter, with their payload (and dense vector when asked) — at most `limit`. */
+    scroll(filter: Filter, limit?: number, withVector?: boolean, timeoutMs?: number): Promise<Array<{
+        id: string;
+        payload: Record<string, unknown>;
+        vector?: number[];
+    }>>;
+    count(filter?: Filter): Promise<number>;
+    /** The items behind `ids` (key points), with their payload — and their dense vector when asked. */
+    retrieve(ids: string[], withVector?: boolean): Promise<Array<{
+        id: string;
+        payload: Record<string, unknown>;
+        vector?: number[];
+    }>>;
+    /** The best items for the query, fused by rank — each with the point that matched best (`point`: its key, a chunk of
+     *  its text, or a cue), so a caller can show the passage that matched. Each channel (a query vector, the keywords)
+     *  ranks up to `per` items (default: `limit`, at least 50); their scores are then fused (CombSUM, above). */
+    fusedItems(q: {
+        dense: number[][];
+        sparse?: SparseVector;
+        filter?: Filter;
+        per?: number;
+        limit?: number;
+    }, timeoutMs?: number): Promise<Array<{
+        item: string;
+        score: number;
+        point?: string;
+    }>>;
+}

package/dist/core/qdrant-client.js

@@ -0,0 +1,327 @@
+/**
+ * The few Qdrant REST calls the memory index makes — plain fetch, no client library. One collection, `memory`, two
+ * tiers (payload `tier`):
+ *
+ *   - `stm` — short-term memory: one point per captured request (its raw text as the key), there until consolidation;
+ *   - `ltm` — long-term memory: per item a KEY point (`part: key` — its content title + ~150-word summary as the dense
+ *     vector, a BM25 sparse vector over its title, summary, body and files, the item itself as payload) and CHILD
+ *     points (`part: child`, ~96-word windows of its body, dense only) pointing at it.
+ *
+ * A query ranks items once per query vector and once by keywords (each hit grouped by item, so an item found through
+ * several children counts once), then fuses those item rankings by rank (RRF). Only `active` items are ever returned
+ * to recall; superseded and archived ones stay in the log, out of the index.
+ *
+ * MANY USERS (a server holding a memory per user): one collection for all of them, its `user` field a
+ * tenant index (`is_tenant`). A client made with `tenant` adds that user to every query, count, scroll, update and
+ * delete, stamps it on every point it writes, and keys each point by user and item, so two users' identical items are
+ * two points. The user comes from the caller's context (whose work this is), never from a query's words; a call made
+ * with no user is refused before it reaches Qdrant.
+ */
+import { createHash } from 'node:crypto';
+import { MEMORY_EMBEDDER } from './embedder.js';
+export const COLLECTION = 'memory';
+/** The payload field a tenant client keys every point by, and the original point id it keeps beside the scoped one. */
+export const TENANT_FIELD = 'user';
+const TENANT_PID = 'pid';
+export const DIMS = MEMORY_EMBEDDER.dims;
+/** What the index is made of; a change empties it and it is written in again from the logs (rebuild.ts).
+ *  2: binary quantisation with rescoring (½ the RAM of int8, recall 0.98–1.00 measured at 1–3M points), keyword terms
+ *  folded and segmented for every language (text.ts); 3: on-disk binary copy, HNSW and payload indexes;
+ *  4: remote EmbeddingGemma GPU vectors (768 dimensions, explicit runtime/pooling tag in embedder.ts); 5: nothing cut —
+ *  every chunk of a body (no 12-chunk limit), short-term entries chunked too, cues numbered below the key, full payloads;
+ *  6: every captured request indexed for good — however old, consolidated or not (the raw record is the truth). */
+export const INDEX_LAYOUT = 6;
+/** The memory-mapped 1-bit copy finds candidates; full vectors on disk re-score 3× as many as asked. */
+const RESCORE = { quantization: { rescore: true, oversampling: 3.0 } };
+/** Two segments: Qdrant's default is one a core, which costs a laptop RAM and per-query work for nothing at memory's
+ *  size. The write-ahead log keeps its 32 MB: a 4 MB one returned wrong
+ *  results in qdrant.integration.test.ts on 1.20, for page cache the system can take back anyway. */
+const SMALL = { optimizers_config: { default_segment_number: 2 } };
+/** Item-level fusion: each channel (a query vector, the keywords) is its own grouped query, its items' scores are
+ *  min-max normalised over its own list, and an item's score is the weighted sum (CombSUM) — the dense channels share
+ *  DENSE_SHARE of the weight, keywords the rest. Each channel is its own query, never a prefetch under a fusion: Qdrant
+ *  1.20 orders a prefetch by the 1-bit scores whatever its `rescore` (42% of the exact top 10 on LoCoMo, any@1 halved).
+ *  Fusing items rather than points keeps one item's many chunk hits from crowding the list; fusing scores rather than
+ *  ranks keeps the gap of a hit that is much closer than the rest. Measured against gold evidence (docs/RESEARCH.md): dense 0.7
+ *  was the best share on both benchmarks, and CombSUM beat rank fusion at the top of both — LoCoMo requests any@3
+ *  71.6 → 74.5, nDCG@10 64.6 → 65.7; LongMemEval-S any@1 63.4 → 64.5, sessions any@1 92.6 → 93.2. */
+const DENSE_SHARE = 0.7;
+/** Cues — the questions an item answers, in the asker's words — search as their own channel at this share of the dense
+ *  weight rather than among the item's points. Measured on LoCoMo's 1,307 held-out questions, together with cues on
+ *  every observation and the seven-days-before calendar: multi-hop 50.6 → 55.2, overall 77.2 → 77.4 (docs/RESEARCH.md). */
+const CUE_SHARE = 0.1;
+/** A point's id from its item and its place (0 = the key, n = the n-th child): the same item writes the same ids. */
+export function pointId(item, n) {
+    const h = createHash('sha1').update(`${item}#${n}`).digest('hex');
+    return `${h.slice(0, 8)}-${h.slice(8, 12)}-5${h.slice(13, 16)}-${((parseInt(h[16], 16) & 0x3) | 0x8).toString(16)}${h.slice(17, 20)}-${h.slice(20, 32)}`;
+}
+export class QdrantError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+export class QdrantClient {
+    endpoint;
+    timeoutMs;
+    opts;
+    collection;
+    /** `endpoint` may start Qdrant (the service runs it only while memory has work): every call waits for it. */
+    constructor(endpoint, timeoutMs = 10_000, opts = {}) {
+        this.endpoint = endpoint;
+        this.timeoutMs = timeoutMs;
+        this.opts = opts;
+        this.collection = opts.collection ?? COLLECTION;
+    }
+    /** The user this call is for (a tenant client), or null (one person's own index). Throws when a tenant client has none. */
+    user() {
+        if (!this.opts.tenant)
+            return null;
+        const u = this.opts.tenant();
+        if (!u)
+            throw new QdrantError('memory index: this call names no user, so it is refused', 403);
+        return u;
+    }
+    /** A filter with the user added — unchanged for one person's own index. */
+    scoped(filter, user) {
+        if (user === null)
+            return filter;
+        return { ...filter, must: [{ key: TENANT_FIELD, match: { value: user } }, ...(filter?.must ?? [])] };
+    }
+    /** A point's id in a shared collection: the user's and the item's (pointId of both), so users never share a point. */
+    scopedId(id, user) {
+        return user === null ? id : pointId(`${user}\u0000${id}`, 0);
+    }
+    /** What a read hands back: the point's own id (as the writer named it) and only this user's points. */
+    own(points, user) {
+        if (user === null)
+            return points;
+        return points.filter((p) => p.payload[TENANT_FIELD] === user).map((p) => {
+            const { [TENANT_FIELD]: _u, [TENANT_PID]: pid, ...payload } = p.payload;
+            return { ...p, id: String(pid ?? p.id), payload };
+        });
+    }
+    async call(method, path, body, timeoutMs = this.timeoutMs) {
+        const e = await this.endpoint();
+        if (!e)
+            throw new QdrantError('memory index is not running', 503);
+        let res = null;
+        const payload = body !== undefined ? JSON.stringify(body) : undefined;
+        // Every call here can be sent again (upserts by id, deletes by filter, reads): a connection the server closed while
+        // it sat idle (keep-alive) fails before the request reaches it, so it is tried again — twice at most.
+        for (let attempt = 0; !res; attempt++) {
+            try {
+                res = await fetch(`${e.url}${path}`, {
+                    method,
+                    headers: { 'api-key': e.apiKey, ...(payload !== undefined ? { 'content-type': 'application/json' } : {}) },
+                    ...(payload !== undefined ? { body: payload } : {}),
+                    signal: AbortSignal.timeout(timeoutMs),
+                });
+            }
+            catch (err) {
+                // fetch says only "fetch failed": the reason is in its cause (refused, reset, closed, timed out).
+                const cause = err.cause;
+                const timedOut = err.name === 'TimeoutError';
+                if (attempt < 2 && !timedOut) {
+                    await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
+                    continue;
+                }
+                throw new QdrantError(`${method} ${path}: ${err.message}${cause ? ` (${cause.code ?? cause.message})` : ''}`, 0);
+            }
+        }
+        const text = await res.text();
+        if (!res.ok)
+            throw new QdrantError(`${method} ${path}: ${res.status} ${text.slice(0, 300)}`, res.status);
+        const json = text ? JSON.parse(text) : {};
+        return json.result;
+    }
+    /** Drop the whole collection: the index is derived, written again from the logs (rebuild.ts). */
+    async dropCollection() {
+        await this.call('DELETE', `/collections/${this.collection}`).catch(() => undefined);
+    }
+    /** The collection and its payload indexes, made if missing — an index added since a collection was made is added to
+     *  it in place. All of it on disk, memory-mapped — the vectors, their 1-bit copy, the graph, the payloads and their
+     *  indexes: RAM holds what queries touch, and the system may take it back. Measured at 30k / 100k items (0.33M /
+     *  1.1M points): 205 → 68 MB and 591 → 144 MB that cannot be given back,
+     *  for 12–16 ms more of Qdrant's CPU a turn (26 → 38, 81 → 97). */
+    async ensureCollection() {
+        let info = null;
+        try {
+            info = (await this.call('GET', `/collections/${this.collection}`)) ?? {};
+        }
+        catch (err) {
+            if (!(err instanceof QdrantError) || err.status !== 404)
+                throw err;
+        }
+        if (!info) {
+            await this.call('PUT', `/collections/${this.collection}`, {
+                vectors: { dense: { size: DIMS, distance: 'Cosine', on_disk: true } },
+                sparse_vectors: { bm25: { modifier: 'idf', index: { on_disk: true } } },
+                quantization_config: { binary: { always_ram: false } },
+                hnsw_config: { m: 16, ef_construct: 100, on_disk: true },
+                on_disk_payload: true,
+                ...SMALL,
+            });
+            info = { payload_schema: {}, config: { optimizer_config: { default_segment_number: SMALL.optimizers_config.default_segment_number } } };
+        }
+        else if (info.config?.optimizer_config?.default_segment_number !== SMALL.optimizers_config.default_segment_number) {
+            // Made before: two segments from now on (the optimiser merges them in the background).
+            await this.call('PATCH', `/collections/${this.collection}`, { optimizers_config: SMALL.optimizers_config });
+        }
+        const have = info.payload_schema ?? {};
+        // Many users: the user field first, as Qdrant's tenant index (its points kept together on disk, searched per user).
+        if (this.opts.tenant && !(TENANT_FIELD in have)) {
+            await this.call('PUT', `/collections/${this.collection}/index?wait=true`, { field_name: TENANT_FIELD, field_schema: { type: 'keyword', is_tenant: true, on_disk: true } });
+        }
+        for (const [field_name, field_schema] of [['item', 'keyword'], ['part', 'keyword'], ['tier', 'keyword'], ['status', 'keyword'],
+            ['kind', 'keyword'], ['scope', 'keyword'], ['project', 'keyword'], ['session', 'keyword'], ['ts', 'integer'],
+            ['synthetic', 'bool'], ['concepts', 'keyword'], ['eventFrom', 'integer'], ['eventTo', 'integer']]) {
+            const schema = field_schema === 'keyword' || field_schema === 'integer' ? { type: field_schema, on_disk: true } : field_schema;
+            if (!(field_name in have))
+                await this.call('PUT', `/collections/${this.collection}/index?wait=true`, { field_name, field_schema: schema });
+        }
+    }
+    async upsert(points, wait = false) {
+        if (!points.length)
+            return;
+        const user = this.user();
+        const out = user === null ? points
+            : points.map((p) => ({ ...p, id: this.scopedId(p.id, user), payload: { ...p.payload, [TENANT_FIELD]: user, [TENANT_PID]: p.id } }));
+        await this.call('PUT', `/collections/${this.collection}/points?wait=${wait}`, { points: out }, 60_000);
+    }
+    /** Every point of these items — before an item is written again (its children may be fewer now). */
+    async deleteItems(items) {
+        if (!items.length)
+            return;
+        await this.call('POST', `/collections/${this.collection}/points/delete?wait=true`, { filter: this.scoped({ must: [{ key: 'item', match: { any: items } }] }, this.user()) }, 60_000);
+    }
+    /** One kind of point of these items (`cue`), before that part is written again. */
+    async deleteParts(items, part) {
+        if (!items.length)
+            return;
+        await this.call('POST', `/collections/${this.collection}/points/delete?wait=true`, {
+            filter: this.scoped({ must: [{ key: 'item', match: { any: items } }, { key: 'part', match: { value: part } }] }, this.user()),
+        }, 60_000);
+    }
+    /** Change some payload fields of these items' points (status, a merged key's display fields). */
+    async setPayload(items, payload, keyOnly = false) {
+        if (!items.length)
+            return;
+        const must = [{ key: 'item', match: { any: items } }];
+        if (keyOnly)
+            must.push({ key: 'part', match: { value: 'key' } });
+        await this.call('POST', `/collections/${this.collection}/points/payload?wait=true`, { payload, filter: this.scoped({ must }, this.user()) }, 60_000);
+    }
+    /** The nearest key points to one vector (consolidation's "do I know this already?"), with their cosine. */
+    async neighbors(vector, filter, limit = 5) {
+        const must = [...(filter.must ?? []), { key: 'part', match: { value: 'key' } }];
+        const r = await this.call('POST', `/collections/${this.collection}/points/query`, {
+            query: vector, using: 'dense', limit, params: RESCORE, filter: this.scoped({ ...filter, must }, this.user()), with_payload: ['item', 'title', 'summary', 'kind', 'scope', 'project', 'status'],
+        });
+        return (r?.points ?? []).map((p) => ({ item: String(p.payload?.item ?? ''), score: p.score, payload: p.payload ?? {} }));
+    }
+    /** Each of these items' best cosine to one vector among its points of one part (key, child, cue) — the ranker's
+     *  per-family similarities. An item with no such point is not in the map. */
+    async bestPerItem(vector, items, part, timeoutMs = 2000) {
+        if (!items.length)
+            return new Map();
+        const r = await this.call('POST', `/collections/${this.collection}/points/query/groups`, {
+            query: vector, using: 'dense', params: RESCORE,
+            filter: this.scoped({ must: [{ key: 'item', match: { any: items } }, { key: 'part', match: { value: part } }] }, this.user()),
+            group_by: 'item', group_size: 1, limit: items.length, with_payload: false,
+        }, timeoutMs);
+        return new Map((r?.groups ?? []).map((g) => [String(g.id), g.hits[0]?.score ?? 0]));
+    }
+    /** `nearest` for several vectors in one call. */
+    /** `fields`: only these payload fields (and the item and its user), for a caller that needs no more — a walk over
+     *  many neighbours reads ids, not the bodies of long requests. */
+    async nearestMany(vectors, filter, limit = 10, timeoutMs = 5000, fields) {
+        if (!vectors.length)
+            return [];
+        const user = this.user();
+        const withPayload = fields ? [...new Set([...fields, 'item', TENANT_FIELD])] : true;
+        const r = await this.call('POST', `/collections/${this.collection}/points/query/batch`, {
+            searches: vectors.map((v) => ({ query: v, using: 'dense', limit, params: RESCORE, filter: this.scoped(filter, user), with_payload: withPayload })),
+        }, timeoutMs);
+        return (r ?? []).map((x) => this.own((x?.points ?? []).map((p) => ({ id: '', payload: p.payload ?? {}, score: p.score })), user)
+            .map((p) => ({ item: String(p.payload.item ?? ''), score: p.score, payload: p.payload })));
+    }
+    /** Points within a filter, with their payload (and dense vector when asked) — at most `limit`. */
+    async scroll(filter, limit = 256, withVector = false, timeoutMs = 5000) {
+        const user = this.user();
+        const r = await this.call('POST', `/collections/${this.collection}/points/scroll`, {
+            filter: this.scoped(filter, user), limit, with_payload: true, with_vector: withVector ? ['dense'] : false,
+        }, timeoutMs);
+        return this.own((r?.points ?? []).map((p) => ({ id: String(p.id), payload: p.payload ?? {}, ...(p.vector?.dense ? { vector: p.vector.dense } : {}) })), user);
+    }
+    async count(filter) {
+        const scoped = this.scoped(filter, this.user());
+        const r = await this.call('POST', `/collections/${this.collection}/points/count`, { exact: false, ...(scoped ? { filter: scoped } : {}) });
+        return r?.count ?? 0;
+    }
+    /** The items behind `ids` (key points), with their payload — and their dense vector when asked. */
+    async retrieve(ids, withVector = false) {
+        if (!ids.length)
+            return [];
+        const user = this.user();
+        const r = await this.call('POST', `/collections/${this.collection}/points`, {
+            ids: ids.map((id) => this.scopedId(id, user)), with_payload: true, with_vector: withVector ? ['dense'] : false,
+        });
+        // A point is fetched by its id alone; another user's (a hash of their id and the item) is dropped all the same.
+        return this.own((r ?? []).map((p) => ({ id: String(p.id), payload: p.payload ?? {}, ...(p.vector?.dense ? { vector: p.vector.dense } : {}) })), user);
+    }
+    /** The best items for the query, fused by rank — each with the point that matched best (`point`: its key, a chunk of
+     *  its text, or a cue), so a caller can show the passage that matched. Each channel (a query vector, the keywords)
+     *  ranks up to `per` items (default: `limit`, at least 50); their scores are then fused (CombSUM, above). */
+    async fusedItems(q, timeoutMs = 3000) {
+        const limit = q.limit ?? 40;
+        const per = q.per ?? Math.max(limit, 50);
+        const user = this.user();
+        const filter = this.scoped(q.filter, user);
+        const channels = [];
+        const sparse = q.sparse?.indices.length ? q.sparse : null;
+        const denseWeight = (sparse ? DENSE_SHARE : 1) / Math.max(1, q.dense.length);
+        const cueOff = { ...(filter ?? {}), must_not: [...(filter?.must_not ?? []), { key: 'part', match: { value: 'cue' } }] };
+        const cueOnly = { ...(filter ?? {}), must: [...(filter?.must ?? []), { key: 'part', match: { value: 'cue' } }] };
+        for (const v of q.dense) {
+            channels.push({ query: v, using: 'dense', params: RESCORE, weight: denseWeight * (1 - CUE_SHARE), filter: cueOff });
+            channels.push({ query: v, using: 'dense', params: RESCORE, weight: denseWeight * CUE_SHARE, filter: cueOnly });
+        }
+        if (sparse)
+            channels.push({ query: sparse, using: 'bm25', weight: 1 - DENSE_SHARE });
+        if (!channels.length)
+            return [];
+        const lists = await Promise.all(channels.map((c) => this.call('POST', `/collections/${this.collection}/points/query/groups`, {
+            query: c.query, using: c.using, ...(c.params ? { params: c.params } : {}),
+            ...(c.filter ?? filter ? { filter: c.filter ?? filter } : {}),
+            group_by: 'item',
+            group_size: 1,
+            limit: per,
+            // A shared collection's point ids are scoped: the writer's own id is in the payload.
+            with_payload: user === null ? false : [TENANT_FIELD, TENANT_PID],
+        }, timeoutMs)));
+        const fused = new Map();
+        lists.forEach((r, c) => {
+            const groups = (r?.groups ?? []).filter((g) => user === null || g.hits[0]?.payload?.[TENANT_FIELD] === user);
+            const scores = groups.map((g) => g.hits[0]?.score ?? 0);
+            const lo = Math.min(...scores);
+            const hi = Math.max(...scores);
+            groups.forEach((g, rank) => {
+                const item = String(g.id);
+                const raw = user === null ? g.hits[0]?.id : g.hits[0]?.payload?.[TENANT_PID];
+                const at = fused.get(item) ?? { score: 0, rank: Infinity };
+                // A channel whose items all score alike (or that found one) counts each of them fully.
+                at.score += channels[c].weight * (hi > lo ? (scores[rank] - lo) / (hi - lo) : 1);
+                // The matched passage: the point from the channel that ranked the item highest.
+                if (rank < at.rank && raw) {
+                    at.rank = rank;
+                    at.point = String(raw);
+                }
+                fused.set(item, at);
+            });
+        });
+        return [...fused.entries()].sort((a, b) => b[1].score - a[1].score).slice(0, limit)
+            .map(([item, f]) => ({ item, score: f.score, ...(f.point ? { point: f.point } : {}) }));
+    }
+}

package/dist/core/qdrant-process.d.ts

@@ -0,0 +1,30 @@
+export interface QdrantEndpoint {
+    url: string;
+    apiKey: string;
+}
+/** The config file: what environment variables cannot say (gRPC off is `grpc_port: null`). */
+export declare function qdrantConfig(storage: string, port: number): string;
+export declare class QdrantProcess {
+    private child;
+    private endpoint;
+    private stopping;
+    private deaths;
+    private restartTimer;
+    private readonly opts;
+    /** `dir` may be relative: Qdrant runs inside it, so every path it is given is made absolute first. */
+    constructor(opts: {
+        binary: string;
+        dir: string;
+        log: (line: string) => void;
+        onDown?: (why: string) => void;
+    });
+    /** Where it answers, once it does; null while it starts or after it stopped. */
+    current(): QdrantEndpoint | null;
+    start(): Promise<QdrantEndpoint>;
+    /** Answering `/readyz` within 30 s, or the start failed. */
+    private ready;
+    private exited;
+    stop(): void;
+    /** Stop and wait for the process to be gone — before its storage is deleted. */
+    stopAndWait(ms?: number): Promise<void>;
+}