dsh-prime-memory 0.11.0

package/dist/store/embedding-source.js

@@ -0,0 +1,443 @@
+/**
+ * 嵌入源状态层 + 活切换管理器。
+ *
+ * - 状态文件 embedding-source.json(写穿持久化:内存态 + tmp/rename 原子写 +
+ *   写队列串行化);无文件 = remote(与历史行为完全一致,老用户无感);
+ * - 生效 = 部署上限 AND 运行时选择(仓库铁律):远程档要求静态四件套配齐
+ *   (运行时覆盖字段由 effectiveCfg 注入后同样计入),本地档受
+ *   embedding.allowLocalModels 上限约束;
+ * - 活切换链(后台执行,RPC 立即返回 accepted,进度靠轮询):
+ *   安装运行时(首次本地)→ 预热加载 → 换服务 + swapProvider(维度变化 drop 向量表)
+ *   → 后台全量重嵌(L1/L0 计数进度,可取消)→ 持久化状态。
+ *   失败语义:异常(安装失败/模型加载失败/db 拒绝)→ 状态不持久化,重启回到旧源;
+ *   重嵌取消/部分失败 → 已切换(物理表即新维度,meta 已同步),缺失向量由周期
+ *   backfill 补齐——不回滚(回滚需要再 drop 一次表,得不偿失)。
+ */
+import { promises as fs } from 'node:fs';
+import * as path from 'node:path';
+import { NoopEmbeddingService, RemoteEmbeddingService } from './embedding.js';
+import { catalogById, MODEL_CATALOG } from './model-catalog.js';
+import { LocalEmbeddingService } from './local-embedding.js';
+// ── 状态存储(写穿持久化) ──
+export class EmbeddingSourceStore {
+    state = { source: 'remote', activeModel: null };
+    file;
+    writeQueue = Promise.resolve();
+    logger;
+    constructor(dataDir, logger) {
+        this.file = path.join(dataDir, 'embedding-source.json');
+        this.logger = logger;
+    }
+    get() {
+        return { ...this.state };
+    }
+    async init() {
+        try {
+            const raw = await fs.readFile(this.file, 'utf8');
+            const parsed = JSON.parse(raw);
+            if ((parsed.source === 'remote' || parsed.source === 'local' || parsed.source === 'off') &&
+                (parsed.activeModel === null || typeof parsed.activeModel === 'string')) {
+                this.state = { source: parsed.source, activeModel: parsed.activeModel };
+            }
+            else {
+                this.logger?.warn('[memory] 嵌入源状态文件损坏,按默认 remote 起步');
+            }
+        }
+        catch {
+            // 无文件 = 历史行为(跟随部署配置的远程嵌入)
+        }
+    }
+    async set(next) {
+        this.state = { source: next.source, activeModel: next.activeModel };
+        this.writeQueue = this.writeQueue.then(() => this.persist()).catch(() => { });
+        await this.writeQueue;
+    }
+    async persist() {
+        const tmp = this.file + '.tmp';
+        await fs.writeFile(tmp, JSON.stringify(this.state, null, 2), 'utf8');
+        await fs.rename(tmp, this.file);
+    }
+}
+/** 远程档部署上限:baseUrl + model + 维度 + enabled。apiKey 可选(本地免 key 自托管
+ *  /embeddings 也放行——与蒸馏 direct 通道的 key 可选语义一致)。cfg.embedding 缺失视为未就绪。 */
+export function remoteCeiling(cfg) {
+    const e = cfg.embedding;
+    if (!e)
+        return false;
+    return e.enabled && !!e.baseUrl && !!e.model && e.dimensions > 0;
+}
+export async function resolveInitialEmbedding(cfg, sourceStore, downloader, makeLocal, logger) {
+    const state = sourceStore.get();
+    if (state.source === 'off') {
+        return { svc: new NoopEmbeddingService(), dims: 0 };
+    }
+    if (state.source === 'local') {
+        if (!cfg.embedding.allowLocalModels) {
+            logger?.warn('[memory] 嵌入源为 local 但部署已禁用本地模型(allowLocalModels=false),本次运行纯 FTS');
+            return { svc: new NoopEmbeddingService(), dims: 0, note: '部署配置已禁用本地嵌入模型' };
+        }
+        const entry = state.activeModel ? catalogById(state.activeModel) : undefined;
+        if (!entry) {
+            logger?.warn(`[memory] 嵌入源 local 的模型 ${state.activeModel} 不在目录,本次运行纯 FTS`);
+            return { svc: new NoopEmbeddingService(), dims: 0, note: '启用的模型不在内置目录' };
+        }
+        if (!(await downloader.isDownloaded(entry.id))) {
+            logger?.warn(`[memory] 本地模型 ${entry.id} 文件缺失(可能被清理),本次运行纯 FTS`);
+            return { svc: new NoopEmbeddingService(), dims: 0, note: '模型文件缺失,请重新下载' };
+        }
+        const svc = makeLocal(entry.id);
+        if (!svc)
+            return { svc: new NoopEmbeddingService(), dims: 0, note: '本地服务构造失败' };
+        return { svc, dims: entry.dims, providerInfo: { provider: 'local', model: entry.id, dimensions: entry.dims } };
+    }
+    // remote(默认)
+    if (!remoteCeiling(cfg)) {
+        return { svc: new NoopEmbeddingService(), dims: 0 };
+    }
+    const svc = new RemoteEmbeddingService({
+        baseUrl: cfg.embedding.baseUrl,
+        apiKey: cfg.embedding.apiKey,
+        model: cfg.embedding.model,
+        dimensions: cfg.embedding.dimensions,
+        maxInputChars: cfg.embedding.maxInputChars,
+        timeoutMs: cfg.embedding.timeoutMs,
+        logger,
+    });
+    return { svc, dims: cfg.embedding.dimensions, providerInfo: svc.getProviderInfo() };
+}
+/** 本地服务构造工厂(index.ts 的初始解析与 Manager 共用一份实现,防漂移)。
+ *  推理在 worker 线程(见 local-embedding.ts);此处只传 runtime 目录与模型目录。 */
+export function makeLocalServiceFactory(installer, downloader, logger, maxInputChars) {
+    return (modelId) => {
+        const entry = catalogById(modelId);
+        if (!entry)
+            return null;
+        return new LocalEmbeddingService(entry, downloader.modelsDir(entry.id), {
+            runtimeDir: installer.runtimeDir,
+            logger,
+            maxInputChars,
+        });
+    };
+}
+export class EmbeddingManager {
+    sourceStore;
+    installer;
+    downloader;
+    deps;
+    current;
+    localSvc = null;
+    applyPhase = 'idle';
+    applyMessage = '';
+    applyStartedAt = 0;
+    applyBusy = false;
+    reindex = {
+        running: false,
+        l1Done: 0,
+        l1Total: 0,
+        l0Done: 0,
+        l0Total: 0,
+        startedAt: 0,
+        cancelled: false,
+    };
+    reindexCancel = false;
+    /** 停机标志:dispose 后应用链不再推进(防卸载后的孤儿重嵌/安装)。 */
+    disposedFlag = false;
+    /** 当前生效目标的 providerInfo(backfill/启动链的 meta 写入用——杜绝陈旧闭包)。 */
+    currentInfo;
+    /** 初始解析的降级说明(活切换成功后清空,防过期提示常驻)。 */
+    activeNote;
+    constructor(deps) {
+        this.deps = deps;
+        this.sourceStore = deps.sourceStore;
+        this.installer = deps.installer;
+        this.downloader = deps.downloader;
+        this.current = deps.initial.svc;
+        this.currentInfo = deps.initial.providerInfo;
+        this.activeNote = deps.initial.note;
+        // 启动即本地档:initial.svc 已是绑定真实运行时 loader 的 LocalEmbeddingService。
+        // 立即后台预热(不阻塞启动)——否则 L1/L0.reindex 与 EmbedHelper.batch 的
+        // vectorReady 短路都不会触发懒加载,缺失向量要等到首次召回才补
+        if (deps.initial.providerInfo?.provider === 'local') {
+            this.localSvc = this.current;
+            this.localSvc.startWarmup();
+        }
+    }
+    /** 当前目标的 providerInfo(index.ts 的启动重嵌链/周期 backfill 写 meta 用)。 */
+    currentProviderInfo() {
+        return this.currentInfo;
+    }
+    /** 当前生效配置(运行时覆盖优先;缺省回退静态 deps.cfg)。远程档 ceiling/换端点都读它。 */
+    eff() {
+        return this.deps.getEff ? this.deps.getEff() : this.deps.cfg;
+    }
+    /** 取消运行时安装(RPC:npm 卡死/用户主动放弃)。 */
+    cancelRuntimeInstall() {
+        return this.installer.cancel();
+    }
+    /** 当前生效服务(index.ts 初始建 store 用)。 */
+    getService() {
+        return this.current;
+    }
+    /** 构造绑定真实运行时 loader 的本地服务(deps.makeLocal 可注入,测试替换)。 */
+    makeLocalService(modelId) {
+        const factory = this.deps.makeLocal ?? makeLocalServiceFactory(this.installer, this.downloader, this.deps.logger, this.deps.cfg.embedding.maxInputChars);
+        return factory(modelId);
+    }
+    /** 活切换请求:验证通过即接受,后台执行应用链(进度轮询可见)。 */
+    requestSource(next) {
+        if (this.applyBusy)
+            return { accepted: false, error: '切换进行中,请等待完成' };
+        if (next.source === 'remote' && !remoteCeiling(this.eff())) {
+            return { accepted: false, error: '未配置远程嵌入(baseUrl/model/dimensions,可在设置页填写),远程档不可选' };
+        }
+        if (next.source === 'local') {
+            if (!this.deps.cfg.embedding.allowLocalModels) {
+                return { accepted: false, error: '部署已禁用本地嵌入模型(allowLocalModels=false)' };
+            }
+            if (!next.activeModel || !catalogById(next.activeModel)) {
+                return { accepted: false, error: '请选择内置目录中的模型' };
+            }
+        }
+        const state = {
+            source: next.source,
+            activeModel: next.source === 'local' ? next.activeModel : null,
+        };
+        this.applyBusy = true;
+        this.applyStartedAt = Date.now();
+        this.applyMessage = '';
+        void this.applyChain(state).finally(() => {
+            this.applyBusy = false;
+        });
+        return { accepted: true };
+    }
+    /** 下载启动(串行队列忙时拒绝);完成后自动做一次可加载性预热验证。 */
+    startDownload(modelId) {
+        if (!this.deps.cfg.embedding.allowLocalModels) {
+            return { ok: false, error: '部署已禁用本地嵌入模型' };
+        }
+        if (!catalogById(modelId))
+            return { ok: false, error: '未知模型' };
+        if (this.downloader.isBusy())
+            return { ok: false, error: '已有下载任务进行中' };
+        void this.downloader
+            .start(modelId)
+            .then(async (p) => {
+            if (p.phase !== 'done')
+                return;
+            // 下载完成自动预热验证:当前启用模型 → 直接 warmup;未启用的临时模型验完即释放
+            if (this.sourceStore.get().source === 'local' && this.sourceStore.get().activeModel === modelId) {
+                this.localSvc?.startWarmup();
+                return;
+            }
+            const scratch = this.makeLocalService(modelId);
+            if (!scratch)
+                return;
+            if (await this.installer.isReady()) {
+                // 预热只为验证可加载性:完成后必须释放(bge-m3 ~550MB,不关就常驻泄漏,
+                // 且 onnxruntime 持文件句柄会卡住该模型的删除)
+                scratch.startWarmup();
+                void scratch.waitForReady().then(() => scratch.close(), () => scratch.close());
+            }
+        })
+            .catch(() => {
+            /* 失败态在 downloader 进度里 */
+        });
+        return { ok: true };
+    }
+    cancelDownload() {
+        return this.downloader.cancel();
+    }
+    async deleteModel(modelId) {
+        const state = this.sourceStore.get();
+        if (state.source === 'local' && state.activeModel === modelId) {
+            return { ok: false, error: '该模型正在使用中,请先切换嵌入源' };
+        }
+        return this.downloader.deleteModel(modelId);
+    }
+    cancelReindex() {
+        if (!this.reindex.running)
+            return false;
+        this.reindexCancel = true;
+        return true;
+    }
+    /** 应用链/后台任务是否在跑(backfill 并发门禁用)。 */
+    isBusy() {
+        return this.applyBusy || this.reindex.running || this.downloader.isBusy();
+    }
+    /** 停机钩子(插件 dispose):取消 npm 安装、下载与重嵌——不留后台孤儿任务。 */
+    dispose() {
+        this.disposedFlag = true;
+        this.installer.cancel();
+        this.downloader.cancel();
+        this.cancelReindex();
+        this.localSvc?.close();
+    }
+    async applyChain(next) {
+        try {
+            let svc;
+            let providerInfo;
+            if (next.source === 'off') {
+                this.applyPhase = 'switching';
+                svc = new NoopEmbeddingService();
+                providerInfo = undefined;
+                this.currentInfo = undefined;
+            }
+            else if (next.source === 'remote') {
+                this.applyPhase = 'switching';
+                const re = this.eff().embedding;
+                svc = new RemoteEmbeddingService({
+                    baseUrl: re.baseUrl,
+                    apiKey: re.apiKey,
+                    model: re.model,
+                    dimensions: re.dimensions,
+                    maxInputChars: re.maxInputChars,
+                    timeoutMs: re.timeoutMs,
+                    logger: this.deps.logger,
+                });
+                providerInfo = svc.getProviderInfo();
+            }
+            else {
+                // local:下载完整性前置校验 → 运行时 → 预热 → 切换
+                if (!(await this.downloader.isDownloaded(next.activeModel))) {
+                    throw new Error('模型文件不完整(未下载或已损坏),请先完成下载');
+                }
+                if (!(await this.installer.isReady())) {
+                    this.applyPhase = 'installing-runtime';
+                    const ok = await this.installer.ensure();
+                    if (!ok) {
+                        throw new Error(`运行时安装失败: ${this.installer.getProgress().error ?? '未知原因'}`);
+                    }
+                }
+                if (this.disposedFlag)
+                    throw new Error('插件已卸载,切换中止');
+                this.applyPhase = 'warming';
+                const local = this.makeLocalService(next.activeModel);
+                if (!local)
+                    throw new Error('模型不在目录');
+                await local.waitForReady();
+                svc = local;
+                providerInfo = local.getProviderInfo();
+                this.applyPhase = 'switching';
+            }
+            // 换服务 + 换表(providerInfo 变化 → drop 向量表按新维度重建)
+            let needsReindex = false;
+            if (providerInfo) {
+                const swap = this.deps.db.swapProvider(providerInfo);
+                if (!swap.ok)
+                    throw new Error(swap.error ?? '切换向量引擎失败');
+                needsReindex = swap.needsReindex;
+                // 物理表在 swap 成功那一刻已是新维度——meta 立即跟上(即使后续重嵌被取消/
+                // 部分失败):meta 的语义是"物理表现状",缺失行由 backfill 按 missing 计数补,
+                // 不依赖 meta。拖着不写会把"meta=旧 provider"留给下次比对埋雷。
+                this.deps.db.markEmbeddingSynced(providerInfo);
+                this.currentInfo = providerInfo;
+            }
+            const oldLocal = this.localSvc;
+            this.localSvc = next.source === 'local' ? svc : null;
+            this.current = svc;
+            this.deps.l0.setEmbeddingService(svc);
+            this.deps.l1.setEmbeddingService(svc);
+            oldLocal?.close();
+            let pendingNote = '';
+            if (needsReindex) {
+                if (this.disposedFlag)
+                    throw new Error('插件已卸载,重嵌入中止');
+                this.applyPhase = 'reindexing';
+                const result = await this.reindexNow();
+                if (result.cancelled) {
+                    pendingNote = ';重嵌入已取消,缺失向量由周期任务补齐';
+                    this.deps.logger.warn('[memory] 重嵌入已取消,缺失向量将由周期补齐(检索暂按关键词降级)');
+                }
+                else if (result.failedTotal > 0) {
+                    pendingNote = `;${result.failedTotal} 条向量待补齐(周期任务会补)`;
+                }
+                if (result.error)
+                    throw new Error(result.error);
+            }
+            await this.sourceStore.set(next);
+            this.activeNote = undefined;
+            this.applyPhase = 'done';
+            this.applyMessage =
+                next.source === 'off'
+                    ? '已切换为关键词检索'
+                    : '切换完成' + pendingNote;
+        }
+        catch (err) {
+            this.applyPhase = 'error';
+            this.applyMessage = err instanceof Error ? err.message : String(err);
+            this.deps.logger.warn(`[memory] 嵌入源切换失败(状态保持不变): ${this.applyMessage}`);
+        }
+    }
+    async reindexNow() {
+        this.reindexCancel = false;
+        this.reindex = { running: true, l1Done: 0, l1Total: 0, l0Done: 0, l0Total: 0, startedAt: Date.now(), cancelled: false };
+        let failedTotal = 0;
+        try {
+            const r1 = await this.deps.l1.reindex({
+                onProgress: (done, total) => {
+                    this.reindex.l1Done = done;
+                    this.reindex.l1Total = total;
+                },
+                shouldCancel: () => this.reindexCancel,
+            });
+            failedTotal += r1.failed;
+            const r0 = await this.deps.l0.reindex({
+                onProgress: (done, total) => {
+                    this.reindex.l0Done = done;
+                    this.reindex.l0Total = total;
+                },
+                shouldCancel: () => this.reindexCancel,
+            });
+            failedTotal += r0.failed;
+            const cancelled = !!(r1.cancelled || r0.cancelled);
+            this.reindex.cancelled = cancelled;
+            this.reindex.running = false;
+            return { cancelled, failedTotal };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            this.reindex.running = false;
+            this.reindex.error = message;
+            return { cancelled: false, failedTotal, error: `重嵌入失败: ${message}` };
+        }
+    }
+    /** RPC 快照(设置页嵌入区块数据源;client 忙时 1s 轮询)。 */
+    async snapshot() {
+        const status = await this.downloader.listStatus();
+        const models = MODEL_CATALOG.map((entry) => {
+            const s = status.find((x) => x.id === entry.id);
+            return {
+                id: entry.id,
+                name: entry.name,
+                dims: entry.dims,
+                contextTokens: entry.contextTokens,
+                tags: entry.tags,
+                description: entry.description,
+                totalBytes: s?.totalBytes ?? 0,
+                bytesOnDisk: s?.bytesOnDisk ?? 0,
+                state: s?.state ?? 'none',
+            };
+        });
+        const state = this.sourceStore.get();
+        const effE = this.eff().embedding;
+        return {
+            source: state.source,
+            activeModel: state.activeModel,
+            ceilings: { remote: remoteCeiling(this.eff()), local: this.deps.cfg.embedding.allowLocalModels },
+            remote: {
+                baseURL: effE.baseUrl ?? '',
+                model: effE.model ?? '',
+                dimensions: effE.dimensions ?? 0,
+                // API key:生效(运行时覆盖 ? 静态配置)非空即视为已配置——明文不回传
+                apiKeySet: Boolean(effE.apiKey),
+            },
+            runtime: this.installer.getProgress(),
+            models,
+            download: this.downloader.getProgress(),
+            apply: { phase: this.applyPhase, message: this.applyMessage, startedAt: this.applyStartedAt, busy: this.applyBusy },
+            local: this.localSvc ? { state: this.localSvc.getState(), error: this.localSvc.getLoadError() } : null,
+            reindex: { ...this.reindex },
+            activeNote: this.activeNote,
+        };
+    }
+}

package/dist/store/embedding.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Embedding 服务基座:类型、远程 OpenAI 兼容实现、降级助手。
+ *
+ * dsh 的 ctx.llm 只有 chat 流式接口、无 embeddings 端点,因此向量能力
+ * 需要用户自备任意 OpenAI 兼容 /embeddings 服务(OpenAI/SiliconFlow/Jina/自托管 BGE 等)。
+ * 默认关闭(Noop)——纯 FTS 模式即可运行,这是官方 provider="none" 的同款语义。
+ *
+ * 重写版远程服务内建两层此前缺失的可靠性纪律:
+ * - 重试退避:5xx/网络错误指数退避带抖动重试(默认 2 次),4xx(请求/维度错误)不重试;
+ * - 并发信号量:全局嵌入调用并发受限(默认 4),防 L1 去重候选一路 Promise.all 打爆上游。
+ * (嵌入源三态活切换、本地 worker、模型下载/运行时安装在嵌入子系统提交中落地。)
+ */
+import type { MemoryLogger } from '../types.js';
+export interface EmbeddingProviderInfo {
+    provider: string;
+    model: string;
+    dimensions: number;
+}
+/** 单次嵌入调用的可选参数:timeoutMs 只允许缩短服务配置的超时(内层钳制),
+ *  永不放大——召回路径用它给 FTS 降级留时间(规格 A 节)。本地实现经 worker
+ *  代理以 Promise.race 钳制(迟到回复丢弃,推理在 worker 线程无法真正取消)。 */
+export interface EmbedCallOptions {
+    timeoutMs?: number;
+}
+export interface EmbeddingService {
+    embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
+    embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
+    getDimensions(): number;
+    getProviderInfo(): EmbeddingProviderInfo;
+    isReady(): boolean;
+    close?(): void;
+}
+/** 空实现:向量能力关闭/不可用时使用,一切调用安全返回。 */
+export declare class NoopEmbeddingService implements EmbeddingService {
+    getDimensions(): number;
+    getProviderInfo(): EmbeddingProviderInfo;
+    isReady(): boolean;
+    embed(): Promise<Float32Array>;
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+}
+export interface RemoteEmbeddingOptions {
+    baseUrl: string;
+    apiKey: string;
+    model: string;
+    /** 必填且须与模型输出一致(vec0 建表需要固定维度)。 */
+    dimensions: number;
+    maxInputChars?: number;
+    timeoutMs?: number;
+    logger?: MemoryLogger;
+    /** 5xx/网络错误的最大重试次数(默认 2;4xx 不重试)。 */
+    maxRetries?: number;
+    /** 全局并发信号量上限(默认 4)。 */
+    concurrency?: number;
+}
+/**
+ * OpenAI 兼容远程 embedding:POST {baseUrl}/embeddings。
+ * 向量在客户端做 L2 归一化(与余弦度量配套)。
+ */
+export declare class RemoteEmbeddingService implements EmbeddingService {
+    private readonly opts;
+    private readonly sem;
+    constructor(opts: RemoteEmbeddingOptions);
+    getDimensions(): number;
+    getProviderInfo(): EmbeddingProviderInfo;
+    isReady(): boolean;
+    embed(text: string, callOpts?: EmbedCallOptions): Promise<Float32Array>;
+    embedBatch(texts: string[], callOpts?: EmbedCallOptions): Promise<Float32Array[]>;
+    private callWithRetry;
+    private callOnce;
+}
+/**
+ * 嵌入调用降级助手(L0/L1 Store 共用):查询向量失败 → undefined(调用方降级 FTS);
+ * 批量嵌入失败 → 全 undefined(跳过向量写入,由周期性 backfill 补齐)。
+ * 同类失败只告警一次,避免刷屏。
+ */
+export declare class EmbedHelper {
+    private embed;
+    private readonly logger?;
+    private warned;
+    constructor(embed: EmbeddingService, logger?: MemoryLogger | undefined);
+    /** 活切换嵌入源:换掉底层服务并复位一次性告警(新服务重新获得告警机会)。 */
+    setService(svc: EmbeddingService): void;
+    vectorReady(): boolean;
+    /** 查询向量;失败或空向量返回 undefined(调用方降级 FTS)。
+     *  timeoutMs 为内层钳制(仅缩短服务超时),召回路径使用。 */
+    query(text: string, timeoutMs?: number): Promise<Float32Array | undefined>;
+    /** 批量嵌入;服务未就绪或失败时返回全 undefined(不阻断元数据/FTS 写入)。 */
+    batch(texts: string[]): Promise<Array<Float32Array | undefined>>;
+    private warn;
+}