@max-null/dsh-memory 0.5.0 → 0.6.0

package/dist/engine.js

@@ -5,13 +5,15 @@
  * is always created `suggested` and becomes effective only through `setStatus`.
  */
 import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
 import { Service } from '@deepseek-ai/cordis';
 import { z } from 'zod';
 import { defineDomain, domainTable, DomainFacility } from '@deepseek-ai/dsh-storage-domain';
 import { JsonStorageBackend } from '@deepseek-ai/dsh-storage-json';
-import { bm25Scores } from "./bm25.js";
+import { bm25FieldScores, cosineSimilarity, rrfFuse } from "./bm25.js";
+import { parsePromptFile, scanPromptDir, writePromptFile } from "./prompt-files.js";
 /** Brand a string as a {@link MemoryId} (compile-time only). */
 export function MemoryId(id) {
     return id;
@@ -24,6 +26,20 @@ const blockSchema = z.object({
     keywords: z.array(z.string()),
     createdAt: z.number(),
     updatedAt: z.number(),
+    lastUsedAt: z.number().optional(),
+    vector: z.array(z.number()).optional(),
+    kind: z.enum(['fact', 'prompt']).optional(),
+    meta: z.object({
+        seq: z.number().optional(),
+        name: z.string(),
+        dimension: z.string().optional(),
+        difficulty: z.string().optional(),
+        tags: z.array(z.string()),
+        summary: z.string(),
+        path: z.string(),
+        mtime: z.number(),
+        source: z.enum(['user', 'agent']),
+    }).optional(),
 });
 /** Shared table shape; the two domains differ only by name and backend route. */
 function memorySpec(name) {
@@ -33,6 +49,30 @@ function memorySpec(name) {
         tables: { blocks: domainTable(blockSchema) },
     });
 }
+/** 读 unit 文件头的 name（文件不存在 → undefined；非法 JSON/结构异常抛错）。 */
+function unitHeaderName(path) {
+    if (!existsSync(path))
+        return undefined;
+    const document = JSON.parse(readFileSync(path, 'utf8'));
+    return typeof document.unit?.name === 'string' ? document.unit.name : undefined;
+}
+/** 改写 unit 文件头的 name（保持 2 空格缩进 + 尾换行，与 storage-json/format 一致）。 */
+function rewriteUnitName(path, name) {
+    const document = JSON.parse(readFileSync(path, 'utf8'));
+    if (document.unit === undefined)
+        throw new Error(`unit '${path}': missing unit header`);
+    document.unit.name = name;
+    writeFileSync(path, `${JSON.stringify(document, null, 2)}\n`);
+}
+/** 迁移旧 unit 文件：校验头名 → 改写为新名并落位规范路径 → 删除旧文件。 */
+function migrateUnit(legacyPath, canonicalPath, legacyName, canonicalName) {
+    const document = JSON.parse(readFileSync(legacyPath, 'utf8'));
+    if (document.unit?.name !== legacyName)
+        throw new Error(`unit '${legacyPath}': unexpected unit header`);
+    document.unit.name = canonicalName;
+    writeFileSync(canonicalPath, `${JSON.stringify(document, null, 2)}\n`);
+    rmSync(legacyPath);
+}
 /**
  * 旧 schema 迁移（0.3.0，设计文档「迁移规则」）：
  * - 旧 `auto` → approved + injected:true（行为不变：仍常驻注入）
@@ -48,7 +88,7 @@ function normalizeBlock(block) {
     return { status: block.status, injected: block.injected ?? false };
 }
 function toRecord(id, block) {
-    return {
+    const base = {
         id: MemoryId(id),
         namespace: block.namespace,
         ...normalizeBlock(block),
@@ -56,6 +96,13 @@ function toRecord(id, block) {
         keywords: block.keywords,
         createdAt: block.createdAt,
         updatedAt: block.updatedAt,
+        ...block.lastUsedAt === undefined ? {} : { lastUsedAt: block.lastUsedAt },
+    };
+    // 0.6.0 模板库字段（缺省 fact；prompt 元数据透传）
+    return {
+        ...base,
+        ...block.kind === undefined ? {} : { kind: block.kind },
+        ...block.meta === undefined ? {} : { meta: block.meta },
     };
 }
 /** Harness-home root for `global` memories. */
@@ -130,13 +177,37 @@ export class MemoryEngine extends Service {
         if (cached !== undefined)
             return cached;
         const backendName = this.projectBackendName(key);
+        // 0.5.2：旧实现 domain 名多一层 memory_project_ 前缀（文件名双重前缀，
+        // 且 unit 文件头 name 同为旧名）。打开前统一迁移到规范名：
+        // ①规范名已存在 + 头为旧名（0.5.2 首版只 rename 未改 header 的中间态）→ 改写头；
+        // ②旧名文件存在且头匹配 → 改写头并落位规范名；③其他情况回退旧域名（不动数据）。
+        const legacyDomainName = `memory_project_${backendName}`;
+        const root = this.projectRootFor(key);
+        const canonicalFile = join(root, `${backendName}.json`);
+        const legacyFile = join(root, `${legacyDomainName}.json`);
+        let domainName = backendName;
+        try {
+            const canonicalHeader = unitHeaderName(canonicalFile);
+            if (canonicalHeader === legacyDomainName) {
+                rewriteUnitName(canonicalFile, backendName);
+            }
+            else if (canonicalHeader === undefined && unitHeaderName(legacyFile) === legacyDomainName) {
+                migrateUnit(legacyFile, canonicalFile, legacyDomainName, backendName);
+            }
+            else if (canonicalHeader === undefined && existsSync(legacyFile)) {
+                domainName = legacyDomainName; // 外域/未知内容：保持旧名路径继续打开，不触碰数据
+            }
+        }
+        catch {
+            domainName = legacyDomainName; // 迁移出错：回退旧名打开，数据不动（后续可重试）
+        }
         if (!this.registeredProjectBackends.has(backendName)) {
-            const backend = new JsonStorageBackend(this.projectRootFor(key));
+            const backend = new JsonStorageBackend(root);
             this.ctx.storage.backend.register(backendName, backend);
             this.registeredProjectBackends.add(backendName);
         }
         const facility = new DomainFacility(this.ctx, { backend: backendName, routes: {} });
-        const domain = await facility.open(memorySpec(`memory_project_${backendName}`));
+        const domain = await facility.open(memorySpec(domainName));
         const table = domain.table('blocks');
         this.projectFacilities.set(key, facility);
         this.projectTables.set(key, table);
@@ -182,16 +253,126 @@ export class MemoryEngine extends Service {
     async list(filter, projectCwd) {
         const records = await this.allRecords(filter?.namespace, projectCwd);
         return records.filter(record => (filter?.status === undefined || record.status === filter.status)
-            && (filter?.injected === undefined || record.injected === filter.injected));
+            && (filter?.injected === undefined || record.injected === filter.injected)
+            && (filter?.kind === undefined || (record.kind ?? 'fact') === filter.kind));
     }
     async search(query, filter, projectCwd) {
         const records = await this.list(filter, projectCwd);
-        const docs = records.map(record => `${record.content} ${record.keywords.join(' ')}`);
-        const scores = bm25Scores(query, docs);
-        return records
+        // 0.5.2：content 与 keywords 分离打加权 BM25（人工关键词命中权重更高）
+        const scores = bm25FieldScores(query, records.map(record => ({
+            body: record.content,
+            tags: record.keywords.join(' '),
+        })));
+        const bm25Ranked = records
             .map((record, index) => ({ record, score: scores[index] ?? 0 }))
             .filter(hit => hit.score > 0)
             .sort((left, right) => right.score - left.score);
+        // 0.5.3：命中即标记使用（冷热追踪）——写回按命中快照之后执行，不影响本次结果
+        await this.markUsed(bm25Ranked.map(hit => hit.record.id), projectCwd);
+        // 0.5.2：配置 embeddings 时与语义结果 RRF 融合；缺省/失败降级回纯 BM25
+        if (this.config.embeddings === undefined)
+            return bm25Ranked;
+        try {
+            const semantic = await this.semanticRanking(query, records.map(record => record.id), projectCwd);
+            if (semantic.length === 0)
+                return bm25Ranked;
+            const byId = new Map(records.map(record => [String(record.id), record]));
+            return rrfFuse([
+                bm25Ranked.map(hit => String(hit.record.id)),
+                semantic.map(hit => String(hit.id)),
+            ]).flatMap(([id, score]) => {
+                const record = byId.get(id);
+                return record === undefined ? [] : [{ record, score }];
+            });
+        }
+        catch (error) {
+            // 语义通道失败降级：不影响检索可用性（BM25 结果照常返回）
+            this.ctx.logger?.warn?.(`dsh-memory: semantic search degraded: ${error instanceof Error ? error.message : String(error)}`);
+            return bm25Ranked;
+        }
+    }
+    /**
+     * 语义排序（0.5.2）：查询向量 + 候选向量的余弦相似度，仅在有
+     * embeddings 配置时被调用。缺失向量（新增/更新过的记录）在此补生成
+     * 并持久化（vector 字段）；生成失败抛给调用方降级。
+     */
+    async semanticRanking(query, ids, projectCwd) {
+        const config = this.config.embeddings;
+        if (config === undefined)
+            return [];
+        const similarity = config.similarity ?? cosineSimilarity;
+        const topK = this.config.semanticTopK ?? 5;
+        const queryVector = (await config.embed([query]))?.[0];
+        if (queryVector === undefined)
+            return [];
+        // 缺失向量批量补写（本轮直接用本地生成的向量参与打分，写回不阻塞结果）
+        const missing = [];
+        for (const id of ids) {
+            const table = this.tableOf(id, projectCwd);
+            const block = table?.get(id);
+            if (table === undefined || block === undefined)
+                continue;
+            if (block.vector === undefined)
+                missing.push({ table, id, content: block.content });
+        }
+        const fresh = new Map();
+        if (missing.length > 0) {
+            const vectors = await config.embed(missing.map(missingRecord => missingRecord.content));
+            for (const [index, missingRecord] of missing.entries()) {
+                const vector = vectors[index];
+                if (vector === undefined)
+                    continue;
+                fresh.set(String(missingRecord.id), [...vector]);
+                const block = missingRecord.table.get(missingRecord.id);
+                if (block !== undefined) {
+                    await missingRecord.table.put(missingRecord.id, { ...block, vector: [...vector] });
+                }
+            }
+        }
+        const scored = [];
+        for (const id of ids) {
+            const block = this.tableOf(id, projectCwd)?.get(id);
+            const vector = fresh.get(String(id)) ?? block?.vector;
+            if (vector === undefined)
+                continue;
+            scored.push({ id, score: similarity(queryVector, vector) });
+        }
+        return scored
+            .filter(hit => hit.score > 0)
+            .sort((left, right) => right.score - left.score)
+            .slice(0, topK);
+    }
+    /** 按已知 id 定位记录所在表（global 优先，其次当前工作区表）。 */
+    tableOf(id, projectCwd) {
+        const global = this.requireTable('global');
+        if (global.get(id) !== undefined)
+            return global;
+        if (projectCwd === undefined || projectCwd === '')
+            return undefined;
+        const project = this.projectTables.get(join(projectCwd));
+        return project !== undefined && project.get(id) !== undefined ? project : undefined;
+    }
+    /** 命中标记：把 lastUsedAt 写回（命中记录不存在时跳过；不 emit 事件——统计性字段）。 */
+    async markUsed(ids, projectCwd) {
+        if (ids.length === 0)
+            return;
+        const now = Date.now();
+        const global = this.requireTable('global');
+        const project = projectCwd === undefined || projectCwd === ''
+            ? undefined
+            : this.projectTables.get(join(projectCwd));
+        for (const id of ids) {
+            const globalBlock = global.get(id);
+            if (globalBlock !== undefined) {
+                await global.put(id, { ...globalBlock, lastUsedAt: now });
+                continue;
+            }
+            if (project !== undefined) {
+                const block = project.get(id);
+                if (block !== undefined)
+                    await project.put(id, { ...block, lastUsedAt: now });
+            }
+        }
     }
     async forget(id, projectCwd) {
         if (await this.requireTable('global').delete(id)) {
@@ -243,36 +424,35 @@ export class MemoryEngine extends Service {
      * 「常驻注入」开关调用（remote.setInjected）。
      */
     async setInjected(id, injected, projectCwd) {
-        const global = this.requireTable('global').get(id);
-        if (global !== undefined) {
-            const updated = {
-                ...global,
-                status: normalizeBlock(global).status,
-                injected,
-                updatedAt: Date.now(),
-            };
-            await this.requireTable('global').put(id, updated);
-            const record = toRecord(id, updated);
-            this.ctx.emit('memory/changed', { operation: 'injected', id, injected });
-            return record;
-        }
-        const project = await this.projectTableFor(projectCwd);
-        if (project !== undefined) {
-            const block = project.get(id);
-            if (block !== undefined) {
-                const updated = {
-                    ...block,
-                    status: normalizeBlock(block).status,
-                    injected,
-                    updatedAt: Date.now(),
-                };
-                await project.put(id, updated);
-                const record = toRecord(id, updated);
-                this.ctx.emit('memory/changed', { operation: 'injected', id, injected });
-                return record;
+        const locate = async () => {
+            const global = this.requireTable('global').get(id);
+            if (global !== undefined)
+                return { table: this.requireTable('global'), block: global };
+            const project = await this.projectTableFor(projectCwd);
+            if (project !== undefined) {
+                const block = project.get(id);
+                if (block !== undefined)
+                    return { table: project, block };
             }
+            return undefined;
+        };
+        const found = await locate();
+        if (found === undefined)
+            throw new Error(`cannot set injected of unknown memory '${id}'`);
+        // 模板永不注入（0.6.0）：prompt 记录拒绝开关（UI 对该类不展示开关，此处兜底）
+        if ((found.block.kind ?? 'fact') === 'prompt') {
+            throw new Error(`cannot set injected of prompt record '${id}': prompt templates are never injected`);
         }
-        throw new Error(`cannot set injected of unknown memory '${id}'`);
+        const updated = {
+            ...found.block,
+            status: normalizeBlock(found.block).status,
+            injected,
+            updatedAt: Date.now(),
+        };
+        await found.table.put(id, updated);
+        const record = toRecord(id, updated);
+        this.ctx.emit('memory/changed', { operation: 'injected', id, injected });
+        return record;
     }
     /**
      * 修改一条记忆的内容/关键词（0.3.1 整理记忆用）。内容被模型改动后
@@ -313,12 +493,32 @@ export class MemoryEngine extends Service {
         throw new Error(`cannot update unknown memory '${id}'`);
     }
     /**
-     * 注入专用（0.3.4）：systemPrompt context 是同步回调、无会话 cwd 可及，
-     * 只返回 global 的 approved+injected（工作区记忆走检索/面板）。
+     * 预热某工作区的 project 表（异步打开并缓存）。system prompt 的
+     * `memory:recall` provider 是同步回调——只读已打开的表；预热让
+     * 会话创建/首次请求后本轮或下一轮即可注入工作区记忆。
      */
-    recallRecords() {
-        return this.recordsOf(this.requireTable('global'))
-            .filter(record => record.status === 'approved' && record.injected);
+    async ensureProjectOpen(projectCwd) {
+        const key = join(projectCwd);
+        if (this.projectTables.has(key))
+            return;
+        await this.projectTableFor(key);
+    }
+    /**
+     * 注入专用（0.3.4 + 0.5.1）：system prompt 的 `memory:recall` context
+     * provider 可经 AssembleContext.agent 拿到当前会话 header.cwd，因此
+     * 注入 = global 的 approved+injected + 当前会话工作区的 approved+injected。
+     * provider 是同步回调：只读已打开/缓存的表，未打开的工作区本轮为空
+     * （`ensureProjectOpen` 提供预热，工具/面板路径也会打开）。
+     */
+    recallRecords(projectCwd) {
+        const injected = (table) => this.recordsOf(table).filter(record => record.status === 'approved' && record.injected);
+        const global = injected(this.requireTable('global'));
+        if (projectCwd === undefined || projectCwd === '')
+            return global;
+        const project = this.projectTables.get(join(projectCwd));
+        if (project === undefined)
+            return global;
+        return [...global, ...injected(project)];
     }
     async allRecords(namespace, projectCwd) {
         if (namespace === 'project') {
@@ -347,4 +547,151 @@ export class MemoryEngine extends Service {
             throw new Error('memory engine is not started yet');
         return table;
     }
+    // ── 提示词模板库（0.6.0）：md 文件是事实源，索引记录 kind='prompt' ──────
+    /** 全局模板根（默认 `$DSH_HOME/prompt-library`，可配置覆盖）。 */
+    promptGlobalDir() {
+        return this.config.promptGlobalRoot ?? join(process.env.DSH_HOME ?? join(homedir(), '.dsh'), 'prompt-library');
+    }
+    /** project 模板根（随工作区 `<workspace>/.dsh/prompt-library`，与记忆同构）。 */
+    promptProjectDir(projectCwd) {
+        if (projectCwd === undefined || projectCwd === '')
+            return '';
+        return join(projectCwd, '.dsh', 'prompt-library');
+    }
+    /** 稳定索引 id（按路径 hash；文件删除后随刷新清理）。 */
+    promptIdOf(path) {
+        let h = 5381;
+        const text = path.toLowerCase();
+        for (let i = 0; i < text.length; i++)
+            h = ((h << 5) + h + text.charCodeAt(i)) | 0;
+        return MemoryId(`prompt-${Math.abs(h).toString(36)}`);
+    }
+    /**
+     * 扫描模板目录并同步索引（mtime 惰性：未变条目零写；目录缺失 = 空库）。
+     * @returns 统计（扫描数/变更数/清理数 + 坏文件警告）。
+     */
+    async refreshPromptIndex(projectCwd) {
+        const globalDir = this.promptGlobalDir();
+        const projectDir = this.promptProjectDir(projectCwd);
+        const globalScan = scanPromptDir(globalDir);
+        const projectScan = scanPromptDir(projectDir);
+        const scanned = [...globalScan.files, ...projectScan.files];
+        const warnings = [...globalScan.warnings, ...projectScan.warnings];
+        const globalTable = this.requireTable('global');
+        const projectTable = projectCwd !== undefined && projectCwd !== ''
+            ? await this.projectTableFor(projectCwd)
+            : undefined;
+        const existing = await this.list({ kind: 'prompt' }, projectCwd);
+        const existingById = new Map(existing.map(record => [String(record.id), record]));
+        const seen = new Set();
+        let changed = 0;
+        let removed = 0;
+        const now = Date.now();
+        for (const file of scanned) {
+            const id = this.promptIdOf(file.path);
+            seen.add(String(id));
+            const isGlobal = file.path.startsWith(globalDir);
+            const table = isGlobal ? globalTable : projectTable;
+            if (table === undefined)
+                continue;
+            const prev = existingById.get(String(id));
+            if (prev?.meta !== undefined && prev.meta.mtime === file.mtime)
+                continue; // mtime 未变：零写
+            const keywords = [
+                file.meta.name,
+                ...(file.meta.dimension !== undefined ? [file.meta.dimension] : []),
+                ...(file.meta.difficulty !== undefined ? [file.meta.difficulty] : []),
+                ...file.meta.tags,
+            ].filter(keyword => keyword !== '').map(keyword => keyword.toLowerCase());
+            const block = {
+                namespace: isGlobal ? 'global' : 'project',
+                status: 'approved', // 文件存在即生效（设计文档 §三：模板不做审核流）
+                injected: false, // 模板永不注入
+                content: [file.meta.name, file.meta.dimension ?? '', file.meta.difficulty ?? '', file.meta.tags.join(' '), file.summary]
+                    .filter(part => part !== '').join(' '),
+                keywords,
+                createdAt: prev?.createdAt ?? now,
+                updatedAt: now,
+                kind: 'prompt',
+                meta: {
+                    seq: file.meta.seq,
+                    name: file.meta.name,
+                    dimension: file.meta.dimension,
+                    difficulty: file.meta.difficulty,
+                    tags: file.meta.tags,
+                    summary: file.summary,
+                    path: file.path,
+                    mtime: file.mtime,
+                    source: file.meta.source,
+                },
+            };
+            await table.put(id, block);
+            changed++;
+        }
+        for (const record of existing) {
+            if (seen.has(String(record.id)))
+                continue;
+            if (record.meta === undefined)
+                continue;
+            const isGlobal = record.meta.path.startsWith(globalDir);
+            const table = isGlobal ? globalTable : projectTable;
+            if (table !== undefined) {
+                await table.delete(record.id);
+                removed++;
+            }
+        }
+        return { scan: scanned.length, changed, removed, warnings };
+    }
+    /**
+     * 取模板（id 或名称/路径片段定位），返回 md 全文（直读文件，忽略索引缓存）。
+     * @throws 未命中或文件解析失败（错误信息可读）。
+     */
+    async promptGet(nameOrId, projectCwd) {
+        const records = await this.list({ kind: 'prompt' }, projectCwd);
+        const needle = nameOrId.trim();
+        const hit = records.find(record => {
+            if (record.meta === undefined)
+                return false;
+            const stem = (record.meta.path.split(/[\\/]/).pop() ?? '').replace(/\.md$/i, '');
+            const afterSeq = stem.replace(/^\d+_/, ''); // 去掉序号前缀：36_名称 → 名称
+            return String(record.id) === needle || afterSeq === needle || afterSeq.includes(needle);
+        });
+        if (hit === undefined || hit.meta === undefined) {
+            throw new Error(`prompt '${nameOrId}' not found（可用 prompt_list 查看全部）`);
+        }
+        const text = readFileSync(hit.meta.path, 'utf8');
+        const parsed = parsePromptFile(hit.meta.path, text);
+        if (!parsed.ok)
+            throw new Error(`prompt file ${hit.meta.path} parse failed: ${parsed.error}`);
+        return parsed.file;
+    }
+    /**
+     * 新增模板：写 md 文件（序号自动分配/文件名安全化/同名防覆盖）+ 刷新索引。
+     * 返回索引记录（模型新增 source='agent'，UI 角标提示）。
+     */
+    async promptAdd(input, projectCwd) {
+        const dir = input.namespace === 'project' ? this.promptProjectDir(projectCwd) : this.promptGlobalDir();
+        if (dir === '')
+            throw new Error('cannot write project prompt without a workspace cwd');
+        const path = writePromptFile(dir, input);
+        await this.refreshPromptIndex(projectCwd);
+        const found = (await this.list({ kind: 'prompt' }, projectCwd))
+            .find(record => String(record.id) === String(this.promptIdOf(path)));
+        if (found === undefined)
+            throw new Error('prompt written but index refresh failed');
+        return found;
+    }
+    /** 删除模板：md 文件与索引一并删除；未命中返回 false。 */
+    async promptRemove(idOrName, projectCwd) {
+        let file = null;
+        try {
+            file = await this.promptGet(idOrName, projectCwd);
+        }
+        catch {
+            return false;
+        }
+        rmSync(file.path, { force: true });
+        await this.refreshPromptIndex(projectCwd);
+        return true;
+    }
 }

package/dist/index.d.ts

@@ -9,7 +9,7 @@ import type { MemoryConfig } from './engine.ts';
 export { MemoryEngine } from './engine.ts';
 export type { MemoryChange, MemoryConfig, MemoryFilter, MemoryHit, MemoryNamespace, MemoryRecord, MemoryStatus, MemoryWrite, } from './engine.ts';
 export { MemoryId } from './engine.ts';
-export { bm25Scores, tokenize } from './bm25.ts';
+export { bm25FieldScores, bm25Scores, cosineSimilarity, rrfFuse, tokenize } from './bm25.ts';
 export declare const name = "dsh-memory";
 export declare const inject: string[];
 export declare function apply(ctx: Context, config?: MemoryConfig): Promise<void>;