@co-engram/core 0.1.0

package/dist/observability/proposal-engine.d.ts

@@ -0,0 +1,224 @@
+/**
+ * 记忆候选引擎（Proposal Engine）
+ *
+ * 实现"候选提示(prompted candidates)"机制——被动观察对话片段,
+ * 当某主题被多次提及但无匹配 engram 时,生成候选提案等待确认。
+ *
+ * 工作流:
+ *   1. observe(message) — host 每次对话都调用,把片段喂进引擎
+ *   2. 内部:向量化 → 在线聚类(余弦 > 阈值归同簇)
+ *   3. cluster.occurrences ≥ threshold → 检查是否已有匹配 engram
+ *   4. 无匹配 → 生成/更新 proposal(status=pending)
+ *   5. 下次会话开始 → system prompt 注入提示
+ *   6. accept / dismiss — LLM 或用户决策
+ *
+ * 设计原则:
+ *   - embedder 注入式: core 不绑定 embedding provider
+ *   - 默认 embedder: hash-based(无 LLM 成本,准确度低但可用于 M1)
+ *   - 查重: 仅用 repository.listEngrams() + 标题子串匹配(简单,M1 够用)
+ *   - 失败静默: 观察失败不阻塞对话流
+ *
+ * @module @co-engram/core/observability
+ */
+import type { EngramRepository } from "../storage/repository.js";
+import type { AuditLog } from "./audit-log.js";
+import type { EngramCreateInput } from "../types/engram.js";
+import { type NecessityEvaluator } from "./necessity-evaluator.js";
+/** Embedder 接口:文本 → 向量 */
+export type Embedder = (text: string) => Promise<readonly number[]>;
+/** 主题簇 */
+export interface TopicCluster {
+    /** 簇唯一 ID(短 hash) */
+    readonly id: string;
+    /** 质心向量(增量平均) */
+    readonly centroid: readonly number[];
+    /** 出现次数 */
+    readonly occurrences: number;
+    /** 最近 N 条样本(原文截断,最多 100 字符) */
+    readonly samples: readonly string[];
+    /** 首次见到时间 */
+    readonly firstSeenAt: string;
+    /** 最后见到时间 */
+    readonly lastSeenAt: string;
+}
+/** 候选提案 */
+export interface Proposal {
+    /** 关联的 cluster id */
+    readonly entityId: string;
+    /** 出现次数(snapshot) */
+    readonly occurrences: number;
+    /** 样本引用(snapshot,最多 3 条) */
+    readonly sampleQuotes: readonly string[];
+    /** 最接近质心的样本(用于预览) */
+    readonly centroidExcerpt: string;
+    /** 首次见到时间 */
+    readonly firstSeenAt: string;
+    /** 最后见到时间 */
+    readonly lastSeenAt: string;
+    /** 创建时间(生成 proposal 时) */
+    readonly createdAt: string;
+    /** 状态 */
+    readonly status: "pending" | "accepted" | "dismissed";
+    /** dismiss 时的截止时间(期间不再提示) */
+    readonly dismissedUntil?: string;
+    /** dismiss 原因 */
+    readonly dismissReason?: string;
+    /** accept 时创建的 engramId */
+    readonly acceptedEngramId?: string;
+    /** 必要性评估理由(展示给用户,辅助审批决策) */
+    readonly necessityReason?: string;
+    /** 必要性评估触发的规则(规则版填充,如 'high_repetition') */
+    readonly necessityRule?: string;
+    /** LLM 建议的标题(可作审批草稿) */
+    readonly suggestedTitle?: string;
+}
+/** Proposal Engine 配置 */
+export interface ProposalEngineConfig {
+    /** 触发阈值(默认 3) */
+    readonly threshold?: number;
+    /** 余弦相似度阈值(默认 0.75) */
+    readonly similarityThreshold?: number;
+    /** 样本保留数(默认 3) */
+    readonly maxSamples?: number;
+    /** 默认 dismiss 天数(默认 30) */
+    readonly defaultDismissDays?: number;
+    /** 短消息过滤阈值(默认 20 字符,更短跳过;Layer 1 已用更严格规则,此字段仅作粗筛) */
+    readonly minMessageLength?: number;
+}
+/** 默认配置 */
+export declare const DEFAULT_PROPOSAL_CONFIG: Required<ProposalEngineConfig>;
+/**
+ * Proposal Engine
+ *
+ * 使用:
+ *   const engine = new ProposalEngine({ repository, embedder, auditLog, dataRoot })
+ *   await engine.observe({ role: 'user', content: '...', at: nowIso })
+ *   const pending = engine.listPending()
+ *   engine.accept(entityId, { title, content, domainTags })
+ */
+export declare class ProposalEngine {
+    private readonly repository;
+    private readonly embedder;
+    private readonly auditLog;
+    private readonly dataRoot;
+    private readonly config;
+    private readonly clustersFile;
+    private readonly proposalsFile;
+    private readonly necessityEvaluator;
+    constructor(deps: {
+        readonly repository: EngramRepository;
+        readonly embedder: Embedder;
+        readonly auditLog: AuditLog;
+        readonly dataRoot: string;
+        readonly config?: ProposalEngineConfig;
+        /**
+         * 必要性评估器(可选)。
+         * 默认 RuleBasedNecessityEvaluator(零依赖,挡机械噪声)。
+         * host 可注入 LlmNecessityEvaluator(需 LlmClient)做语义判断。
+         */
+        readonly necessityEvaluator?: NecessityEvaluator;
+    });
+    /**
+     * 观察一条对话消息
+     *
+     * 流程:
+     *   1. Layer 1 规则预过滤(零成本挡机械噪声:trivial/短消息/低密度)
+     *   2. 向量化
+     *   3. 找最相似的现有 cluster
+     *   4. 归簇 or 新建
+     *   5. 若 occurrences 达到阈值 → Layer 2 必要性评估 → 查重 → 生成 proposal
+     */
+    observe(message: {
+        readonly role: "user" | "assistant" | "system";
+        readonly content: string;
+        readonly at?: string;
+    }): Promise<void>;
+    /** 列出 status=pending 的提案 */
+    listPending(): readonly Proposal[];
+    /** 列出所有提案(包含 accepted/dismissed,调试用) */
+    listAll(): readonly Proposal[];
+    /**
+     * 接受提案 → 创建 engram
+     *
+     * @returns 新建的 engram id
+     */
+    accept(entityId: string, input: {
+        readonly title: string;
+        readonly content: string;
+        readonly domainTags: readonly string[];
+        readonly createdBy?: string;
+        readonly kind?: EngramCreateInput["kind"];
+    }): string;
+    /**
+     * 拒绝提案
+     *
+     * @param entityId 簇 id
+     * @param reason 拒绝原因(可选,便于元学习)
+     * @param dismissDays 多少天内不再提示(默认 30)
+     */
+    dismiss(entityId: string, reason?: string, dismissDays?: number): void;
+    /** 清理过期/已处理数据(测试用) */
+    clear(): void;
+    /**
+     * 检查是否已有相关 engram;若无则创建/更新 proposal
+     *
+     * 流程:
+     *   1. 查重:已有相似 engram → 跳过
+     *   2. 已 accepted → 跳过;已 dismissed 且未过期 → 跳过
+     *   3. Layer 2 必要性评估:necessity=false → 记 audit 不晋升
+     *   4. necessity=true → 生成/更新 proposal(带 necessityReason)
+     */
+    private maybePromoteToProposal;
+    /** 查重:简单子串匹配 */
+    private hasSimilarEngram;
+    private readClusters;
+    private writeClusters;
+    private readProposals;
+    private writeProposals;
+}
+/**
+ * 创建新 cluster
+ */
+export declare function newCluster(vector: readonly number[], firstSample: string, at: string): TopicCluster;
+/**
+ * 把新样本加入现有 cluster,更新质心
+ *
+ * 质心更新公式: new = old + (sample - old) / (n+1)
+ */
+export declare function addToCluster(cluster: TopicCluster, sample: string, vector: readonly number[], at: string, maxSamples: number): TopicCluster;
+/**
+ * 找最相似的 cluster(余弦)
+ *
+ * 返回 null 表示无匹配(需新建)
+ */
+export declare function findBestMatch(vector: readonly number[], clusters: readonly TopicCluster[], threshold: number): {
+    readonly cluster: TopicCluster;
+    readonly similarity: number;
+} | null;
+/** 余弦相似度 */
+export declare function cosineSimilarity(a: readonly number[], b: readonly number[]): number;
+/** 从向量派生短 id(便于调试 + 唯一索引) */
+export declare function clusterId(vector: readonly number[]): string;
+/**
+ * 默认 embedder: hash-based(无 LLM)
+ *
+ * 把文本拆词,每个词 hash 到某维度,符号累加。CJK 字符连续段会进一步切成
+ * 字符 bigram(我们/以后/所有…)——不加这一步,中文整段会被当成一个超大 token,
+ * 导致相似 Chinese 句子的余弦接近正交,proposal pipeline 在中文场景下完全失灵。
+ * 准确度低但零成本,可用于 M1 验证机制。M2 由宿主替换为真实 embedding。
+ */
+export declare const DEFAULT_HASHER_EMBEDDER: Embedder;
+/**
+ * Hash-based embedder 配套的相似度阈值。
+ *
+ * `DEFAULT_PROPOSAL_CONFIG.similarityThreshold = 0.75` 是为真实 LLM embedding
+ * 设计的;hash-based fallback 的余弦分布完全不同(同义改写最多 ~0.4,完全无关
+ * 的文本 ~0.05-0.1),用 0.75 会导致 proposal pipeline 在 hash fallback 下
+ * 完全无法成簇——任何话题被反复提及都不会生成 proposal。
+ *
+ * 0.35 是经验值:在 256 维 + CJK bigram 切分下,足以让"arrow function"
+ * 这样的核心词在 4-5 次提及后成簇,又不会把"今天天气真好"和"我们去爬山"
+ * 这种无关句子误聚到一起。宿主替换为真实 embedder 时应使用更高阈值(如 0.75)。
+ */
+export declare const DEFAULT_HASHER_SIMILARITY_THRESHOLD = 0.35;
+//# sourceMappingURL=proposal-engine.d.ts.map

package/dist/observability/proposal-engine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proposal-engine.d.ts","sourceRoot":"","sources":["../../src/observability/proposal-engine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAYH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAGL,KAAK,kBAAkB,EAExB,MAAM,0BAA0B,CAAC;AAElC,0BAA0B;AAC1B,MAAM,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;AAEpE,UAAU;AACV,MAAM,WAAW,YAAY;IAC3B,qBAAqB;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,iBAAiB;IACjB,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,WAAW;IACX,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,+BAA+B;IAC/B,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,aAAa;IACb,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,aAAa;IACb,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAED,WAAW;AACX,MAAM,WAAW,QAAQ;IACvB,qBAAqB;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,qBAAqB;IACrB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,4BAA4B;IAC5B,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,qBAAqB;IACrB,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,aAAa;IACb,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,aAAa;IACb,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,0BAA0B;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,SAAS;IACT,QAAQ,CAAC,MAAM,EAAE,SAAS,GAAG,UAAU,GAAG,WAAW,CAAC;IACtD,6BAA6B;IAC7B,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,iBAAiB;IACjB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,2BAA2B;IAC3B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,4BAA4B;IAC5B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,4CAA4C;IAC5C,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,wBAAwB;IACxB,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;CAClC;AAED,yBAAyB;AACzB,MAAM,WAAW,oBAAoB;IACnC,iBAAiB;IACjB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,uBAAuB;IACvB,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IACtC,kBAAkB;IAClB,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,2BAA2B;IAC3B,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,qDAAqD;IACrD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,WAAW;AACX,eAAO,MAAM,uBAAuB,EAAE,QAAQ,CAAC,oBAAoB,CAMlE,CAAC;AAEF;;;;;;;;GAQG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAmB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiC;IACxD,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqB;gBAE5C,IAAI,EAAE;QAChB,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;QACtC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC5B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,oBAAoB,CAAC;QACvC;;;;WAIG;QACH,QAAQ,CAAC,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KAClD;IAgBD;;;;;;;;;OASG;IACG,OAAO,CAAC,OAAO,EAAE;QACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;QAC/C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC;KACtB,GAAG,OAAO,CAAC,IAAI,CAAC;IA0EjB,4BAA4B;IAC5B,WAAW,IAAI,SAAS,QAAQ,EAAE;IAWlC,wCAAwC;IACxC,OAAO,IAAI,SAAS,QAAQ,EAAE;IAI9B;;;;OAIG;IACH,MAAM,CACJ,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE;QACL,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,CAAC;QACvC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,MAAM,CAAC,CAAC;KAC3C,GACA,MAAM;IAqCT;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI;IA6BtE,sBAAsB;IACtB,KAAK,IAAI,IAAI;IAUb;;;;;;;;OAQG;YACW,sBAAsB;IAyFpC,gBAAgB;IAChB,OAAO,CAAC,gBAAgB;IAkBxB,OAAO,CAAC,YAAY;IAqBpB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,aAAa;IAIrB,OAAO,CAAC,cAAc;CAGvB;AAMD;;GAEG;AACH,wBAAgB,UAAU,CACxB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,WAAW,EAAE,MAAM,EACnB,EAAE,EAAE,MAAM,GACT,YAAY,CASd;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,EAAE,EAAE,MAAM,EACV,UAAU,EAAE,MAAM,GACjB,YAAY,CAkBd;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,QAAQ,EAAE,SAAS,YAAY,EAAE,EACjC,SAAS,EAAE,MAAM,GAChB;IAAE,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAgBxE;AAED,YAAY;AACZ,wBAAgB,gBAAgB,CAC9B,CAAC,EAAE,SAAS,MAAM,EAAE,EACpB,CAAC,EAAE,SAAS,MAAM,EAAE,GACnB,MAAM,CAcR;AAED,6BAA6B;AAC7B,wBAAgB,SAAS,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAU3D;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB,EAAE,QAmBrC,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,mCAAmC,OAAO,CAAC"}