@jack200714/mafw 4.5.1

package/gateway/dist/memory/harmonic-file-store.js

@@ -0,0 +1,338 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HarmonicUnitFileStore = void 0;
+exports.onMemoryWritten = onMemoryWritten;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const harmonic_index_1 = require("../core/memory/harmonic-index");
+const cognitive_graph_1 = require("../core/memory/cognitive-graph");
+const okf_writer_1 = require("./okf-writer");
+const okf_parser_1 = require("./okf-parser");
+const event_log_1 = require("./event-log");
+const write_queue_1 = require("./write-queue");
+const anchor_graph_1 = require("../graph/anchor-graph");
+const derived_terms_1 = require("./derived-terms");
+const minhash_merger_1 = require("../core/memory/minhash-merger");
+const globalWriteQueues = new Map();
+const writeListeners = [];
+function onMemoryWritten(cb) {
+    writeListeners.push(cb);
+}
+class HarmonicUnitFileStore {
+    baseDir;
+    indexManager;
+    graphManager;
+    eventLog;
+    writeQueue;
+    anchorGraph;
+    minHashMerger;
+    anchorGraphStore;
+    constructor(baseDir, indexManager, anchorGraphStore) {
+        this.baseDir = baseDir;
+        this.indexManager = indexManager || new harmonic_index_1.HarmonicIndexManager(baseDir);
+        this.graphManager = new cognitive_graph_1.CognitiveGraphManager(baseDir);
+        this.eventLog = new event_log_1.EventLog(baseDir);
+        const normalizedDir = path.resolve(baseDir);
+        let queue = globalWriteQueues.get(normalizedDir);
+        if (!queue) {
+            queue = new write_queue_1.WriteQueue();
+            globalWriteQueues.set(normalizedDir, queue);
+        }
+        this.writeQueue = queue;
+        this.anchorGraph = new anchor_graph_1.AnchorGraph();
+        this.minHashMerger = new minhash_merger_1.MinHashMerger();
+        this.anchorGraphStore = anchorGraphStore;
+    }
+    async write(unit, tier, opts) {
+        let fileName = '';
+        let writtenUnit = unit;
+        // C: auto-fill missing cue_anchors from the abstraction so every memory
+        // has retrievable anchors (only 13% of entries had them previously).
+        if (!unit.cue_anchors || unit.cue_anchors.length === 0) {
+            unit.cue_anchors = extractAnchors(unit.primary_abstraction);
+        }
+        await this.writeQueue.enqueue(async () => {
+            // Cross-tier merge check: if the incoming unit is highly similar to an
+            // existing memory, fold the existing one into it (skipped when the unit
+            // is itself the product of a merge, or the caller opts out — e.g. the
+            // LongMemEval benchmark ingests one deterministic unit per round and must
+            // not be collapsed).
+            let targetUnit = unit;
+            if (!opts?.skipMerge) {
+                const merged = await this.minHashMerger.merge(unit, this.indexManager, {
+                    read: (id) => this.read(id),
+                    deleteSync: (id) => this.deleteSync(id),
+                    markSuperseded: (id, byId) => this.markSuperseded(id, byId),
+                });
+                targetUnit = merged.merged_from?.length ? merged : unit;
+            }
+            writtenUnit = targetUnit;
+            fileName = (0, okf_writer_1.writeOKFFile)(this.baseDir, targetUnit);
+            const entryTier = tier || (targetUnit.type === 'semantic' && targetUnit.granularity ? 'knowledge'
+                : targetUnit.type === 'procedural' ? 'procedural'
+                    : targetUnit.type === 'episodic' ? 'episodic'
+                        : targetUnit.type === 'global' ? 'global'
+                            : 'semantic');
+            this.indexManager.addEntry({
+                id: targetUnit.id,
+                type: targetUnit.type,
+                primary_abstraction: targetUnit.primary_abstraction,
+                cue_anchors: targetUnit.cue_anchors,
+                tier: entryTier,
+                energy: targetUnit.energy,
+                salience: targetUnit.salience,
+                superseded_by: targetUnit.superseded_by,
+                pinned: targetUnit.pinned,
+                sticky_until: targetUnit.sticky_until,
+                merged_from: targetUnit.merged_from,
+                // filePath must reflect the ACTUAL on-disk directory (getOKFDirectory
+                // keys off unit.type), not the tier label — a tier arg such as 'tier3'
+                // does not relocate the file. read() resolves via this path.
+                filePath: path.join('memory', (0, okf_writer_1.getOKFDirectory)(targetUnit), fileName).replace(/\\/g, '/'),
+                source_session_id: targetUnit.source_session_id,
+            }, entryTier);
+            // 锚点图（多跳检索）增量更新——失败不影响记忆写入（降级）
+            try {
+                this.anchorGraphStore?.upsertUnit(targetUnit.id, targetUnit.cue_anchors ?? []);
+            }
+            catch { /* non-fatal */ }
+            const linkRegex = /\[\[([^\]]+)\]\]/g;
+            let match;
+            while ((match = linkRegex.exec(targetUnit.memory_value)) !== null) {
+                this.graphManager.addConnection(targetUnit.id, match[1]);
+            }
+        });
+        // Extract derived terms (index-only, not written to frontmatter)
+        const derivedTerms = (0, derived_terms_1.extractDerivedTerms)(unit.memory_value);
+        // Add to anchor graph for implicit edge computation
+        this.anchorGraph.addUnit(unit.id, unit.cue_anchors);
+        // Log via event log
+        this.eventLog.appendWrite(unit.id, {
+            primary: unit.primary_abstraction,
+            anchors: unit.cue_anchors,
+            terms: (0, derived_terms_1.tokenize)(unit.primary_abstraction + ' ' + unit.cue_anchors.join(' ')),
+            derived: derivedTerms,
+        });
+        // Static post-write listeners (dense embedding indexing, consolidation).
+        for (const cb of writeListeners) {
+            try {
+                cb(writtenUnit);
+            }
+            catch { /* non-fatal */ }
+        }
+        return fileName;
+    }
+    async read(id) {
+        const index = this.indexManager.getIndex();
+        const entry = index.entries.find(e => e.id === id);
+        if (!entry || !entry.filePath)
+            return null;
+        const fullPath = path.join(this.baseDir, entry.filePath);
+        if (!fs.existsSync(fullPath))
+            return null;
+        const { unit, body } = (0, okf_parser_1.readOKFFile)(fullPath);
+        return {
+            ...unit,
+            memory_value: body,
+            updated_at: new Date().toISOString(),
+        };
+    }
+    indexManager_() {
+        return this.indexManager;
+    }
+    /**
+     * Mark an existing memory as superseded by a newer unit instead of deleting
+     * it. Lowers its energy and writes `superseded_by` into the OKF frontmatter.
+     * Called from inside the serialized write path (MinHash merge).
+     */
+    markSuperseded(id, byId) {
+        const entry = this.indexManager.getIndex().entries.find(e => e.id === id);
+        if (!entry || !entry.filePath)
+            return false;
+        const fullPath = path.join(this.baseDir, entry.filePath);
+        if (!fs.existsSync(fullPath))
+            return false;
+        const { unit, body } = (0, okf_parser_1.readOKFFile)(fullPath);
+        unit.superseded_by = byId;
+        unit.energy = Math.max(0.1, (unit.energy ?? entry.energy) * 0.5);
+        unit.updated_at = new Date().toISOString();
+        const yaml = require('js-yaml');
+        const yamlStr = yaml.dump(unit, { lineWidth: -1, quotingType: '"' });
+        const tmpPath = fullPath + '.tmp';
+        fs.writeFileSync(tmpPath, `---\n${yamlStr}---\n${body}\n`, 'utf-8');
+        fs.renameSync(tmpPath, fullPath);
+        entry.energy = unit.energy;
+        entry.superseded_by = byId;
+        this.indexManager.save();
+        try {
+            this.anchorGraphStore?.removeUnit(id);
+        }
+        catch { /* non-fatal */ }
+        return true;
+    }
+    /**
+     * Pin/unpin an existing memory for the disclosure layer. Rewrites the OKF
+     * frontmatter and updates the index entry. Returns false when id is unknown.
+     */
+    setPinned(id, pinned) {
+        const entry = this.indexManager.getIndex().entries.find(e => e.id === id);
+        if (!entry || !entry.filePath)
+            return false;
+        const fullPath = path.join(this.baseDir, entry.filePath);
+        if (!fs.existsSync(fullPath))
+            return false;
+        const { unit, body } = (0, okf_parser_1.readOKFFile)(fullPath);
+        unit.pinned = pinned;
+        unit.updated_at = new Date().toISOString();
+        const yaml = require('js-yaml');
+        const yamlStr = yaml.dump(unit, { lineWidth: -1, quotingType: '"' });
+        const tmpPath = fullPath + '.tmp';
+        fs.writeFileSync(tmpPath, `---\n${yamlStr}---\n${body}\n`, 'utf-8');
+        fs.renameSync(tmpPath, fullPath);
+        entry.pinned = pinned;
+        this.indexManager.save();
+        return true;
+    }
+    /**
+     * Stick/unstick an existing memory on the note board. Rewrites the OKF
+     * frontmatter and updates the index entry. `until === null` removes the
+     * sticky marker (memory itself is untouched). Returns false when id is
+     * unknown. */
+    setSticky(id, until) {
+        const entry = this.indexManager.getIndex().entries.find(e => e.id === id);
+        if (!entry || !entry.filePath)
+            return false;
+        const fullPath = path.join(this.baseDir, entry.filePath);
+        if (!fs.existsSync(fullPath))
+            return false;
+        const { unit, body } = (0, okf_parser_1.readOKFFile)(fullPath);
+        if (until === null) {
+            delete unit.sticky_until;
+            delete entry.sticky_until;
+        }
+        else {
+            unit.sticky_until = until;
+            entry.sticky_until = until;
+        }
+        unit.updated_at = new Date().toISOString();
+        const yaml = require('js-yaml');
+        const yamlStr = yaml.dump(unit, { lineWidth: -1, quotingType: '"' });
+        const tmpPath = fullPath + '.tmp';
+        fs.writeFileSync(tmpPath, `---\n${yamlStr}---\n${body}\n`, 'utf-8');
+        fs.renameSync(tmpPath, fullPath);
+        this.indexManager.save();
+        return true;
+    }
+    /**
+     * Synchronous delete used inside the serialized write path (merge) and
+     * exposed for tests. Removes the index entry and the OKF file.
+     */
+    deleteSync(id) {
+        const entry = this.indexManager.getIndex().entries.find(e => e.id === id);
+        if (!entry)
+            return false;
+        if (entry.filePath) {
+            const fullPath = path.join(this.baseDir, entry.filePath);
+            if (fs.existsSync(fullPath))
+                fs.unlinkSync(fullPath);
+        }
+        this.indexManager.removeEntry(id);
+        try {
+            this.anchorGraphStore?.removeUnit(id);
+        }
+        catch { /* non-fatal */ }
+        return true;
+    }
+    async delete(id) {
+        let removed = false;
+        await this.writeQueue.enqueue(async () => {
+            const index = this.indexManager.getIndex();
+            const entry = index.entries.find(e => e.id === id);
+            if (!entry)
+                return;
+            removed = true;
+            if (entry.filePath) {
+                const fullPath = path.join(this.baseDir, entry.filePath);
+                if (fs.existsSync(fullPath))
+                    fs.unlinkSync(fullPath);
+            }
+            this.indexManager.removeEntry(id);
+            try {
+                this.anchorGraphStore?.removeUnit(id);
+            }
+            catch { /* non-fatal */ }
+        });
+        return removed;
+    }
+    async archive(id) {
+        const index = this.indexManager.getIndex();
+        const entry = index.entries.find(e => e.id === id);
+        if (!entry || !entry.filePath)
+            return;
+        const fullPath = path.join(this.baseDir, entry.filePath);
+        if (!fs.existsSync(fullPath))
+            return;
+        const { unit, body } = (0, okf_parser_1.readOKFFile)(fullPath);
+        unit.archived = true;
+        unit.updated_at = new Date().toISOString();
+        const yaml = require('js-yaml');
+        const yamlStr = yaml.dump(unit, { lineWidth: -1, quotingType: '"' });
+        const tmpPath = fullPath + '.tmp';
+        fs.writeFileSync(tmpPath, `---\n${yamlStr}---\n${body}\n`, 'utf-8');
+        fs.renameSync(tmpPath, fullPath);
+    }
+}
+exports.HarmonicUnitFileStore = HarmonicUnitFileStore;
+/** Extract 2-5 cue anchors from a primary_abstraction when none were provided.
+ *  Picks the most frequent lowercase words (len>=4) plus notable tokens. */
+function extractAnchors(abstraction) {
+    if (!abstraction)
+        return [];
+    const words = abstraction.toLowerCase().split(/[^a-z0-9]+/).filter(w => w.length >= 4);
+    const freq = new Map();
+    for (const w of words)
+        freq.set(w, (freq.get(w) || 0) + 1);
+    const sorted = [...freq.entries()].sort((a, b) => b[1] - a[1]);
+    const stop = new Set(['with', 'this', 'that', 'from', 'have', 'must', 'should', 'when', 'into', 'will', 'also', 'they', 'them', 'then', 'than', 'which', 'their', 'there']);
+    const anchors = sorted.filter(([w]) => !stop.has(w)).map(([w]) => w);
+    // ensure coverage: pick first 5 distinct tokens (word-level), fall back to CJK
+    if (anchors.length < 2) {
+        const cjk = abstraction.match(/[\u4e00-\u9fff]/g) || [];
+        for (const c of [...new Set(cjk)].slice(0, 3))
+            anchors.push(c);
+    }
+    return anchors.slice(0, 5);
+}