@jack200714/mafw 4.5.1

package/gateway/dist/core/compression/hybrid-compressor.js

@@ -0,0 +1,104 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HybridCompressor = void 0;
+const zero_token_compressor_1 = require("./zero-token-compressor");
+const harmonic_types_1 = require("../memory/harmonic-types");
+const salience_perceptor_1 = require("../memory/salience-perceptor");
+const harmonic_file_store_1 = require("../../memory/harmonic-file-store");
+class HybridCompressor {
+    zeroToken;
+    energyThreshold = 0.6;
+    gatewayUrl;
+    harmonicIndex;
+    baseDir;
+    constructor(gatewayUrl, harmonicIndex, baseDir) {
+        this.zeroToken = new zero_token_compressor_1.ZeroTokenCompressor();
+        this.gatewayUrl = gatewayUrl || process.env.MAFW_GATEWAY_URL || 'http://127.0.0.1:3000';
+        this.harmonicIndex = harmonicIndex;
+        this.baseDir = baseDir;
+    }
+    async compress(observations) {
+        if (observations.length === 0) {
+            return this.makeUnit(observations, [], 0);
+        }
+        const highEnergy = observations.filter(o => (o.energy ?? 0.5) > this.energyThreshold);
+        const useHighEnergy = highEnergy.length > 0 ? highEnergy : observations;
+        const energy = useHighEnergy.reduce((s, o) => s + (o.energy ?? 0.5), 0) / useHighEnergy.length;
+        const memoryType = this.detectMemoryType(useHighEnergy);
+        try {
+            const res = await fetch(`${this.gatewayUrl}/api/llm/compress`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ observations: useHighEnergy.map(o => o.content || '') }),
+                signal: AbortSignal.timeout(10000)
+            });
+            if (res.ok) {
+                const result = await res.json();
+                const unit = this.makeUnit(useHighEnergy, result.concepts || [], result.energy ?? energy);
+                unit.type = memoryType;
+                unit.primary_abstraction = (result.narrative || 'compressed').slice(0, 50);
+                unit.cue_anchors = Array.isArray(result.concepts) ? result.concepts.slice(0, 8) : [];
+                unit.memory_value = result.narrative || useHighEnergy.map(o => o.content).join('\n');
+                await this.persistUnit(unit);
+                return unit;
+            }
+        }
+        catch { }
+        const unit = this.makeUnit(useHighEnergy, [], energy);
+        unit.type = memoryType;
+        unit.primary_abstraction = (useHighEnergy[0]?.content || 'rule-compressed').slice(0, 50);
+        unit.cue_anchors = this.extractConcepts(useHighEnergy).slice(0, 8);
+        unit.memory_value = useHighEnergy.map(o => o.content).join('\n').slice(0, 1000);
+        await this.persistUnit(unit);
+        return unit;
+    }
+    detectMemoryType(observations) {
+        const allContent = observations.map(o => o.content || '').join(' ');
+        if (observations.every(o => o.type === 'file_edit' || (o.content && o.content.includes('File modified'))))
+            return 'procedural';
+        if (/plan|wave|task|goal/i.test(allContent))
+            return 'episodic';
+        return 'semantic';
+    }
+    makeUnit(observations, concepts, energy) {
+        const allContent = observations.map(o => o.content || '').join(' ');
+        const memoryType = this.detectMemoryType(observations);
+        return {
+            id: (0, harmonic_types_1.generateHarmonicId)(),
+            type: memoryType,
+            primary_abstraction: 'compressed',
+            cue_anchors: concepts.slice(0, 8),
+            memory_value: '',
+            energy: Math.max(0, Math.min(1, energy)),
+            salience: (0, salience_perceptor_1.calculateSalience)(allContent),
+            abstraction_level: memoryType === 'procedural' ? 2 : memoryType === 'episodic' ? 1 : 2,
+            created_at: new Date().toISOString(),
+            updated_at: new Date().toISOString()
+        };
+    }
+    async persistUnit(unit) {
+        if (!this.baseDir || !this.harmonicIndex)
+            return;
+        const store = new harmonic_file_store_1.HarmonicUnitFileStore(this.baseDir, this.harmonicIndex);
+        await store.write(unit);
+    }
+    extractConcepts(observations) {
+        const concepts = new Set();
+        const patterns = [
+            { regex: /(jwt|oauth|bcrypt|auth)/i, concept: 'authentication' },
+            { regex: /(error|fail|exception)/i, concept: 'error' },
+            { regex: /(coverage|test|jest)/i, concept: 'testing' },
+            { regex: /(api|endpoint|route)/i, concept: 'api' },
+            { regex: /(config|setting|env)/i, concept: 'configuration' }
+        ];
+        for (const obs of observations) {
+            const content = obs.content || '';
+            for (const { regex, concept } of patterns) {
+                if (regex.test(content))
+                    concepts.add(concept);
+            }
+        }
+        return Array.from(concepts);
+    }
+}
+exports.HybridCompressor = HybridCompressor;

package/gateway/dist/core/compression/lesson-compactor.js

@@ -0,0 +1,114 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LessonCompactor = void 0;
+/**
+ * Lesson Compactor — L2 结构化压实器
+ *
+ * 职责：将自然语言 Lesson 压缩为结构化 YAML，减少约 75% token。
+ *
+ * 压缩规则（从文档 §3.2 提取）：
+ *   1. 保留结构化字段：loop, trigger, result, domain, task, violation, root_cause, details, lesson, energy, files
+ *   2. 移除冗余描述：自然语言中的背景说明、重复信息
+ *   3. 转为 YAML 格式，便于后续 MemoryExtractor 解析
+ *   4. 压缩阈值：自然语言 lesson > 300 tokens 时触发
+ *
+ * 压缩前 (自然语言，约 500 tokens) → 压缩后 (YAML，约 120 tokens，压缩率 76%)
+ */
+class LessonCompactor {
+    COMPACTION_THRESHOLD = 300;
+    /**
+     * 判断是否需要压缩
+     */
+    shouldCompact(raw) {
+        const tokens = this.estimateTokens(raw.lesson);
+        return tokens > this.COMPACTION_THRESHOLD;
+    }
+    /**
+     * 执行压缩：RawLesson → CompactedLesson
+     */
+    compact(raw) {
+        const originalTokens = this.estimateTokens(raw.lesson);
+        const compacted = {
+            loop: raw.loop,
+            trigger: raw.trigger,
+            result: raw.result,
+            domain: raw.domain,
+            task: raw.task,
+            violation: raw.violation,
+            root_cause: raw.root_cause,
+            details: raw.details,
+            lesson: this.summarizeLesson(raw.lesson),
+            energy: raw.energy,
+            files: raw.files,
+            metrics: raw.metrics,
+            applied_deltas: raw.applied_deltas,
+            created_at: raw.created_at,
+            _compacted: true,
+            _originalTokens: originalTokens,
+            _compactedTokens: this.estimateTokens(this.summarizeLesson(raw.lesson))
+        };
+        const errors = this.verifyCompaction(raw, compacted);
+        return {
+            success: errors.length === 0,
+            original: raw,
+            compacted,
+            errors
+        };
+    }
+    /**
+     * 将自然语言 lesson 提炼为简洁的陈述句。
+     * 规则：
+     *   - 移除"在实现...时"、"这导致..."等背景
+     *   - 保留"必须"、"禁止"、"应该"等动作词
+     *   - 控制在 30 个中文词或 50 个英文词以内
+     */
+    summarizeLesson(lesson) {
+        // 简单启发式：提取包含"必须/禁止/应该/不得/需要"的句子
+        const sentences = lesson.split(/[。\.!！?？]/);
+        const core = sentences.filter(s => /(必须|禁止|应该|不得|需要|需|务必|严禁|确保|检查|验证|引用|查阅|对比|禁止|不能|不准|不允许|must|must not|should|need to|ensure|check|verify|reference|compare)/i.test(s));
+        if (core.length > 0) {
+            return core.map(s => s.trim()).join('。').substring(0, 200);
+        }
+        //  fallback：直接截断
+        return lesson.substring(0, 120).trim() + (lesson.length > 120 ? '...' : '');
+    }
+    /**
+     * 压缩完整性验证（5 项检查清单）
+     */
+    verifyCompaction(original, compacted) {
+        const errors = [];
+        // 1. loop 不能丢失
+        if (compacted.loop !== original.loop)
+            errors.push('loop 字段丢失');
+        // 2. result 不能丢失
+        if (compacted.result !== original.result)
+            errors.push('result 字段丢失');
+        // 3. domain 不能丢失
+        if (compacted.domain !== original.domain)
+            errors.push('domain 字段丢失');
+        // 4. lesson 核心动作不能丢失（检查是否包含关键动词）
+        const originalVerbs = this.extractActionVerbs(original.lesson);
+        const compactedVerbs = this.extractActionVerbs(compacted.lesson);
+        if (originalVerbs.length > 0 && compactedVerbs.length === 0) {
+            errors.push('lesson 核心动作丢失');
+        }
+        // 5. violation 若存在，rule 不能丢失
+        if (original.violation && (!compacted.violation || !compacted.violation.rule)) {
+            errors.push('violation.rule 字段丢失');
+        }
+        return errors;
+    }
+    extractActionVerbs(text) {
+        const verbs = ['必须', '禁止', '应该', '不得', '需要', '务必', '确保', '检查', '验证', '引用', '查阅', '对比', 'must', 'must not', 'should', 'need to', 'ensure', 'check', 'verify', 'reference', 'compare'];
+        return verbs.filter(v => text.toLowerCase().includes(v.toLowerCase()));
+    }
+    /**
+     * 粗略 token 估算
+     */
+    estimateTokens(text) {
+        const chineseChars = (text.match(/[\u4e00-\u9fff]/g) || []).length;
+        const englishWords = text.split(/\s+/).filter(w => /[a-zA-Z]/.test(w)).length;
+        return Math.ceil(chineseChars + englishWords * 0.75);
+    }
+}
+exports.LessonCompactor = LessonCompactor;

package/gateway/dist/core/compression/memory-index.js

@@ -0,0 +1,190 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryIndexManager = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const bm25_index_1 = require("./bm25-index");
+/**
+ * Memory Index — L2 倒排索引检索器
+ *
+ * 职责：为 lessons 文件建立轻量级倒排索引，Plan Agent 启动时
+ * 根据 Goal Charter 关键词检索最多 3 条相关 lessons，总 token ≤ 1200。
+ *
+ * 不需要向量数据库。单个 Goal 内 lessons 通常 < 20 条。
+ *
+ * 索引结构：
+ *   {
+ *     "version": "1",
+ *     "entries": [...],
+ *     "inverted_index": { "jwt": ["id1"], "coverage": ["id1", "id2"] },
+ *     "domain_index": { "auth": ["id1"], "viz": ["id2"] }
+ *   }
+ */
+class MemoryIndexManager {
+    indexPath;
+    index;
+    bm25;
+    constructor(indexPath = '.mafw/memory-index.json') {
+        this.indexPath = indexPath;
+        this.index = this.load();
+        this.bm25 = new bm25_index_1.BM25Index();
+    }
+    load() {
+        if (!fs.existsSync(this.indexPath)) {
+            return { version: '1', entries: [], inverted_index: {}, domain_index: {} };
+        }
+        return JSON.parse(fs.readFileSync(this.indexPath, 'utf-8'));
+    }
+    save() {
+        fs.writeFileSync(this.indexPath, JSON.stringify(this.index, null, 2), 'utf-8');
+    }
+    /**
+     * 添加一个 lesson 条目到索引
+     */
+    add(entry) {
+        // 去重：相同 ID 更新
+        const existingIdx = this.index.entries.findIndex(e => e.id === entry.id);
+        if (existingIdx >= 0) {
+            this.index.entries[existingIdx] = entry;
+        }
+        else {
+            this.index.entries.push(entry);
+        }
+        // 更新倒排索引
+        for (const tag of entry.tags) {
+            if (!this.index.inverted_index[tag])
+                this.index.inverted_index[tag] = [];
+            if (!this.index.inverted_index[tag].includes(entry.id)) {
+                this.index.inverted_index[tag].push(entry.id);
+            }
+        }
+        // 更新 domain 索引 (使用 goal 字段)
+        if (entry.goal) {
+            if (!this.index.domain_index[entry.goal])
+                this.index.domain_index[entry.goal] = [];
+            if (!this.index.domain_index[entry.goal].includes(entry.id)) {
+                this.index.domain_index[entry.goal].push(entry.id);
+            }
+        }
+        // Also index into BM25 for keyword search
+        this.bm25.addDocument(entry.id, entry.tags.join(' '));
+    }
+    /**
+     * 根据关键词检索相关 lessons，最多 3 条，按 energy 排序。
+     */
+    search(keywords, domain, maxResults = 3) {
+        const scores = new Map();
+        // 1. 关键词匹配（倒排索引）
+        for (const kw of keywords) {
+            const lowerKw = kw.toLowerCase();
+            for (const [tag, ids] of Object.entries(this.index.inverted_index)) {
+                if (tag.toLowerCase().includes(lowerKw)) {
+                    for (const id of ids) {
+                        scores.set(id, (scores.get(id) || 0) + 1);
+                    }
+                }
+            }
+        }
+        // 2. domain 匹配（加权）
+        if (domain && this.index.domain_index[domain]) {
+            for (const id of this.index.domain_index[domain]) {
+                scores.set(id, (scores.get(id) || 0) + 2);
+            }
+        }
+        // 3. 按分数 + energy 排序，取前 N
+        const scored = Array.from(scores.entries())
+            .map(([id, score]) => {
+            const entry = this.index.entries.find(e => e.id === id);
+            return { id, score, entry };
+        })
+            .filter(x => x.entry)
+            .sort((a, b) => {
+            if (b.score !== a.score)
+                return b.score - a.score;
+            return (b.entry.energy || 0) - (a.entry.energy || 0);
+        })
+            .slice(0, maxResults)
+            .map(x => x.entry);
+        return scored;
+    }
+    /**
+     * BM25 关键词搜索
+     */
+    searchBM25(query, topK = 10) {
+        return this.bm25.search(query, topK);
+    }
+    /**
+     * 从关键词自动提取 tags（简单分词）
+     */
+    static extractTags(text) {
+        const words = text
+            .toLowerCase()
+            .replace(/[^\u4e00-\u9fff\w\s]/g, ' ')
+            .split(/\s+/)
+            .filter(w => w.length >= 2);
+        return Array.from(new Set(words));
+    }
+    /**
+     * 重建完整索引（用于手动清理后）
+     */
+    rebuild(lessonsDir = '.mafw/lessons') {
+        this.index = { version: '1', entries: [], inverted_index: {}, domain_index: {} };
+        if (!fs.existsSync(lessonsDir))
+            return;
+        const files = fs.readdirSync(lessonsDir).filter(f => f.endsWith('.md'));
+        for (const f of files) {
+            const content = fs.readFileSync(path.join(lessonsDir, f), 'utf-8');
+            // 简单解析：按 "### Loop N" 分割
+            const loops = content.split(/###\s+Loop\s+\d+/).slice(1);
+            // 这里只做占位，实际需更复杂的 YAML front-matter 解析
+            // 为简化，假设每个 lesson 文件已包含 YAML 块
+            const tags = MemoryIndexManager.extractTags(content);
+            const entry = {
+                id: `${f.replace('.md', '')}-loop-1`,
+                file: path.join(lessonsDir, f),
+                anchor: 'L1',
+                tags,
+                energy: 0.5,
+                type: 'constraint_source',
+                loop: 1,
+                goal: f.replace('.md', '')
+            };
+            this.add(entry);
+        }
+        this.save();
+    }
+}
+exports.MemoryIndexManager = MemoryIndexManager;

package/gateway/dist/core/compression/observation-classifier.js

@@ -0,0 +1,41 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObservationClassifier = void 0;
+class ObservationClassifier {
+    classify(observation) {
+        const knownTypes = ['tool_use', 'file_edit', 'llm_call', 'error', 'state_change'];
+        if (observation.type && knownTypes.includes(observation.type)) {
+            return observation.type;
+        }
+        const content = observation.content || '';
+        const metadata = observation.metadata || {};
+        if (content.includes('Tool:') || metadata.toolName) {
+            return 'tool_use';
+        }
+        if (content.includes('File modified') || metadata.filePath) {
+            return 'file_edit';
+        }
+        if (content.includes('LLM call') || content.includes('llm_call')) {
+            return 'llm_call';
+        }
+        if (content.includes('Error:') || metadata.errorType || metadata.success === false) {
+            return 'error';
+        }
+        if (content.includes('state_change') || content.includes('phase')) {
+            return 'state_change';
+        }
+        return 'tool_use';
+    }
+    classifyBatch(observations) {
+        const groups = new Map();
+        for (const obs of observations) {
+            const type = this.classify(obs);
+            if (!groups.has(type)) {
+                groups.set(type, []);
+            }
+            groups.get(type).push(obs);
+        }
+        return groups;
+    }
+}
+exports.ObservationClassifier = ObservationClassifier;

package/gateway/dist/core/compression/observation-deduplicator.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObservationDeduplicator = void 0;
+const crypto_1 = require("crypto");
+class ObservationDeduplicator {
+    recentHashes;
+    windowMs;
+    constructor(windowMs = 5 * 60 * 1000) {
+        this.recentHashes = new Map();
+        this.windowMs = windowMs;
+    }
+    deduplicate(observations) {
+        this.cleanExpired();
+        const result = [];
+        const now = Date.now();
+        for (const obs of observations) {
+            const hash = this.computeHash(obs);
+            const existing = this.recentHashes.get(hash);
+            if (existing !== undefined && (now - existing) < this.windowMs) {
+                continue;
+            }
+            this.recentHashes.set(hash, now);
+            result.push(obs);
+        }
+        return result;
+    }
+    clear() {
+        this.recentHashes.clear();
+    }
+    computeHash(obs) {
+        const content = `${obs.type}:${obs.phase}:${obs.content}`;
+        return (0, crypto_1.createHash)('sha256').update(content).digest('hex').substring(0, 16);
+    }
+    cleanExpired() {
+        const now = Date.now();
+        for (const [hash, timestamp] of this.recentHashes) {
+            if ((now - timestamp) >= this.windowMs) {
+                this.recentHashes.delete(hash);
+            }
+        }
+    }
+}
+exports.ObservationDeduplicator = ObservationDeduplicator;

package/gateway/dist/core/compression/rrf-fusion.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.reciprocalRankFusion = reciprocalRankFusion;
+exports.diversifyByLoop = diversifyByLoop;
+function reciprocalRankFusion(first, ...rest) {
+    let k = 60;
+    let resultLists;
+    if (first === undefined) {
+        return [];
+    }
+    if (typeof first === 'number') {
+        k = first;
+        resultLists = rest;
+    }
+    else {
+        resultLists = [first, ...rest];
+    }
+    const fused = new Map();
+    for (const list of resultLists) {
+        for (let rank = 0; rank < list.length; rank++) {
+            const item = list[rank];
+            const contribution = 1 / (k + rank);
+            if (fused.has(item.id)) {
+                const existing = fused.get(item.id);
+                existing.score += contribution;
+                if (item.text !== undefined)
+                    existing.text = item.text;
+                if (item.metadata !== undefined)
+                    existing.metadata = item.metadata;
+                if (item.loopNum !== undefined)
+                    existing.loopNum = item.loopNum;
+            }
+            else {
+                fused.set(item.id, {
+                    score: contribution,
+                    text: item.text,
+                    metadata: item.metadata,
+                    loopNum: item.loopNum
+                });
+            }
+        }
+    }
+    return Array.from(fused.entries())
+        .map(([id, data]) => ({ id, ...data }))
+        .sort((a, b) => b.score - a.score);
+}
+function diversifyByLoop(results, maxPerLoop = 3) {
+    const counts = new Map();
+    const output = [];
+    for (const item of results) {
+        if (item.loopNum === undefined) {
+            output.push(item);
+        }
+        else {
+            const count = counts.get(item.loopNum) ?? 0;
+            if (count < maxPerLoop) {
+                counts.set(item.loopNum, count + 1);
+                output.push(item);
+            }
+        }
+    }
+    return output;
+}