@jack200714/mafw 4.5.1

package/gateway/dist/core/memory/observation-service.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObservationService = exports.SESSION_TIMEOUT_MS = void 0;
+exports.SESSION_TIMEOUT_MS = 60 * 60 * 1000; // 1 hour
+class ObservationService {
+    t1Store;
+    compressor;
+    maxContent;
+    sessionTurns;
+    sessionStarted;
+    constructor(config) {
+        this.t1Store = config.t1Store;
+        this.compressor = config.compressor;
+        this.maxContent = config.maxContentLength || 2000;
+        this.sessionTurns = new Map();
+        this.sessionStarted = new Map();
+    }
+    captureUserInput(sessionID, text, goalId) {
+        if (!text || !sessionID)
+            return;
+        const turn = (this.sessionTurns.get(sessionID) || 0) + 1;
+        this.sessionTurns.set(sessionID, turn);
+        if (!this.sessionStarted.has(sessionID)) {
+            this.sessionStarted.set(sessionID, Date.now());
+        }
+        this.t1Store.append({
+            content: this.truncate(text),
+            source: 'user_input',
+            sessionID,
+            turnID: turn,
+            goalId,
+            timestamp: Date.now(),
+            energy: 0.8,
+        }, goalId);
+    }
+    captureAssistantReply(sessionID, text, goalId) {
+        if (!text || !sessionID)
+            return;
+        const turn = this.sessionTurns.get(sessionID) || 0;
+        this.t1Store.append({
+            content: this.truncate(text),
+            source: 'assistant_reply',
+            sessionID,
+            turnID: turn,
+            goalId,
+            timestamp: Date.now(),
+            energy: 0.75,
+        }, goalId);
+    }
+    captureToolResult(sessionID, toolName, output, args, goalId, loopNum) {
+        if (!sessionID || (!output && !args))
+            return;
+        const turn = this.sessionTurns.get(sessionID) || 0;
+        const content = args
+            ? `[${toolName}] args: ${this.truncate(args || '')}\noutput: ${this.truncate(output || '')}`
+            : `[${toolName}] ${this.truncate(output || '')}`;
+        this.t1Store.append({
+            content,
+            source: 'tool_result',
+            toolName,
+            sessionID,
+            turnID: turn,
+            loopNum,
+            goalId,
+            timestamp: Date.now(),
+            energy: 0.7,
+        }, goalId);
+    }
+    async endSession(sessionID, goalId) {
+        if (!sessionID)
+            return;
+        const started = this.sessionStarted.get(sessionID);
+        const elapsed = started ? Date.now() - started : 0;
+        if (elapsed <= exports.SESSION_TIMEOUT_MS && this.t1Store.hasSession(sessionID, goalId)) {
+            await this.compressor.compressSession(sessionID, goalId);
+        }
+        this.sessionTurns.delete(sessionID);
+        this.sessionStarted.delete(sessionID);
+    }
+    truncate(text) {
+        if (text.length <= this.maxContent)
+            return text;
+        const half = Math.floor((this.maxContent - 20) / 2);
+        return text.substring(0, half) + '\n...[truncated]...\n' + text.substring(text.length - half);
+    }
+}
+exports.ObservationService = ObservationService;

package/gateway/dist/core/memory/privacy-filter.js

@@ -0,0 +1,163 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PrivacyFilter = void 0;
+const logger_1 = require("../utils/logger");
+const DEFAULT_RULES = [
+    {
+        name: 'api_key',
+        pattern: /(api[_-]?key|apikey)\s*[:=]\s*["']?[a-zA-Z0-9_-]{20,}["']?/gi,
+        replacement: '$1: [REDACTED_API_KEY]',
+        severity: 'critical',
+    },
+    {
+        name: 'aws_access_key',
+        pattern: /AKIA[0-9A-Z]{16}/g,
+        replacement: '[REDACTED_AWS_KEY]',
+        severity: 'critical',
+    },
+    {
+        name: 'password',
+        pattern: /(password|passwd|pwd)\s*[:=]\s*["']?[^\s"']+["']?/gi,
+        replacement: '$1: [REDACTED_PASSWORD]',
+        severity: 'critical',
+    },
+    {
+        name: 'bearer_token',
+        pattern: /Bearer\s+[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+/g,
+        replacement: 'Bearer [REDACTED_JWT]',
+        severity: 'high',
+    },
+    {
+        name: 'private_tag',
+        pattern: /<private>.*?<\/private>/gs,
+        replacement: '<private>[REDACTED]</private>',
+        severity: 'high',
+    },
+    {
+        name: 'email',
+        pattern: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g,
+        replacement: '[REDACTED_EMAIL]',
+        severity: 'medium',
+    },
+    {
+        name: 'ip_address',
+        pattern: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g,
+        replacement: '[REDACTED_IP]',
+        severity: 'low',
+    },
+    {
+        name: 'credit_card',
+        pattern: /\b(?:\d{4}[-\s]?){3}\d{4}\b/g,
+        replacement: '[REDACTED_CC]',
+        severity: 'critical',
+    },
+];
+class PrivacyFilter {
+    rules;
+    enabled;
+    logRedactions;
+    activeRules;
+    constructor(config) {
+        this.enabled = config?.enabled ?? true;
+        this.logRedactions = config?.logRedactions ?? false;
+        this.activeRules = config?.rules ?? [];
+        const rules = [...DEFAULT_RULES];
+        if (config?.customPatterns) {
+            for (const cp of config.customPatterns) {
+                const existing = rules.findIndex(r => r.name === cp.name);
+                const rule = {
+                    name: cp.name,
+                    pattern: new RegExp(cp.pattern, 'g'),
+                    replacement: cp.replacement,
+                    severity: 'high',
+                };
+                if (existing >= 0) {
+                    rules[existing] = rule;
+                }
+                else {
+                    rules.push(rule);
+                }
+            }
+        }
+        this.rules = rules;
+    }
+    filter(text) {
+        if (!this.enabled) {
+            return { filtered: text, redactions: [] };
+        }
+        const redactions = [];
+        const rulesToApply = this.activeRules.length === 0
+            ? this.rules
+            : this.rules.filter(r => this.activeRules.includes(r.name));
+        let filtered = text;
+        for (const rule of rulesToApply) {
+            const regex = new RegExp(rule.pattern.source, rule.pattern.flags);
+            const replacements = [];
+            let m;
+            while ((m = regex.exec(filtered)) !== null) {
+                const match = m;
+                let replacement = rule.replacement;
+                replacement = replacement.replace(/\$(\d+)/g, (_, n) => match[parseInt(n)] ?? '');
+                replacements.push({
+                    offset: m.index,
+                    original: m[0],
+                    replacement,
+                });
+                if (m.index === regex.lastIndex) {
+                    regex.lastIndex++;
+                }
+            }
+            for (let i = replacements.length - 1; i >= 0; i--) {
+                const r = replacements[i];
+                redactions.push({
+                    rule: rule.name,
+                    severity: rule.severity,
+                    position: { start: r.offset, end: r.offset + r.original.length },
+                    originalLength: r.original.length,
+                    timestamp: new Date().toISOString(),
+                });
+                filtered = filtered.slice(0, r.offset) + r.replacement + filtered.slice(r.offset + r.original.length);
+            }
+        }
+        if (this.logRedactions && redactions.length > 0) {
+            logger_1.log.info(`[PrivacyFilter] Redacted ${redactions.length} items`);
+        }
+        return { filtered, redactions };
+    }
+    filterObservation(obs) {
+        if (!obs || typeof obs !== 'object')
+            return obs;
+        const result = Array.isArray(obs) ? [...obs] : { ...obs };
+        if (typeof result.content === 'string') {
+            const { filtered, redactions } = this.filter(result.content);
+            result.content = filtered;
+            if (redactions.length > 0) {
+                result.metadata = {
+                    ...(result.metadata || {}),
+                    redactions,
+                };
+            }
+        }
+        return result;
+    }
+    setEnabledRules(rules) {
+        this.activeRules = rules;
+    }
+    enable() {
+        this.enabled = true;
+    }
+    disable() {
+        this.enabled = false;
+    }
+    getStats() {
+        const names = this.activeRules.length === 0
+            ? this.rules.map(r => r.name)
+            : this.activeRules;
+        return {
+            enabled: this.enabled,
+            rules: this.rules.length,
+            activeRules: names,
+        };
+    }
+}
+exports.PrivacyFilter = PrivacyFilter;

package/gateway/dist/core/memory/reranker.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrossEncoderReranker = exports.HeuristicReranker = void 0;
+exports.createReranker = createReranker;
+exports.applyReranker = applyReranker;
+const os = __importStar(require("os"));
+const path = __importStar(require("path"));
+// Prefer domestic mirror on Windows/China networks unless explicitly configured.
+if (typeof process !== 'undefined') {
+    process.env.HF_ENDPOINT = process.env.HF_ENDPOINT || 'https://hf-mirror.com';
+    process.env.HF_HOME = process.env.HF_HOME || path.join(os.homedir(), '.mafw', 'models', 'huggingface');
+}
+function minMax(values) {
+    return { min: Math.min(...values), max: Math.max(...values) };
+}
+function normalize(values) {
+    const { min, max } = minMax(values);
+    if (max === min)
+        return values.map(() => 0.5);
+    return values.map(v => (v - min) / (max - min));
+}
+function parseDate(d) {
+    if (!d)
+        return null;
+    const parsed = new Date(d);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+/**
+ * Zero-dependency reranker that fuses BM25 score with intra-candidate recency,
+ * energy, and salience. Recency is computed relative to the candidate set so it
+ * still works when every memory is old (e.g. LongMemEval haystacks from 2023).
+ */
+class HeuristicReranker {
+    name = 'heuristic';
+    weights;
+    cutoffRatio;
+    constructor(opts = {}) {
+        this.weights = {
+            bm25: opts.weights?.bm25 ?? 0.6,
+            recency: opts.weights?.recency ?? 0.2,
+            energy: opts.weights?.energy ?? 0.1,
+            salience: opts.weights?.salience ?? 0.1,
+        };
+        this.cutoffRatio = opts.cutoffRatio ?? 0.3;
+    }
+    rerank(_query, candidates, topK) {
+        if (candidates.length === 0)
+            return [];
+        const bm25Scores = candidates.map(c => c.score);
+        const normBm25 = normalize(bm25Scores);
+        const dates = candidates.map(c => parseDate(c.entry.created_at));
+        const validDates = dates.filter((d) => d !== null);
+        let recency;
+        if (validDates.length >= 2) {
+            const minT = Math.min(...validDates.map(d => d.getTime()));
+            const maxT = Math.max(...validDates.map(d => d.getTime()));
+            recency = dates.map(d => {
+                if (!d)
+                    return 0.5;
+                if (maxT === minT)
+                    return 0.5;
+                return (d.getTime() - minT) / (maxT - minT);
+            });
+        }
+        else {
+            recency = candidates.map(() => 0.5);
+        }
+        const energies = candidates.map(c => c.entry.energy ?? 0.5);
+        const saliences = candidates.map(c => {
+            const s = c.entry.salience;
+            return s === undefined ? 1.0 : s;
+        });
+        const normSalience = normalize(saliences);
+        const fused = candidates.map((c, i) => ({
+            ...c,
+            fusedScore: this.weights.bm25 * normBm25[i] +
+                this.weights.recency * recency[i] +
+                this.weights.energy * energies[i] +
+                this.weights.salience * normSalience[i],
+        }));
+        fused.sort((a, b) => b.fusedScore - a.fusedScore);
+        const threshold = this.cutoffRatio > 0 && fused.length > 0
+            ? fused[0].fusedScore * this.cutoffRatio
+            : -Infinity;
+        const kept = fused.filter(c => c.fusedScore >= threshold);
+        return kept.slice(0, topK).map(({ entry, score }) => ({ entry, score }));
+    }
+}
+exports.HeuristicReranker = HeuristicReranker;
+/**
+ * Optional local cross-encoder reranker (bge-reranker-base via @huggingface/transformers v4).
+ * Uses text-classification pipeline with proper pair semantics for query×memory scoring.
+ * Falls back to HeuristicReranker when the dependency or model is unavailable.
+ *
+ * v4 supports HF_ENDPOINT env var natively (no manual env.remoteHost hack needed).
+ */
+class CrossEncoderReranker {
+    name = 'cross-encoder';
+    modelName = 'Xenova/bge-reranker-base';
+    classifier = null;
+    fallback = new HeuristicReranker();
+    loading = null;
+    constructor() { }
+    async ensureClassifier() {
+        if (this.classifier)
+            return;
+        if (this.loading)
+            return this.loading;
+        this.loading = (async () => {
+            try {
+                const mod = await new Function('spec', 'return import(spec)')('@huggingface/transformers');
+                // v4 has env.remoteHost but doesn't auto-read HF_ENDPOINT; set it manually
+                mod.env.remoteHost = process.env.HF_ENDPOINT || 'https://hf-mirror.com/';
+                this.classifier = await mod.pipeline('text-classification', this.modelName);
+            }
+            catch (err) {
+                // eslint-disable-next-line no-console
+                console.warn(`[CrossEncoderReranker] failed to load ${this.modelName}, falling back to heuristic: ${err.message}`);
+            }
+        })();
+        return this.loading;
+    }
+    async rerank(query, candidates, topK) {
+        await this.ensureClassifier();
+        if (!this.classifier) {
+            return this.fallback.rerank(query, candidates, topK);
+        }
+        if (candidates.length === 0)
+            return [];
+        // v4 text-classification supports pair semantics: [[query, doc], ...]
+        const pairs = candidates.map(c => [query, c.entry.primary_abstraction + ' ' + (c.entry.cue_anchors || []).join(' ')]);
+        const outputs = await this.classifier(pairs);
+        // Extract scores from outputs (v4 returns array of {label, score} objects)
+        const ceScores = outputs.map((o) => {
+            // bge-reranker outputs LABEL_0 (irrelevant) and LABEL_1 (relevant)
+            // We want the relevance score
+            if (Array.isArray(o)) {
+                // Multi-label output: find LABEL_1 score
+                const label1 = o.find((x) => x.label === 'LABEL_1');
+                return label1 ? label1.score : 0;
+            }
+            // Single label output: score is the confidence
+            return o.label === 'LABEL_1' ? o.score : (1 - o.score);
+        });
+        // Fuse cross-encoder score with original BM25 score (both min-max normalized).
+        const normBm25 = normalize(candidates.map(c => c.score));
+        const normCe = normalize(ceScores);
+        const fused = candidates.map((c, i) => ({
+            ...c,
+            fusedScore: 0.5 * normBm25[i] + 0.5 * normCe[i],
+        }));
+        fused.sort((a, b) => b.fusedScore - a.fusedScore);
+        return fused.slice(0, topK).map(({ entry, score }) => ({ entry, score }));
+    }
+}
+exports.CrossEncoderReranker = CrossEncoderReranker;
+function createReranker(name, weights) {
+    if (name === 'off')
+        return null;
+    if (name === 'heuristic')
+        return new HeuristicReranker({ weights });
+    return new CrossEncoderReranker();
+}
+/**
+ * Apply a (possibly async) reranker to scored candidates and truncate.
+ * Handles optional post-rerank cutoff.
+ */
+async function applyReranker(query, candidates, reranker, topK, cutoffRatio = 0) {
+    let scored = await Promise.resolve(reranker.rerank(query, candidates, topK));
+    if (cutoffRatio > 0 && scored.length > 0) {
+        const threshold = scored[0].score * cutoffRatio;
+        scored = scored.filter(s => s.score >= threshold);
+    }
+    return scored.slice(0, topK);
+}

package/gateway/dist/core/memory/review-scheduler.js

@@ -0,0 +1,100 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReviewScheduler = void 0;
+exports.getNextInterval = getNextInterval;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+function getNextInterval(reviewCount) {
+    return Math.pow(2, reviewCount);
+}
+class ReviewScheduler {
+    indexManager;
+    baseDir;
+    timer = null;
+    queue = [];
+    constructor(indexManager, baseDir) {
+        this.indexManager = indexManager;
+        this.baseDir = baseDir;
+    }
+    start(periodMs = 3600000) {
+        this.timer = setInterval(() => this.tick(), periodMs);
+    }
+    stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+    }
+    getReviewQueue() {
+        return [...this.queue];
+    }
+    tick() {
+        const index = this.indexManager.getIndex();
+        const nowTime = this.now();
+        const tasks = [];
+        for (const entry of index.entries) {
+            if (entry.energy < 0.5)
+                continue;
+            const reviewCount = entry.review_count || 0;
+            const interval = getNextInterval(reviewCount);
+            const daysSince = this.calcDaysSince(entry, nowTime);
+            if (daysSince >= interval) {
+                tasks.push({
+                    unitId: entry.id,
+                    primaryAbstraction: entry.primary_abstraction,
+                    overdue: daysSince - interval
+                });
+            }
+        }
+        tasks.sort((a, b) => b.overdue - a.overdue);
+        this.queue = tasks.slice(0, 3);
+        const queuePath = path.join(this.baseDir, 'memory', '.review_queue.json');
+        const tmpPath = queuePath + '.tmp';
+        fs.writeFileSync(tmpPath, JSON.stringify(this.queue, null, 2), 'utf-8');
+        fs.renameSync(tmpPath, queuePath);
+    }
+    now() {
+        return Date.now();
+    }
+    calcDaysSince(entry, nowTime) {
+        const lastReviewed = entry.last_reviewed || entry.created_at;
+        if (!lastReviewed)
+            return 0;
+        const lastTime = new Date(lastReviewed).getTime();
+        return (nowTime - lastTime) / (24 * 60 * 60 * 1000);
+    }
+}
+exports.ReviewScheduler = ReviewScheduler;

package/gateway/dist/core/memory/salience-perceptor.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.calculateSalience = calculateSalience;
+exports.importanceToSalience = importanceToSalience;
+function calculateSalience(text) {
+    if (!text)
+        return 1.0;
+    const HIGH = [/error|crash|fail|丢失|数据损坏|宕机|critical|emergency|严重/i];
+    const LOW = [/info|success|完成|正常|debug|trace|verbose/i];
+    if (HIGH.some(p => p.test(text)))
+        return 1.5;
+    if (LOW.some(p => p.test(text)))
+        return 0.5;
+    return 1.0;
+}
+/** Map an explicit importance score (1-10) to salience, bridging the regex
+ *  three-tier scale (0.5 / 1.0 / 1.5). Out-of-range or missing → 1.0. */
+function importanceToSalience(importance) {
+    if (typeof importance !== 'number' || !Number.isFinite(importance))
+        return 1.0;
+    if (importance < 1 || importance > 10)
+        return 1.0;
+    return 0.5 + ((importance - 1) / 9);
+}