@yeaft/webchat-agent 0.1.645 → 0.1.647

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.645",
+  "version": "0.1.647",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/memory/ams.js

@@ -0,0 +1,197 @@
+/**
+ * memory/ams.js — DESIGN-H2-AMS §5. Active Memory Set.
+ *
+ * Per-group session state. Three layers:
+ *
+ *   resident   summaries of all relevant scopes — always-on, high precision
+ *   recent     LRU of segments touched in last N turns — warm cache
+ *   onDemand   segments the pre-flow FTS pulled in this turn — hot recall
+ *
+ * AMS is in-memory; it's rebuilt at session start. The disk source of
+ * truth is `<scope>/memory.md` + `<scope>/summary.md`. AMS itself
+ * doesn't write to disk — that's Dream's job.
+ *
+ * Privacy (DESIGN-v2 §2.2): `vp/<other>` scopes are ALWAYS filtered out
+ * for any worker that isn't `<other>`. The owning code passes its own
+ * vpId at construction.
+ */
+
+import { approxTokens, packWithinBudget } from './budget.js';
+
+const RECENT_DEFAULT_CAPACITY = 64;
+
+/**
+ * @typedef {object} AmsLayers
+ * @property {Map<string, string>}                resident
+ * @property {Array<{ id: string, seg: import('./segment.js').Segment, ts: number }>} recent
+ * @property {Map<string, import('./segment.js').Segment>} onDemand
+ */
+
+/**
+ * @typedef {object} AmsSnapshot
+ * @property {Array<{ scope: string, summary: string }>} resident
+ * @property {import('./segment.js').Segment[]} recent
+ * @property {import('./segment.js').Segment[]} onDemand
+ * @property {{ resident: number, recent: number, onDemand: number, total: number }} usage
+ */
+
+export class ActiveMemorySet {
+  /**
+   * @param {{
+   *   ownVpId?: string | null,
+   *   budget: import('./budget.js').BudgetSplit,
+   *   recentCapacity?: number,
+   * }} opts
+   */
+  constructor(opts) {
+    if (!opts || !opts.budget) throw new Error('ActiveMemorySet: budget required');
+    this.ownVpId = opts.ownVpId || null;
+    this.budget = opts.budget;
+    this.recentCapacity = opts.recentCapacity || RECENT_DEFAULT_CAPACITY;
+    /** @type {Map<string, string>} */
+    this._resident = new Map();          // scope → summaryText
+    /** @type {Map<string, { seg: import('./segment.js').Segment, ts: number }>} */
+    this._recent = new Map();            // segId → entry (insertion-order is LRU order)
+    /** @type {Map<string, import('./segment.js').Segment>} */
+    this._onDemand = new Map();          // segId → segment
+  }
+
+  // ────────────────────────── resident ──────────────────────────
+
+  /**
+   * Replace the resident layer with a fresh set of scope→summary
+   * pairs. Foreign VP scopes are silently dropped.
+   *
+   * @param {Array<{ scope: string, summary: string }>} entries
+   */
+  setResident(entries) {
+    this._resident.clear();
+    for (const e of entries) {
+      if (this._isForeignVp(e.scope)) continue;
+      if (!e.summary) continue;
+      this._resident.set(e.scope, e.summary);
+    }
+  }
+
+  // ────────────────────────── recent ──────────────────────────
+
+  /**
+   * Touch a segment as "used this turn". LRU semantics: most-recent at
+   * the end. Trims to capacity automatically.
+   *
+   * @param {import('./segment.js').Segment} seg
+   */
+  touchRecent(seg) {
+    if (!seg || !seg.id) return;
+    if (this._isForeignVp(seg.scope)) return;
+    if (this._recent.has(seg.id)) this._recent.delete(seg.id);
+    this._recent.set(seg.id, { seg, ts: Date.now() });
+    while (this._recent.size > this.recentCapacity) {
+      const firstKey = this._recent.keys().next().value;
+      this._recent.delete(firstKey);
+    }
+  }
+
+  // ────────────────────────── onDemand ──────────────────────────
+
+  /**
+   * Replace the onDemand layer with this turn's FTS hits.
+   *
+   * @param {import('./segment.js').Segment[]} segments
+   */
+  setOnDemand(segments) {
+    this._onDemand.clear();
+    for (const seg of segments) {
+      if (this._isForeignVp(seg.scope)) continue;
+      this._onDemand.set(seg.id, seg);
+    }
+  }
+
+  /**
+   * Add segments to onDemand without clearing (used by adjustMemory).
+   *
+   * @param {import('./segment.js').Segment[]} segments
+   */
+  addOnDemand(segments) {
+    for (const seg of segments) {
+      if (this._isForeignVp(seg.scope)) continue;
+      this._onDemand.set(seg.id, seg);
+    }
+  }
+
+  /**
+   * Remove segment ids from onDemand (used by adjustMemory eviction).
+   *
+   * @param {string[]} ids
+   */
+  removeOnDemand(ids) {
+    for (const id of ids) this._onDemand.delete(id);
+  }
+
+  // ────────────────────────── snapshot ──────────────────────────
+
+  /**
+   * Produce a budget-aware snapshot that can be injected into the
+   * system prompt. Each layer is greedily packed within its budget;
+   * overflow is dropped from this turn but not from disk.
+   *
+   * @returns {AmsSnapshot}
+   */
+  snapshot() {
+    // Resident: pack scopes by priority order (caller provides via insert
+    // order — current group's own vp first, then user, etc.).
+    const resEntries = [...this._resident.entries()].map(([scope, summary]) => ({
+      scope, summary,
+    }));
+    const { picked: resPicked, cost: resCost } = packWithinBudget(
+      resEntries, this.budget.resident,
+      e => approxTokens(e.summary),
+    );
+
+    // Recent: insertion order is oldest-first; we want newest first.
+    const recentArr = [...this._recent.values()]
+      .reverse()
+      .map(e => e.seg);
+    const { picked: recPicked, cost: recCost } = packWithinBudget(
+      recentArr, this.budget.recent,
+      seg => approxTokens(seg.body),
+    );
+
+    // OnDemand: insertion order from caller (already FTS-ranked).
+    const odArr = [...this._onDemand.values()];
+    const { picked: odPicked, cost: odCost } = packWithinBudget(
+      odArr, this.budget.onDemand,
+      seg => approxTokens(seg.body),
+    );
+
+    return {
+      resident: resPicked,
+      recent: recPicked,
+      onDemand: odPicked,
+      usage: {
+        resident: resCost,
+        recent: recCost,
+        onDemand: odCost,
+        total: resCost + recCost + odCost,
+      },
+    };
+  }
+
+  /**
+   * Read-only inspectors (for tests / observability / adjustMemory input).
+   */
+  residentScopes() { return [...this._resident.keys()]; }
+  recentIds()      { return [...this._recent.keys()]; }
+  onDemandIds()    { return [...this._onDemand.keys()]; }
+  onDemandSegments() { return [...this._onDemand.values()]; }
+  size() { return this._resident.size + this._recent.size + this._onDemand.size; }
+
+  // ────────────────────────── privacy ──────────────────────────
+
+  _isForeignVp(scope) {
+    if (!scope || !scope.startsWith('vp/')) return false;
+    if (!this.ownVpId) return false;        // no own id → no filtering
+    const other = scope.slice(3).split('/')[0];
+    return other !== this.ownVpId;
+  }
+}

package/unify/memory/budget.js

@@ -0,0 +1,111 @@
+/**
+ * memory/budget.js — DESIGN-H2-AMS §5.2.
+ *
+ * Memory budget = `min(50_000, modelMaxContext * 0.10)`.
+ *
+ * Then split across the three AMS layers (resident / recent / onDemand)
+ * with a configurable ratio. The defaults are tuned for ~200k context
+ * models (Claude / GPT-5):
+ *
+ *   resident   40%   →  20k   (all relevant scope summaries)
+ *   recent     25%   →  12.5k (LRU of recently-used segments)
+ *   onDemand   35%   →  17.5k (this turn's FTS recall)
+ *
+ * Token counting here is approximate (chars / 4) — accurate enough for
+ * budget enforcement. The engine has a real tokenizer for prompt
+ * assembly; budget here is a guard rail, not the source of truth.
+ */
+
+export const ABSOLUTE_CAP = 50_000;
+export const MODEL_FRACTION = 0.10;
+
+export const DEFAULT_RATIO = {
+  resident: 0.40,
+  recent:   0.25,
+  onDemand: 0.35,
+};
+
+/**
+ * @typedef {object} BudgetSplit
+ * @property {number} total
+ * @property {number} resident
+ * @property {number} recent
+ * @property {number} onDemand
+ */
+
+/**
+ * @param {number} modelMaxContext   tokens of the model's full context window
+ * @param {Partial<typeof DEFAULT_RATIO>} [ratio]
+ * @returns {BudgetSplit}
+ */
+export function computeBudget(modelMaxContext, ratio = {}) {
+  const ctx = Number.isFinite(modelMaxContext) && modelMaxContext > 0
+    ? modelMaxContext : 200_000;
+  const total = Math.min(ABSOLUTE_CAP, Math.floor(ctx * MODEL_FRACTION));
+
+  const r = { ...DEFAULT_RATIO, ...ratio };
+  // Normalise so ratios sum to 1 (defensive).
+  const sum = r.resident + r.recent + r.onDemand;
+  const norm = sum > 0 ? sum : 1;
+  return {
+    total,
+    resident: Math.floor(total * (r.resident / norm)),
+    recent:   Math.floor(total * (r.recent   / norm)),
+    onDemand: Math.floor(total * (r.onDemand / norm)),
+  };
+}
+
+/**
+ * Approximate token count of a string. Avg English ≈ 4 chars / token,
+ * Chinese ≈ 1 char / token. We use a conservative blended estimate.
+ *
+ * @param {string} text
+ * @returns {number}
+ */
+export function approxTokens(text) {
+  if (!text) return 0;
+  // Count CJK chars as ~1 token each, the rest as char/4.
+  let cjk = 0;
+  let other = 0;
+  for (const ch of text) {
+    const c = ch.codePointAt(0) || 0;
+    if (
+      (c >= 0x4e00 && c <= 0x9fff) ||
+      (c >= 0x3040 && c <= 0x309f) ||
+      (c >= 0x30a0 && c <= 0x30ff) ||
+      (c >= 0xac00 && c <= 0xd7af)
+    ) {
+      cjk += 1;
+    } else {
+      other += 1;
+    }
+  }
+  return Math.ceil(cjk + other / 4);
+}
+
+/**
+ * Greedy pack: pick items in order until adding the next would exceed
+ * the budget. Returns the picked list and the total cost. Does NOT
+ * sort — caller decides ordering.
+ *
+ * @template T
+ * @param {T[]} items
+ * @param {number} budget
+ * @param {(item: T) => number} costFn
+ * @returns {{ picked: T[], cost: number, dropped: T[] }}
+ */
+export function packWithinBudget(items, budget, costFn) {
+  const picked = [];
+  const dropped = [];
+  let cost = 0;
+  for (const it of items) {
+    const c = costFn(it);
+    if (cost + c <= budget) {
+      picked.push(it);
+      cost += c;
+    } else {
+      dropped.push(it);
+    }
+  }
+  return { picked, cost, dropped };
+}

package/unify/memory/keywords.js

@@ -0,0 +1,57 @@
+/**
+ * keywords.js — pure-rule keyword extraction shared by memory recall paths.
+ *
+ * Extracted from the legacy R5 recall.js so recall-v2.js (and any future
+ * recall path) can use it without dragging in the rest of the R5 module.
+ * Pure CPU, no LLM, <1ms.
+ */
+
+/** Common stop words filtered out before frequency counting. */
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+  'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
+  'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
+  'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
+  'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
+  'where', 'why', 'how', 'all', 'both', 'each', 'few', 'more', 'most',
+  'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same',
+  'so', 'than', 'too', 'very', 'just', 'because', 'but', 'and', 'or',
+  'if', 'while', 'about', 'up', 'it', 'its', 'my', 'me', 'i', 'you',
+  'your', 'we', 'our', 'they', 'them', 'their', 'this', 'that', 'what',
+  'which', 'who', 'whom', 'these', 'those',
+  // Chinese stop words
+  '的', '了', '在', '是', '我', '有', '和', '就',
+  '不', '人', '都', '一', '一个', '上', '也',
+  '很', '到', '说', '要', '去', '你', '会',
+  '着', '没有', '看', '好', '自己', '这',
+  '他', '她', '吗', '呢', '吧', '把', '被',
+  '那', '它', '让', '给', '可以', '什么',
+  '怎么', '帮', '帮我', '请', '能', '想',
+]);
+
+/**
+ * Extract keywords from a prompt (pure rules, no LLM).
+ *
+ * @param {string} prompt
+ * @returns {string[]} keywords sorted by frequency descending then alpha.
+ */
+export function extractKeywords(prompt) {
+  if (!prompt || !prompt.trim()) return [];
+
+  // Tokenize: split on whitespace + punctuation, keep CJK chars.
+  const tokens = prompt
+    .toLowerCase()
+    .replace(/[^\w\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]+/g, ' ')
+    .split(/\s+/)
+    .filter(t => t.length > 1 && !STOP_WORDS.has(t));
+
+  const freq = new Map();
+  for (const t of tokens) {
+    freq.set(t, (freq.get(t) || 0) + 1);
+  }
+
+  return [...freq.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([word]) => word);
+}

package/unify/memory/preflow.js

@@ -0,0 +1,173 @@
+/**
+ * memory/preflow.js — DESIGN-H2-AMS §6. Pre-turn memory recall.
+ *
+ * Pure-CPU pipeline (no LLM). Runs on every turn:
+ *
+ *   userMsg
+ *     → extractKeywords (rule-based tokeniser)
+ *     → SQLite FTS5 MATCH (scope-filtered, bm25 ranked)
+ *     → rerank by (scope match + tag overlap + recency)
+ *     → onDemand layer (budget-clamped)
+ *
+ * Latency target: < 10ms p95 on a 10k-segment index.
+ *
+ * Honours `vp/<other>` privacy: caller passes `ownVpId` and the scope
+ * filter excludes foreign VP scopes.
+ */
+
+import { extractKeywords } from './keywords.js';
+import { approxTokens } from './budget.js';
+
+/**
+ * @typedef {object} PreflowOptions
+ * @property {string}        userMsg
+ * @property {string[]}      relevantScopes      e.g. ['user', 'group/g1', 'vp/alice']
+ * @property {string|null}  [ownVpId]
+ * @property {string[]}     [currentTags]        tags from the current group/feature context
+ * @property {number}       [topK]               max FTS rows to fetch (default 50)
+ * @property {number}       [budgetTokens]       onDemand budget (caller-supplied)
+ */
+
+/**
+ * @typedef {object} PreflowResult
+ * @property {string[]}                                  keywords
+ * @property {string}                                     ftsQuery
+ * @property {import('./index-db.js').SearchHit[]}       hits
+ * @property {import('./segment.js').Segment[]}          picked
+ * @property {number}                                     pickedTokens
+ * @property {number}                                     droppedCount
+ */
+
+/**
+ * Run the pre-flow against a segment index.
+ *
+ * @param {import('./index-db.js').SegmentIndex} index
+ * @param {PreflowOptions} opts
+ * @returns {PreflowResult}
+ */
+export function runPreflow(index, opts) {
+  const userMsg = (opts.userMsg || '').trim();
+  const relevantScopes = Array.isArray(opts.relevantScopes) ? opts.relevantScopes : [];
+  const ownVpId = opts.ownVpId || null;
+  const currentTags = Array.isArray(opts.currentTags) ? opts.currentTags : [];
+  const topK = Number.isFinite(opts.topK) && opts.topK > 0 ? opts.topK : 50;
+  const budgetTokens = Number.isFinite(opts.budgetTokens) && opts.budgetTokens > 0
+    ? opts.budgetTokens : Infinity;
+
+  const keywords = extractKeywords(userMsg);
+  if (keywords.length === 0) {
+    return {
+      keywords: [], ftsQuery: '', hits: [],
+      picked: [], pickedTokens: 0, droppedCount: 0,
+    };
+  }
+
+  const ftsQuery = buildFtsQuery(keywords);
+  const scopeFilter = filterScopes(relevantScopes, ownVpId);
+  if (scopeFilter.length === 0) {
+    return {
+      keywords, ftsQuery, hits: [],
+      picked: [], pickedTokens: 0, droppedCount: 0,
+    };
+  }
+
+  const hits = index.search({ query: ftsQuery, scopeFilter, limit: topK });
+  const reranked = rerank(hits, { currentTags });
+
+  const picked = [];
+  let cost = 0;
+  let dropped = 0;
+  for (const h of reranked) {
+    const tk = approxTokens(h.body);
+    if (cost + tk <= budgetTokens) {
+      picked.push(toSegment(h));
+      cost += tk;
+    } else {
+      dropped += 1;
+    }
+  }
+
+  return {
+    keywords, ftsQuery, hits: reranked,
+    picked, pickedTokens: cost, droppedCount: dropped,
+  };
+}
+
+/**
+ * Compose an FTS5 MATCH query from keywords. Each keyword is OR'd with
+ * a prefix wildcard so morphological variants match. We escape any
+ * FTS5-special characters by quoting tokens.
+ *
+ * @param {string[]} keywords
+ * @returns {string}
+ */
+export function buildFtsQuery(keywords) {
+  const cleaned = keywords
+    .map(k => k.replace(/"/g, ''))
+    .filter(k => k.length > 1)
+    .slice(0, 8);   // top-8 keywords — avoid query bloat
+  if (cleaned.length === 0) return '';
+  return cleaned.map(k => `"${k}"*`).join(' OR ');
+}
+
+/**
+ * Strip foreign VP scopes from the filter list (privacy).
+ *
+ * @param {string[]} scopes
+ * @param {string|null} ownVpId
+ * @returns {string[]}
+ */
+export function filterScopes(scopes, ownVpId) {
+  return scopes.filter(s => {
+    if (!s.startsWith('vp/')) return true;
+    if (!ownVpId) return true;
+    const other = s.slice(3).split('/')[0];
+    return other === ownVpId;
+  });
+}
+
+/**
+ * Rerank FTS hits with two soft signals on top of bm25:
+ *   - tag overlap with the current group/feature context (subtract penalty)
+ *   - recency: recent items get a small bonus
+ *
+ * SQLite FTS5 bm25 returns NEGATIVE numbers (more negative = better
+ * match). We treat lower score as better. To make overlap & recency
+ * push hits ahead, we SUBTRACT bonuses from the bm25 base (making the
+ * score more negative).
+ *
+ * @param {import('./index-db.js').SearchHit[]} hits
+ * @param {{ currentTags: string[] }} ctx
+ * @returns {import('./index-db.js').SearchHit[]}
+ */
+export function rerank(hits, ctx) {
+  const tagSet = new Set((ctx.currentTags || []).map(t => String(t).toLowerCase()));
+  const now = Date.now();
+  return [...hits]
+    .map(h => {
+      const overlap = (h.tags || []).reduce(
+        (n, t) => n + (tagSet.has(String(t).toLowerCase()) ? 1 : 0), 0,
+      );
+      const tagBonus = Math.min(2, overlap * 0.5);   // up to 2 points
+      const ageDays = Math.max(0, (now - Date.parse(h.updatedAt || h.createdAt || '')) / 86400000);
+      const recencyBonus = Math.min(0.5, 0.2 / Math.max(0.5, ageDays + 1));
+      const base = h.rank ?? 0;
+      const score = base - tagBonus - recencyBonus;
+      return { ...h, _score: score };
+    })
+    .sort((a, b) => a._score - b._score)
+    .map(({ _score, ...rest }) => rest);
+}
+
+function toSegment(h) {
+  return {
+    id: h.id,
+    scope: h.scope,
+    kind: h.kind,
+    tags: h.tags,
+    sourceMessages: h.sourceMessages,
+    body: h.body,
+    createdAt: h.createdAt,
+    updatedAt: h.updatedAt,
+  };
+}

package/unify/memory/summary-store.js

@@ -0,0 +1,63 @@
+/**
+ * memory/summary-store.js — DESIGN-H2-AMS §3.
+ *
+ * `summary.md` is a bounded, per-scope prose digest derived from all
+ * segments in that scope. Resident AMS layer = concatenation of all
+ * relevant scope summaries. Regenerated by Dream after segments change.
+ *
+ * Layout:
+ *   ~/.yeaft/memory/<scope>/summary.md
+ *
+ * Format: plain prose, optionally prefixed with a one-line metadata
+ * header `<!-- updatedAt: ISO -->`. No required schema beyond non-empty
+ * text.
+ */
+
+import {
+  readFileSync, writeFileSync, existsSync, mkdirSync, renameSync,
+} from 'node:fs';
+import { join, dirname } from 'node:path';
+
+/**
+ * @param {string} memoryRoot
+ * @param {string} scope
+ * @returns {string}
+ */
+export function summaryPath(memoryRoot, scope) {
+  return join(memoryRoot, scope, 'summary.md');
+}
+
+/**
+ * Read summary text for a scope (empty string if missing).
+ *
+ * @param {string} memoryRoot
+ * @param {string} scope
+ * @returns {string}
+ */
+export function readSummary(memoryRoot, scope) {
+  const p = summaryPath(memoryRoot, scope);
+  if (!existsSync(p)) return '';
+  return stripHeader(readFileSync(p, 'utf8')).trim();
+}
+
+/**
+ * Atomic write summary text. Empty string = wipe content but keep file.
+ *
+ * @param {string} memoryRoot
+ * @param {string} scope
+ * @param {string} text
+ */
+export function writeSummary(memoryRoot, scope, text) {
+  const p = summaryPath(memoryRoot, scope);
+  mkdirSync(dirname(p), { recursive: true });
+  const header = `<!-- updatedAt: ${new Date().toISOString()} -->\n`;
+  const body = (text || '').trim();
+  const final = body ? `${header}${body}\n` : header;
+  const tmp = `${p}.tmp`;
+  writeFileSync(tmp, final, 'utf8');
+  renameSync(tmp, p);
+}
+
+function stripHeader(text) {
+  return text.replace(/^<!--[^>]*-->\s*/m, '');
+}