@yeaft/webchat-agent 0.1.646 → 0.1.648

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.646",
+  "version": "0.1.648",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/memory/adjust.js

@@ -0,0 +1,303 @@
+/**
+ * memory/adjust.js — DESIGN-H2-AMS §7. Post-turn LLM AMS adjustment.
+ *
+ * Pre-flow uses FTS keyword recall — fast but coarse. Some semantically
+ * relevant segments will be missed; some FTS hits won't be relevant to
+ * THIS turn. adjustMemory is the LLM-grade correction step:
+ *
+ *   - Sees the full visible memory (all reachable scopes, privacy-filtered).
+ *   - Outputs `add` (segments to pull into AMS.onDemand that pre-flow missed)
+ *     and `evict` (segments currently in AMS that this turn didn't need).
+ *   - Does NOT modify segment bodies. Does NOT create new segments. Only
+ *     manipulates AMS membership.
+ *
+ * Triggered conditionally — typical session shape is "hot turn skips,
+ * adjust runs every 5–10 turns or on first turn":
+ *
+ *   shouldRunAdjust =
+ *        (newMemoryWritten && onDemand.size >= 5)
+ *     || (turnTokenUsage    > totalBudget * 0.9)
+ *     || (!session.adjustRanThisSession)        // first-turn guarantee
+ *
+ * The trigger lives at the call site (engine post-turn hook); this
+ * module just exposes the policy + the LLM round-trip.
+ */
+
+import { approxTokens } from './budget.js';
+
+/**
+ * @typedef {object} AdjustTriggerInput
+ * @property {boolean} newMemoryWritten
+ * @property {number}  onDemandSize
+ * @property {number}  turnTokenUsage
+ * @property {number}  totalBudget
+ * @property {boolean} adjustRanThisSession
+ */
+
+/**
+ * Pure decision function: should adjustMemory run this turn?
+ *
+ * @param {AdjustTriggerInput} input
+ * @returns {{ run: boolean, reason: string }}
+ */
+export function shouldRunAdjust(input) {
+  if (!input) return { run: false, reason: 'no input' };
+  if (!input.adjustRanThisSession) {
+    return { run: true, reason: 'first-turn-guarantee' };
+  }
+  if (input.turnTokenUsage > input.totalBudget * 0.9) {
+    return { run: true, reason: 'budget-pressure' };
+  }
+  if (input.newMemoryWritten && input.onDemandSize >= 5) {
+    return { run: true, reason: 'new-memory+onDemand-saturated' };
+  }
+  return { run: false, reason: 'no-trigger' };
+}
+
+/**
+ * Build the candidate visible-segments list for the LLM, applying
+ * privacy and a per-scope summarisation cap so prompts stay bounded.
+ *
+ * If a scope holds more than `bodyCap` segments, we replace each
+ * segment body with its first sentence + tags (cheap summary). This
+ * keeps the adjust prompt < ~10k tokens even when the user has
+ * thousands of segments.
+ *
+ * @param {{
+ *   index: import('./index-db.js').SegmentIndex,
+ *   scopes: string[],
+ *   ownVpId: string|null,
+ *   currentAmsIds: Set<string>,
+ *   bodyCap?: number,
+ * }} args
+ * @returns {Array<{
+ *   id: string, scope: string, kind: string, tags: string[],
+ *   body: string, inAMS: boolean, summarised: boolean,
+ * }>}
+ */
+export function buildVisibleSegments(args) {
+  const bodyCap = Number.isFinite(args.bodyCap) && args.bodyCap > 0
+    ? args.bodyCap : 200;
+  const visibleScopes = args.scopes.filter(s => isOwnOrNonVp(s, args.ownVpId));
+  const out = [];
+  for (const scope of visibleScopes) {
+    const segs = args.index.listByScope(scope);
+    const summarise = segs.length > bodyCap;
+    for (const s of segs) {
+      out.push({
+        id: s.id,
+        scope: s.scope,
+        kind: s.kind,
+        tags: s.tags || [],
+        body: summarise ? firstSentence(s.body) : s.body,
+        inAMS: args.currentAmsIds.has(s.id),
+        summarised: summarise,
+      });
+    }
+  }
+  return out;
+}
+
+function firstSentence(body) {
+  if (!body) return '';
+  const m = /^([^.!?。！？\n]+[.!?。！？]?)/.exec(body.trim());
+  return m ? m[1].trim() : body.slice(0, 200);
+}
+
+function isOwnOrNonVp(scope, ownVpId) {
+  if (!scope.startsWith('vp/')) return true;
+  if (!ownVpId) return true;
+  const other = scope.slice(3).split('/')[0];
+  return other === ownVpId;
+}
+
+/**
+ * Build the prompt the LLM sees. Bilingual-friendly — the engine's
+ * regular system prompt provides language; this is just the user-turn
+ * payload.
+ *
+ * @param {object} args
+ * @returns {string}
+ */
+export function buildAdjustPrompt(args) {
+  const {
+    userMsg, assistantReply, residentScopes, recentIds, onDemandIds,
+    visibleSegments,
+  } = args;
+
+  const lines = [];
+  lines.push('# AMS Adjustment Task');
+  lines.push('');
+  lines.push('You are managing the Active Memory Set (AMS) for the current session.');
+  lines.push('Decide which memory segments should be ADDED to AMS.onDemand and which');
+  lines.push('should be EVICTED, based on what this turn actually needed.');
+  lines.push('');
+  lines.push('## Current turn');
+  lines.push('### user');
+  lines.push(truncate(userMsg, 4000));
+  lines.push('### assistant');
+  lines.push(truncate(assistantReply, 4000));
+  lines.push('');
+  lines.push('## Current AMS state');
+  lines.push(`resident scopes: ${residentScopes.join(', ') || '(none)'}`);
+  lines.push(`recent ids: ${recentIds.slice(0, 50).join(', ') || '(none)'}`);
+  lines.push(`onDemand ids: ${onDemandIds.join(', ') || '(none)'}`);
+  lines.push('');
+  lines.push('## Visible memory segments');
+  lines.push('Each row: [inAMS] id | scope | kind | tags | body');
+  for (const seg of visibleSegments) {
+    lines.push(
+      `[${seg.inAMS ? 'X' : ' '}] ${seg.id} | ${seg.scope} | ${seg.kind} | ` +
+      `${(seg.tags || []).join(',')} | ${truncate(seg.body, 240)}`,
+    );
+  }
+  lines.push('');
+  lines.push('## Output format');
+  lines.push('Reply with a single JSON object on its own line:');
+  lines.push('```json');
+  lines.push('{ "add": ["seg_..."], "evict": ["seg_..."], "reason": "<one line>" }');
+  lines.push('```');
+  lines.push('Rules: use only ids from the visible list; never repeat an id in both');
+  lines.push('arrays; keep evict ⊆ current onDemand; keep add ∩ current onDemand = ∅.');
+  return lines.join('\n');
+}
+
+function truncate(text, max) {
+  if (!text) return '';
+  if (text.length <= max) return text;
+  return `${text.slice(0, max)}…`;
+}
+
+/**
+ * Parse the LLM's reply. Tolerant: extracts the first JSON object,
+ * coerces missing arrays to []. Returns null on hard parse failure.
+ *
+ * @param {string} replyText
+ * @returns {{ add: string[], evict: string[], reason: string } | null}
+ */
+export function parseAdjustReply(replyText) {
+  if (!replyText) return null;
+  // Strip markdown fences
+  const cleaned = replyText.replace(/^```(?:json)?\s*|\s*```$/gm, '').trim();
+  // Find first { ... } JSON object
+  const start = cleaned.indexOf('{');
+  const end = cleaned.lastIndexOf('}');
+  if (start < 0 || end <= start) return null;
+  const json = cleaned.slice(start, end + 1);
+  let obj;
+  try { obj = JSON.parse(json); } catch { return null; }
+  if (!obj || typeof obj !== 'object') return null;
+  const add = Array.isArray(obj.add) ? obj.add.filter(s => typeof s === 'string') : [];
+  const evict = Array.isArray(obj.evict) ? obj.evict.filter(s => typeof s === 'string') : [];
+  const reason = typeof obj.reason === 'string' ? obj.reason : '';
+  return { add, evict, reason };
+}
+
+/**
+ * Apply an adjustment to AMS membership, with safety guards:
+ *  - Drop add ids that aren't in the visible segment set.
+ *  - Drop evict ids that aren't currently in onDemand.
+ *  - Reject pathological replies (huge add / huge evict).
+ *
+ * @param {object} args
+ * @param {import('./ams.js').ActiveMemorySet} args.ams
+ * @param {import('./index-db.js').SegmentIndex} args.index
+ * @param {{ add: string[], evict: string[] }} args.decision
+ * @param {Set<string>} args.visibleIds
+ * @param {number} [args.maxAdd]
+ * @param {number} [args.maxEvict]
+ * @returns {{ added: number, evicted: number, skipped: number }}
+ */
+export function applyAdjustment(args) {
+  const maxAdd = args.maxAdd ?? 32;
+  const maxEvict = args.maxEvict ?? 32;
+  const currentOnDemand = new Set(args.ams.onDemandIds());
+  const addIds = (args.decision.add || [])
+    .filter(id => args.visibleIds.has(id) && !currentOnDemand.has(id))
+    .slice(0, maxAdd);
+  const evictIds = (args.decision.evict || [])
+    .filter(id => currentOnDemand.has(id))
+    .slice(0, maxEvict);
+  const skipped =
+    (args.decision.add?.length || 0) - addIds.length +
+    (args.decision.evict?.length || 0) - evictIds.length;
+
+  // Resolve add segments via the index
+  const addSegs = [];
+  for (const id of addIds) {
+    const s = args.index.get(id);
+    if (s) addSegs.push(s);
+  }
+  if (addSegs.length > 0) args.ams.addOnDemand(addSegs);
+  if (evictIds.length > 0) args.ams.removeOnDemand(evictIds);
+
+  return {
+    added: addSegs.length,
+    evicted: evictIds.length,
+    skipped: Math.max(0, skipped),
+  };
+}
+
+/**
+ * Full round-trip: decide whether to run, build prompt, call LLM,
+ * parse, apply. Returns telemetry counts.
+ *
+ * The caller supplies the LLM via `runLLM(prompt) → text` so this
+ * module stays adapter-agnostic.
+ *
+ * @param {object} args
+ * @param {AdjustTriggerInput} args.trigger
+ * @param {import('./ams.js').ActiveMemorySet} args.ams
+ * @param {import('./index-db.js').SegmentIndex} args.index
+ * @param {string[]} args.scopes
+ * @param {string|null} args.ownVpId
+ * @param {string} args.userMsg
+ * @param {string} args.assistantReply
+ * @param {(prompt: string) => Promise<string>} args.runLLM
+ * @returns {Promise<{
+ *   ran: boolean, reason: string,
+ *   added: number, evicted: number, skipped: number,
+ *   promptTokens: number,
+ * }>}
+ */
+export async function runAdjust(args) {
+  const decision = shouldRunAdjust(args.trigger);
+  if (!decision.run) {
+    return { ran: false, reason: decision.reason, added: 0, evicted: 0, skipped: 0, promptTokens: 0 };
+  }
+  const currentAmsIds = new Set([
+    ...args.ams.onDemandIds(),
+    ...args.ams.recentIds(),
+  ]);
+  const visibleSegments = buildVisibleSegments({
+    index: args.index, scopes: args.scopes, ownVpId: args.ownVpId,
+    currentAmsIds,
+  });
+  const prompt = buildAdjustPrompt({
+    userMsg: args.userMsg,
+    assistantReply: args.assistantReply,
+    residentScopes: args.ams.residentScopes(),
+    recentIds: args.ams.recentIds(),
+    onDemandIds: args.ams.onDemandIds(),
+    visibleSegments,
+  });
+  const reply = await args.runLLM(prompt);
+  const parsed = parseAdjustReply(reply);
+  if (!parsed) {
+    return {
+      ran: true, reason: decision.reason + '+parse-fail',
+      added: 0, evicted: 0, skipped: 0,
+      promptTokens: approxTokens(prompt),
+    };
+  }
+  const visibleIds = new Set(visibleSegments.map(s => s.id));
+  const apply = applyAdjustment({
+    ams: args.ams, index: args.index,
+    decision: parsed, visibleIds,
+  });
+  return {
+    ran: true, reason: decision.reason,
+    ...apply,
+    promptTokens: approxTokens(prompt),
+  };
+}

package/unify/memory/keywords.js

@@ -0,0 +1,57 @@
+/**
+ * keywords.js — pure-rule keyword extraction shared by memory recall paths.
+ *
+ * Extracted from the legacy R5 recall.js so recall-v2.js (and any future
+ * recall path) can use it without dragging in the rest of the R5 module.
+ * Pure CPU, no LLM, <1ms.
+ */
+
+/** Common stop words filtered out before frequency counting. */
+const STOP_WORDS = new Set([
+  'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+  'should', 'may', 'might', 'can', 'shall', 'to', 'of', 'in', 'for',
+  'on', 'with', 'at', 'by', 'from', 'as', 'into', 'through', 'during',
+  'before', 'after', 'above', 'below', 'between', 'out', 'off', 'over',
+  'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
+  'where', 'why', 'how', 'all', 'both', 'each', 'few', 'more', 'most',
+  'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same',
+  'so', 'than', 'too', 'very', 'just', 'because', 'but', 'and', 'or',
+  'if', 'while', 'about', 'up', 'it', 'its', 'my', 'me', 'i', 'you',
+  'your', 'we', 'our', 'they', 'them', 'their', 'this', 'that', 'what',
+  'which', 'who', 'whom', 'these', 'those',
+  // Chinese stop words
+  '的', '了', '在', '是', '我', '有', '和', '就',
+  '不', '人', '都', '一', '一个', '上', '也',
+  '很', '到', '说', '要', '去', '你', '会',
+  '着', '没有', '看', '好', '自己', '这',
+  '他', '她', '吗', '呢', '吧', '把', '被',
+  '那', '它', '让', '给', '可以', '什么',
+  '怎么', '帮', '帮我', '请', '能', '想',
+]);
+
+/**
+ * Extract keywords from a prompt (pure rules, no LLM).
+ *
+ * @param {string} prompt
+ * @returns {string[]} keywords sorted by frequency descending then alpha.
+ */
+export function extractKeywords(prompt) {
+  if (!prompt || !prompt.trim()) return [];
+
+  // Tokenize: split on whitespace + punctuation, keep CJK chars.
+  const tokens = prompt
+    .toLowerCase()
+    .replace(/[^\w\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff]+/g, ' ')
+    .split(/\s+/)
+    .filter(t => t.length > 1 && !STOP_WORDS.has(t));
+
+  const freq = new Map();
+  for (const t of tokens) {
+    freq.set(t, (freq.get(t) || 0) + 1);
+  }
+
+  return [...freq.entries()]
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([word]) => word);
+}

package/unify/memory/preflow.js

@@ -0,0 +1,173 @@
+/**
+ * memory/preflow.js — DESIGN-H2-AMS §6. Pre-turn memory recall.
+ *
+ * Pure-CPU pipeline (no LLM). Runs on every turn:
+ *
+ *   userMsg
+ *     → extractKeywords (rule-based tokeniser)
+ *     → SQLite FTS5 MATCH (scope-filtered, bm25 ranked)
+ *     → rerank by (scope match + tag overlap + recency)
+ *     → onDemand layer (budget-clamped)
+ *
+ * Latency target: < 10ms p95 on a 10k-segment index.
+ *
+ * Honours `vp/<other>` privacy: caller passes `ownVpId` and the scope
+ * filter excludes foreign VP scopes.
+ */
+
+import { extractKeywords } from './keywords.js';
+import { approxTokens } from './budget.js';
+
+/**
+ * @typedef {object} PreflowOptions
+ * @property {string}        userMsg
+ * @property {string[]}      relevantScopes      e.g. ['user', 'group/g1', 'vp/alice']
+ * @property {string|null}  [ownVpId]
+ * @property {string[]}     [currentTags]        tags from the current group/feature context
+ * @property {number}       [topK]               max FTS rows to fetch (default 50)
+ * @property {number}       [budgetTokens]       onDemand budget (caller-supplied)
+ */
+
+/**
+ * @typedef {object} PreflowResult
+ * @property {string[]}                                  keywords
+ * @property {string}                                     ftsQuery
+ * @property {import('./index-db.js').SearchHit[]}       hits
+ * @property {import('./segment.js').Segment[]}          picked
+ * @property {number}                                     pickedTokens
+ * @property {number}                                     droppedCount
+ */
+
+/**
+ * Run the pre-flow against a segment index.
+ *
+ * @param {import('./index-db.js').SegmentIndex} index
+ * @param {PreflowOptions} opts
+ * @returns {PreflowResult}
+ */
+export function runPreflow(index, opts) {
+  const userMsg = (opts.userMsg || '').trim();
+  const relevantScopes = Array.isArray(opts.relevantScopes) ? opts.relevantScopes : [];
+  const ownVpId = opts.ownVpId || null;
+  const currentTags = Array.isArray(opts.currentTags) ? opts.currentTags : [];
+  const topK = Number.isFinite(opts.topK) && opts.topK > 0 ? opts.topK : 50;
+  const budgetTokens = Number.isFinite(opts.budgetTokens) && opts.budgetTokens > 0
+    ? opts.budgetTokens : Infinity;
+
+  const keywords = extractKeywords(userMsg);
+  if (keywords.length === 0) {
+    return {
+      keywords: [], ftsQuery: '', hits: [],
+      picked: [], pickedTokens: 0, droppedCount: 0,
+    };
+  }
+
+  const ftsQuery = buildFtsQuery(keywords);
+  const scopeFilter = filterScopes(relevantScopes, ownVpId);
+  if (scopeFilter.length === 0) {
+    return {
+      keywords, ftsQuery, hits: [],
+      picked: [], pickedTokens: 0, droppedCount: 0,
+    };
+  }
+
+  const hits = index.search({ query: ftsQuery, scopeFilter, limit: topK });
+  const reranked = rerank(hits, { currentTags });
+
+  const picked = [];
+  let cost = 0;
+  let dropped = 0;
+  for (const h of reranked) {
+    const tk = approxTokens(h.body);
+    if (cost + tk <= budgetTokens) {
+      picked.push(toSegment(h));
+      cost += tk;
+    } else {
+      dropped += 1;
+    }
+  }
+
+  return {
+    keywords, ftsQuery, hits: reranked,
+    picked, pickedTokens: cost, droppedCount: dropped,
+  };
+}
+
+/**
+ * Compose an FTS5 MATCH query from keywords. Each keyword is OR'd with
+ * a prefix wildcard so morphological variants match. We escape any
+ * FTS5-special characters by quoting tokens.
+ *
+ * @param {string[]} keywords
+ * @returns {string}
+ */
+export function buildFtsQuery(keywords) {
+  const cleaned = keywords
+    .map(k => k.replace(/"/g, ''))
+    .filter(k => k.length > 1)
+    .slice(0, 8);   // top-8 keywords — avoid query bloat
+  if (cleaned.length === 0) return '';
+  return cleaned.map(k => `"${k}"*`).join(' OR ');
+}
+
+/**
+ * Strip foreign VP scopes from the filter list (privacy).
+ *
+ * @param {string[]} scopes
+ * @param {string|null} ownVpId
+ * @returns {string[]}
+ */
+export function filterScopes(scopes, ownVpId) {
+  return scopes.filter(s => {
+    if (!s.startsWith('vp/')) return true;
+    if (!ownVpId) return true;
+    const other = s.slice(3).split('/')[0];
+    return other === ownVpId;
+  });
+}
+
+/**
+ * Rerank FTS hits with two soft signals on top of bm25:
+ *   - tag overlap with the current group/feature context (subtract penalty)
+ *   - recency: recent items get a small bonus
+ *
+ * SQLite FTS5 bm25 returns NEGATIVE numbers (more negative = better
+ * match). We treat lower score as better. To make overlap & recency
+ * push hits ahead, we SUBTRACT bonuses from the bm25 base (making the
+ * score more negative).
+ *
+ * @param {import('./index-db.js').SearchHit[]} hits
+ * @param {{ currentTags: string[] }} ctx
+ * @returns {import('./index-db.js').SearchHit[]}
+ */
+export function rerank(hits, ctx) {
+  const tagSet = new Set((ctx.currentTags || []).map(t => String(t).toLowerCase()));
+  const now = Date.now();
+  return [...hits]
+    .map(h => {
+      const overlap = (h.tags || []).reduce(
+        (n, t) => n + (tagSet.has(String(t).toLowerCase()) ? 1 : 0), 0,
+      );
+      const tagBonus = Math.min(2, overlap * 0.5);   // up to 2 points
+      const ageDays = Math.max(0, (now - Date.parse(h.updatedAt || h.createdAt || '')) / 86400000);
+      const recencyBonus = Math.min(0.5, 0.2 / Math.max(0.5, ageDays + 1));
+      const base = h.rank ?? 0;
+      const score = base - tagBonus - recencyBonus;
+      return { ...h, _score: score };
+    })
+    .sort((a, b) => a._score - b._score)
+    .map(({ _score, ...rest }) => rest);
+}
+
+function toSegment(h) {
+  return {
+    id: h.id,
+    scope: h.scope,
+    kind: h.kind,
+    tags: h.tags,
+    sourceMessages: h.sourceMessages,
+    body: h.body,
+    createdAt: h.createdAt,
+    updatedAt: h.updatedAt,
+  };
+}