@yeaft/webchat-agent 0.1.531 → 0.1.532

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yeaft/webchat-agent",
-  "version": "0.1.531",
+  "version": "0.1.532",
   "description": "Remote agent for Yeaft WebChat — connects worker machines to the central server",
   "main": "index.js",
   "type": "module",

package/unify/engine.js

@@ -310,6 +310,9 @@ export class Engine {
       memoryInjection,
       compactSummary,
       skillContent,
+      // task-334f: memory_trace tool is now registered (49 → 51 tools), so
+      // unlock the core_memory meta-line behind 334e's feature flag.
+      memoryTraceAvailable: true,
     });
   }
 

package/unify/memory/migrate-r5-to-r6.js

@@ -0,0 +1,61 @@
+/**
+ * migrate-r5-to-r6.js — task-334f §Δ23 migration stub.
+ *
+ * Legacy (R5): `~/.yeaft/memory/entries/<slug>.md` plus numeric shard files
+ *              `memory-001.md`, `memory-002.md`, ...
+ *
+ * R6:          `~/.yeaft/memory/vp/<vpId>/memory-<semantic>.md`
+ *              Semantic shards: skill / relations / lessons / preferences /
+ *                               project-<slug>
+ *
+ * This slice (334f) only DEFINES the API surface and a dry-run classifier.
+ * The actual batch migration runs in 334i; 334f does not mutate disk.
+ */
+
+import { existsSync, readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { parseEntry } from './store.js';
+import { classifyLegacyEntryToShard } from './shard-store.js';
+
+/**
+ * Produce a migration plan without applying it.
+ *
+ * @param {string} legacyEntriesDir  e.g. ~/.yeaft/memory/entries
+ * @returns {{
+ *   totalEntries: number,
+ *   plan: Array<{ slug: string, shard: string, kind: string, tags: string[] }>,
+ *   byShard: Record<string, number>
+ * }}
+ */
+export function planR5ToR6Migration(legacyEntriesDir) {
+  if (!legacyEntriesDir || !existsSync(legacyEntriesDir)) {
+    return { totalEntries: 0, plan: [], byShard: {} };
+  }
+  const files = readdirSync(legacyEntriesDir).filter(f => f.endsWith('.md'));
+  const plan = [];
+  const byShard = {};
+  for (const file of files) {
+    const raw = readFileSync(join(legacyEntriesDir, file), 'utf8');
+    const entry = parseEntry(raw);
+    if (!entry) continue;
+    const shard = classifyLegacyEntryToShard(entry);
+    plan.push({
+      slug: file.replace(/\.md$/, ''),
+      shard,
+      kind: entry.kind || 'fact',
+      tags: entry.tags || [],
+    });
+    byShard[shard] = (byShard[shard] || 0) + 1;
+  }
+  return { totalEntries: plan.length, plan, byShard };
+}
+
+/**
+ * Apply the migration. STUB — 334i will fill in the body writer. 334f keeps
+ * this function exported so downstream tests can assert the hook exists.
+ *
+ * @param {object} _opts  { legacyEntriesDir, targetDir, vpId, dryRun }
+ */
+export async function applyR5ToR6Migration(_opts) {
+  throw new Error('applyR5ToR6Migration: not yet implemented (task-334i)');
+}

package/unify/memory/recall-r6.js

@@ -0,0 +1,291 @@
+/**
+ * recall-r6.js — task-334f R6 4-step recall pipeline (§Δ24.2).
+ *
+ * Pipeline:
+ *   Step 1: Shard classifier   — fastModel OR keyword heuristic → top-1~2 shards
+ *   Step 2: In-shard candidate — scan only the selected shards, filter by kind/tags/pinned
+ *   Step 3: LLM rerank         — pick top-7 from candidates
+ *   Step 4: Inject body only   — return entries without sourceRef (§Δ23)
+ *
+ * The legacy R5 recall (agent/unify/memory/recall.js) remains for back-compat
+ * of callers still on MemoryStore. R6 callers should use `recallR6()` below.
+ */
+
+import { createHash } from 'crypto';
+import { pickEffort } from '../effort.js';
+import {
+  VP_DEFAULT_SHARDS,
+  TASK_SHARDS,
+  USER_SHARDS,
+} from './schema.js';
+
+const MAX_RECALL_RESULTS = 7;
+const MAX_CANDIDATES = 15;
+
+// ─── Step 1: Shard Classifier ───────────────────────────────────
+
+/**
+ * Keyword → shard heuristic. Zero-cost fallback when fastModel is unavailable.
+ *
+ * The lexicons are intentionally small: classifier output only needs to point
+ * at the most likely shard; LLM rerank in Step 3 catches misses. Empirically
+ * this covers > 70% of queries with zero LLM cost.
+ */
+const SHARD_LEXICON = {
+  skill:       ['code', 'api', 'library', 'framework', 'implement', 'debug', 'syntax', 'typescript', 'vue', 'pattern', '代码', '实现', '调试', '语法', '模式', '技术'],
+  lessons:     ['mistake', 'avoid', 'lesson', 'pitfall', 'gotcha', 'bug', 'regression', '教训', '避坑', '坑', '踩坑', '反模式'],
+  preferences: ['prefer', 'like', 'style', 'convention', 'favorite', '偏好', '风格', '习惯', '喜欢'],
+  relations:   ['colleague', 'partner', 'team', 'user', 'collaborator', 'vp', '同事', '队友', '协作', '关系'],
+  // Task-memory shards
+  decision:    ['decide', 'decision', 'chose', 'picked', 'resolved', '决定', '决策', '选择'],
+  progress:    ['done', 'progress', 'milestone', 'shipped', 'finished', '完成', '进度', '交付'],
+  context:     ['background', 'context', 'requirement', 'scope', '背景', '需求', '范围'],
+  blocker:     ['block', 'stuck', 'blocker', 'issue', 'waiting', '阻塞', '卡住', '等待'],
+  artifact:    ['pr', 'commit', 'doc', 'file', 'link', 'artifact', 'deliverable', 'commit', '产出', '文档'],
+  // User-memory shards
+  profile:     ['name', 'role', 'background', 'who', 'identity', '身份', '角色', '背景'],
+  projects:    ['project', 'repo', '项目', '仓库'],
+  goals:       ['goal', 'target', 'okr', 'plan', '目标', '计划'],
+};
+
+/**
+ * Pick top-N shards via keyword heuristic.
+ * @param {string} prompt
+ * @param {string[]} availableShards  all shards present in this store
+ * @param {number} [topN=2]
+ */
+export function classifyShardsByKeyword(prompt, availableShards, topN = 2) {
+  if (!prompt || !availableShards || availableShards.length === 0) return [];
+  const lower = prompt.toLowerCase();
+  const scores = new Map();
+  for (const shard of availableShards) {
+    // project-<slug> shards score via their slug
+    const lex = SHARD_LEXICON[shard] || [];
+    let score = 0;
+    for (const kw of lex) {
+      if (lower.includes(kw)) score += 1;
+    }
+    if (shard.startsWith('project-')) {
+      const slug = shard.slice('project-'.length);
+      if (slug && lower.includes(slug.toLowerCase())) score += 3;
+    }
+    if (score > 0) scores.set(shard, score);
+  }
+  const ranked = [...scores.entries()].sort((a, b) => b[1] - a[1]).map(([s]) => s);
+  if (ranked.length >= 1) return ranked.slice(0, topN);
+  // Fallback: first N defaults in the shard set
+  return availableShards.slice(0, topN);
+}
+
+/**
+ * Shard classifier — calls fastModel if adapter provided, else falls back to
+ * the keyword heuristic. Budget < 200 tokens (§Δ24.2).
+ */
+export async function classifyShards({
+  prompt,
+  availableShards,
+  adapter,
+  fastModel,
+  topN = 2,
+}) {
+  if (!adapter || !fastModel) {
+    return classifyShardsByKeyword(prompt, availableShards, topN);
+  }
+  const system = `You classify user queries into memory shards. Return ONLY a JSON array of up to ${topN} shard names from the provided list. No prose.`;
+  const user = `Available shards: ${JSON.stringify(availableShards)}
+Query: ${JSON.stringify(prompt)}
+Return JSON array of up to ${topN} most relevant shard names.`;
+  try {
+    const res = await adapter.call({
+      model: fastModel,
+      system,
+      messages: [{ role: 'user', content: user }],
+      maxTokens: 64,
+      effort: pickEffort({ scenario: 'recall' }),
+    });
+    const m = (res.text || '').match(/\[[\s\S]*?\]/);
+    if (!m) return classifyShardsByKeyword(prompt, availableShards, topN);
+    const arr = JSON.parse(m[0]);
+    const valid = arr.filter(s => typeof s === 'string' && availableShards.includes(s));
+    if (valid.length === 0) return classifyShardsByKeyword(prompt, availableShards, topN);
+    return valid.slice(0, topN);
+  } catch {
+    return classifyShardsByKeyword(prompt, availableShards, topN);
+  }
+}
+
+// ─── Step 2: In-shard candidate generation ──────────────────────
+
+function collectCandidates(memoryShardStore, { shards, kind, tags, pinned }) {
+  const filter = {};
+  if (shards && shards.length) filter.shard = shards.length === 1 ? shards[0] : shards;
+  if (kind) filter.kind = kind;
+  if (tags && tags.length) filter.tags = tags;
+  if (pinned !== undefined) filter.pinned = pinned;
+  const { results } = memoryShardStore.query(filter);
+  return results
+    // Drop superseded entries from the candidate pool (they stay on disk
+    // for memory_trace but should not compete for recall slots).
+    .filter(rec => !rec.supersededBy)
+    .slice(0, MAX_CANDIDATES);
+}
+
+// ─── Step 3: LLM Rerank ─────────────────────────────────────────
+
+async function llmRerank({ adapter, fastModel, prompt, candidates, memoryShardStore }) {
+  if (candidates.length <= MAX_RECALL_RESULTS) {
+    return candidates.map(c => c.id);
+  }
+  const lines = candidates.map((c, i) => {
+    return `${i + 1}. [id=${c.id}] shard=${c.shard} kind=${c.kind || '?'} tags=[${(c.tags || []).join(',')}]`;
+  }).join('\n');
+  const system = `You pick the most relevant memories for the user's prompt. Return ONLY a JSON array of entry ids (up to ${MAX_RECALL_RESULTS}). No prose.`;
+  const user = `User prompt: ${JSON.stringify(prompt)}
+
+Candidate memories:
+${lines}
+
+Return JSON array of up to ${MAX_RECALL_RESULTS} ids.`;
+  try {
+    const res = await adapter.call({
+      model: fastModel,
+      system,
+      messages: [{ role: 'user', content: user }],
+      maxTokens: 256,
+      effort: pickEffort({ scenario: 'recall' }),
+    });
+    const m = (res.text || '').match(/\[[\s\S]*\]/);
+    if (!m) return candidates.slice(0, MAX_RECALL_RESULTS).map(c => c.id);
+    const arr = JSON.parse(m[0]).filter(x => typeof x === 'string');
+    const valid = arr.filter(id => candidates.find(c => c.id === id));
+    if (valid.length === 0) return candidates.slice(0, MAX_RECALL_RESULTS).map(c => c.id);
+    return valid.slice(0, MAX_RECALL_RESULTS);
+  } catch {
+    return candidates.slice(0, MAX_RECALL_RESULTS).map(c => c.id);
+  }
+}
+
+// ─── Step 4: Inject (no sourceRef) ──────────────────────────────
+
+/**
+ * Produce the body-only injection payload (§Δ24.5). Prefix lines with
+ * `[mem:<shard>]` so the LLM knows the category without seeing the id.
+ */
+export function formatForInjection(entries) {
+  return entries.map(e => {
+    const prefix = `[mem:${e.shard}]`;
+    const body = (e.body || '').trim();
+    return `${prefix} ${body}`;
+  }).join('\n\n');
+}
+
+// ─── Fingerprint cache ──────────────────────────────────────────
+
+const _cache = new Map();
+const CACHE_TTL = 5 * 60 * 1000;
+
+function computeFingerprint({ shards, prompt, taskId }) {
+  const head = prompt.slice(0, 200);
+  const input = `${shards.join(',')}|${head}|${taskId || ''}`;
+  return createHash('sha256').update(input).digest('hex').slice(0, 16);
+}
+
+// ─── Entry point ────────────────────────────────────────────────
+
+/**
+ * Run the R6 4-step recall pipeline.
+ *
+ * @param {{
+ *   prompt: string,
+ *   memoryShardStore: object,           // openMemoryShardStore() handle
+ *   adapter?: object,                   // LLM adapter (null = keyword-only)
+ *   fastModel?: string,                 // fast model for classifier + rerank
+ *   availableShards?: string[],         // defaults to store stats
+ *   taskId?: string,
+ *   kind?: string, tags?: string[], pinned?: boolean,
+ * }} params
+ * @returns {Promise<{entries: object[], shards: string[], fingerprint: string, cached: boolean}>}
+ */
+export async function recallR6(params) {
+  const {
+    prompt,
+    memoryShardStore,
+    adapter,
+    fastModel,
+    availableShards,
+    taskId,
+    kind,
+    tags,
+    pinned,
+  } = params;
+
+  if (!prompt || !prompt.trim() || !memoryShardStore) {
+    return { entries: [], shards: [], fingerprint: '', cached: false };
+  }
+
+  const shardsFromStore = availableShards
+    || Object.keys(memoryShardStore.stats().shards);
+
+  if (shardsFromStore.length === 0) {
+    return { entries: [], shards: [], fingerprint: '', cached: false };
+  }
+
+  // Step 1
+  const chosenShards = await classifyShards({
+    prompt,
+    availableShards: shardsFromStore,
+    adapter,
+    fastModel,
+    topN: 2,
+  });
+
+  const fingerprint = computeFingerprint({ shards: chosenShards, prompt, taskId });
+  const cached = _cache.get(fingerprint);
+  if (cached && Date.now() - cached.t < CACHE_TTL) {
+    return { entries: cached.entries, shards: chosenShards, fingerprint, cached: true };
+  }
+
+  // Step 2
+  const candidates = collectCandidates(memoryShardStore, {
+    shards: chosenShards,
+    kind,
+    tags,
+    pinned,
+  });
+  if (candidates.length === 0) {
+    _cache.set(fingerprint, { entries: [], t: Date.now() });
+    return { entries: [], shards: chosenShards, fingerprint, cached: false };
+  }
+
+  // Step 3
+  const selectedIds = await llmRerank({
+    adapter, fastModel, prompt, candidates, memoryShardStore,
+  });
+
+  // Step 4 — load full bodies (minus sourceRef for injection).
+  const entries = [];
+  for (const id of selectedIds) {
+    const full = memoryShardStore.get(id);
+    if (!full) continue;
+    entries.push({
+      id: full.id,
+      shard: full.shard,
+      kind: full.kind,
+      body: full.body,
+      tags: full.tags || [],
+      // deliberately do NOT expose sourceRef here (§Δ23)
+    });
+  }
+  _cache.set(fingerprint, { entries, t: Date.now() });
+  return { entries, shards: chosenShards, fingerprint, cached: false };
+}
+
+export function clearR6RecallCache() {
+  _cache.clear();
+}
+
+export const R6_DEFAULTS = {
+  VP_DEFAULT_SHARDS,
+  TASK_SHARDS,
+  USER_SHARDS,
+};

package/unify/memory/schema.js

@@ -0,0 +1,166 @@
+/**
+ * schema.js — task-334f R6 Memory schema constants.
+ *
+ * References:
+ *   §Δ22 Memory 本质定性 (6 公理)
+ *   §Δ23 Memory Entry Schema 扩字段
+ *   §Δ25 Shard 语义初始分类
+ *   §Δ26.3 Shard 软上限
+ *
+ * This module is **data only** — no I/O, no LLM calls. It is safe to import
+ * from any layer (store, recall, tools) without circular hazards.
+ */
+
+// ─── §Δ25.1 VP-memory default shard set ─────────────────────────
+export const VP_DEFAULT_SHARDS = Object.freeze([
+  'skill',
+  'relations',
+  'lessons',
+  'preferences',
+]);
+
+// ─── §Δ25.2 Task-memory fixed 5 shards ──────────────────────────
+export const TASK_SHARDS = Object.freeze([
+  'decision',
+  'progress',
+  'context',
+  'blocker',
+  'artifact',
+]);
+
+// ─── §Δ25.3 User-memory default shard set ───────────────────────
+export const USER_SHARDS = Object.freeze([
+  'profile',
+  'preferences',
+  'projects',
+  'goals',
+  'relations',
+]);
+
+/**
+ * §Δ26.3 soft-cap table. `project-<slug>` is matched via the dedicated
+ * helper `softCapFor()` below because its key is dynamic.
+ *
+ * Shape: { entries: number, bytes: number }
+ */
+export const SOFT_CAPS = Object.freeze({
+  // VP
+  skill:       { entries: 80,  bytes: 64 * 1024 },
+  lessons:     { entries: 80,  bytes: 64 * 1024 },
+  preferences: { entries: 80,  bytes: 64 * 1024 },
+  relations:   { entries: 50,  bytes: 32 * 1024 },
+  // Task (Δ26.3 task-memory row)
+  decision: { entries: 40, bytes: 24 * 1024 },
+  progress: { entries: 40, bytes: 24 * 1024 },
+  context:  { entries: 40, bytes: 24 * 1024 },
+  blocker:  { entries: 40, bytes: 24 * 1024 },
+  artifact: { entries: 40, bytes: 24 * 1024 },
+  // User (Δ26.3 user-memory row — 60 entries / 48 KiB per shard)
+  profile:  { entries: 60, bytes: 48 * 1024 },
+  projects: { entries: 60, bytes: 48 * 1024 },
+  goals:    { entries: 60, bytes: 48 * 1024 },
+});
+
+/** Project shards are dynamic: `project-<slug>` → 150 entries / 128 KiB. */
+export const PROJECT_SHARD_SOFT_CAP = Object.freeze({
+  entries: 150,
+  bytes: 128 * 1024,
+});
+
+/** Default cap if callers ask for a shard not in the canonical set. */
+export const DEFAULT_SOFT_CAP = Object.freeze({
+  entries: 80,
+  bytes: 64 * 1024,
+});
+
+/**
+ * Trigger threshold: when a dream sweep finds ≥ PROJECT_DERIVE_THRESHOLD
+ * memory entries tagged with the same groupId, it may derive a
+ * `project-<slug>` shard. (§Δ25.1)
+ */
+export const PROJECT_DERIVE_THRESHOLD = 30;
+
+/**
+ * Max number of VP-memory shards — re-compression kicks in when exceeded
+ * (§Δ25.1: "最大 shard 数软上限 12").
+ */
+export const MAX_VP_SHARDS = 12;
+
+/**
+ * Return the soft cap for a given shard name.
+ * Handles the dynamic `project-<slug>` case.
+ */
+export function softCapFor(shardName) {
+  if (typeof shardName !== 'string' || !shardName) return DEFAULT_SOFT_CAP;
+  if (shardName.startsWith('project-')) return PROJECT_SHARD_SOFT_CAP;
+  return SOFT_CAPS[shardName] || DEFAULT_SOFT_CAP;
+}
+
+/**
+ * Build a schema object suitable for `openShardStore(dir, schema)` (334o).
+ *
+ * @param {'vp'|'task'|'user'} kind
+ * @param {{ extraShards?: string[] }} [opts]  e.g. existing project shards
+ * @returns {{ shards: string[], softCap: Record<string,{entries:number,bytes:number}>, defaultSoftCap: object }}
+ */
+export function buildShardSchema(kind, opts = {}) {
+  let shards;
+  switch (kind) {
+    case 'vp':   shards = [...VP_DEFAULT_SHARDS]; break;
+    case 'task': shards = [...TASK_SHARDS]; break;
+    case 'user': shards = [...USER_SHARDS]; break;
+    default: throw new Error(`buildShardSchema: unknown kind "${kind}"`);
+  }
+  if (Array.isArray(opts.extraShards)) {
+    for (const s of opts.extraShards) {
+      if (typeof s === 'string' && s && !shards.includes(s)) shards.push(s);
+    }
+  }
+  const softCap = {};
+  for (const s of shards) softCap[s] = softCapFor(s);
+  return {
+    shards,
+    softCap,
+    defaultSoftCap: DEFAULT_SOFT_CAP,
+  };
+}
+
+/**
+ * R6 memory-entry authored-by enum (§Δ23).
+ * Free-form strings are allowed; these are canonical examples.
+ */
+export const AUTHORED_BY = Object.freeze({
+  VP:             (vpId) => `vp:${vpId}`,
+  USER:           (uid)  => `user:${uid}`,
+  SUMMARY:        'system:summary-extractor',
+  DREAM:          'system:dream',
+});
+
+/**
+ * Validate an R6 entry shape (schema-level, no I/O).
+ * Throws on structural violation. Callers that want soft warnings should
+ * wrap in try/catch.
+ */
+export function validateR6Entry(entry) {
+  if (!entry || typeof entry !== 'object') throw new Error('entry must be an object');
+  if (!entry.id || typeof entry.id !== 'string') throw new Error('entry.id required');
+  if (!entry.shard || typeof entry.shard !== 'string') throw new Error('entry.shard required');
+  if (!entry.kind || typeof entry.kind !== 'string') throw new Error('entry.kind required');
+  // sourceRef is required except for identity/preference pure-declaration entries.
+  const needsSourceRef = !(entry.kind === 'identity' || entry.kind === 'preference');
+  if (needsSourceRef) {
+    if (!entry.sourceRef || typeof entry.sourceRef !== 'object') {
+      throw new Error('entry.sourceRef required for kind=' + entry.kind);
+    }
+    if (!Array.isArray(entry.sourceRef.msgIds) || entry.sourceRef.msgIds.length === 0) {
+      throw new Error('entry.sourceRef.msgIds required (non-empty array)');
+    }
+  }
+  if (entry.supersedes != null && !Array.isArray(entry.supersedes)) {
+    throw new Error('entry.supersedes must be an array when present');
+  }
+  if (entry.supersededBy != null && typeof entry.supersededBy !== 'string') {
+    throw new Error('entry.supersededBy must be a string when present');
+  }
+  return true;
+}

package/unify/memory/shard-store.js

@@ -0,0 +1,373 @@
+/**
+ * shard-store.js — task-334f R6 semantic-shard memory store (VP/task/user).
+ *
+ * This sits ON TOP OF 334o's `storage/shard-store.js` primitive. It adds:
+ *   - R6 entry schema (shard / sourceRef / supersedes / supersededBy / authoredBy)
+ *   - Frontmatter body serialisation (markdown-friendly, grep-able on disk)
+ *   - Supersede chain management (Δ26.2 Phase B)
+ *   - Atomic re-compression handoff (`memory-<shard>.md.compacting`)
+ *   - Migration stub from legacy R5 `memory-NNN.md` → semantic shards (334i)
+ *
+ * Hard boundaries (task-334f guardrails):
+ *   - does NOT touch 334o's jsonl-log layer
+ *   - does NOT run dream extract / re-compression decisions (334g)
+ *   - does NOT implement user-memory business logic (334l reuses this lib)
+ */
+
+import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import { openShardStore, writeAtomic } from '../storage/index.js';
+import {
+  rebuildShardIndexFromDisk,
+  saveShardIndex,
+} from '../storage/shard-index.js';
+import {
+  buildShardSchema,
+  softCapFor,
+  PROJECT_DERIVE_THRESHOLD,
+  MAX_VP_SHARDS,
+  validateR6Entry,
+  AUTHORED_BY,
+} from './schema.js';
+
+const COMPACTING_SUFFIX = '.compacting';
+
+/**
+ * Open (or create) an R6 memory shard store rooted at `dir`.
+ *
+ * @param {string} dir   filesystem directory (e.g. `~/.yeaft/memory/vp/<vpId>`)
+ * @param {'vp'|'task'|'user'} kind
+ * @param {{ extraShards?: string[] }} [opts]
+ * @returns {object} handle with put/get/query/remove/compact/...
+ */
+export function openMemoryShardStore(dir, kind = 'vp', opts = {}) {
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+
+  // Hydrate any pre-existing project-* shards from disk so the schema
+  // allow-list recognises them on this open.
+  const extraShards = new Set(opts.extraShards || []);
+  for (const name of discoverOnDiskShards(dir)) {
+    if (!extraShards.has(name)) extraShards.add(name);
+  }
+  const schema = buildShardSchema(kind, { extraShards: [...extraShards] });
+  const inner = openShardStore(dir, schema);
+
+  /**
+   * Put an R6 entry. Merges the rich entry schema into `body` (frontmatter)
+   * and surfaces the 5 filter-worthy fields through `meta`.
+   */
+  function put(entry) {
+    validateR6Entry(entry);
+    const body = serialiseR6Body(entry);
+    const meta = pickMeta(entry);
+    const res = inner.put({
+      id: entry.id,
+      shard: entry.shard,
+      body,
+      meta,
+    });
+    return res;
+  }
+
+  /** Retrieve an R6 entry (returns the parsed frontmatter + body). */
+  function get(id) {
+    const raw = inner.get(id);
+    if (!raw) return null;
+    const parsed = parseR6Body(raw.body);
+    return {
+      ...parsed,
+      id: raw.id,
+      shard: raw.shard,
+      _meta: raw.meta || {},
+    };
+  }
+
+  /** Query. See storage/shard-store.js for the filter shape. */
+  function query(filter = {}) {
+    const res = inner.query(filter);
+    return {
+      results: res.results.map(mapRecordToThinEntry),
+      needsRecompression: res.needsRecompression.slice(),
+    };
+  }
+
+  /** Remove an entry; underlying shard compacts immediately. */
+  function remove(id) { return inner.remove(id); }
+
+  /**
+   * Supersede: create a new entry N that replaces olds M[...].
+   * Writes N with supersedes=M[...], then marks each M.supersededBy=N.
+   * Old entries are NOT removed — they stay for audit / memory_trace.
+   */
+  function supersede({ newEntry, oldIds }) {
+    validateR6Entry(newEntry);
+    if (!Array.isArray(oldIds) || oldIds.length === 0) {
+      throw new Error('supersede: oldIds required (non-empty)');
+    }
+    const supersedes = oldIds.slice();
+    const write = { ...newEntry, supersedes };
+    const r = put(write);
+
+    for (const oldId of oldIds) {
+      const existing = get(oldId);
+      if (!existing) continue;
+      const updated = { ...existing, supersededBy: newEntry.id };
+      put(updated);
+    }
+    return r;
+  }
+
+  /**
+   * Atomic re-compression handoff:
+   * caller writes the new shard body to `memory-<shard>.md.compacting`,
+   * then calls `commitRecompression(shard)` which atomically renames it
+   * over the live file. Readers always see either the old or the new file.
+   *
+   * Consumers (334g dream) build the new body themselves; we just manage
+   * the rename + stats recomputation.
+   */
+  function stageRecompression(shardName, newBody) {
+    const tmpPath = join(dir, `memory-${shardName}.md${COMPACTING_SUFFIX}`);
+    writeAtomic(tmpPath, newBody);
+    return tmpPath;
+  }
+
+  function commitRecompression(shardName) {
+    const livePath = join(dir, `memory-${shardName}.md`);
+    const tmpPath = join(dir, `memory-${shardName}.md${COMPACTING_SUFFIX}`);
+    if (!existsSync(tmpPath)) {
+      throw new Error(`commitRecompression: no tmp file at ${tmpPath}`);
+    }
+    renameSync(tmpPath, livePath);
+    // The caller-supplied body replaced the whole shard. The old index rows
+    // for this shard are stale (new ids / offsets). Rebuild that shard's
+    // index rows from disk by delegating to the storage primitive — it walks
+    // START/END markers and recomputes offsets. We keep other shards intact.
+    const schema = buildShardSchema(kind, { extraShards: [...extraShards] });
+    const rebuilt = rebuildShardIndexFromDisk(dir, schema);
+    const innerIndex = inner.getIndex();
+    // Swap this shard's rows + bucket.
+    innerIndex.entries = innerIndex.entries.filter(e => e.shard !== shardName)
+      .concat(rebuilt.entries.filter(e => e.shard === shardName));
+    if (rebuilt.shards[shardName]) {
+      innerIndex.shards[shardName] = rebuilt.shards[shardName];
+    }
+    saveShardIndex(dir, innerIndex);
+  }
+
+  function abortRecompression(shardName) {
+    const tmpPath = join(dir, `memory-${shardName}.md${COMPACTING_SUFFIX}`);
+    if (existsSync(tmpPath)) unlinkSync(tmpPath);
+  }
+
+  /** Return shard-level stats (byte count, entry count, soft cap). */
+  function stats() {
+    const raw = inner.stats();
+    const shards = {};
+    for (const [name, bucket] of Object.entries(raw.shards)) {
+      shards[name] = { ...bucket, softCap: softCapFor(name) };
+    }
+    return { shards, count: raw.count };
+  }
+
+  /**
+   * Hint for the dream layer — returns `project-<slug>` candidate when
+   * ≥ PROJECT_DERIVE_THRESHOLD entries share a groupId and no project
+   * shard exists for that slug yet. Returns `null` otherwise.
+   *
+   * The actual derive (creating the shard file + re-compressing entries
+   * into it) is 334g dream work; this slice only advertises the hint so
+   * dream can schedule.
+   */
+  function projectDeriveHint() {
+    const groupCounts = new Map();
+    const { results } = inner.query({});
+    for (const rec of results) {
+      const gid = rec.meta?.groupId;
+      if (!gid) continue;
+      groupCounts.set(gid, (groupCounts.get(gid) || 0) + 1);
+    }
+    for (const [gid, count] of groupCounts) {
+      if (count < PROJECT_DERIVE_THRESHOLD) continue;
+      const slug = slugify(gid);
+      const shardName = `project-${slug}`;
+      const shardNames = Object.keys(inner.stats().shards);
+      if (shardNames.includes(shardName)) continue;
+      if (shardNames.length >= MAX_VP_SHARDS) continue;
+      return { groupId: gid, shard: shardName, count };
+    }
+    return null;
+  }
+
+  function close() { /* underlying store is stateless (index saved on every op) */ }
+
+  return {
+    put,
+    get,
+    query,
+    remove,
+    supersede,
+    stageRecompression,
+    commitRecompression,
+    abortRecompression,
+    stats,
+    projectDeriveHint,
+    close,
+    _innerForTest: inner,
+  };
+}
+
+// ─── Serialisation ──────────────────────────────────────────────
+
+function serialiseR6Body(entry) {
+  const fm = ['---'];
+  fm.push(`id: ${entry.id}`);
+  if (entry.vp) fm.push(`vp: ${entry.vp}`);
+  if (entry.taskId) fm.push(`taskId: ${entry.taskId}`);
+  fm.push(`kind: ${entry.kind}`);
+  fm.push(`shard: ${entry.shard}`);
+  if (entry.sourceRef) {
+    fm.push('sourceRef:');
+    if (entry.sourceRef.groupId)     fm.push(`  groupId: ${entry.sourceRef.groupId}`);
+    if (entry.sourceRef.taskId)      fm.push(`  taskId: ${entry.sourceRef.taskId}`);
+    if (Array.isArray(entry.sourceRef.msgIds) && entry.sourceRef.msgIds.length) {
+      fm.push(`  msgIds: [${entry.sourceRef.msgIds.join(', ')}]`);
+    }
+    if (entry.sourceRef.timeWindow)  fm.push(`  timeWindow: ${entry.sourceRef.timeWindow}`);
+    if (entry.sourceRef.hint)        fm.push(`  hint: ${JSON.stringify(entry.sourceRef.hint)}`);
+  }
+  if (Array.isArray(entry.supersedes) && entry.supersedes.length) {
+    fm.push(`supersedes: [${entry.supersedes.join(', ')}]`);
+  }
+  if (entry.supersededBy) fm.push(`supersededBy: ${entry.supersededBy}`);
+  if (entry.pinned != null) fm.push(`pinned: ${entry.pinned ? 'true' : 'false'}`);
+  if (Array.isArray(entry.tags) && entry.tags.length) {
+    fm.push(`tags: [${entry.tags.join(', ')}]`);
+  }
+  if (entry.authoredBy) fm.push(`authoredBy: ${entry.authoredBy}`);
+  const now = new Date().toISOString();
+  fm.push(`createdAt: ${entry.createdAt || now}`);
+  fm.push(`updatedAt: ${now}`);
+  fm.push('---');
+  fm.push('');
+  fm.push(entry.body || entry.content || '');
+  return fm.join('\n');
+}
+
+function parseR6Body(raw) {
+  if (!raw || !raw.startsWith('---')) return { body: raw || '' };
+  const endIdx = raw.indexOf('\n---', 3);
+  if (endIdx === -1) return { body: raw };
+  const fm = raw.slice(4, endIdx).trim();
+  const body = raw.slice(endIdx + 4).replace(/^\n+/, '');
+  const out = { body };
+  let inSourceRef = false;
+  const sourceRef = {};
+  for (const line of fm.split('\n')) {
+    if (/^sourceRef:\s*$/.test(line)) { inSourceRef = true; continue; }
+    if (inSourceRef && /^\s+/.test(line)) {
+      const m = line.match(/^\s+(\w+):\s*(.*)$/);
+      if (!m) continue;
+      const [, k, v] = m;
+      if (k === 'msgIds') {
+        sourceRef.msgIds = v.replace(/^\[|\]$/g, '').split(',').map(s => s.trim()).filter(Boolean);
+      } else if (k === 'hint') {
+        try { sourceRef.hint = JSON.parse(v); } catch { sourceRef.hint = v; }
+      } else {
+        sourceRef[k] = v;
+      }
+      continue;
+    }
+    inSourceRef = false;
+    const m = line.match(/^(\w+):\s*(.*)$/);
+    if (!m) continue;
+    const [, k, v] = m;
+    switch (k) {
+      case 'id':            out.id = v; break;
+      case 'vp':            out.vp = v; break;
+      case 'taskId':        out.taskId = v; break;
+      case 'kind':          out.kind = v; break;
+      case 'shard':         out.shard = v; break;
+      case 'supersededBy':  out.supersededBy = v; break;
+      case 'pinned':        out.pinned = v === 'true'; break;
+      case 'authoredBy':    out.authoredBy = v; break;
+      case 'createdAt':     out.createdAt = v; break;
+      case 'updatedAt':     out.updatedAt = v; break;
+      case 'supersedes':
+        out.supersedes = v.replace(/^\[|\]$/g, '').split(',').map(s => s.trim()).filter(Boolean);
+        break;
+      case 'tags':
+        out.tags = v.replace(/^\[|\]$/g, '').split(',').map(s => s.trim()).filter(Boolean);
+        break;
+      default: break;
+    }
+  }
+  if (Object.keys(sourceRef).length > 0) out.sourceRef = sourceRef;
+  return out;
+}
+
+function pickMeta(entry) {
+  // Meta is only the fields the 334o query() layer filters on.
+  const meta = {};
+  if (entry.kind)   meta.kind = entry.kind;
+  if (entry.tags)   meta.tags = entry.tags.slice();
+  if (entry.pinned) meta.pinned = true;
+  if (entry.sourceRef?.groupId) meta.groupId = entry.sourceRef.groupId;
+  if (entry.sourceRef?.taskId)  meta.taskId  = entry.sourceRef.taskId;
+  if (entry.supersededBy) meta.supersededBy = entry.supersededBy;
+  return meta;
+}
+
+function mapRecordToThinEntry(rec) {
+  return {
+    id: rec.id,
+    shard: rec.shard,
+    kind: rec.meta?.kind,
+    tags: rec.meta?.tags || [],
+    pinned: Boolean(rec.meta?.pinned),
+    groupId: rec.meta?.groupId,
+    taskId: rec.meta?.taskId,
+    supersededBy: rec.meta?.supersededBy || null,
+  };
+}
+
+function slugify(s) {
+  return String(s || '')
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 40);
+}
+
+function discoverOnDiskShards(dir) {
+  try {
+    return readdirSync(dir)
+      .filter(f => /^memory-[A-Za-z0-9_-]+\.md$/.test(f))
+      .map(f => f.replace(/^memory-/, '').replace(/\.md$/, ''));
+  } catch { return []; }
+}
+
+// ─── Migration stub (§Δ23 / 334i) ───────────────────────────────
+
+/**
+ * Migration stub: map a legacy R5 `memory-NNN.md` shard file path into a
+ * semantic shard assignment. Actual batch migration runs in 334i; this
+ * slice only defines the classifier API so dependent code can stub it.
+ *
+ * @param {object} legacyEntry  parsed legacy entry { kind, scope, tags, ... }
+ * @returns {string}            semantic shard name ("skill" / "lessons" / ...)
+ */
+export function classifyLegacyEntryToShard(legacyEntry) {
+  if (!legacyEntry || typeof legacyEntry !== 'object') return 'skill';
+  const kind = legacyEntry.kind || 'fact';
+  switch (kind) {
+    case 'lesson':     return 'lessons';
+    case 'preference': return 'preferences';
+    case 'identity':   return 'preferences';
+    case 'relation':   return 'relations';
+    case 'skill':      return 'skill';
+    default:           return 'skill';
+  }
+}
+
+export { AUTHORED_BY };

package/unify/tools/index.js

@@ -22,6 +22,8 @@ import memoryRead from './memory-read.js';
 import memoryWrite from './memory-write.js';
 import memorySearch, { memorySearchAlias } from './memory-search.js';
 import memoryQuery from './memory-query.js';
+import memoryTrace from './memory-trace.js';
+import openSourceMessage from './open-source-message.js';
 import webSearch from './web-search.js';
 import webFetch from './web-fetch.js';
 import historySearch from './history-search.js';
@@ -98,6 +100,8 @@ export const allTools = [
   memorySearch,
   memorySearchAlias,
   memoryQuery,
+  memoryTrace,
+  openSourceMessage,
   webSearch,
   webFetch,
   historySearch,

package/unify/tools/memory-trace.js

@@ -0,0 +1,135 @@
+/**
+ * memory-trace.js — task-334f R6 §Δ24.3.
+ *
+ * Given a memory id, return the full entry (including sourceRef) plus the
+ * original source messages referenced by sourceRef.msgIds / timeWindow.
+ *
+ * Hard guardrails (task-334f):
+ *   - Results are returned to the current turn ONLY. Nothing is written back
+ *     to memory; the extraction lane sees its own copy.
+ *   - Does not do cross-group fan-out. A trace is anchored to one groupId.
+ */
+
+import { defineTool } from './types.js';
+
+const MAX_BYTES = 64 * 1024;
+
+export default defineTool({
+  name: 'memory_trace',
+  description: `Trace a memory entry back to its original source messages.
+
+Use this when a recalled memory body is insufficient and you need the raw
+discussion. Returns the full memory entry (with sourceRef) plus the source
+messages from the group jsonl log.
+
+Parameters:
+  - memId (required): the memory id (from recall)
+  - expand: "full" (default, exact msgIds) | "window" (expand around timeWindow)
+
+Returns JSON: { memory, messages[], truncated? }.
+The result is NOT written back to memory — it is context for the current turn
+only.`,
+  parameters: {
+    type: 'object',
+    properties: {
+      memId:  { type: 'string', description: 'Memory entry id' },
+      expand: { type: 'string', enum: ['full', 'window'], default: 'full' },
+    },
+    required: ['memId'],
+  },
+  isConcurrencySafe: () => true,
+  isReadOnly: () => true,
+  async execute(input, ctx) {
+    const memId = input?.memId;
+    if (!memId || typeof memId !== 'string') {
+      return JSON.stringify({ error: 'memId required (string)' });
+    }
+    const expand = input?.expand === 'window' ? 'window' : 'full';
+
+    const store = ctx?.memoryShardStore;
+    if (!store) {
+      return JSON.stringify({ error: 'R6 memory shard store not initialised' });
+    }
+    const entry = store.get(memId);
+    if (!entry) {
+      return JSON.stringify({ error: `memory entry not found: ${memId}` });
+    }
+
+    const sourceRef = entry.sourceRef || null;
+    if (!sourceRef) {
+      return JSON.stringify({
+        memory: entry,
+        messages: [],
+        note: 'entry has no sourceRef (pure declaration)',
+      });
+    }
+
+    const coordinator = ctx?.coordinator;
+    const groupId = sourceRef.groupId;
+    if (!coordinator || !groupId) {
+      return JSON.stringify({
+        memory: entry,
+        messages: [],
+        note: 'no group coordinator available',
+      });
+    }
+
+    const group = typeof coordinator.openGroup === 'function'
+      ? coordinator.openGroup(groupId)
+      : null;
+    if (!group) {
+      return JSON.stringify({
+        memory: entry,
+        messages: [],
+        note: `group ${groupId} not resolvable`,
+      });
+    }
+
+    const messages = [];
+    let bytes = 0;
+    let truncated = false;
+
+    if (expand === 'full' && Array.isArray(sourceRef.msgIds) && sourceRef.msgIds.length) {
+      const targetSet = new Set(sourceRef.msgIds);
+      // Walk only the smallest overlapping range instead of streaming all.
+      const first = sourceRef.msgIds[0];
+      const last  = sourceRef.msgIds[sourceRef.msgIds.length - 1];
+      const iter = typeof group.readMessageRange === 'function'
+        ? group.readMessageRange(first, last)
+        : group.streamMessages();
+      for (const msg of iter) {
+        if (!targetSet.has(msg.id)) continue;
+        const chunk = estimateBytes(msg);
+        if (bytes + chunk > MAX_BYTES) { truncated = true; break; }
+        messages.push(msg);
+        bytes += chunk;
+      }
+    } else if (expand === 'window' && sourceRef.timeWindow) {
+      // timeWindow is "ISO..ISO"; best-effort textual compare works for ULIDs/ISO.
+      const [t0, t1] = String(sourceRef.timeWindow).split('..');
+      for (const msg of group.streamMessages()) {
+        const ts = msg.ts || '';
+        if (t0 && ts < t0) continue;
+        if (t1 && ts > t1) break;
+        const chunk = estimateBytes(msg);
+        if (bytes + chunk > MAX_BYTES) { truncated = true; break; }
+        messages.push(msg);
+        bytes += chunk;
+      }
+    }
+
+    return JSON.stringify({
+      memory: entry,
+      messages,
+      ...(truncated ? { truncated: true } : {}),
+    });
+  },
+});
+
+function estimateBytes(msg) {
+  try {
+    return Buffer.byteLength(JSON.stringify(msg), 'utf8');
+  } catch {
+    return 512;
+  }
+}

package/unify/tools/open-source-message.js

@@ -0,0 +1,49 @@
+/**
+ * open-source-message.js — task-334f R6 §Δ24.4.
+ *
+ * Low-level random access: given a (groupId, msgId), fetch the raw message
+ * from the group's jsonl log. Used when a VP has an exact pointer but does
+ * not want to run the memory_trace wrapper (5% case: audit / debug).
+ */
+
+import { defineTool } from './types.js';
+
+export default defineTool({
+  name: 'open_source_message',
+  description: `Open a single source message by (groupId, msgId).
+
+This is the low-level random-access primitive. Prefer memory_trace if you are
+starting from a memory entry. Returns JSON: { message } or { error }.`,
+  parameters: {
+    type: 'object',
+    properties: {
+      groupId: { type: 'string', description: 'Group id' },
+      msgId:   { type: 'string', description: 'Message id' },
+    },
+    required: ['groupId', 'msgId'],
+  },
+  isConcurrencySafe: () => true,
+  isReadOnly: () => true,
+  async execute(input, ctx) {
+    const { groupId, msgId } = input || {};
+    if (!groupId || !msgId) {
+      return JSON.stringify({ error: 'groupId and msgId required' });
+    }
+    const coordinator = ctx?.coordinator;
+    if (!coordinator || typeof coordinator.openGroup !== 'function') {
+      return JSON.stringify({ error: 'group coordinator not available' });
+    }
+    const group = coordinator.openGroup(groupId);
+    if (!group) return JSON.stringify({ error: `group not found: ${groupId}` });
+
+    const iter = typeof group.readMessageRange === 'function'
+      ? group.readMessageRange(msgId, msgId)
+      : group.streamMessages();
+    for (const msg of iter) {
+      if (msg.id === msgId) {
+        return JSON.stringify({ message: msg });
+      }
+    }
+    return JSON.stringify({ error: `message not found: ${msgId} in ${groupId}` });
+  },
+});