@openviking/dsh-memory-plugin 0.1.0

package/shared/profile-inject.mjs

@@ -0,0 +1,294 @@
+// GENERATED FROM examples/memory-plugin-shared/lib. DO NOT EDIT.
+/**
+ * Session-start profile injection helper.
+ *
+ * Builds a <user-profile> + <available-memories> block from
+ *   viking://user/<space>/memories/profile.md
+ *   viking://user/<space>/memories/preferences/   (ls with abstracts)
+ *   viking://user/<space>/memories/entities/      (ls with abstracts)
+ *
+ * Budget enforced via the CJK-aware estimateTokens() below — codepoint >=
+ * 0x3000 counts at 1.5 tokens, else chars/4. The estimator is exported so
+ * callers (e.g. session-start.mjs) can log token counts that match what
+ * the budget logic actually sees.
+ *
+ * Returned block is the *inner* content only (no outer <openviking-context>);
+ * session-start.mjs composes the outer wrapper so the archive block can sit
+ * alongside in a single context envelope.
+ */
+
+const USER_RESERVED_DIRS = new Set(["memories"]);
+let _userSpaceCache = null;
+
+/**
+ * Mirrors auto-recall.mjs resolveScopeSpace for scope="user" (lines 123-147).
+ * Duplicated rather than imported to avoid coupling session-start to
+ * auto-recall's module-level fetchJSON closure.
+ */
+async function resolveUserSpace(fetchJSON, actorPeerId = "") {
+  if (_userSpaceCache) return _userSpaceCache;
+
+  let fallbackSpace = "default";
+  const status = await fetchJSON("/api/v1/system/status");
+  if (status.ok && typeof status.result?.user === "string" && status.result.user.trim()) {
+    fallbackSpace = status.result.user.trim();
+  }
+
+  const lsRes = await fetchJSON(
+    `/api/v1/fs/ls?uri=${encodeURIComponent("viking://user")}&output=original`,
+    {},
+    { actorPeerId },
+  );
+  if (lsRes.ok && Array.isArray(lsRes.result)) {
+    const spaces = lsRes.result
+      .filter((e) => e?.isDir)
+      .map((e) => (typeof e.name === "string" ? e.name.trim() : ""))
+      .filter((n) => n && !n.startsWith(".") && !USER_RESERVED_DIRS.has(n));
+    if (spaces.length > 0) {
+      if (spaces.includes(fallbackSpace)) { _userSpaceCache = fallbackSpace; return fallbackSpace; }
+      if (spaces.includes("default")) { _userSpaceCache = "default"; return "default"; }
+      if (spaces.length === 1) { _userSpaceCache = spaces[0]; return spaces[0]; }
+    }
+  }
+  _userSpaceCache = fallbackSpace;
+  return fallbackSpace;
+}
+
+/**
+ * Token estimate that splits CJK from the rest. The rest of the plugin uses a
+ * flat chars/4 heuristic, which silently undercounts CJK content by 4-6× and
+ * makes a "5000 token budget" really worth ~1k real tokens for Chinese text.
+ *
+ * Rule:
+ *   - codepoint >= 0x3000 → CJK / Hiragana / Katakana / Hangul / CJK
+ *     punctuation / fullwidth ASCII; counted at 1.5 tokens/char (empirical
+ *     average for cl100k_base on Chinese; Claude's tokenizer is in the same
+ *     ballpark).
+ *   - everything else → chars/4 (the standard English-ish heuristic).
+ *
+ * Single linear pass, no dependencies. Errs on the side of overcounting CJK
+ * by ~10-20% in the worst case (some common Chinese phrases compress better
+ * than 1.5 tokens/char), which is the safe direction for budget enforcement.
+ */
+export function estimateTokens(text) {
+  if (!text) return 0;
+  let cjk = 0;
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) >= 0x3000) cjk++;
+  }
+  const other = text.length - cjk;
+  return Math.ceil(cjk * 1.5 + other / 4);
+}
+
+/**
+ * Convert a token budget to a max-chars budget that respects this content's
+ * actual CJK density. Avoids the chars/4 trap where a "5000 token" sub-cap
+ * yields 20000 chars of pure-CJK text → 30000 actual tokens (6× over).
+ *
+ * For an empty/missing string, returns 0.
+ */
+function tokensToCharsBudget(content, maxTokens) {
+  if (!content) return 0;
+  let cjk = 0;
+  for (let i = 0; i < content.length; i++) {
+    if (content.charCodeAt(i) >= 0x3000) cjk++;
+  }
+  const ratio = cjk / content.length;
+  const tokensPerChar = ratio * 1.5 + (1 - ratio) * 0.25;
+  return Math.floor(maxTokens / Math.max(tokensPerChar, 0.25));
+}
+
+async function readProfile(fetchJSON, profileUri, actorPeerId = "") {
+  const res = await fetchJSON(
+    `/api/v1/content/read?uri=${encodeURIComponent(profileUri)}`,
+    {},
+    { actorPeerId },
+  );
+  if (!res.ok || typeof res.result !== "string") return null;
+  const trimmed = res.result.trim();
+  return trimmed || null;
+}
+
+/**
+ * Recursive ls of a memory directory, flattening to .md leaves.
+ *
+ * Memory layout under preferences/ and entities/ is two-level:
+ *   <dir>/<owner_name>/<topic>.md
+ * so we use the server's recursive=true flag and filter to leaf .md files.
+ * `rel_path` (e.g. "zhengxiao.wu/pr_workflow.md") is preserved as display name
+ * to keep owner-namespacing visible and unambiguous when multiple owners exist.
+ */
+async function lsDir(fetchJSON, dirUri, actorPeerId = "") {
+  const url = `/api/v1/fs/ls?uri=${encodeURIComponent(dirUri)}&output=agent&recursive=true&abs_limit=512&node_limit=512`;
+  const res = await fetchJSON(url, {}, { actorPeerId });
+  if (!res.ok || !Array.isArray(res.result)) return [];
+  return res.result
+    .filter((e) => !e.isDir)
+    .map((e) => {
+      const rel = typeof e.rel_path === "string" && e.rel_path
+        ? e.rel_path
+        : (typeof e.name === "string" ? e.name : "");
+      return {
+        name: rel,
+        abstract: typeof e.abstract === "string" ? e.abstract.trim() : "",
+      };
+    })
+    .filter((e) => e.name && e.name.endsWith(".md"))
+    .sort((a, b) => a.name.localeCompare(b.name));
+}
+
+/**
+ * When profile exceeds its sub-cap, keep the head (identity block + first
+ * timeline events) and the tail (most-recent events), drop the middle. This
+ * preserves both stable identity facts (top of file) and most-recent activity
+ * (bottom of file) — only the noisy middle timeline is sacrificed.
+ *
+ * Layout:
+ *   <first HEAD_LINES lines>
+ *   ... [profile middle elided] ...
+ *   <as many trailing lines as fit in remaining budget>
+ *
+ * Falls back to head-only truncate when the file is too short to elide
+ * meaningfully (<HEAD_LINES + 4 lines) or the budget is too tight to fit
+ * both head and a useful tail.
+ */
+function elideProfile(content, maxTokens) {
+  // Compute char budget from token budget using *this* content's CJK density,
+  // not chars/4 — otherwise the truncated string can still blow the token cap
+  // for CJK-heavy profiles (Copilot review point).
+  const maxChars = Math.max(400, tokensToCharsBudget(content, maxTokens));
+  if (estimateTokens(content) <= maxTokens) return content;
+
+  const HEAD_LINES = 8;
+  const ELLIPSIS = "\n... [profile middle elided] ...\n";
+  const lines = content.split("\n");
+
+  const fallbackHeadTruncate = () =>
+    content.slice(0, maxChars).trimEnd() + "\n... [profile truncated]";
+
+  if (lines.length <= HEAD_LINES + 4) return fallbackHeadTruncate();
+
+  const head = lines.slice(0, HEAD_LINES).join("\n");
+  const reserveForTail = maxChars - head.length - ELLIPSIS.length;
+  if (reserveForTail < 200) return fallbackHeadTruncate();
+
+  let tailChars = 0;
+  let tailStart = lines.length;
+  for (let i = lines.length - 1; i > HEAD_LINES; i--) {
+    const lineLen = lines[i].length + 1;
+    if (tailChars + lineLen > reserveForTail) break;
+    tailChars += lineLen;
+    tailStart = i;
+  }
+  if (tailStart >= lines.length - 1) return fallbackHeadTruncate();
+
+  return `${head}${ELLIPSIS}${lines.slice(tailStart).join("\n")}`;
+}
+
+function formatListing(headerUri, entries, budgetTokens) {
+  if (entries.length === 0) return { lines: [], used: 0, dropped: 0 };
+  // Header is the full directory URI; child lines are relative paths so the
+  // agent can reconstruct each leaf's full URI by concatenation while the
+  // listing itself stays compact.
+  const header = `  ${headerUri}/`;
+  const headerTokens = estimateTokens(header);
+  // If the header alone busts the budget, emit just a one-line stub instead
+  // of silently violating the cap (Copilot review point).
+  if (headerTokens > budgetTokens) {
+    const stub = `  ${headerUri}/  (${entries.length} entries, budget too tight; use \`memory_recall\`)`;
+    return { lines: [stub], used: estimateTokens(stub), dropped: entries.length };
+  }
+  const lines = [header];
+  let used = headerTokens;
+  for (let i = 0; i < entries.length; i++) {
+    const e = entries[i];
+    const desc = e.abstract
+      ? ` — ${e.abstract.replace(/\s+/g, " ").slice(0, 200)}`
+      : "";
+    const line = `    - ${e.name}${desc}`;
+    const tokens = estimateTokens(line);
+    if (used + tokens > budgetTokens) {
+      const remaining = entries.length - i;
+      const tail = `    ... +${remaining} more, use \`memory_recall\``;
+      const tailTokens = estimateTokens(tail);
+      // Only emit the tail if it fits; otherwise the listing closes silently
+      // rather than violating the cap to advertise its own truncation.
+      if (used + tailTokens <= budgetTokens) {
+        lines.push(tail);
+        return { lines, used: used + tailTokens, dropped: remaining };
+      }
+      return { lines, used, dropped: remaining };
+    }
+    lines.push(line);
+    used += tokens;
+  }
+  return { lines, used, dropped: 0 };
+}
+
+/**
+ * Build the profile injection block.
+ *
+ * Returns null when neither profile.md nor either listing has any content.
+ * The returned `block` is just the inner <user-profile>/<available-memories>
+ * payload — the caller wraps it in <openviking-context source="...">.
+ *
+ * @param {Function} fetchJSON  ov-session.mjs:makeFetchJSON closure
+ * @param {number} totalBudgetTokens  chars/4 budget, total for the whole block
+ * @returns {Promise<null | {
+ *   block: string, chars: number, tokens: number, profileUri: string,
+ *   profileChars: number, prefCount: number, entCount: number,
+ *   droppedPref: number, droppedEnt: number,
+ * }>}
+ */
+export async function buildProfileBlock(fetchJSON, totalBudgetTokens, actorPeerId = "") {
+  const space = await resolveUserSpace(fetchJSON, actorPeerId);
+  const profileUri = `viking://user/${space}/memories/profile.md`;
+  const prefUri = `viking://user/${space}/memories/preferences`;
+  const entUri = `viking://user/${space}/memories/entities`;
+
+  const [profile, prefs, ents] = await Promise.all([
+    readProfile(fetchJSON, profileUri, actorPeerId),
+    lsDir(fetchJSON, prefUri, actorPeerId),
+    lsDir(fetchJSON, entUri, actorPeerId),
+  ]);
+
+  if (!profile && prefs.length === 0 && ents.length === 0) return null;
+
+  // Profile gets up to half the total budget; listings split the rest.
+  // Sub-cap protects against a runaway profile blowing the listing budget.
+  const profileBudget = Math.floor(totalBudgetTokens / 2);
+  const profileTrunc = profile ? elideProfile(profile, profileBudget) : null;
+  const profileTokens = estimateTokens(profileTrunc || "");
+
+  const listingBudget = Math.max(0, totalBudgetTokens - profileTokens);
+  const halfListing = Math.floor(listingBudget / 2);
+  const prefBlock = formatListing(prefUri, prefs, halfListing);
+  const entBudget = Math.max(0, listingBudget - prefBlock.used);
+  const entBlock = formatListing(entUri, ents, entBudget);
+
+  const lines = [];
+  if (profileTrunc) {
+    lines.push(`<user-profile uri="${profileUri}">`);
+    lines.push(profileTrunc);
+    lines.push(`</user-profile>`);
+  }
+  if (prefBlock.lines.length > 0 || entBlock.lines.length > 0) {
+    lines.push(`<available-memories>`);
+    lines.push(...prefBlock.lines);
+    lines.push(...entBlock.lines);
+    lines.push(`</available-memories>`);
+  }
+
+  const block = lines.join("\n");
+  return {
+    block,
+    chars: block.length,
+    tokens: estimateTokens(block),
+    profileUri,
+    profileChars: profile?.length ?? 0,
+    prefCount: prefs.length,
+    entCount: ents.length,
+    droppedPref: prefBlock.dropped,
+    droppedEnt: entBlock.dropped,
+  };
+}

package/shared/recall-compress-core.mjs

@@ -0,0 +1,195 @@
+// GENERATED FROM examples/memory-plugin-shared/lib. DO NOT EDIT.
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+
+export const NO_RELEVANT_MEMORY = "NO_RELEVANT_MEMORY";
+export const DIGEST_HEADER = "OpenViking memory digest:";
+const COMPRESS_OK = "ok";
+const COMPRESS_EMPTY = "empty";
+const COMPRESS_FAILED = "failed";
+
+// Medium-constraint prompt: state the goal and two structural floors, but no hard
+// bullet-length contract. Hard per-bullet limits pin the digest to headline
+// density; leaving it unconstrained lets small models rewrite long URIs into dead
+// links, which the URI repair below cleans up.
+export function buildRecallCompressionPrompt({ query, rendered, maxBullets = 6 }) {
+  return `You are a memory relevance compressor utility.
+Do not use any tools. Do not investigate. Only transform the given text.
+
+User query:
+${query}
+
+Retrieved OpenViking context fragments:
+${rendered}
+
+Write a memory digest for a coding agent about to answer that query. Keep the
+concrete facts (paths, identifiers, decisions, constraints); drop pleasantries
+and conversational filler.
+
+Format rules:
+- Group related facts by topic, one bullet per topic, at most ${maxBullets} bullets.
+- Start every bullet with "- ".
+- End every bullet with its source, copied verbatim from the fragments above:
+  "来源：viking://..." or "source: viking://...". Never edit, shorten, or invent a URI.
+- Output the digest body only. No preamble, no closing remark.
+
+If nothing above is relevant to the query, output exactly: ${NO_RELEVANT_MEMORY}`;
+}
+
+function editDistance(a, b) {
+  if (a === b) return 0;
+  const rows = a.length + 1;
+  const cols = b.length + 1;
+  let prev = Array.from({ length: cols }, (_, i) => i);
+  for (let i = 1; i < rows; i += 1) {
+    const cur = [i];
+    for (let j = 1; j < cols; j += 1) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      cur[j] = Math.min(cur[j - 1] + 1, prev[j] + 1, prev[j - 1] + cost);
+    }
+    prev = cur;
+  }
+  return prev[cols - 1];
+}
+
+function nearestUri(candidate, validUris) {
+  let best = "";
+  let bestDistance = Infinity;
+  for (const uri of validUris) {
+    const distance = editDistance(candidate, uri);
+    if (distance < bestDistance) {
+      best = uri;
+      bestDistance = distance;
+    }
+  }
+  // Only repair near-misses; an unrelated hallucination is dropped instead.
+  const tolerance = Math.max(4, Math.floor(candidate.length * 0.25));
+  return bestDistance <= tolerance ? best : "";
+}
+
+/**
+ * Small models occasionally mangle long URIs. Snap every cited URI back onto the
+ * set the server actually returned, and drop bullets whose citation cannot be
+ * recovered so the digest never carries a dead link.
+ */
+export function repairDigestUris(digest, validUris = []) {
+  const text = String(digest || "");
+  if (!text) return "";
+  const valid = validUris.map((uri) => String(uri || "").trim()).filter(Boolean);
+  if (!valid.length) return text;
+  const validSet = new Set(valid);
+
+  const lines = [];
+  for (const line of text.split(/\r?\n/)) {
+    if (!line.trim().startsWith("- ")) {
+      lines.push(line);
+      continue;
+    }
+    let dropped = false;
+    const repaired = line.replace(/viking:\/\/[^\s<>"')\]]+/g, (uri) => {
+      if (validSet.has(uri)) return uri;
+      const nearest = nearestUri(uri, valid);
+      if (nearest) return nearest;
+      dropped = true;
+      return uri;
+    });
+    if (!dropped) lines.push(repaired);
+  }
+  return lines.join("\n").trim();
+}
+
+export function normalizeCompressedContext(raw, maxChars = 4000, maxBullets = 6) {
+  const text = String(raw || "").trim();
+  if (!text) return null;
+  if (text.toUpperCase() === NO_RELEVANT_MEMORY) return "";
+  const bullets = text.split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => /^[-*]\s+/.test(line) && line.includes("viking://"))
+    .slice(0, Math.max(1, maxBullets))
+    .map((line) => `- ${line.replace(/^[-*]\s+/, "").slice(0, 500).trim()}`);
+  if (!bullets.length) return null;
+  return (`${DIGEST_HEADER}\n${bullets.join("\n")}`).slice(0, Math.max(100, maxChars));
+}
+
+export function recallDigestCacheKey({
+  query = "",
+  rendered = "",
+  entries = [],
+  maxInputChars = 18000,
+  maxBullets = 6,
+} = {}) {
+  const uris = entries.map((entry) => String(entry?.uri || "").trim()).filter(Boolean).sort();
+  const source = JSON.stringify({
+    version: 2,
+    query: String(query),
+    rendered: String(rendered).slice(0, maxInputChars),
+    uris,
+    maxInputChars,
+    maxBullets,
+  });
+  return createHash("sha256").update(source).digest("hex");
+}
+
+async function readCache(path) {
+  if (!path) return null;
+  try { return JSON.parse(await readFile(path, "utf8")); } catch { return null; }
+}
+
+async function writeCache(path, value) {
+  if (!path) return;
+  try {
+    await mkdir(dirname(path), { recursive: true });
+    const tmp = `${path}.tmp`;
+    await writeFile(tmp, JSON.stringify(value));
+    await rename(tmp, path);
+  } catch { /* best effort */ }
+}
+
+export async function compressRecallContext({
+  query,
+  rendered,
+  entries = [],
+  cfg = {},
+  runCompressor,
+  cachePath = "",
+  now = 0,
+}) {
+  const input = String(rendered || "").trim();
+  if (!input) return { status: COMPRESS_EMPTY, context: "" };
+  const minChars = Math.max(0, Number(cfg.recallCompressMinInputChars ?? 1500));
+  if (input.length < minChars) return { status: COMPRESS_OK, context: input };
+
+  const maxInputChars = Math.max(1000, Number(cfg.recallCompressMaxInputChars || 18000));
+  const maxBullets = Math.max(1, Number(cfg.recallCompressMaxBullets || 6));
+  const key = recallDigestCacheKey({
+    query,
+    rendered: input,
+    entries,
+    maxInputChars,
+    maxBullets,
+  });
+  const cached = await readCache(cachePath);
+  if (cached?.key === key && typeof cached.digest === "string") {
+    return { status: COMPRESS_OK, context: cached.digest };
+  }
+
+  const prompt = buildRecallCompressionPrompt({
+    query,
+    rendered: input.slice(0, maxInputChars),
+    maxBullets,
+  });
+  const raw = await runCompressor(prompt);
+  const normalized = normalizeCompressedContext(raw, 4000, maxBullets);
+  if (normalized === null) return { status: COMPRESS_FAILED, context: "" };
+  if (!normalized) return { status: COMPRESS_EMPTY, context: "" };
+
+  const validUris = entries.map((entry) => entry?.uri).filter(Boolean);
+  const digest = repairDigestUris(normalized, validUris.length
+    ? validUris
+    : (input.match(/viking:\/\/[^\s<>"']+/g) || []));
+  if (!digest) return { status: COMPRESS_FAILED, context: "" };
+
+  await writeCache(cachePath, { key, digest, updatedAt: now || 0 });
+  return { status: COMPRESS_OK, context: digest };
+}