@openviking/dsh-memory-plugin 0.3.0 → 0.3.2

package/shared/input-filters.mjs

@@ -0,0 +1,219 @@
+// GENERATED FROM examples/memory-plugin-shared/lib. DO NOT EDIT.
+/**
+ * Ordered, operator-configured regex filters for memory plugin input.
+ *
+ * Rules are sed-style strings, applied in order to a single piece of text:
+ *   s/^\s*ultrathink\s+//i   substitute (add `g` to replace every match)
+ *   d|^\s*[/!]|              drop the text when the pattern matches
+ *   k/^\?ov\b/               keep the text only when the pattern matches
+ *   user:d/^\s*\/clear\b/    apply to one role only
+ *
+ * A malformed rule becomes an entry in `errors` and is skipped rather than
+ * throwing, so a typo in a config file cannot take a hook down.
+ */
+
+const OPS = new Set(["s", "d", "k"]);
+const SCOPES = ["user", "assistant"];
+const ALLOWED_FLAGS = "imsug";
+
+export const INPUT_FILTER_KNOBS = [
+  {
+    key: "recallQueryFilters",
+    env: "OPENVIKING_RECALL_QUERY_FILTERS",
+    label: "recall query filters",
+  },
+  {
+    key: "captureFilters",
+    env: "OPENVIKING_CAPTURE_FILTERS",
+    label: "capture filters",
+  },
+];
+
+/**
+ * Read one delimited field. `\` always consumes the next character, and only
+ * `\<delim>` is un-escaped, so `\d` / `\b` / `\x2c` reach RegExp verbatim.
+ * Returns the index of the closing delimiter, or -1 when it is missing.
+ */
+function scanField(source, start, delim) {
+  let out = "";
+  let i = start;
+  while (i < source.length) {
+    const c = source[i];
+    if (c === "\\") {
+      const next = source[i + 1];
+      if (next === undefined) {
+        out += "\\";
+        i += 1;
+        continue;
+      }
+      out += next === delim ? next : "\\" + next;
+      i += 2;
+      continue;
+    }
+    if (c === delim) return { value: out, end: i };
+    out += c;
+    i += 1;
+  }
+  return { value: out, end: -1 };
+}
+
+export function parseInputFilterRule(source) {
+  if (typeof source !== "string") {
+    return { source: String(source ?? ""), error: "rule must be a string" };
+  }
+  const raw = source.trim();
+  if (!raw) return { source: raw, error: "rule is empty" };
+
+  let scope = "";
+  let rest = raw;
+  for (const candidate of SCOPES) {
+    if (rest.startsWith(`${candidate}:`)) {
+      scope = candidate;
+      rest = rest.slice(candidate.length + 1);
+      break;
+    }
+  }
+
+  const op = rest[0] || "";
+  if (!OPS.has(op)) {
+    return {
+      source: raw,
+      error: `unknown operation "${op}" — expected s (substitute), d (drop) or k (keep)`,
+    };
+  }
+  const delim = rest[1];
+  if (delim === undefined) {
+    return { source: raw, error: `missing delimiter after "${op}"` };
+  }
+  if (/[A-Za-z0-9\s\\]/.test(delim)) {
+    return {
+      source: raw,
+      error: `invalid delimiter ${JSON.stringify(delim)} — use punctuation such as / | # or :`,
+    };
+  }
+
+  const first = scanField(rest, 2, delim);
+  const pattern = first.value;
+  let replacement = null;
+  let flagsRaw = "";
+  if (op === "s") {
+    if (first.end < 0) {
+      return { source: raw, error: `missing ${JSON.stringify(delim)} after the pattern` };
+    }
+    const second = scanField(rest, first.end + 1, delim);
+    if (second.end < 0) {
+      return {
+        source: raw,
+        error: `missing ${JSON.stringify(delim)} after the replacement — write s${delim}pattern${delim}replacement${delim}`,
+      };
+    }
+    replacement = second.value;
+    flagsRaw = rest.slice(second.end + 1);
+  } else {
+    flagsRaw = first.end < 0 ? "" : rest.slice(first.end + 1);
+  }
+
+  if (!pattern) return { source: raw, error: "pattern is empty" };
+
+  const flags = new Set();
+  for (const flag of flagsRaw) {
+    if (!ALLOWED_FLAGS.includes(flag)) {
+      return { source: raw, error: `unknown flag "${flag}" — allowed flags are i, m, s, u, g` };
+    }
+    if (flags.has(flag)) return { source: raw, error: `duplicate flag "${flag}"` };
+    flags.add(flag);
+  }
+  // d/k decide with .test(), which advances lastIndex on a global regex and
+  // would then alternate between calls. `g` means nothing there, so drop it.
+  if (op !== "s") flags.delete("g");
+
+  let re;
+  try {
+    re = new RegExp(pattern, [...flags].join(""));
+  } catch (err) {
+    // V8 already says "Invalid regular expression: <pattern>: <why>"; keep that
+    // detail without stuttering the prefix the doctors key off.
+    const message = err?.message || String(err);
+    return {
+      source: raw,
+      error: /^invalid regular expression/i.test(message)
+        ? `invalid${message.slice("Invalid".length)}`
+        : `invalid regular expression: ${message}`,
+    };
+  }
+
+  return { op, scope, re, replacement, source: raw };
+}
+
+export function compileInputFilters(rules) {
+  const compiled = [];
+  const errors = [];
+  const list = Array.isArray(rules) ? rules : [];
+  list.forEach((entry, index) => {
+    if (typeof entry !== "string") {
+      errors.push({ index, source: String(entry ?? ""), message: "rule must be a string" });
+      return;
+    }
+    if (!entry.trim()) return;
+    const parsed = parseInputFilterRule(entry);
+    if (parsed.error) errors.push({ index, source: parsed.source, message: parsed.error });
+    else compiled.push({ ...parsed, index });
+  });
+  return { rules: compiled, errors };
+}
+
+/**
+ * Run compiled rules over `text`.
+ *
+ * `substituteOnly` skips the d/k operations, so a caller that already took one
+ * drop decision for a turn can rewrite its individual pieces without taking a
+ * second, possibly contradictory one.
+ */
+export function applyInputFilters(text, compiled, { role = "", substituteOnly = false } = {}) {
+  const original = typeof text === "string" ? text : String(text ?? "");
+  let current = original;
+  for (const rule of Array.isArray(compiled) ? compiled : []) {
+    if (rule.scope && rule.scope !== role) continue;
+    if (rule.op === "s") {
+      current = current.replace(rule.re, rule.replacement);
+      continue;
+    }
+    if (substituteOnly) continue;
+    const matched = rule.re.test(current);
+    if ((rule.op === "d" && matched) || (rule.op === "k" && !matched)) {
+      return { text: "", changed: false, dropped: true, ruleIndex: rule.index, op: rule.op };
+    }
+  }
+  const finalText = current.trim();
+  return {
+    text: finalText,
+    changed: finalText !== original,
+    dropped: false,
+    ruleIndex: -1,
+    op: "",
+  };
+}
+
+/** Doctor-facing summary of both filter knobs on a resolved plugin config. */
+export function describeInputFilters(cfg = {}) {
+  return INPUT_FILTER_KNOBS.map(({ key, env, label }) => {
+    const configured = Array.isArray(cfg?.[key]) ? cfg[key] : [];
+    const compiled = compileInputFilters(configured);
+    const ops = { s: 0, d: 0, k: 0 };
+    for (const rule of compiled.rules) ops[rule.op] += 1;
+    const bits = [];
+    if (ops.s) bits.push(`${ops.s} substitute`);
+    if (ops.d) bits.push(`${ops.d} drop`);
+    if (ops.k) bits.push(`${ops.k} keep-only`);
+    const active = compiled.rules.length;
+    return {
+      key,
+      env,
+      label,
+      total: configured.length,
+      active,
+      summary: `${active} rule${active === 1 ? "" : "s"}${bits.length ? ` (${bits.join(", ")})` : ""}`,
+      errors: compiled.errors,
+    };
+  });
+}

package/shared/mcp-proxy-config.mjs

@@ -45,24 +45,34 @@ export function defaultCredentialPaths(env = process.env) {
  * Resolve the actor peer header for a long-lived MCP proxy process.
  *
  * MCP servers may start in the plugin directory rather than the active
- * workspace, so their process cwd is not a reliable peer identity. Broad
- * recall intentionally spans the authenticated user's peer workspaces and
- * therefore leaves the actor header unset. Actor-scoped recall requires an
- * explicit peer so isolation never depends on the proxy launch directory.
+ * workspace, so their process cwd is not a reliable peer identity — and unlike
+ * a hook, a long-lived proxy cannot re-derive one per turn. Broad recall
+ * intentionally spans the authenticated user's peer workspaces and therefore
+ * leaves the actor header unset. Actor-scoped recall needs an explicit peer so
+ * isolation never depends on the proxy's launch directory.
+ *
+ * A missing one degrades to broad recall with a warning rather than refusing to
+ * start: the proxy is what carries every memory tool, and taking those away
+ * because a scope preference cannot be honoured costs the user far more than
+ * the wider search does. Only the harness whose parent process injects the peer
+ * (dsh, `mcp.mjs:27`) can satisfy this without the user setting it.
  */
 export function resolveMcpActorPeerId({
   peerId = "",
   recallPeerScope = "all",
+  onWarn = null,
 } = {}) {
   if (recallPeerScope !== "actor") return "";
 
   const explicitPeerId = String(peerId || "").trim();
   if (explicitPeerId) return explicitPeerId;
 
-  throw new Error(
-    "OpenViking MCP actor-scoped recall requires an explicit peer ID. "
-    + "Set actor_peer_id in ovcli.conf or OPENVIKING_PEER_ID in the MCP environment.",
-  );
+  const message = "OpenViking MCP: actor-scoped recall needs an explicit peer id, which an MCP proxy cannot "
+    + "derive from its launch directory. Falling back to broad recall across every peer under this user. "
+    + "Set actor_peer_id in ovcli.conf, or OPENVIKING_PEER_ID in the MCP server environment, to scope it.";
+  if (typeof onWarn === "function") onWarn(message);
+  else process.stderr.write(`${message}\n`);
+  return "";
 }
 
 function uniq(values) {

package/shared/mcp-proxy-core.mjs

@@ -292,6 +292,18 @@ export function createOpenVikingMcpProxy({
       );
     }
     const msg = err instanceof Error ? err.message : String(err);
+    if (err && err.name === "AbortError") {
+      // Client-side timeout, not an outage: the server may be healthy and
+      // still computing (rerank-inclusive find/search can legitimately take
+      // longer than the default budget). Keep -32001 for genuine
+      // connection failures so the two are not misdiagnosed as each other.
+      return errorResponse(
+        id,
+        -32004,
+        `OpenViking MCP request timed out after ${proxyConfig.timeoutMs}ms (${proxyConfig.mcpUrl}). The server may still be processing (rerank can be slow) — check /health or raise OPENVIKING_TIMEOUT_MS.`,
+        { timeoutMs: proxyConfig.timeoutMs, mcpUrl: proxyConfig.mcpUrl, cause: msg },
+      );
+    }
     return errorResponse(
       id,
       -32001,

package/shared/pending-queue.mjs

@@ -4,9 +4,10 @@
  *
  * When the OpenViking server is temporarily unreachable, write operations
  * (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
  *
  * Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
  *
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
   }
 }
 
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+  if (!claimedFilename.endsWith(".processing")) return null;
+  const dir = getPendingDir();
+  const restored = pendingFromProcessingFilename(claimedFilename);
+  try {
+    await rename(join(dir, claimedFilename), join(dir, restored));
+    return restored;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Remove a pending entry after successful replay.
  */
@@ -350,9 +368,16 @@ export async function cleanStale() {
  *
  * @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
  * @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ *   failures release their claim instead of incrementing the retry count, so
+ *   a background drainer can keep retrying a transient failure without
+ *   burning the session-start retry budget. Exhausted and non-retryable
+ *   entries are deleted exactly as in the default mode.
  * @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
  */
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+  const consumeRetries = options.consumeRetries !== false;
   const pending = await listPending();
 
   if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
       await dequeue(claimedFilename);
       skipped++;
     } else {
-      await incrementRetry(claimedFilename, entry);
+      if (consumeRetries) {
+        await incrementRetry(claimedFilename, entry);
+      } else {
+        const released = await releaseClaim(claimedFilename);
+        log("pending-queue", {
+          action: released ? "replay-deferred" : "release-failed",
+          sessionId: entry.sessionId,
+          type: entry.type,
+          status: res?.result?.status || res?.status,
+          retries: entry.retries || 0,
+        });
+      }
       failed++;
       if (entry.type === "addMessage") {
         deferred += Math.max(0, pending.length - processed);

package/shared/recall-compress-core.mjs

@@ -86,10 +86,16 @@ export function repairDigestUris(digest, validUris = []) {
       lines.push(line);
       continue;
     }
+    let tokenized = line;
+    for (const uri of [...valid].sort((a, b) => b.length - a.length)) {
+      if (/\s/.test(uri)) tokenized = tokenized.split(uri).join(encodeURI(uri));
+    }
     let dropped = false;
-    const repaired = line.replace(/viking:\/\/[^\s<>"')\]]+/g, (uri) => {
-      if (validSet.has(uri)) return uri;
-      const nearest = nearestUri(uri, valid);
+    const repaired = tokenized.replace(/viking:\/\/[^\s<>"')\]]+/g, (uri) => {
+      let decoded = uri;
+      try { decoded = decodeURI(uri); } catch { /* keep the original candidate */ }
+      if (validSet.has(decoded)) return decoded;
+      const nearest = nearestUri(decoded, valid);
       if (nearest) return nearest;
       dropped = true;
       return uri;
@@ -149,6 +155,7 @@ async function writeCache(path, value) {
 export async function compressRecallContext({
   query,
   rendered,
+  shortContext = rendered,
   entries = [],
   cfg = {},
   runCompressor,
@@ -158,7 +165,9 @@ export async function compressRecallContext({
   const input = String(rendered || "").trim();
   if (!input) return { status: COMPRESS_EMPTY, context: "" };
   const minChars = Math.max(0, Number(cfg.recallCompressMinInputChars ?? 1500));
-  if (input.length < minChars) return { status: COMPRESS_OK, context: input };
+  if (input.length < minChars) {
+    return { status: COMPRESS_OK, context: String(shortContext ?? input).trim() };
+  }
 
   const maxInputChars = Math.max(1000, Number(cfg.recallCompressMaxInputChars || 18000));
   const maxBullets = Math.max(1, Number(cfg.recallCompressMaxBullets || 6));
@@ -171,7 +180,8 @@ export async function compressRecallContext({
   });
   const cached = await readCache(cachePath);
   if (cached?.key === key && typeof cached.digest === "string") {
-    return { status: COMPRESS_OK, context: cached.digest };
+    const digest = normalizeCompressedContext(cached.digest, 4000, maxBullets);
+    if (digest) return { status: COMPRESS_OK, context: digest };
   }
 
   const prompt = buildRecallCompressionPrompt({
@@ -185,9 +195,10 @@ export async function compressRecallContext({
   if (!normalized) return { status: COMPRESS_EMPTY, context: "" };
 
   const validUris = entries.map((entry) => entry?.uri).filter(Boolean);
-  const digest = repairDigestUris(normalized, validUris.length
+  const repaired = repairDigestUris(normalized, validUris.length
     ? validUris
     : (input.match(/viking:\/\/[^\s<>"']+/g) || []));
+  const digest = normalizeCompressedContext(repaired, 4000, maxBullets);
   if (!digest) return { status: COMPRESS_FAILED, context: "" };
 
   await writeCache(cachePath, { key, digest, updatedAt: now || 0 });

package/shared/recall-core.mjs

@@ -417,6 +417,31 @@ export async function markContextFaceLegacy(path = stateFile("context-face.json"
   await writeJsonFile(path, { legacyUntil: now + LEGACY_CACHE_TTL_MS });
 }
 
+/**
+ * A `peer_scope` the server rejects is remembered the same way, but unlike the
+ * context face this one is not a silent capability probe: dropping the field
+ * widens recall from the caller's own peer to the whole user root, so doctor
+ * reads this file back and warns.
+ */
+export function peerScopeMemoPath() {
+  return stateFile("peer-scope.json");
+}
+
+export async function readPeerScopeDowngrade(path = peerScopeMemoPath(), now = Date.now()) {
+  const cached = await readJsonFile(path);
+  if (!cached?.legacyUntil || Number(cached.legacyUntil) <= now) return null;
+  return cached;
+}
+
+export async function markPeerScopeDowngrade(scope, status, path = peerScopeMemoPath(), now = Date.now()) {
+  await writeJsonFile(path, {
+    legacyUntil: now + LEGACY_CACHE_TTL_MS,
+    scope: String(scope || ""),
+    status: Number(status) || 0,
+    at: now,
+  });
+}
+
 function looksLikeUnknownField(res) {
   const text = JSON.stringify(res?.error ?? res?.result ?? res?.detail ?? "").toLowerCase();
   return text.includes("extra") || text.includes("mode") || text.includes("unexpected");
@@ -558,7 +583,15 @@ async function recallViaEndpoint(fetchJSON, cfg, query, actorPeerId = "", log =
 export async function postRecall(fetchJSON, body, opts = {}) {
   const actorPeerId = opts.actorPeerId || "";
   const log = opts.log || (() => {});
+  const memoPath = opts.peerScopeMemoPath;
   const request = { ...body };
+
+  // A remembered downgrade is still a downgrade: recall runs wider than the
+  // caller asked for, so the memo doubles as the doctor's evidence.
+  if (request.peer_scope && await readPeerScopeDowngrade(memoPath)) {
+    delete request.peer_scope;
+  }
+
   const res = await fetchJSON("/api/v1/search/recall", {
     method: "POST",
     body: JSON.stringify(request),
@@ -566,9 +599,17 @@ export async function postRecall(fetchJSON, body, opts = {}) {
   if (!request.peer_scope || (res.status !== 400 && res.status !== 422)) {
     return res;
   }
+  // Only an unknown-field rejection means "this server predates peer_scope".
+  // Retrying every other 400/422 without it silently widens the search from
+  // the caller's own peer to the whole user root.
+  if (!looksLikeUnknownField(res)) {
+    log("recall_peer_scope_error", { status: res.status || 0 });
+    return res;
+  }
 
   const downgraded = { ...request };
   delete downgraded.peer_scope;
+  await markPeerScopeDowngrade(String(request.peer_scope), res.status || 0, memoPath);
   log("recall_peer_scope_downgrade", { status: res.status || 0 });
   return fetchJSON("/api/v1/search/recall", {
     method: "POST",
@@ -576,7 +617,31 @@ export async function postRecall(fetchJSON, body, opts = {}) {
   }, { actorPeerId });
 }
 
+/**
+ * Recall, plus whatever is still filed under the peer this workspace used
+ * before the identity rule changed.
+ *
+ * Under the default `peer_scope: "all"` this costs nothing: the server's own
+ * sweep of `{user_root}/peers` already reaches the old peer's memories. Under
+ * `"actor"` that sweep is off by definition, so the old peer is asked for
+ * separately — as itself, which is both cheaper and wider than a bare
+ * cross-peer read (that would need the user id, and reaches memories only).
+ */
 export async function buildRecallBlock(fetchJSON, cfg, query, options = {}) {
+  const primary = await recallForPeer(fetchJSON, cfg, query, options);
+
+  const legacyPeerId = String(options.legacyPeerId || "").trim();
+  const actorPeerId = options.actorPeerId ?? cfg.peerId ?? "";
+  if (cfg.recallPeerScope !== "actor" || !legacyPeerId || legacyPeerId === actorPeerId) return primary;
+
+  const log = options.log || (() => {});
+  const legacy = await recallForPeer(fetchJSON, cfg, query, { ...options, actorPeerId: legacyPeerId });
+  if (!legacy) return primary;
+  log("recall_legacy_peer_hit", { legacyPeerId });
+  return primary ? `${primary}\n${legacy}` : legacy;
+}
+
+async function recallForPeer(fetchJSON, cfg, query, options = {}) {
   const actorPeerId = options.actorPeerId ?? cfg.peerId ?? "";
   const log = options.log || (() => {});
   const trimmed = String(query || "").trim();