@openviking/dsh-memory-plugin 0.3.0 → 0.4.3

package/shared/mcp-proxy-config.mjs

@@ -12,9 +12,56 @@
 import { homedir } from "node:os";
 import { join, resolve as resolvePath } from "node:path";
 
+import { CONNECTION_ENV_VARS, CREDENTIAL_ENV_VARS } from "./credentials.mjs";
+
 export const DEFAULT_PROXY_TIMEOUT_MS = 15000;
 const MIN_PROXY_TIMEOUT_MS = 1000;
 
+/**
+ * Every variable that changes what an MCP proxy sends. Hooks inherit their
+ * host's whole environment; a host that hands its MCP servers an allowlist
+ * instead (Codex's `env_vars`) has to name each of these, or the proxy
+ * resolves a different connection from the hooks beside it.
+ */
+export const MCP_PROXY_ENV_VARS = [
+  ...CONNECTION_ENV_VARS,
+  "OPENVIKING_HOME",
+  "OPENVIKING_STATE_DIR",
+  "OPENVIKING_RECALL_PEER_SCOPE",
+  "OPENVIKING_TIMEOUT_MS",
+  "OPENVIKING_DEBUG",
+  "OPENVIKING_DEBUG_LOG",
+  "OPENVIKING_EXTRA_HEADERS",
+];
+
+/**
+ * A resolved connection as the environment of a child proxy process.
+ *
+ * The other way across the process boundary, for a host that cannot hand its
+ * MCP servers the inputs (DSH drops credential-shaped names from what a child
+ * inherits): the parent resolves once and the child reads the answer back. The
+ * forced `env` source makes that exact — the child reads no file, so a key the
+ * parent left empty stays empty instead of falling through to
+ * `server.root_api_key`. Every credential variable is written, the empty ones
+ * too, because the rest of the parent's environment still reaches the child:
+ * an `OPENVIKING_ACCOUNT` the parent's pinned chain ignored must not fill the
+ * gap there.
+ */
+export function forwardConnectionEnv(connection) {
+  const env = Object.fromEntries(CREDENTIAL_ENV_VARS.map((name) => [name, ""]));
+  return {
+    ...env,
+    OPENVIKING_CREDENTIAL_SOURCE: "env",
+    OPENVIKING_URL: connection.baseUrl,
+    OPENVIKING_MCP_URL: connection.mcpUrl,
+    OPENVIKING_API_KEY: connection.apiKey || "",
+    OPENVIKING_ACCOUNT: connection.account || "",
+    OPENVIKING_USER: connection.user || "",
+    OPENVIKING_PEER_ID: connection.peerId || "",
+    OPENVIKING_AUTH_MODE: connection.authMode,
+  };
+}
+
 export function trimSlash(value) {
   return String(value || "").replace(/\/+$/, "");
 }
@@ -45,30 +92,101 @@ export function defaultCredentialPaths(env = process.env) {
  * Resolve the actor peer header for a long-lived MCP proxy process.
  *
  * MCP servers may start in the plugin directory rather than the active
- * workspace, so their process cwd is not a reliable peer identity. Broad
- * recall intentionally spans the authenticated user's peer workspaces and
- * therefore leaves the actor header unset. Actor-scoped recall requires an
- * explicit peer so isolation never depends on the proxy launch directory.
+ * workspace, so their process cwd is not a reliable peer identity — and unlike
+ * a hook, a long-lived proxy cannot re-derive one per turn. Broad recall
+ * intentionally spans the authenticated user's peer workspaces and therefore
+ * leaves the actor header unset. Actor-scoped recall needs an explicit peer so
+ * isolation never depends on the proxy's launch directory.
+ *
+ * A missing one degrades to broad recall with a warning rather than refusing to
+ * start: the proxy is what carries every memory tool, and taking those away
+ * because a scope preference cannot be honoured costs the user far more than
+ * the wider search does. Only the harness whose parent process injects the peer
+ * (dsh, `mcp-env.mjs`) can satisfy this without the user setting it.
  */
 export function resolveMcpActorPeerId({
   peerId = "",
   recallPeerScope = "all",
+  onWarn = null,
 } = {}) {
   if (recallPeerScope !== "actor") return "";
 
   const explicitPeerId = String(peerId || "").trim();
   if (explicitPeerId) return explicitPeerId;
 
-  throw new Error(
-    "OpenViking MCP actor-scoped recall requires an explicit peer ID. "
-    + "Set actor_peer_id in ovcli.conf or OPENVIKING_PEER_ID in the MCP environment.",
-  );
+  const message = "OpenViking MCP: actor-scoped recall needs an explicit peer id, which an MCP proxy cannot "
+    + "derive from its launch directory. Falling back to broad recall across every peer under this user. "
+    + "Set actor_peer_id in ovcli.conf, or OPENVIKING_PEER_ID in the MCP server environment, to scope it.";
+  if (typeof onWarn === "function") onWarn(message);
+  else process.stderr.write(`${message}\n`);
+  return "";
 }
 
 function uniq(values) {
   return [...new Set(values.filter(Boolean))];
 }
 
+/**
+ * Parse `OPENVIKING_EXTRA_HEADERS` into a plain string→string map.
+ *
+ * Some private deployments sit behind gateways that require custom headers
+ * (a tenant/vault name, a signed token, a region hint) on every request. The
+ * proxy has no other seam to inject those: the credential file's own
+ * `extra_headers` field is intentionally not exported to MCP callers (see
+ * `workspace-config.mjs` `FORBIDDEN_KEYS`), so this env-only escape hatch keeps
+ * that boundary while letting an operator unblock a strict upstream.
+ *
+ * The value is a JSON object; anything else is ignored with a stderr warning
+ * rather than crashing the proxy — a bad env var must not take down the whole
+ * memory tool surface. Reserved names (`Content-Type`, `Accept`,
+ * `MCP-Protocol-Version`, `Mcp-Session-Id`, `Authorization`) are dropped: they
+ * are set by the proxy itself and letting the env override them would break
+ * session negotiation or expose the wrong credential.
+ */
+const RESERVED_HEADER_NAMES = new Set([
+  "content-type",
+  "accept",
+  "mcp-protocol-version",
+  "mcp-session-id",
+  "authorization",
+]);
+
+export function parseExtraHeaders(raw, { onWarn = null } = {}) {
+  const source = String(raw || "").trim();
+  if (!source) return {};
+  const warn = (message) => {
+    if (typeof onWarn === "function") onWarn(message);
+    else process.stderr.write(`${message}\n`);
+  };
+  let parsed;
+  try {
+    parsed = JSON.parse(source);
+  } catch (err) {
+    warn(`OPENVIKING_EXTRA_HEADERS is not valid JSON; ignoring (${err.message})`);
+    return {};
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    warn("OPENVIKING_EXTRA_HEADERS must be a JSON object of string values; ignoring");
+    return {};
+  }
+  const out = {};
+  for (const [key, value] of Object.entries(parsed)) {
+    const name = String(key || "").trim();
+    if (!name) continue;
+    if (RESERVED_HEADER_NAMES.has(name.toLowerCase())) {
+      warn(`OPENVIKING_EXTRA_HEADERS: ignoring reserved header "${name}"`);
+      continue;
+    }
+    if (value === null || value === undefined) continue;
+    if (typeof value === "object") {
+      warn(`OPENVIKING_EXTRA_HEADERS: header "${name}" must be a scalar; ignoring`);
+      continue;
+    }
+    out[name] = String(value);
+  }
+  return out;
+}
+
 /**
  * Shape a harness's loaded config into the object `createOpenVikingMcpProxy`
  * expects.
@@ -76,7 +194,9 @@ function uniq(values) {
  * `mcpUrl` wins over `baseUrl` when both are given, so a harness that lets the
  * user pin an explicit MCP URL keeps that behavior. `watchedPaths` are the
  * harness's own extra files; the shared defaults are appended and the result
- * deduplicated.
+ * deduplicated. `sendIdentityHeaders` carries the harness's resolved auth mode
+ * so an `api_key` server, which reads the identity out of the key and ignores
+ * the headers, never sees the operator's account and user on the wire.
  */
 export function buildMcpProxyConfig({
   baseUrl = "",
@@ -84,6 +204,7 @@ export function buildMcpProxyConfig({
   apiKey = "",
   account = "",
   user = "",
+  sendIdentityHeaders = false,
   peerId = "",
   userAgent = "",
   timeoutMs,
@@ -92,13 +213,18 @@ export function buildMcpProxyConfig({
   credentialSource = "auto",
   credentialPath = "",
   watchedPaths = [],
+  extraHeaders = null,
   env = process.env,
 } = {}) {
+  const resolvedExtraHeaders = extraHeaders && typeof extraHeaders === "object" && !Array.isArray(extraHeaders)
+    ? { ...extraHeaders }
+    : parseExtraHeaders(env.OPENVIKING_EXTRA_HEADERS);
   return {
     mcpUrl: mcpUrl || `${trimSlash(baseUrl)}/mcp`,
     apiKey: apiKey || "",
     account: account || "",
     user: user || "",
+    sendIdentityHeaders: sendIdentityHeaders === true,
     peerId: peerId || "",
     userAgent: userAgent || "",
     timeoutMs: Math.max(
@@ -110,5 +236,41 @@ export function buildMcpProxyConfig({
     credentialSource: credentialSource || "auto",
     credentialPath: credentialPath || "",
     watchedPaths: uniq([...watchedPaths, ...defaultCredentialPaths(env)]),
+    extraHeaders: resolvedExtraHeaders,
   };
 }
+
+/**
+ * The proxy config for a resolved harness config, mapped in one place.
+ *
+ * Every proxy used to copy fields out of its loader by hand, and the copies
+ * drifted: two dropped `mcpUrl`, so `OPENVIKING_MCP_URL` reached the hooks and
+ * not the tools. `cfg` is what `buildPluginConfig` or `buildProxyConnection`
+ * returned. `peerId` replaces the actor peer only for a harness whose parent
+ * process resolved it (dsh); everywhere else a long-lived proxy may send one
+ * only when actor-scoped recall asks for it.
+ */
+export function toMcpProxyConfig(cfg, {
+  env = process.env,
+  peerId = undefined,
+  debug = cfg.debug,
+  debugLogPath = cfg.debugLogPath,
+} = {}) {
+  return buildMcpProxyConfig({
+    baseUrl: cfg.baseUrl,
+    mcpUrl: cfg.mcpUrl,
+    apiKey: cfg.apiKey,
+    account: cfg.account,
+    user: cfg.user,
+    sendIdentityHeaders: cfg.sendIdentityHeaders,
+    peerId: peerId === undefined ? resolveMcpActorPeerId(cfg) : peerId,
+    userAgent: cfg.userAgent,
+    timeoutMs: cfg.timeoutMs,
+    debug,
+    debugLogPath,
+    credentialSource: cfg.credentialSource,
+    credentialPath: cfg.credentialPath || "",
+    watchedPaths: [cfg.cliPath, cfg.ovPath],
+    env,
+  });
+}

package/shared/mcp-proxy-core.mjs

@@ -9,6 +9,8 @@
 import { statSync } from "node:fs";
 import { createInterface } from "node:readline";
 
+import { buildOvHeaders } from "./ov-http.mjs";
+
 const DEFAULT_PROTOCOL_VERSION = "2025-06-18";
 const DELETE_TIMEOUT_MS = 2000;
 const MAX_CONCURRENT_REQUESTS = 16;
@@ -240,21 +242,27 @@ export function createOpenVikingMcpProxy({
   }
 
   function headersForRequest(includeSession = true) {
-    const headers = {
-      "Content-Type": "application/json",
-      "Accept": "application/json, text/event-stream",
-      // Always the proxy's current version (default, then server-negotiated) —
-      // never the client's un-negotiated ask, which strict upstreams reject
-      // with HTTP 400 before initialize negotiation can run.
-      "MCP-Protocol-Version": protocolVersion,
-    };
-    if (includeSession && sessionId) headers["Mcp-Session-Id"] = sessionId;
-    if (proxyConfig.apiKey) headers.Authorization = `Bearer ${proxyConfig.apiKey}`;
-    if (proxyConfig.account) headers["X-OpenViking-Account"] = proxyConfig.account;
-    if (proxyConfig.user) headers["X-OpenViking-User"] = proxyConfig.user;
-    if (proxyConfig.peerId) headers["X-OpenViking-Actor-Peer"] = proxyConfig.peerId;
-    if (proxyConfig.userAgent) headers["User-Agent"] = proxyConfig.userAgent;
-    return headers;
+    const built = buildOvHeaders(proxyConfig, {
+      actorPeerId: proxyConfig.peerId,
+      extraHeaders: {
+        "Accept": "application/json, text/event-stream",
+        // Always the proxy's current version (default, then server-negotiated) —
+        // never the client's un-negotiated ask, which strict upstreams reject
+        // with HTTP 400 before initialize negotiation can run.
+        "MCP-Protocol-Version": protocolVersion,
+        ...(includeSession && sessionId ? { "Mcp-Session-Id": sessionId } : {}),
+      },
+    });
+    // Operator-supplied extras (OPENVIKING_EXTRA_HEADERS) merged with the
+    // proxy's own headers winning — the env-only escape hatch fills gaps for
+    // strict private-gateway upstreams (tenant/vault names, region hints)
+    // without letting a stray env var override authentication or session
+    // negotiation.
+    const opExtras = proxyConfig.extraHeaders;
+    if (opExtras && typeof opExtras === "object") {
+      return { ...opExtras, ...built };
+    }
+    return built;
   }
 
   function writeMessage(obj) {
@@ -292,6 +300,18 @@ export function createOpenVikingMcpProxy({
       );
     }
     const msg = err instanceof Error ? err.message : String(err);
+    if (err && err.name === "AbortError") {
+      // Client-side timeout, not an outage: the server may be healthy and
+      // still computing (rerank-inclusive find/search can legitimately take
+      // longer than the default budget). Keep -32001 for genuine
+      // connection failures so the two are not misdiagnosed as each other.
+      return errorResponse(
+        id,
+        -32004,
+        `OpenViking MCP request timed out after ${proxyConfig.timeoutMs}ms (${proxyConfig.mcpUrl}). The server may still be processing (rerank can be slow) — check /health or raise OPENVIKING_TIMEOUT_MS.`,
+        { timeoutMs: proxyConfig.timeoutMs, mcpUrl: proxyConfig.mcpUrl, cause: msg },
+      );
+    }
     return errorResponse(
       id,
       -32001,

package/shared/ov-http.mjs

@@ -0,0 +1,112 @@
+// GENERATED FROM examples/memory-plugin-shared/lib. DO NOT EDIT.
+/**
+ * The one path from a hook to the OpenViking server.
+ *
+ * Every harness used to own its own AbortController, header block and envelope
+ * parser, and they drifted: one sent the api key twice, one named the operator
+ * to every proxy on the path whether or not the server was trusted with it, one
+ * read a different field for the same account. A server-side change now lands
+ * in one place instead of eight.
+ *
+ * `X-API-Key` is deliberately absent. The server takes `Authorization: Bearer`,
+ * and the identity headers go out only when the resolved config says this
+ * server is trusted with the operator's name.
+ */
+
+// A request that aborts the instant it starts reads as a dead server. A config
+// that asks for less than a second gets a second.
+const MIN_TIMEOUT_MS = 1000;
+
+/**
+ * The headers every harness puts on every OpenViking request.
+ *
+ * `identityHeaders` overrides `cfg.sendIdentityHeaders` for the callers that
+ * decide per request rather than per config — the doctor probes both ways.
+ */
+export function buildOvHeaders(cfg = {}, { actorPeerId = "", identityHeaders, extraHeaders } = {}) {
+  const identity = identityHeaders === undefined ? Boolean(cfg.sendIdentityHeaders) : Boolean(identityHeaders);
+  const headers = { "Content-Type": "application/json" };
+  if (cfg.apiKey) headers["Authorization"] = `Bearer ${cfg.apiKey}`;
+  if (identity && cfg.account) headers["X-OpenViking-Account"] = cfg.account;
+  if (identity && cfg.user) headers["X-OpenViking-User"] = cfg.user;
+  if (actorPeerId) headers["X-OpenViking-Actor-Peer"] = actorPeerId;
+  if (cfg.userAgent) headers["User-Agent"] = cfg.userAgent;
+  return { ...headers, ...extraHeaders };
+}
+
+function responseTraceId(body) {
+  return body?.result?.trace_id || body?.error?.trace_id || body?.trace_id || undefined;
+}
+
+/**
+ * Build the `fetchJSON(path, init, { timeoutMs, actorPeerId })` a harness talks
+ * through. It resolves to `{ ok, status, result, error, traceId }` and never
+ * throws: a hook that crashes on a closed port would take its host's turn with
+ * it, so a network error is `{ ok: false, status: 0 }` and the retry rules in
+ * `retryable.mjs` decide what happens next.
+ *
+ * `resolveActorPeerId` is a callback because Codex only knows its peer after
+ * loading session state under the lock, well after the client is built.
+ * `requireJsonBody` is for the capture hooks, which would rather retry a body
+ * they cannot parse than record the turn as sent.
+ */
+export function createOvHttp(cfg = {}, {
+  defaultTimeoutMs = 0,
+  resolveActorPeerId = () => "",
+  identityHeaders,
+  extraHeaders,
+  requireJsonBody = false,
+} = {}) {
+  // dsh and pi name this field `endpoint`; every other harness names it `baseUrl`.
+  const baseUrl = cfg.baseUrl || cfg.endpoint || "";
+  return async function fetchJSON(path, init = {}, options = {}) {
+    const timeoutMs = Math.max(MIN_TIMEOUT_MS, Number(options.timeoutMs) || Number(defaultTimeoutMs) || 0);
+    const controller = new AbortController();
+    let timedOut = false;
+    const timer = setTimeout(() => {
+      timedOut = true;
+      controller.abort();
+    }, timeoutMs);
+    try {
+      const headers = {
+        ...buildOvHeaders(cfg, {
+          actorPeerId: options.actorPeerId ?? resolveActorPeerId(),
+          identityHeaders,
+          extraHeaders,
+        }),
+        ...(init.headers || {}),
+      };
+      const response = await fetch(`${baseUrl}${path}`, { ...init, headers, signal: controller.signal });
+      const body = await response.json().catch(() => null);
+      if (requireJsonBody && !body) {
+        return {
+          ok: false,
+          status: response.status,
+          result: null,
+          error: { message: "empty or invalid JSON response" },
+        };
+      }
+      const traceId = responseTraceId(body);
+      if (!response.ok || body?.status === "error") {
+        return {
+          ok: false,
+          status: response.status,
+          result: null,
+          error: body?.error || { message: `HTTP ${response.status}` },
+          traceId,
+        };
+      }
+      return { ok: true, status: response.status, result: body?.result ?? body ?? {}, traceId };
+    } catch (error) {
+      const message = error?.message || String(error);
+      // This module owns the controller, so it says a timeout was a timeout
+      // rather than leaving callers to guess it from whatever fetch threw.
+      const failure = timedOut
+        ? { name: "AbortError", aborted: true, message }
+        : { message };
+      return { ok: false, status: 0, result: null, error: failure };
+    } finally {
+      clearTimeout(timer);
+    }
+  };
+}

package/shared/pending-queue.mjs

@@ -4,9 +4,10 @@
  *
  * When the OpenViking server is temporarily unreachable, write operations
  * (addMessage, commitSession) serialize their payloads to
- * `~/.openviking/pending/` as JSON files. On the next session-start, the
- * queue is replayed in small batches. This is a session-start-triggered retry
- * path with maxRetries/TTL, not a long-running background worker.
+ * `~/.openviking/pending/` as JSON files. The queue is replayed in small
+ * batches, either at session-start (consuming retry budgets, with
+ * maxRetries/TTL) or by a long-running drainer that passes
+ * `consumeRetries: false` so transient failures stay retryable.
  *
  * Each file contains: { type, sessionId, payload, createdAt, retries, dedupKey }
  *
@@ -274,6 +275,23 @@ export async function claimForReplay(filename) {
   }
 }
 
+/**
+ * Release a claimed file back to the queue without consuming a retry. The
+ * entry keeps its original filename, retry count, and createdAt position, so a
+ * later run retries it as if this attempt never happened.
+ */
+export async function releaseClaim(claimedFilename) {
+  if (!claimedFilename.endsWith(".processing")) return null;
+  const dir = getPendingDir();
+  const restored = pendingFromProcessingFilename(claimedFilename);
+  try {
+    await rename(join(dir, claimedFilename), join(dir, restored));
+    return restored;
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Remove a pending entry after successful replay.
  */
@@ -350,9 +368,16 @@ export async function cleanStale() {
  *
  * @param {Function} fetchJSON - the configured fetchJSON from makeFetchJSON
  * @param {Function} log - logger function
+ * @param {object} [options]
+ * @param {boolean} [options.consumeRetries=true] - when false, retryable
+ *   failures release their claim instead of incrementing the retry count, so
+ *   a background drainer can keep retrying a transient failure without
+ *   burning the session-start retry budget. Exhausted and non-retryable
+ *   entries are deleted exactly as in the default mode.
  * @returns {{ replayed: number, failed: number, skipped: number, deferred: number }}
  */
-export async function replayPending(fetchJSON, log) {
+export async function replayPending(fetchJSON, log, options = {}) {
+  const consumeRetries = options.consumeRetries !== false;
   const pending = await listPending();
 
   if (pending.length === 0) {
@@ -427,7 +452,18 @@ export async function replayPending(fetchJSON, log) {
       await dequeue(claimedFilename);
       skipped++;
     } else {
-      await incrementRetry(claimedFilename, entry);
+      if (consumeRetries) {
+        await incrementRetry(claimedFilename, entry);
+      } else {
+        const released = await releaseClaim(claimedFilename);
+        log("pending-queue", {
+          action: released ? "replay-deferred" : "release-failed",
+          sessionId: entry.sessionId,
+          type: entry.type,
+          status: res?.result?.status || res?.status,
+          retries: entry.retries || 0,
+        });
+      }
       failed++;
       if (entry.type === "addMessage") {
         deferred += Math.max(0, pending.length - processed);