@0xgeorgemathew/zcode 0.3.2 → 0.4.0

package/lib/isolation.js

@@ -1,93 +1,71 @@
 'use strict';
-// Scratch-HOME isolation (DESIGN §6): the engine runs with HOME pointed at a
-// copy we own (~/.zcode/home, mode 700) so the live ~/.zcode — which the
+// Scratch-HOME isolation (DESIGN §6, reworked in 0.4): the engine runs with
+// HOME pointed at a directory we own so the live ~/.zcode — which the
 // desktop app is actively using — is never written by us, and the user's
 // OAuth credentials.json is never copied.
 //
-// Invariant (§6.4): zcode performs no writes to the live ~/.zcode. The
-// fallback ladder NEVER re-adds credentials.json.
+// 0.4: one engine HOME per zcode session (<root>/sessions/<id>/home) instead
+// of one shared HOME. The shared HOME was rm -rf'd and rebuilt whenever a
+// volatile desktop file (agents-state.json, bot-config.v3.json, …) changed
+// mtime — which deleted the engine's session database under every other
+// session and made --session resume 404 — and per-run MCP/skills overlays
+// rewrote files that concurrent runs were reading. Per-session HOMEs give
+// each session its own engine DB, config and skills; config files are
+// refreshed in place on every invocation and engine state (cli/db,
+// cli/agents, rollout, logs) is never deleted.
+//
+// Invariant (§6.4): zcode performs no writes to the live ~/.zcode, and
+// credentials.json is never copied.
 const fs = require('node:fs');
 const path = require('node:path');
 const os = require('node:os');
-const crypto = require('node:crypto');
 
-const LIVE_ZCODE = path.join(os.homedir(), '.zcode');
+// The desktop app's live directory. ZAI_CODE_LIVE_DIR points elsewhere
+// (e.g. a second desktop profile, or a fixture in tests).
+function liveDir(env = process.env) {
+  return env.ZAI_CODE_LIVE_DIR || path.join(os.homedir(), '.zcode');
+}
+
+const LIVE_ZCODE = liveDir({});
 const LIVE_V2 = path.join(LIVE_ZCODE, 'v2');
 
 // v2 top-level entries we deliberately never copy.
 const V2_EXCLUDE_FILES = new Set(['credentials.json', '.DS_Store']);
 function isV2Excluded(name) {
-  return (
-    V2_EXCLUDE_FILES.has(name) || name.startsWith('tasks-index.sqlite')
-  );
+  return V2_EXCLUDE_FILES.has(name) || name.startsWith('tasks-index.sqlite');
 }
 
-const keySha256 = (key) =>
-  crypto.createHash('sha256').update(key).digest('hex');
+// Root of zcode's own state. MUST NOT be ~/.zcode: that is the desktop
+// app's live directory (0.3.0/0.3.1 regression — the package rename made
+// rootDir collide with it). Ours is ~/.zcode-cli.
+function rootDir(env = process.env) {
+  return env.ZAI_CODE_HOME || path.join(os.homedir(), '.zcode-cli');
+}
 
-// --- source inventory -------------------------------------------------------
+// --- small fs helpers ---------------------------------------------------------
 
-// Files the scratch copy depends on: every source path + mtimeMs. A change
-// in the set, an mtime, or the key fingerprint invalidates the stamp.
-function sourceInventory() {
-  const sources = [];
-  const pushFile = (abs) => {
-    try {
-      sources.push({ file: abs, mtimeMs: fs.statSync(abs).mtimeMs });
-    } catch {
-      /* source vanished — not copied */
-    }
-  };
-  try {
-    for (const entry of fs.readdirSync(LIVE_V2, { withFileTypes: true })) {
-      if (!entry.isFile() || isV2Excluded(entry.name)) continue;
-      pushFile(path.join(LIVE_V2, entry.name));
-    }
-  } catch {
-    /* no live v2 — synthesized path */
-  }
-  pushFile(path.join(LIVE_ZCODE, 'cli', 'config.json'));
-  pushFile(path.join(LIVE_ZCODE, 'AGENTS.md'));
+function writeJson600(file, value) {
+  fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', { mode: 0o600 });
+  fs.chmodSync(file, 0o600);
+}
+
+function readJson(file) {
   try {
-    for (const entry of fs.readdirSync(path.join(LIVE_ZCODE, 'skills'), {
-      withFileTypes: true,
-    })) {
-      if (entry.isDirectory()) {
-        pushFile(path.join(LIVE_ZCODE, 'skills', entry.name, 'SKILL.md'));
-      }
-    }
+    return JSON.parse(fs.readFileSync(file, 'utf8'));
   } catch {
-    /* no skills dir */
+    return null;
   }
+}
+
+function listDir(dir) {
   try {
-    for (const entry of fs.readdirSync(path.join(LIVE_ZCODE, 'agents'), {
-      withFileTypes: true,
-    })) {
-      if (entry.isFile() && entry.name.endsWith('.md')) {
-        pushFile(path.join(LIVE_ZCODE, 'agents', entry.name));
-      }
-    }
+    return fs.readdirSync(dir, { withFileTypes: true });
   } catch {
-    /* no agents dir */
+    return [];
   }
-  return sources;
 }
 
-// --- build ------------------------------------------------------------------
-
-function writeJson600(file, value) {
-  fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n', {
-    mode: 0o600,
-  });
-  fs.chmodSync(file, 0o600);
-}
-
-function copyFileTo(src, destDir) {
-  fs.mkdirSync(destDir, { recursive: true });
-  const dest = path.join(destDir, path.basename(src));
-  fs.copyFileSync(src, dest);
-  return dest;
-}
+// --- provider/key rewriting -----------------------------------------------------
 
 // Prune to exactly the zai-api rule and swap in the package key (§6.2 item 2).
 function rewriteProviderConfig(target, key) {
@@ -101,32 +79,16 @@ function rewriteProviderConfig(target, key) {
   }
   const zaiRules = rules.filter((r) => r && r.providerId === 'zai-api');
   if (zaiRules.length === 0) {
-    throw new Error(
-      `no zai-api rule found in provider_config.json at ${target}`
-    );
+    throw new Error(`no zai-api rule found in provider_config.json at ${target}`);
   }
   const rule = zaiRules[0];
   rule.config = rule.config || {};
   rule.config.group = rule.config.group || 'standard-personal';
-  rule.config.access = {
-    type: 'api-key',
-    apiKey: key,
-  };
+  rule.config.access = { type: 'api-key', apiKey: key };
   parsed.config.providerConfigRules.providerRules = [rule];
   writeJson600(target, parsed);
 }
 
-// Overwrite builtin:zai apiKey with the package key (§6.2 item 3).
-function rewriteCliConfig(target, key) {
-  const parsed = JSON.parse(fs.readFileSync(target, 'utf8'));
-  const zai = parsed.provider && parsed.provider['builtin:zai'];
-  if (zai) {
-    zai.options = zai.options || {};
-    zai.options.apiKey = key;
-  }
-  writeJson600(target, parsed);
-}
-
 function synthesizedProviderConfig(key) {
   return {
     schemaVersion: 1,
@@ -147,130 +109,231 @@ function synthesizedProviderConfig(key) {
   };
 }
 
-// Tier (a), the review-proven recipe: top-level v2 files, no directories, no
-// credentials.json, no sqlite. Fresh machines (no live v2) synthesize instead.
-function buildScratch(home, key) {
-  fs.rmSync(home, { recursive: true, force: true });
-  const scratchV2 = path.join(home, '.zcode', 'v2');
-  fs.mkdirSync(scratchV2, { recursive: true });
+// --- skills -------------------------------------------------------------------
 
-  let mode = 'synthesized';
-  let copied = 0;
-  try {
-    const entries = fs.readdirSync(LIVE_V2, { withFileTypes: true });
-    const files = entries.filter(
-      (e) => e.isFile() && !isV2Excluded(e.name)
-    );
-    if (files.length > 0 || fs.existsSync(path.join(LIVE_V2, 'provider_config.json'))) {
-      mode = 'copied-v2-top-level';
-    }
-    for (const entry of files) {
-      const dest = copyFileTo(path.join(LIVE_V2, entry.name), scratchV2);
-      if (entry.name === 'provider_config.json') {
-        rewriteProviderConfig(dest, key);
+// A --skills dir is either one skill (<dir>/SKILL.md) or a skills root
+// (<dir>/<name>/SKILL.md). Returns [{name, dir}] of copyable skills.
+function discoverSkillDirs(dir) {
+  if (fs.existsSync(path.join(dir, 'SKILL.md'))) {
+    return [{ name: path.basename(path.resolve(dir)), dir }];
+  }
+  return listDir(dir)
+    .filter((e) => e.isDirectory() && fs.existsSync(path.join(dir, e.name, 'SKILL.md')))
+    .map((e) => ({ name: e.name, dir: path.join(dir, e.name) }));
+}
+
+// --- MCP ----------------------------------------------------------------------
+
+// The live desktop MCP servers, read-only, honoring `enabled`.
+function liveMcpServers(live = LIVE_ZCODE) {
+  const cfg = readJson(path.join(live, 'cli', 'config.json'));
+  const servers = cfg && cfg.mcp && cfg.mcp.servers;
+  if (!servers || typeof servers !== 'object') return {};
+  const out = {};
+  for (const [name, def] of Object.entries(servers)) {
+    if (!def || typeof def !== 'object' || def.enabled === false) continue;
+    out[name] = def;
+  }
+  return out;
+}
+
+// Effective set for one invocation. Hermetic by default: ONLY the caller's
+// --mcp servers. --mcp-inherit pulls in the desktop's enabled servers as a
+// base (specs still win by name). Hermetic default is deliberate: the
+// desktop set includes OAuth servers that can never complete headless and
+// stall the engine's connect pool.
+function effectiveMcpServers({ inherit = false, mcpSpecs = [], live = LIVE_ZCODE } = {}) {
+  const servers = inherit ? liveMcpServers(live) : {};
+  for (const spec of mcpSpecs) {
+    for (const [name, def] of Object.entries(spec)) servers[name] = def;
+  }
+  return servers;
+}
+
+// --- shared caches ------------------------------------------------------------
+
+// Engine-populated caches that are expensive to recreate per session (the
+// plugin marketplace cache is ~30 MB) are shared through symlinks into
+// <root>/shared. A pre-0.4 shared HOME donates its caches once.
+const SHARED_CACHES = [
+  { name: 'plugins', rel: path.join('.zcode', 'cli', 'plugins') },
+  { name: 'runtime', rel: path.join('.zcode', 'v2', 'runtime') },
+];
+
+function linkSharedCaches(home, root) {
+  for (const cache of SHARED_CACHES) {
+    const shared = path.join(root, 'shared', cache.name);
+    if (!fs.existsSync(shared)) {
+      const legacy = path.join(root, 'home', cache.rel);
+      fs.mkdirSync(path.dirname(shared), { recursive: true, mode: 0o700 });
+      try {
+        if (fs.existsSync(legacy) && !fs.lstatSync(legacy).isSymbolicLink()) {
+          fs.renameSync(legacy, shared);
+        }
+      } catch {
+        /* fall through to an empty cache */
       }
-      copied++;
+      fs.mkdirSync(shared, { recursive: true, mode: 0o700 });
+    }
+    const target = path.join(home, cache.rel);
+    if (fs.existsSync(target) || isDanglingLink(target)) continue;
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    try {
+      fs.symlinkSync(shared, target, 'dir');
+    } catch {
+      /* engine recreates the cache itself */
     }
+  }
+}
+
+function isDanglingLink(p) {
+  try {
+    return fs.lstatSync(p).isSymbolicLink() && !fs.existsSync(p);
   } catch {
-    /* no live v2 at all */
+    return false;
   }
+}
 
-  const scratchProviderConfig = path.join(scratchV2, 'provider_config.json');
-  if (!fs.existsSync(scratchProviderConfig)) {
-    writeJson600(scratchProviderConfig, synthesizedProviderConfig(key));
-  } else {
-    // Ensure key swap even when the copy step found an unexpected shape.
-    rewriteProviderConfig(scratchProviderConfig, key);
+// --- build ------------------------------------------------------------------
+
+// Creates or refreshes one engine HOME. Config is (re)written from the live
+// sources + this invocation's flags; engine state under the HOME (cli/db,
+// cli/agents, cli/rollout, logs, memories) is never touched, so a session
+// stays resumable across invocations. Returns facts for reporting.
+function buildEngineHome({
+  home,
+  root,
+  key,
+  live = LIVE_ZCODE,
+  mcpSpecs = [],
+  mcpInherit = false,
+  skillsDirs = [],
+  noSkills = false,
+}) {
+  const zdir = path.join(home, '.zcode');
+  const v2 = path.join(zdir, 'v2');
+  const cliDir = path.join(zdir, 'cli');
+  fs.mkdirSync(v2, { recursive: true, mode: 0o700 });
+  fs.mkdirSync(cliDir, { recursive: true, mode: 0o700 });
+  fs.chmodSync(home, 0o700);
+
+  // v2: top-level files only, no credentials.json, no sqlite (§6.2 item 1).
+  const liveV2 = path.join(live, 'v2');
+  const v2Files = listDir(liveV2).filter((e) => e.isFile() && !isV2Excluded(e.name));
+  for (const entry of v2Files) {
+    fs.copyFileSync(path.join(liveV2, entry.name), path.join(v2, entry.name));
   }
+  const providerConfig = path.join(v2, 'provider_config.json');
+  const mode = v2Files.some((e) => e.name === 'provider_config.json')
+    ? 'copied-v2-top-level'
+    : 'synthesized';
+  if (mode === 'synthesized') writeJson600(providerConfig, synthesizedProviderConfig(key));
+  else rewriteProviderConfig(providerConfig, key);
 
-  const liveCliConfig = path.join(LIVE_ZCODE, 'cli', 'config.json');
-  if (fs.existsSync(liveCliConfig)) {
-    const dest = copyFileTo(liveCliConfig, path.join(home, '.zcode', 'cli'));
-    rewriteCliConfig(dest, key);
+  // cli/config.json: live copy (builtin:zai key swapped) + this run's MCP set.
+  const cfg = readJson(path.join(live, 'cli', 'config.json')) || {};
+  const zai = cfg.provider && cfg.provider['builtin:zai'];
+  if (zai) {
+    zai.options = zai.options || {};
+    zai.options.apiKey = key;
   }
+  const servers = effectiveMcpServers({ inherit: mcpInherit, mcpSpecs, live });
+  cfg.mcp = { ...(cfg.mcp || {}), servers };
+  writeJson600(path.join(cliDir, 'config.json'), cfg);
 
-  const liveAgents = path.join(LIVE_ZCODE, 'agents');
-  if (fs.existsSync(liveAgents)) {
-    for (const entry of fs.readdirSync(liveAgents, { withFileTypes: true })) {
-      if (entry.isFile() && entry.name.endsWith('.md')) {
-        copyFileTo(path.join(liveAgents, entry.name), path.join(home, '.zcode', 'agents'));
-      }
-    }
+  // Agent profiles + AGENTS.md: mirror the live set exactly.
+  const agentsDir = path.join(zdir, 'agents');
+  fs.rmSync(agentsDir, { recursive: true, force: true });
+  for (const entry of listDir(path.join(live, 'agents'))) {
+    if (!entry.isFile() || !entry.name.endsWith('.md')) continue;
+    fs.mkdirSync(agentsDir, { recursive: true });
+    fs.copyFileSync(path.join(live, 'agents', entry.name), path.join(agentsDir, entry.name));
   }
-  const liveAgentsMd = path.join(LIVE_ZCODE, 'AGENTS.md');
-  if (fs.existsSync(liveAgentsMd)) {
-    copyFileTo(liveAgentsMd, path.join(home, '.zcode'));
+  const agentsMd = path.join(zdir, 'AGENTS.md');
+  if (fs.existsSync(path.join(live, 'AGENTS.md'))) {
+    fs.copyFileSync(path.join(live, 'AGENTS.md'), agentsMd);
+  } else {
+    fs.rmSync(agentsMd, { force: true });
   }
 
-  // Skills: copy each <name>/SKILL.md dir from the live user scope so the
-  // engine (running under this scratch HOME) sees the same skills as the
-  // desktop. Project skills (<cwd>/.zcode/skills) need no copy — the engine
-  // reads them from the real workspace.
+  // Skills: live user skills (unless --no-skills) + this run's --skills.
+  // Project skills (<cwd>/.zcode/skills) need no copy — the engine reads
+  // them from the real workspace.
+  const skillsDir = path.join(zdir, 'skills');
+  fs.rmSync(skillsDir, { recursive: true, force: true });
+  fs.mkdirSync(skillsDir, { recursive: true });
   let skillsCopied = 0;
-  const liveSkills = path.join(LIVE_ZCODE, 'skills');
-  if (fs.existsSync(liveSkills)) {
-    for (const entry of fs.readdirSync(liveSkills, { withFileTypes: true })) {
+  if (!noSkills) {
+    const liveSkills = path.join(live, 'skills');
+    for (const entry of listDir(liveSkills)) {
       if (!entry.isDirectory()) continue;
       if (!fs.existsSync(path.join(liveSkills, entry.name, 'SKILL.md'))) continue;
-      fs.cpSync(
-        path.join(liveSkills, entry.name),
-        path.join(home, '.zcode', 'skills', entry.name),
-        { recursive: true }
-      );
+      fs.cpSync(path.join(liveSkills, entry.name), path.join(skillsDir, entry.name), {
+        recursive: true,
+      });
       skillsCopied++;
     }
   }
+  const skillsInjected = [];
+  if (!noSkills) {
+    for (const dir of skillsDirs) {
+      for (const { name, dir: skillDir } of discoverSkillDirs(dir)) {
+        fs.rmSync(path.join(skillsDir, name), { recursive: true, force: true });
+        fs.cpSync(skillDir, path.join(skillsDir, name), { recursive: true });
+        skillsInjected.push(name);
+      }
+    }
+  }
 
-  return { mode, copied, skillsCopied };
-}
-
-// --- public API -------------------------------------------------------------
+  if (root) linkSharedCaches(home, root);
 
-// Root of zcode's own state. MUST NOT be ~/.zcode: that is the desktop
-// app's live directory (0.3.0/0.3.1 regression — the package rename made
-// rootDir collide with it, so login/scratch files landed inside the live
-// config). Ours is ~/.zcode-cli.
-function rootDir(env = process.env) {
-  return env.ZAI_CODE_HOME || path.join(os.homedir(), '.zcode-cli');
+  return {
+    home,
+    mode,
+    copiedFiles: v2Files.length,
+    skillsCopied,
+    skillsInjected,
+    mcp: {
+      servers,
+      // Names the caller explicitly added — the CLI waits for these to
+      // settle before sending the prompt. Inherited servers are best-effort.
+      added: mcpSpecs.flatMap((spec) => Object.keys(spec)),
+    },
+  };
 }
 
-// The only file/dir names this package ever creates at its root.
-// migrateRoot moves exactly these out of legacy locations and never
-// touches anything else.
+// --- migration ----------------------------------------------------------------
+
+// The only names pre-0.4 versions created at a root. migrateRoot moves
+// exactly these out of legacy locations and never touches anything else.
 const OUR_ARTIFACTS = ['credentials.json', 'build-stamp.json', 'injected-skills.json', 'home'];
 
 // 0.3.2 migration: vacate our artifacts from (a) the live ~/.zcode, where
 // 0.3.0/0.3.1 wrongly wrote them, and (b) the pre-rename ~/.zai-code root,
-// into the new ~/.zcode-cli root. Idempotent; never writes or deletes
-// anything else in the live directory.
+// into the ~/.zcode-cli root. Idempotent; never writes or deletes anything
+// else in the live directory.
 function migrateRoot(env = process.env) {
   const root = rootDir(env);
   try {
-    fs.mkdirSync(root, { recursive: true });
+    fs.mkdirSync(root, { recursive: true, mode: 0o700 });
   } catch {
-    return; // unwritable home — leave everything as-is; ensureIsolation surfaces it
+    return; // unwritable home — callers surface it
   }
-  const sources = [
-    LIVE_ZCODE, // live dir: move ours out, keep everything else
-    path.join(os.homedir(), '.zai-code'), // pre-rename root: wholly ours
-  ];
+  const sources = [liveDir(env), path.join(os.homedir(), '.zai-code')];
   for (const dir of sources) {
     let entries;
     try {
       entries = new Set(fs.readdirSync(dir));
     } catch {
-      continue; // nothing to migrate from this source
+      continue;
     }
     for (const name of OUR_ARTIFACTS) {
       if (!entries.has(name)) continue;
       const src = path.join(dir, name);
       const dest = path.join(root, name);
       try {
-        if (fs.existsSync(dest)) {
-          fs.rmSync(src, { recursive: true, force: true });
-        } else {
-          fs.renameSync(src, dest);
-        }
+        if (fs.existsSync(dest)) fs.rmSync(src, { recursive: true, force: true });
+        else fs.renameSync(src, dest);
       } catch {
         /* keep going; a missing artifact just rebuilds */
       }
@@ -278,64 +341,8 @@ function migrateRoot(env = process.env) {
   }
 }
 
-// Ensures the scratch HOME exists and is current. Returns
-// {root, home, mode, built, stamp}.
-function ensureIsolation(key, { fresh = false, env = process.env } = {}) {
-  migrateRoot(env);
-  const root = rootDir(env);
-  fs.mkdirSync(root, { recursive: true });
-  fs.chmodSync(root, 0o700);
-  const home = path.join(root, 'home');
-  const stampFile = path.join(root, 'build-stamp.json');
-
-  const sources = sourceInventory();
-  const fingerprint = keySha256(key);
-  const desired = {
-    version: 1,
-    keySha256: fingerprint,
-    sources,
-  };
-
-  if (!fresh && fs.existsSync(home) && fs.existsSync(stampFile)) {
-    try {
-      const stamp = JSON.parse(fs.readFileSync(stampFile, 'utf8'));
-      const sameSources =
-        Array.isArray(stamp.sources) &&
-        stamp.sources.length === desired.sources.length &&
-        stamp.sources.every(
-          (s, i) =>
-            s.file === desired.sources[i].file && s.mtimeMs === desired.sources[i].mtimeMs
-        );
-      if (stamp.keySha256 === fingerprint && sameSources) {
-        return {
-          root,
-          home,
-          mode: stamp.mode,
-          built: false,
-          stamp,
-        };
-      }
-    } catch {
-      /* corrupt stamp — rebuild */
-    }
-  }
-
-  const { mode, copied, skillsCopied } = buildScratch(home, key);
-  fs.chmodSync(home, 0o700);
-  const stamp = {
-    ...desired,
-    mode,
-    copiedFiles: copied,
-    skillsCopied,
-    builtAt: new Date().toISOString(),
-  };
-  writeJson600(stampFile, stamp);
-  return { root, home, mode, built: true, stamp };
-}
-
 // Env for the engine child process (§6.3): HOME=scratch, pinned builtin
-// config, and the uncontrolled overrides deleted. cwd is untouched here —
-// the caller sets it to the user's real workspace.
+// config, and the uncontrolled overrides deleted. cwd is set by the caller.
 function engineEnv({ home, builtinProviderConfig, parentEnv = process.env }) {
   const childEnv = { ...parentEnv };
   childEnv.HOME = home;
@@ -353,11 +360,14 @@ function engineEnv({ home, builtinProviderConfig, parentEnv = process.env }) {
 }
 
 module.exports = {
-  ensureIsolation,
+  buildEngineHome,
   engineEnv,
   rootDir,
+  liveDir,
   migrateRoot,
-  keySha256,
+  effectiveMcpServers,
+  liveMcpServers,
+  discoverSkillDirs,
   LIVE_ZCODE,
   LIVE_V2,
 };

package/lib/key.js

@@ -5,16 +5,7 @@
 // in this package.
 const fs = require('node:fs');
 const path = require('node:path');
-const os = require('node:os');
-const { migrateRoot } = require('./isolation.js');
-
-// Root of zcode's own state (scratch HOME lives under <root>/home).
-// MUST NOT be ~/.zcode — that is the desktop app's live directory the
-// engine itself owns (0.3.0/0.3.1 regression: the rename made the CLI
-// write credentials and the scratch HOME straight into it).
-function rootDir(env = process.env) {
-  return env.ZAI_CODE_HOME || path.join(os.homedir(), '.zcode-cli');
-}
+const { migrateRoot, rootDir } = require('./isolation.js');
 
 function credentialsPath(env = process.env) {
   return path.join(rootDir(env), 'credentials.json');