nemoris 0.1.0

package/src/setup.js

@@ -0,0 +1,369 @@
+/**
+ * First-run setup script for Nemoris V2.
+ *
+ * Validates the environment, detects API keys, checks external services,
+ * validates config, and ensures state directories exist.
+ *
+ * No npm dependencies — uses native fetch, native fs, ANSI codes only.
+ * Idempotent and non-interactive (no prompts).
+ * Exit code 0 if all critical checks pass, 1 otherwise.
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// ── ANSI helpers ────────────────────────────────────────────────────
+
+const BOLD = "\x1b[1m";
+const RESET = "\x1b[0m";
+const GREEN = "\x1b[32m";
+const RED = "\x1b[31m";
+const YELLOW = "\x1b[33m";
+const DIM = "\x1b[2m";
+
+const PASS = `${GREEN}\u2713${RESET}`;
+const FAIL = `${RED}\u2717${RESET}`;
+const WARN = `${YELLOW}!${RESET}`;
+
+function bold(text) {
+  return `${BOLD}${text}${RESET}`;
+}
+
+// ── Path resolution ─────────────────────────────────────────────────
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const projectRoot = path.resolve(__dirname, "..");
+const configRoot = path.join(projectRoot, "config");
+const stateRoot = path.join(projectRoot, "state", "memory");
+
+// ── Env loader (mirrors cli.js) ─────────────────────────────────────
+
+function loadParentEnv(envPath) {
+  try {
+    const content = fs.readFileSync(envPath, "utf8");
+    for (const line of content.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith("#")) continue;
+      const eqIdx = trimmed.indexOf("=");
+      if (eqIdx < 1) continue;
+      const key = trimmed.slice(0, eqIdx).trim();
+      if (process.env[key] !== undefined) continue;
+      let value = trimmed.slice(eqIdx + 1).trim();
+      if (
+        (value.startsWith('"') && value.endsWith('"')) ||
+        (value.startsWith("'") && value.endsWith("'"))
+      ) {
+        value = value.slice(1, -1);
+      }
+      process.env[key] = value;
+    }
+  } catch {
+    // .env file is optional
+  }
+}
+
+// ── Helpers ─────────────────────────────────────────────────────────
+
+function maskKey(key) {
+  if (!key || key.length < 7) return "***";
+  return key.slice(0, 6) + "...";
+}
+
+function readPackageJson() {
+  const raw = fs.readFileSync(path.join(projectRoot, "package.json"), "utf8");
+  return JSON.parse(raw);
+}
+
+// ── Step 1: Welcome banner ──────────────────────────────────────────
+
+function printBanner(pkg) {
+  console.log("");
+  console.log(
+    bold("── Nemoris V2 Setup ") +
+      DIM +
+      `v${pkg.version}` +
+      RESET +
+      bold(" ─────────────────────")
+  );
+  console.log("");
+}
+
+// ── Step 2: Node.js version check ───────────────────────────────────
+
+function checkNodeVersion() {
+  const full = process.version; // e.g. "v25.8.0"
+  const major = parseInt(full.slice(1), 10);
+  const ok = major >= 22;
+  if (ok) {
+    console.log(`  Node.js        ${full}          ${PASS}`);
+  } else {
+    console.log(
+      `  Node.js        ${full}          ${FAIL}  (need >= 22.5)`
+    );
+  }
+  return ok;
+}
+
+// ── Step 3: Detect API keys ─────────────────────────────────────────
+
+function detectApiKeys() {
+  // Load parent .env first
+  const parentEnvPath = path.resolve(
+    process.env.HOME || os.homedir(),
+    ".openclaw",
+    ".env"
+  );
+  loadParentEnv(parentEnvPath);
+
+  const keys = {};
+
+  // OpenRouter (primary for battle-testing)
+  const orKey = process.env.OPENROUTER_API_KEY;
+  if (orKey) {
+    console.log(`  OpenRouter     ${maskKey(orKey)}  ${PASS}  (primary)`);
+    keys.openrouter = orKey;
+  } else {
+    console.log(`  OpenRouter     not found       ${FAIL}`);
+    keys.openrouter = null;
+  }
+
+  // Anthropic (optional — routes through OpenRouter when unavailable)
+  const antKey = process.env.NEMORIS_ANTHROPIC_API_KEY;
+  if (antKey) {
+    console.log(`  Anthropic      ${maskKey(antKey)}  ${PASS}`);
+    keys.anthropic = antKey;
+  } else {
+    console.log(`  Anthropic      not found       ${WARN}  (optional)`);
+    keys.anthropic = null;
+  }
+
+  return keys;
+}
+
+// ── Step 4: Validate OpenRouter API key ─────────────────────────────
+
+async function validateOpenRouterKey(apiKey) {
+  if (!apiKey) return false;
+
+  try {
+    const res = await fetch("https://openrouter.ai/api/v1/models", {
+      headers: {
+        "Authorization": `Bearer ${apiKey}`,
+      },
+      signal: AbortSignal.timeout(10000),
+    });
+
+    if (res.ok) {
+      console.log(`  API check      verified        ${PASS}`);
+      return true;
+    }
+
+    console.log(`  API check      invalid         ${FAIL}  (HTTP ${res.status})`);
+    return false;
+  } catch (err) {
+    console.log(`  API check      error           ${FAIL}  (${err.message})`);
+    return false;
+  }
+}
+
+// ── Step 5: Check Ollama ────────────────────────────────────────────
+
+async function checkOllama() {
+  try {
+    const res = await fetch("http://localhost:11434/api/tags", {
+      signal: AbortSignal.timeout(5000),
+    });
+    if (!res.ok) {
+      console.log(
+        `  Ollama         unreachable      ${WARN}  (HTTP ${res.status})`
+      );
+      return { ok: false, modelCount: 0 };
+    }
+    const data = await res.json();
+    const models = data.models || [];
+    if (models.length === 0) {
+      console.log(`  Ollama         0 models        ${WARN}  (no models pulled)`);
+    } else {
+      console.log(
+        `  Ollama         ${models.length} model${models.length === 1 ? "" : "s"}${models.length < 10 ? "         " : "        "} ${PASS}`
+      );
+      for (const m of models) {
+        console.log(`${DIM}    - ${m.name}${RESET}`);
+      }
+    }
+    return { ok: true, modelCount: models.length };
+  } catch {
+    console.log(
+      `  Ollama         not running      ${WARN}  (remote providers still work)`
+    );
+    return { ok: false, modelCount: 0 };
+  }
+}
+
+// ── Step 6: Validate config ─────────────────────────────────────────
+
+async function checkConfig() {
+  try {
+    const { ConfigLoader } = await import("./config/loader.js");
+    const { validateAllConfigs } = await import(
+      "./config/schema-validator.js"
+    );
+
+    const loader = new ConfigLoader({ rootDir: configRoot });
+    const config = await loader.loadAll();
+    const result = validateAllConfigs(config, configRoot);
+
+    if (result.ok) {
+      console.log(`  Config         all valid        ${PASS}`);
+    } else {
+      console.log(
+        `  Config         ${result.errors.length} error${result.errors.length === 1 ? "" : "s"}        ${FAIL}`
+      );
+      for (const e of result.errors) {
+        console.log(`${DIM}    - ${e}${RESET}`);
+      }
+    }
+    return result.ok;
+  } catch (err) {
+    console.log(
+      `  Config         load error       ${FAIL}  (${err.message})`
+    );
+    return false;
+  }
+}
+
+// ── Step 7: Ensure state directories ────────────────────────────────
+
+async function ensureStateDirs() {
+  try {
+    // Read agent configs to know which dirs are needed
+    const agentsDir = path.join(configRoot, "agents");
+    const agentFiles = fs.readdirSync(agentsDir).filter((f) => f.endsWith(".toml"));
+    const agentIds = agentFiles.map((f) => path.basename(f, ".toml"));
+
+    let created = 0;
+    let _existed = 0;
+
+    for (const id of agentIds) {
+      const dir = path.join(stateRoot, id);
+      if (fs.existsSync(dir)) {
+        _existed++;
+      } else {
+        fs.mkdirSync(dir, { recursive: true });
+        created++;
+      }
+    }
+
+    if (created > 0) {
+      console.log(
+        `  State dirs     ${created} created       ${PASS}`
+      );
+    } else {
+      console.log(
+        `  State dirs     all present      ${PASS}`
+      );
+    }
+    return true;
+  } catch (err) {
+    console.log(
+      `  State dirs     error            ${FAIL}  (${err.message})`
+    );
+    return false;
+  }
+}
+
+// ── Summary ─────────────────────────────────────────────────────────
+
+function printSummary(results) {
+  console.log("");
+  console.log(bold("── Setup Complete ") + bold("─".repeat(30)));
+
+  const nodeLabel = results.nodeVersion
+    ? `${process.version}`
+    : process.version;
+  console.log(
+    `  Node.js:     ${nodeLabel.padEnd(17)}${results.nodeVersion ? PASS : FAIL}`
+  );
+
+  const orLabel = results.keys.openrouter
+    ? `${maskKey(results.keys.openrouter)}`
+    : "missing";
+  const orStatus = results.apiKeyValid
+    ? `${PASS} (verified)`
+    : results.keys.openrouter
+      ? `${FAIL} (unverified)`
+      : FAIL;
+  console.log(`  OpenRouter:  ${orLabel.padEnd(17)}${orStatus}`);
+
+  const antLabel = results.keys.anthropic
+    ? `${maskKey(results.keys.anthropic)}`
+    : "not set";
+  const antStatus = results.keys.anthropic ? PASS : `${WARN} (optional)`;
+  console.log(`  Anthropic:   ${antLabel.padEnd(17)}${antStatus}`);
+
+  const ollamaLabel = results.ollama.ok
+    ? `${results.ollama.modelCount} model${results.ollama.modelCount === 1 ? "" : "s"}`
+    : "offline";
+  const ollamaStatus = results.ollama.ok ? PASS : WARN;
+  console.log(
+    `  Ollama:      ${ollamaLabel.padEnd(17)}${ollamaStatus}`
+  );
+
+  const configLabel = results.configValid ? "all valid" : "errors";
+  console.log(
+    `  Config:      ${configLabel.padEnd(17)}${results.configValid ? PASS : FAIL}`
+  );
+
+  const stateLabel = results.stateDirs ? "ready" : "error";
+  console.log(
+    `  State dirs:  ${stateLabel.padEnd(17)}${results.stateDirs ? PASS : FAIL}`
+  );
+
+  console.log("");
+  console.log(bold("  Ready to run:"));
+  console.log(
+    `    ${DIM}npm run run:heartbeat${RESET}           # dry-run (no API calls)`
+  );
+  console.log(
+    `    ${DIM}npm run run:heartbeat:provider${RESET}  # live run (uses API)`
+  );
+  console.log("");
+}
+
+// ── Main ────────────────────────────────────────────────────────────
+
+export async function runSetup() {
+  const pkg = readPackageJson();
+  printBanner(pkg);
+
+  // Critical checks
+  const nodeVersion = checkNodeVersion();
+  const keys = detectApiKeys();
+  const apiKeyValid = await validateOpenRouterKey(keys.openrouter);
+
+  // Non-blocking check
+  const ollama = await checkOllama();
+
+  // Critical checks (continued)
+  const configValid = await checkConfig();
+  const stateDirs = await ensureStateDirs();
+
+  const results = {
+    nodeVersion,
+    keys,
+    apiKeyValid,
+    ollama,
+    configValid,
+    stateDirs,
+  };
+
+  printSummary(results);
+
+  // Exit code: fail if any critical check failed
+  const criticalPass =
+    nodeVersion && keys.openrouter && apiKeyValid && configValid && stateDirs;
+
+  return criticalPass ? 0 : 1;
+}

package/src/shadow/bridge.js

@@ -0,0 +1,303 @@
+import path from "node:path";
+import fs from "node:fs/promises";
+import { listFilesRecursive, readJson, readJsonLines, readText } from "../utils/fs.js";
+
+function sortNewestFirst(items) {
+  return [...items].sort((a, b) => b.mtimeMs - a.mtimeMs);
+}
+
+export class OpenClawShadowBridge {
+  constructor({ liveRoot }) {
+    this.liveRoot = liveRoot;
+  }
+
+  get available() {
+    return Boolean(this.liveRoot);
+  }
+
+  async loadConfig() {
+    if (!this.available) return {};
+    return readJson(path.join(this.liveRoot, "openclaw.json"), {});
+  }
+
+  async loadCronJobs() {
+    if (!this.available) return [];
+    const data = await readJson(path.join(this.liveRoot, "cron", "jobs.json"), { jobs: [] });
+    return data.jobs || [];
+  }
+
+  async loadCronRunHistory(jobId, limit = 3) {
+    if (!this.available) return [];
+    const filePath = path.join(this.liveRoot, "cron", "runs", `${jobId}.jsonl`);
+    const entries = await readJsonLines(filePath);
+    return entries.slice(-limit).reverse();
+  }
+
+  async listAgents() {
+    const config = await this.loadConfig();
+    return (config.agents?.list || []).map((agent) => ({
+      id: agent.id,
+      name: agent.name,
+      workspace: agent.workspace,
+      agentDir: agent.agentDir || path.join(this.liveRoot, "agents", agent.id, "agent"),
+      primaryModel: agent.model?.primary || null,
+      fallbackModels: agent.model?.fallbacks || [],
+      deniedTools: agent.tools?.deny || [],
+      skills: agent.skills || []
+    }));
+  }
+
+  async loadAgentProfiles(agentId) {
+    if (!this.available) return {};
+    return readJson(path.join(this.liveRoot, "agents", agentId, "agent", "auth-profiles.json"), {});
+  }
+
+  async loadAgentModels(agentId) {
+    if (!this.available) return {};
+    return readJson(path.join(this.liveRoot, "agents", agentId, "agent", "models.json"), {});
+  }
+
+  async loadSessionIndex(agentId) {
+    if (!this.available) return {};
+    return readJson(path.join(this.liveRoot, "agents", agentId, "sessions", "sessions.json"), {});
+  }
+
+  async resolveWorkspacePath(agentId, workspaceOverride = null) {
+    const agents = await this.listAgents();
+    const agent = agents.find((item) => item.id === agentId);
+    if (!agent) {
+      if (workspaceOverride) return workspaceOverride;
+      throw new Error(`Unknown agent: ${agentId}`);
+    }
+    return agent.workspace;
+  }
+
+  /**
+   * Read an identity file (SOUL.md, IDENTITY.md, etc.) with priority:
+   * 1. agent.workspace/fileName
+   * 2. agent.agentDir/fileName
+   * 3. null (caller falls back to stub)
+   *
+   * Whitespace-only files are treated as absent (same as missing).
+   */
+  async readIdentityFile(agent, fileName) {
+    // Priority 1: workspace dir
+    const wsPath = path.join(agent.workspace, fileName);
+    try {
+      const content = await fs.readFile(wsPath, "utf8");
+      if (content.trim()) return content;
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+    }
+
+    // Priority 2: agent dir
+    const agentDirPath = path.join(
+      agent.agentDir || path.join(this.liveRoot, "agents", agent.id, "agent"),
+      fileName
+    );
+    try {
+      const content = await fs.readFile(agentDirPath, "utf8");
+      if (content.trim()) return content;
+    } catch (err) {
+      if (err.code !== "ENOENT") throw err;
+    }
+
+    return null;
+  }
+
+  /**
+   * Read session messages from OpenClaw JSONL files for FTS5 indexing.
+   * Only reads active sessions (skips *.reset.jsonl, *.deleted.jsonl).
+   * Returns up to `limit` message events (message_in / message_out).
+   *
+   * @param {string} agentId
+   * @param {number} limit  Max messages to return (default 1000)
+   * Note: each session file is fully loaded before line filtering; for large
+   * installations with many messages, memory usage scales with the largest file.
+   */
+  async readSessionMessages(agentId, limit = 1000) {
+    if (!this.available) return [];
+    const sessionsDir = path.join(this.liveRoot, "agents", agentId, "sessions");
+    let fileNames;
+    try {
+      const entries = await fs.readdir(sessionsDir, { withFileTypes: true });
+      fileNames = entries
+        .filter(e => e.isFile() && e.name.endsWith(".jsonl")
+          && !e.name.endsWith(".reset.jsonl")
+          && !e.name.endsWith(".deleted.jsonl"))
+        .map(e => e.name);
+    } catch (err) {
+      if (err.code === "ENOENT" || err.code === "ENOTDIR") return [];
+      throw err;
+    }
+
+    const messages = [];
+    for (const fileName of fileNames) {
+      if (messages.length >= limit) break;
+      const filePath = path.join(sessionsDir, fileName);
+      const lines = await readJsonLines(filePath);
+      for (let lineIdx = 0; lineIdx < lines.length; lineIdx++) {
+        if (messages.length >= limit) break;
+        const line = lines[lineIdx];
+        // Accept events with kind message_in/message_out (Nemoris format)
+        // or role user/assistant (OpenClaw legacy format)
+        const kind = line.kind || (line.role === "user" ? "message_in" : line.role === "assistant" ? "message_out" : null);
+        if (kind !== "message_in" && kind !== "message_out") continue;
+        messages.push({
+          id: line.id || `oc-${path.basename(fileName, ".jsonl")}-${lineIdx}`,
+          session_id: line.session_id || path.basename(fileName, ".jsonl"),
+          kind,
+          ts: line.ts || 0,
+          payload_json: line.payload_json || JSON.stringify({ content: line.content || "" }),
+        });
+      }
+    }
+    return messages;
+  }
+
+  async readWorkspaceDocs(agentId, workspaceOverride = null) {
+    const workspaceRoot = await this.resolveWorkspacePath(agentId, workspaceOverride);
+    const MAX_FILES = 50;
+    const MAX_BYTES = 100 * 1024;
+
+    let fileNames;
+    try {
+      const entries = await fs.readdir(workspaceRoot, { withFileTypes: true });
+      const IDENTITY_FILES = new Set(["SOUL.md", "IDENTITY.md"]);
+      fileNames = entries
+        .filter(e => e.isFile() && e.name.endsWith(".md") && !IDENTITY_FILES.has(e.name))
+        .map(e => e.name)
+        .slice(0, MAX_FILES);
+    } catch (err) {
+      if (err.code === "ENOENT" || err.code === "ENOTDIR") return [];
+      throw err;
+    }
+
+    const docs = [];
+    for (const fileName of fileNames) {
+      const fullPath = path.join(workspaceRoot, fileName);
+      try {
+        const content = await fs.readFile(fullPath, "utf8");
+        if (content.length <= MAX_BYTES) {
+          docs.push({ fileName, fullPath, content });
+        }
+      } catch (err) {
+        if (err.code !== "ENOENT") throw err;
+        continue;
+      }
+    }
+    return docs;
+  }
+
+  async readRecentWorkspaceMemory(agentId, limit = 5, workspaceOverride = null) {
+    const workspaceRoot = await this.resolveWorkspacePath(agentId, workspaceOverride);
+    const memoryDir = path.join(workspaceRoot, "memory");
+    const allFiles = (await listFilesRecursive(memoryDir)).filter((filePath) => filePath.endsWith(".md"));
+    const stats = await Promise.all(
+      allFiles.map(async (filePath) => {
+        const content = await readText(filePath, "");
+        const match = /(\d{4}-\d{2}-\d{2})/.exec(path.basename(filePath));
+        const dateHint = match ? new Date(`${match[1]}T00:00:00Z`).getTime() : 0;
+        return {
+          filePath,
+          content,
+          mtimeMs: dateHint
+        };
+      })
+    );
+
+    return sortNewestFirst(stats).slice(0, limit);
+  }
+
+  async buildWorkspaceSnapshot(agentId, options = {}) {
+    const workspaceOverride = options.workspaceOverride || null;
+    const canUseLiveAgent = (await this.listAgents()).some((agent) => agent.id === agentId);
+    const [docs, recentMemory, sessionIndex, authProfiles, models] = await Promise.all([
+      this.readWorkspaceDocs(agentId, workspaceOverride),
+      this.readRecentWorkspaceMemory(agentId, 6, workspaceOverride),
+      canUseLiveAgent ? this.loadSessionIndex(agentId) : {},
+      canUseLiveAgent ? this.loadAgentProfiles(agentId) : {},
+      canUseLiveAgent ? this.loadAgentModels(agentId) : {}
+    ]);
+
+    const sessionEntries = Object.entries(sessionIndex || {}).map(([sessionKey, value]) => ({
+      sessionKey,
+      model: value.model || null,
+      modelProvider: value.modelProvider || null,
+      updatedAt: value.updatedAt || null,
+      origin: value.origin || null,
+      lastChannel: value.lastChannel || null
+    }));
+
+    return {
+      agentId,
+      docs,
+      recentMemory,
+      sessionEntries,
+      authProfiles,
+      models
+    };
+  }
+
+  async importWorkspaceSnapshot(agentId, memoryStore, policy, options = {}) {
+    const snapshot = await this.buildWorkspaceSnapshot(agentId, options);
+    const memoryImportLimit = options.memoryImportLimit ?? 5;
+    const memorySnippetChars = options.memorySnippetChars ?? 2000;
+    let writesThisRun = 0;
+    let importedFacts = 0;
+    let skippedFacts = 0;
+
+    await memoryStore.writeSummary(agentId, {
+      title: `shadow snapshot ${agentId}`,
+      summary: `Imported ${snapshot.docs.length} workspace docs and ${snapshot.recentMemory.length} recent memory files from live OpenClaw in read-only mode.`,
+      content: snapshot.docs.map((doc) => `${doc.fileName}: ${doc.fullPath}`).join("\n"),
+      category: "artifact_summary",
+      sourceKind: "shadow_snapshot",
+      salience: 0.75,
+      sourceRefs: snapshot.docs.map((doc) => doc.fullPath)
+    });
+
+    for (const entry of snapshot.recentMemory.slice(0, memoryImportLimit)) {
+      const result = await memoryStore.writeFact(
+        agentId,
+        {
+          title: path.basename(entry.filePath),
+          content: entry.content.slice(0, memorySnippetChars),
+          category: "artifact_summary",
+          sourceKind: "shadow_snapshot",
+          reason: "Shadow import of recent workspace memory for retrieval and continuity.",
+          sourceRefs: [entry.filePath]
+        },
+        policy,
+        {
+          writesThisRun
+        }
+      );
+      if (result.accepted && !result.skipped) {
+        writesThisRun += 1;
+        importedFacts += 1;
+      } else if (result.skipped) {
+        skippedFacts += 1;
+      }
+    }
+
+    await memoryStore.appendEvent(agentId, {
+      title: `shadow import complete:${agentId}`,
+      content: `Imported live workspace snapshot for ${agentId} without modifying source files.`,
+      category: "shadow_import",
+      salience: 0.66,
+      dedupeKey: `shadow_import:${agentId}`
+    });
+
+    return {
+      ...snapshot,
+      importStats: {
+        importedFacts,
+        skippedFacts,
+        writesThisRun,
+        memoryImportLimit,
+        memorySnippetChars
+      }
+    };
+  }
+}