@evomap/evolver 1.80.6 → 1.80.8

package/src/webui/index.js

@@ -0,0 +1,14 @@
+'use strict';
+
+const { WebUiServer } = require('./server/http');
+
+async function startWebUi(opts = {}) {
+  const server = new WebUiServer(opts);
+  const info = await server.start();
+  return { server, ...info };
+}
+
+module.exports = {
+  startWebUi,
+  WebUiServer,
+};

package/src/webui/observer/assets.js

@@ -0,0 +1,146 @@
+'use strict';
+
+const { getObserverPaths } = require('./paths');
+const { readJsonSafe, readJsonl, paginate } = require('./jsonl');
+const { redactValue } = require('./redact');
+
+function getAssetOverview(query = {}) {
+  const paths = getObserverPaths();
+  const genes = readGenes(paths);
+  const capsules = readCapsules(paths);
+  const failedCapsules = readFailedCapsules(paths);
+  const events = readJsonl(paths.eventsPath, { last: query.eventsLast || 500 }).map(redactValue);
+  const candidates = readJsonl(paths.candidatesPath, { last: query.candidatesLast || 200 }).map(redactValue);
+  const externalCandidates = readJsonl(paths.externalCandidatesPath, { last: query.candidatesLast || 200 }).map(redactValue);
+  const assetCalls = readJsonl(paths.assetCallLogPath, { last: query.assetCallsLast || 500 }).map(redactValue);
+
+  return {
+    counts: {
+      genes: genes.length,
+      capsules: capsules.length,
+      failedCapsules: failedCapsules.length,
+      events: events.length,
+      candidates: candidates.length,
+      externalCandidates: externalCandidates.length,
+      assetCalls: assetCalls.length,
+    },
+    genesByCategory: countBy(genes, 'category'),
+    capsulesByOutcome: countBy(capsules, (c) => c.outcome && c.outcome.status || 'unknown'),
+    assetCallsByAction: countBy(assetCalls, 'action'),
+    recentEvents: events.slice(-20).reverse(),
+    recentAssetCalls: assetCalls.slice(-20).reverse(),
+  };
+}
+
+function listGenes(query = {}) {
+  const genes = readGenes(getObserverPaths());
+  return {
+    ...paginate(filterText(genes, query.q), query),
+    categories: countBy(genes, 'category'),
+  };
+}
+
+function listCapsules(query = {}) {
+  const paths = getObserverPaths();
+  const all = readCapsules(paths).concat(readFailedCapsules(paths).map((c) => ({ ...c, failed_store: true })));
+  return {
+    ...paginate(filterText(all, query.q), query),
+    outcomes: countBy(all, (c) => c.outcome && c.outcome.status || 'unknown'),
+  };
+}
+
+function listEvents(query = {}) {
+  const events = readJsonl(getObserverPaths().eventsPath).map(redactValue).reverse();
+  return paginate(filterText(events, query.q), query);
+}
+
+function listCandidates(query = {}) {
+  const paths = getObserverPaths();
+  const local = readJsonl(paths.candidatesPath).map((entry) => ({ ...entry, source: 'local' }));
+  const external = readJsonl(paths.externalCandidatesPath).map((entry) => ({ ...entry, source: 'external' }));
+  return paginate(filterText(local.concat(external).map(redactValue).reverse(), query.q), query);
+}
+
+function listAssetCalls(query = {}) {
+  const calls = readJsonl(getObserverPaths().assetCallLogPath).map(redactValue).reverse();
+  return paginate(filterText(calls, query.q), query);
+}
+
+function getLineage(id) {
+  const paths = getObserverPaths();
+  const genes = readGenes(paths);
+  const capsules = readCapsules(paths).concat(readFailedCapsules(paths));
+  const events = readJsonl(paths.eventsPath).map(redactValue);
+  const assetCalls = readJsonl(paths.assetCallLogPath).map(redactValue);
+  return {
+    id,
+    genes: genes.filter((g) => matchesAsset(g, id)),
+    capsules: capsules.filter((c) => matchesAsset(c, id) || c.gene === id),
+    events: events.filter((e) => eventMentions(e, id)),
+    assetCalls: assetCalls.filter((entry) => matchesAsset(entry, id)),
+  };
+}
+
+function readGenes(paths) {
+  const fromJson = readJsonSafe(paths.genesPath, { genes: [] }).genes || [];
+  const fromJsonl = readJsonl(paths.genesJsonlPath).filter((g) => g && g.type === 'Gene');
+  return dedupeById(fromJson.concat(fromJsonl)).map(redactValue);
+}
+
+function readCapsules(paths) {
+  const fromJson = readJsonSafe(paths.capsulesPath, { capsules: [] }).capsules || [];
+  const fromJsonl = readJsonl(paths.capsulesJsonlPath).filter((c) => c && c.type === 'Capsule');
+  return dedupeById(fromJson.concat(fromJsonl)).map(redactValue);
+}
+
+function readFailedCapsules(paths) {
+  const failed = readJsonSafe(paths.failedCapsulesPath, { failed_capsules: [] }).failed_capsules || [];
+  return failed.map(redactValue);
+}
+
+function dedupeById(items) {
+  const out = new Map();
+  for (const item of items) {
+    if (item && item.id) out.set(String(item.id), item);
+  }
+  return Array.from(out.values());
+}
+
+function countBy(items, keyOrFn) {
+  const counts = {};
+  for (const item of items) {
+    const key = typeof keyOrFn === 'function' ? keyOrFn(item) : item && item[keyOrFn];
+    const safeKey = key || 'unknown';
+    counts[safeKey] = (counts[safeKey] || 0) + 1;
+  }
+  return counts;
+}
+
+function filterText(items, q) {
+  if (!q) return items;
+  const needle = String(q).toLowerCase();
+  return items.filter((item) => JSON.stringify(item).toLowerCase().includes(needle));
+}
+
+function matchesAsset(item, id) {
+  if (!item || !id) return false;
+  return item.id === id || item.asset_id === id || item.asset_id === `sha256:${id}`;
+}
+
+function eventMentions(event, id) {
+  if (matchesAsset(event, id)) return true;
+  if (event.capsule_id === id || event.mutation_id === id) return true;
+  return Array.isArray(event.genes_used) && event.genes_used.includes(id);
+}
+
+module.exports = {
+  getAssetOverview,
+  listGenes,
+  listCapsules,
+  listEvents,
+  listCandidates,
+  listAssetCalls,
+  getLineage,
+  readGenes,
+  readCapsules,
+};

package/src/webui/observer/index.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const { getStatus } = require('./status');
+const { getSafetyState } = require('./safety');
+const { listRuns, getRun } = require('./runs');
+const { getAssetOverview, listGenes, listCapsules, listEvents, listCandidates, listAssetCalls, getLineage } = require('./assets');
+const { getInteractions } = require('./interactions');
+const { getPersonality, getMemoryGraph } = require('./personality');
+const { listSkills } = require('./skills');
+const { getObserverPaths } = require('./paths');
+const { tailText } = require('./jsonl');
+const { redactText } = require('./redact');
+
+function getEvolverLog(query = {}) {
+  const tail = query.tail || 200;
+  const text = tailText(getObserverPaths().evolverLogPath, tail);
+  return { text: redactText(text), tail: Number(tail) || 200 };
+}
+
+module.exports = {
+  getStatus,
+  getSafetyState,
+  listRuns,
+  getRun,
+  getAssetOverview,
+  listGenes,
+  listCapsules,
+  listEvents,
+  listCandidates,
+  listAssetCalls,
+  getLineage,
+  getInteractions,
+  getPersonality,
+  getMemoryGraph,
+  listSkills,
+  getEvolverLog,
+};

package/src/webui/observer/interactions.js

@@ -0,0 +1,120 @@
+'use strict';
+
+const http = require('http');
+const os = require('os');
+const path = require('path');
+const { getProxySettings } = require('./status');
+const { readJsonl, paginate } = require('./jsonl');
+const { redactValue } = require('./redact');
+
+const DEFAULT_MAILBOX_DIR = path.join(os.homedir(), '.evomap', 'mailbox');
+
+async function getInteractions(query = {}) {
+  const proxy = getProxySettings();
+  const proxySnapshots = proxy.url ? await readProxySnapshots(proxy.url) : {};
+  const mailbox = readMailboxMessages(query);
+  return {
+    proxy,
+    proxySnapshots,
+    mailbox: paginate(mailbox, query),
+  };
+}
+
+function readMailboxMessages(query = {}) {
+  const mailboxDir = query.mailboxDir || process.env.EVOMAP_MAILBOX_DIR || DEFAULT_MAILBOX_DIR;
+  const messages = readJsonl(path.join(mailboxDir, 'messages.jsonl'), { last: query.last || 500 })
+    .filter((row) => row && !row._op)
+    .map((row) => normalizeMessage(redactValue(row)));
+  return filterMessages(messages, query);
+}
+
+async function readProxySnapshots(baseUrl) {
+  const endpoints = {
+    status: '/proxy/status',
+    hubStatus: '/proxy/hub-status',
+    taskMetrics: '/task/metrics',
+    assetSubmissions: '/asset/submissions?limit=20',
+    sessions: '/session/list?limit=20',
+    dms: '/dm/list?limit=20',
+    atpPolicy: '/atp/policy',
+    atpProofs: '/atp/proofs?limit=20',
+  };
+  // Issue all proxy requests concurrently. Sequential await meant the
+  // worst-case (every endpoint timing out at 1500ms each) blocked the
+  // /webui/interactions response for ~12s; parallel keeps it ~1.5s.
+  const entries = await Promise.all(
+    Object.entries(endpoints).map(async ([key, endpoint]) =>
+      [key, await requestJson(baseUrl + endpoint)],
+    ),
+  );
+  return redactValue(Object.fromEntries(entries));
+}
+
+function requestJson(url) {
+  return new Promise((resolve) => {
+    const req = http.get(url, { timeout: 1500 }, (res) => {
+      const chunks = [];
+      res.on('data', (chunk) => chunks.push(chunk));
+      res.on('end', () => {
+        const raw = Buffer.concat(chunks).toString();
+        try {
+          resolve({ ok: res.statusCode >= 200 && res.statusCode < 300, status: res.statusCode, body: JSON.parse(raw) });
+        } catch {
+          resolve({ ok: false, status: res.statusCode, body: null });
+        }
+      });
+    });
+    req.on('timeout', () => {
+      req.destroy();
+      resolve({ ok: false, status: 0, error: 'timeout' });
+    });
+    req.on('error', (err) => resolve({ ok: false, status: 0, error: err.message }));
+  });
+}
+
+function normalizeMessage(msg) {
+  return {
+    id: msg.id || null,
+    source: 'mailbox',
+    direction: msg.direction || null,
+    type: msg.type || null,
+    status: msg.status || null,
+    timestamp: toIso(msg.created_at || msg.updated_at || msg.synced_at),
+    refId: msg.ref_id || msg.payload && msg.payload.ref_id || null,
+    priority: msg.priority || null,
+    summary: summarizePayload(msg),
+    payload: msg.payload || null,
+  };
+}
+
+function filterMessages(messages, query) {
+  return messages.filter((msg) => {
+    if (query.type && msg.type !== query.type) return false;
+    if (query.direction && msg.direction !== query.direction) return false;
+    if (query.status && msg.status !== query.status) return false;
+    return true;
+  }).reverse();
+}
+
+function summarizePayload(msg) {
+  const payload = msg.payload || {};
+  if (payload.task_id) return `Task ${payload.task_id}`;
+  if (payload.asset_id) return `Asset ${payload.asset_id}`;
+  if (payload.session_id) return `Session ${payload.session_id}`;
+  if (payload.title) return String(payload.title);
+  if (msg.type) return String(msg.type).replace(/_/g, ' ');
+  return 'message';
+}
+
+function toIso(value) {
+  if (!value) return null;
+  if (typeof value === 'string') return value;
+  const n = Number(value);
+  return Number.isFinite(n) ? new Date(n).toISOString() : null;
+}
+
+module.exports = {
+  getInteractions,
+  readMailboxMessages,
+  readProxySnapshots,
+};

package/src/webui/observer/jsonl.js

@@ -0,0 +1,75 @@
+'use strict';
+
+const fs = require('fs');
+
+function readJsonSafe(filePath, fallback) {
+  try {
+    if (!filePath || !fs.existsSync(filePath)) return fallback;
+    const raw = fs.readFileSync(filePath, 'utf8');
+    if (!raw.trim()) return fallback;
+    return JSON.parse(raw);
+  } catch {
+    return fallback;
+  }
+}
+
+function readJsonl(filePath, opts = {}) {
+  try {
+    if (!filePath || !fs.existsSync(filePath)) return [];
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const lines = raw.split('\n').map((line) => line.trim()).filter(Boolean);
+    const selected = opts.last ? lines.slice(-toPositiveInt(opts.last, lines.length)) : lines;
+    const rows = [];
+    for (const line of selected) {
+      try {
+        rows.push(JSON.parse(line));
+      } catch {
+        // Corrupt JSONL rows should not break the dashboard.
+      }
+    }
+    return rows;
+  } catch {
+    return [];
+  }
+}
+
+function tailText(filePath, maxLines) {
+  try {
+    if (!filePath || !fs.existsSync(filePath)) return '';
+    const raw = fs.readFileSync(filePath, 'utf8');
+    const lines = raw.split('\n');
+    return lines.slice(-toPositiveInt(maxLines, 200)).join('\n');
+  } catch {
+    return '';
+  }
+}
+
+function paginate(items, query = {}) {
+  const limit = Math.min(toPositiveInt(query.limit, 50), 200);
+  const offset = Math.max(0, toPositiveInt(query.offset || query.cursor, 0));
+  const data = items.slice(offset, offset + limit);
+  const nextOffset = offset + data.length;
+  return {
+    data,
+    pagination: {
+      limit,
+      offset,
+      totalItems: items.length,
+      nextCursor: nextOffset < items.length ? String(nextOffset) : null,
+    },
+  };
+}
+
+function toPositiveInt(value, fallback) {
+  const n = Number(value);
+  if (!Number.isFinite(n) || n < 0) return fallback;
+  return Math.floor(n);
+}
+
+module.exports = {
+  readJsonSafe,
+  readJsonl,
+  tailText,
+  paginate,
+  toPositiveInt,
+};

package/src/webui/observer/paths.js

@@ -0,0 +1,46 @@
+'use strict';
+
+const path = require('path');
+const {
+  getRepoRoot,
+  getWorkspaceRoot,
+  getEvolutionDir,
+  getGepAssetsDir,
+  getSkillsDir,
+  getEvolverLogPath,
+  getAgentSessionsDir,
+  getNarrativePath,
+  getReflectionLogPath,
+} = require('../../gep/paths');
+
+function getObserverPaths() {
+  const evolutionDir = getEvolutionDir();
+  const gepAssetsDir = getGepAssetsDir();
+  return {
+    repoRoot: getRepoRoot(),
+    workspaceRoot: getWorkspaceRoot(),
+    evolutionDir,
+    gepAssetsDir,
+    skillsDir: getSkillsDir(),
+    agentSessionsDir: getAgentSessionsDir(),
+    evolverLogPath: getEvolverLogPath(),
+    cycleProgressPath: path.join(evolutionDir, 'cycle_progress.json'),
+    solidifyStatePath: path.join(evolutionDir, 'evolution_solidify_state.json'),
+    evolutionStatePath: path.join(evolutionDir, 'evolution_state.json'),
+    pipelineEventsPath: path.join(evolutionDir, 'pipeline_events.jsonl'),
+    assetCallLogPath: path.join(evolutionDir, 'asset_call_log.jsonl'),
+    reflectionLogPath: getReflectionLogPath(),
+    narrativePath: getNarrativePath(),
+    memoryGraphPath: path.join(evolutionDir, 'memory_graph.jsonl'),
+    genesPath: path.join(gepAssetsDir, 'genes.json'),
+    genesJsonlPath: path.join(gepAssetsDir, 'genes.jsonl'),
+    capsulesPath: path.join(gepAssetsDir, 'capsules.json'),
+    capsulesJsonlPath: path.join(gepAssetsDir, 'capsules.jsonl'),
+    eventsPath: path.join(gepAssetsDir, 'events.jsonl'),
+    candidatesPath: path.join(gepAssetsDir, 'candidates.jsonl'),
+    externalCandidatesPath: path.join(gepAssetsDir, 'external_candidates.jsonl'),
+    failedCapsulesPath: path.join(gepAssetsDir, 'failed_capsules.json'),
+  };
+}
+
+module.exports = { getObserverPaths };

package/src/webui/observer/personality.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const path = require('path');
+const fs = require('fs');
+const { getMemoryDir, getEvolutionDir } = require('../../gep/paths');
+const { readJsonSafe, readJsonl } = require('./jsonl');
+const { redactValue } = require('./redact');
+
+function getPersonality() {
+  const evoDir = getEvolutionDir();
+  const memDir = getMemoryDir();
+  const candidates = [
+    path.join(evoDir, 'personality_state.json'),
+    path.join(memDir, 'personality_state.json'),
+  ];
+  for (const candidate of candidates) {
+    const data = readJsonSafe(candidate, null);
+    if (data) return { exists: true, ...redactValue(data) };
+  }
+  return { exists: false, current: null, history: [] };
+}
+
+function getMemoryGraph(query = {}) {
+  const limit = Math.min(Number(query.limit) || 50, 500);
+  const graphPath = path.join(getEvolutionDir(), 'memory_graph.jsonl');
+  if (!fs.existsSync(graphPath)) return { exists: false, items: [] };
+  const rows = readJsonl(graphPath, { last: limit }).map(redactValue);
+  return {
+    exists: true,
+    total: countLines(graphPath),
+    items: rows.reverse(),
+  };
+}
+
+function countLines(filePath) {
+  try {
+    return fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean).length;
+  } catch {
+    return 0;
+  }
+}
+
+module.exports = { getPersonality, getMemoryGraph };

package/src/webui/observer/pipelineEvents.js

@@ -0,0 +1,58 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { getObserverPaths } = require('./paths');
+const { readJsonl } = require('./jsonl');
+const { redactValue } = require('./redact');
+
+const VALID_PHASE_STATUSES = new Set(['pending', 'running', 'success', 'failed', 'skipped', 'blocked']);
+
+function readPipelineEvents(opts = {}) {
+  return readJsonl(getObserverPaths().pipelineEventsPath, opts).map(normalizePipelineEvent);
+}
+
+function logPipelineEvent(event) {
+  const normalized = normalizePipelineEvent(event);
+  const target = getObserverPaths().pipelineEventsPath;
+  fs.mkdirSync(path.dirname(target), { recursive: true });
+  fs.appendFileSync(target, JSON.stringify(normalized) + '\n', 'utf8');
+  return normalized;
+}
+
+function normalizePipelineEvent(event) {
+  const input = event && typeof event === 'object' ? event : {};
+  const status = VALID_PHASE_STATUSES.has(input.status) ? input.status : 'running';
+  return redactValue({
+    run_id: stringOrNull(input.run_id),
+    cycle_id: stringOrNull(input.cycle_id),
+    phase: stringOrDefault(input.phase, 'unknown'),
+    status,
+    started_at: stringOrNull(input.started_at),
+    finished_at: stringOrNull(input.finished_at),
+    summary: stringOrDefault(input.summary, ''),
+    evidence_refs: arrayOfObjects(input.evidence_refs),
+    asset_refs: arrayOfObjects(input.asset_refs),
+    validation_refs: arrayOfObjects(input.validation_refs),
+    requires_confirmation: Boolean(input.requires_confirmation),
+    timestamp: input.timestamp || new Date().toISOString(),
+  });
+}
+
+function stringOrNull(value) {
+  return typeof value === 'string' && value ? value : null;
+}
+
+function stringOrDefault(value, fallback) {
+  return typeof value === 'string' ? value : fallback;
+}
+
+function arrayOfObjects(value) {
+  return Array.isArray(value) ? value.filter((entry) => entry && typeof entry === 'object') : [];
+}
+
+module.exports = {
+  readPipelineEvents,
+  logPipelineEvent,
+  normalizePipelineEvent,
+};

package/src/webui/observer/redact.js

@@ -0,0 +1,63 @@
+'use strict';
+
+const SECRET_KEY_RE = /(secret|token|api[_-]?key|authorization|cookie|oauth|password|private[_-]?key|node_secret)/i;
+// Three alternation arms, each with one capturing group so the replace()
+// callback can keep the prefix and only mask the value:
+//   1) HTTP Bearer auth header
+//   2) Env/INI/CLI style NAME=value (or NAME: value) where NAME ends in
+//      SECRET / TOKEN / KEY / PASSWORD / PASSWD / PWD. The original
+//      regex only matched SECRET / TOKEN; without KEY / PASSWORD a line
+//      like `OPENAI_API_KEY=sk-...` or `DB_PASSWORD=hunter2` would slip
+//      through to the /webui/logs/evolver tail, leaking live credentials.
+//   3) Bare OpenAI/Anthropic-shape API keys (`sk-...` / `sk-ant-...`) for
+//      cases where they appear as standalone tokens in log lines without
+//      a NAME= prefix.
+// Flagged /gi so lower-case variants (`password=...`, `bearer ...`,
+// `sk-...`) are also caught; over-redacting harmless prose tokens like
+// "monkey=" is acceptable, leaking a real key is not.
+const SECRET_TEXT_RE = new RegExp([
+  '(Bearer\\s+)[A-Za-z0-9._~+/-]+=*',
+  '([A-Za-z0-9_]*(?:SECRET|TOKEN|KEY|PASSWORD|PASSWD|PWD)[A-Za-z0-9_]*\\s*[:=]\\s*)["\']?[^\\s,"\'`]+',
+  '(sk-(?:ant-)?[A-Za-z0-9_-]{20,})',
+].join('|'), 'gi');
+
+function redactValue(value, depth = 0) {
+  if (depth > 8) return '[REDACTED_DEPTH]';
+  if (Array.isArray(value)) return value.map((entry) => redactValue(entry, depth + 1));
+  if (!value || typeof value !== 'object') {
+    return typeof value === 'string' ? redactText(value) : value;
+  }
+
+  const out = {};
+  for (const [key, child] of Object.entries(value)) {
+    if (SECRET_KEY_RE.test(key)) {
+      out[key] = '[REDACTED]';
+    } else if (isLargeSensitiveField(key, child)) {
+      out[key] = '[REDACTED_SENSITIVE_DETAIL]';
+    } else {
+      out[key] = redactValue(child, depth + 1);
+    }
+  }
+  return out;
+}
+
+function redactText(text) {
+  if (typeof text !== 'string') return text;
+  return text.replace(SECRET_TEXT_RE, (match, bearer, namedPrefix, bareKey) => {
+    if (bearer) return `${bearer}[REDACTED]`;
+    if (namedPrefix) return `${namedPrefix}[REDACTED]`;
+    if (bareKey) return '[REDACTED]';
+    return '[REDACTED]';
+  });
+}
+
+function isLargeSensitiveField(key, value) {
+  if (typeof value !== 'string') return false;
+  if (!/(prompt|diff|content|transcript|message|raw)/i.test(key)) return false;
+  return value.length > 500;
+}
+
+module.exports = {
+  redactValue,
+  redactText,
+};