@evomap/evolver 1.90.0 → 2.0.0-beta.0

package/src/webui/index.js

@@ -1,14 +0,0 @@
-'use strict';
-
-const { WebUiServer } = require('./server/http');
-
-async function startWebUi(opts = {}) {
-  const server = new WebUiServer(opts);
-  const info = await server.start();
-  return { server, ...info };
-}
-
-module.exports = {
-  startWebUi,
-  WebUiServer,
-};

package/src/webui/observer/assets.js

@@ -1,146 +0,0 @@
-'use strict';
-
-const { getObserverPaths } = require('./paths');
-const { readJsonSafe, readJsonl, paginate } = require('./jsonl');
-const { redactValue } = require('./redact');
-
-function getAssetOverview(query = {}) {
-  const paths = getObserverPaths();
-  const genes = readGenes(paths);
-  const capsules = readCapsules(paths);
-  const failedCapsules = readFailedCapsules(paths);
-  const events = readJsonl(paths.eventsPath, { last: query.eventsLast || 500 }).map(redactValue);
-  const candidates = readJsonl(paths.candidatesPath, { last: query.candidatesLast || 200 }).map(redactValue);
-  const externalCandidates = readJsonl(paths.externalCandidatesPath, { last: query.candidatesLast || 200 }).map(redactValue);
-  const assetCalls = readJsonl(paths.assetCallLogPath, { last: query.assetCallsLast || 500 }).map(redactValue);
-
-  return {
-    counts: {
-      genes: genes.length,
-      capsules: capsules.length,
-      failedCapsules: failedCapsules.length,
-      events: events.length,
-      candidates: candidates.length,
-      externalCandidates: externalCandidates.length,
-      assetCalls: assetCalls.length,
-    },
-    genesByCategory: countBy(genes, 'category'),
-    capsulesByOutcome: countBy(capsules, (c) => c.outcome && c.outcome.status || 'unknown'),
-    assetCallsByAction: countBy(assetCalls, 'action'),
-    recentEvents: events.slice(-20).reverse(),
-    recentAssetCalls: assetCalls.slice(-20).reverse(),
-  };
-}
-
-function listGenes(query = {}) {
-  const genes = readGenes(getObserverPaths());
-  return {
-    ...paginate(filterText(genes, query.q), query),
-    categories: countBy(genes, 'category'),
-  };
-}
-
-function listCapsules(query = {}) {
-  const paths = getObserverPaths();
-  const all = readCapsules(paths).concat(readFailedCapsules(paths).map((c) => ({ ...c, failed_store: true })));
-  return {
-    ...paginate(filterText(all, query.q), query),
-    outcomes: countBy(all, (c) => c.outcome && c.outcome.status || 'unknown'),
-  };
-}
-
-function listEvents(query = {}) {
-  const events = readJsonl(getObserverPaths().eventsPath).map(redactValue).reverse();
-  return paginate(filterText(events, query.q), query);
-}
-
-function listCandidates(query = {}) {
-  const paths = getObserverPaths();
-  const local = readJsonl(paths.candidatesPath).map((entry) => ({ ...entry, source: 'local' }));
-  const external = readJsonl(paths.externalCandidatesPath).map((entry) => ({ ...entry, source: 'external' }));
-  return paginate(filterText(local.concat(external).map(redactValue).reverse(), query.q), query);
-}
-
-function listAssetCalls(query = {}) {
-  const calls = readJsonl(getObserverPaths().assetCallLogPath).map(redactValue).reverse();
-  return paginate(filterText(calls, query.q), query);
-}
-
-function getLineage(id) {
-  const paths = getObserverPaths();
-  const genes = readGenes(paths);
-  const capsules = readCapsules(paths).concat(readFailedCapsules(paths));
-  const events = readJsonl(paths.eventsPath).map(redactValue);
-  const assetCalls = readJsonl(paths.assetCallLogPath).map(redactValue);
-  return {
-    id,
-    genes: genes.filter((g) => matchesAsset(g, id)),
-    capsules: capsules.filter((c) => matchesAsset(c, id) || c.gene === id),
-    events: events.filter((e) => eventMentions(e, id)),
-    assetCalls: assetCalls.filter((entry) => matchesAsset(entry, id)),
-  };
-}
-
-function readGenes(paths) {
-  const fromJson = readJsonSafe(paths.genesPath, { genes: [] }).genes || [];
-  const fromJsonl = readJsonl(paths.genesJsonlPath).filter((g) => g && g.type === 'Gene');
-  return dedupeById(fromJson.concat(fromJsonl)).map(redactValue);
-}
-
-function readCapsules(paths) {
-  const fromJson = readJsonSafe(paths.capsulesPath, { capsules: [] }).capsules || [];
-  const fromJsonl = readJsonl(paths.capsulesJsonlPath).filter((c) => c && c.type === 'Capsule');
-  return dedupeById(fromJson.concat(fromJsonl)).map(redactValue);
-}
-
-function readFailedCapsules(paths) {
-  const failed = readJsonSafe(paths.failedCapsulesPath, { failed_capsules: [] }).failed_capsules || [];
-  return failed.map(redactValue);
-}
-
-function dedupeById(items) {
-  const out = new Map();
-  for (const item of items) {
-    if (item && item.id) out.set(String(item.id), item);
-  }
-  return Array.from(out.values());
-}
-
-function countBy(items, keyOrFn) {
-  const counts = {};
-  for (const item of items) {
-    const key = typeof keyOrFn === 'function' ? keyOrFn(item) : item && item[keyOrFn];
-    const safeKey = key || 'unknown';
-    counts[safeKey] = (counts[safeKey] || 0) + 1;
-  }
-  return counts;
-}
-
-function filterText(items, q) {
-  if (!q) return items;
-  const needle = String(q).toLowerCase();
-  return items.filter((item) => JSON.stringify(item).toLowerCase().includes(needle));
-}
-
-function matchesAsset(item, id) {
-  if (!item || !id) return false;
-  return item.id === id || item.asset_id === id || item.asset_id === `sha256:${id}`;
-}
-
-function eventMentions(event, id) {
-  if (matchesAsset(event, id)) return true;
-  if (event.capsule_id === id || event.mutation_id === id) return true;
-  return Array.isArray(event.genes_used) && event.genes_used.includes(id);
-}
-
-module.exports = {
-  getAssetOverview,
-  listGenes,
-  listCapsules,
-  listEvents,
-  listCandidates,
-  listAssetCalls,
-  getLineage,
-  readGenes,
-  readCapsules,
-};

package/src/webui/observer/index.js

@@ -1,37 +0,0 @@
-'use strict';
-
-const { getStatus } = require('./status');
-const { getSafetyState } = require('./safety');
-const { listRuns, getRun } = require('./runs');
-const { getAssetOverview, listGenes, listCapsules, listEvents, listCandidates, listAssetCalls, getLineage } = require('./assets');
-const { getInteractions } = require('./interactions');
-const { getPersonality, getMemoryGraph } = require('./personality');
-const { listSkills } = require('./skills');
-const { getObserverPaths } = require('./paths');
-const { tailText } = require('./jsonl');
-const { redactText } = require('./redact');
-
-function getEvolverLog(query = {}) {
-  const tail = query.tail || 200;
-  const text = tailText(getObserverPaths().evolverLogPath, tail);
-  return { text: redactText(text), tail: Number(tail) || 200 };
-}
-
-module.exports = {
-  getStatus,
-  getSafetyState,
-  listRuns,
-  getRun,
-  getAssetOverview,
-  listGenes,
-  listCapsules,
-  listEvents,
-  listCandidates,
-  listAssetCalls,
-  getLineage,
-  getInteractions,
-  getPersonality,
-  getMemoryGraph,
-  listSkills,
-  getEvolverLog,
-};

package/src/webui/observer/interactions.js

@@ -1,127 +0,0 @@
-'use strict';
-
-const http = require('http');
-const path = require('path');
-const { getProxySettings } = require('./status');
-const { getProxyToken } = require('../../proxy/server/settings');
-const { readJsonl, paginate } = require('./jsonl');
-const { redactValue } = require('./redact');
-const { getEvomapPath } = require('../../gep/paths');
-
-// Lazy via paths.getEvomapPath() — honors EVOLVER_HOME (#114).
-function _defaultMailboxDir() { return getEvomapPath('mailbox'); }
-
-async function getInteractions(query = {}) {
-  const proxy = getProxySettings();
-  const proxySnapshots = proxy.url ? await readProxySnapshots(proxy.url) : {};
-  const mailbox = readMailboxMessages(query);
-  return {
-    proxy,
-    proxySnapshots,
-    mailbox: paginate(mailbox, query),
-  };
-}
-
-function readMailboxMessages(query = {}) {
-  const mailboxDir = query.mailboxDir || process.env.EVOMAP_MAILBOX_DIR || _defaultMailboxDir();
-  const messages = readJsonl(path.join(mailboxDir, 'messages.jsonl'), { last: query.last || 500 })
-    .filter((row) => row && !row._op)
-    .map((row) => normalizeMessage(redactValue(row)));
-  return filterMessages(messages, query);
-}
-
-async function readProxySnapshots(baseUrl) {
-  const endpoints = {
-    status: '/proxy/status',
-    hubStatus: '/proxy/hub-status',
-    taskMetrics: '/task/metrics',
-    assetSubmissions: '/asset/submissions?limit=20',
-    sessions: '/session/list?limit=20',
-    dms: '/dm/list?limit=20',
-    atpPolicy: '/atp/policy',
-    atpProofs: '/atp/proofs?limit=20',
-  };
-  // Issue all proxy requests concurrently. Sequential await meant the
-  // worst-case (every endpoint timing out at 1500ms each) blocked the
-  // /webui/interactions response for ~12s; parallel keeps it ~1.5s.
-  const entries = await Promise.all(
-    Object.entries(endpoints).map(async ([key, endpoint]) =>
-      [key, await requestJson(baseUrl + endpoint)],
-    ),
-  );
-  return redactValue(Object.fromEntries(entries));
-}
-
-function requestJson(url) {
-  return new Promise((resolve) => {
-    const u = new URL(url);
-    const token = getProxyToken();
-    const headers = {};
-    if (token) headers['Authorization'] = 'Bearer ' + token;
-    const req = http.request(
-      { hostname: u.hostname, port: u.port, path: u.pathname + (u.search || ''), method: 'GET', headers, timeout: 1500 },
-      (res) => {
-        const chunks = [];
-        res.on('data', (chunk) => chunks.push(chunk));
-        res.on('end', () => {
-          const raw = Buffer.concat(chunks).toString();
-          try {
-            resolve({ ok: res.statusCode >= 200 && res.statusCode < 300, status: res.statusCode, body: JSON.parse(raw) });
-          } catch {
-            resolve({ ok: false, status: res.statusCode, body: null });
-          }
-        });
-      },
-    );
-    req.on('timeout', () => { req.destroy(); resolve({ ok: false, status: 0, error: 'timeout' }); });
-    req.on('error', (err) => resolve({ ok: false, status: 0, error: err.message }));
-    req.end();
-  });
-}
-
-function normalizeMessage(msg) {
-  return {
-    id: msg.id || null,
-    source: 'mailbox',
-    direction: msg.direction || null,
-    type: msg.type || null,
-    status: msg.status || null,
-    timestamp: toIso(msg.created_at || msg.updated_at || msg.synced_at),
-    refId: msg.ref_id || msg.payload && msg.payload.ref_id || null,
-    priority: msg.priority || null,
-    summary: summarizePayload(msg),
-    payload: msg.payload || null,
-  };
-}
-
-function filterMessages(messages, query) {
-  return messages.filter((msg) => {
-    if (query.type && msg.type !== query.type) return false;
-    if (query.direction && msg.direction !== query.direction) return false;
-    if (query.status && msg.status !== query.status) return false;
-    return true;
-  }).reverse();
-}
-
-function summarizePayload(msg) {
-  const payload = msg.payload || {};
-  if (payload.task_id) return `Task ${payload.task_id}`;
-  if (payload.asset_id) return `Asset ${payload.asset_id}`;
-  if (payload.session_id) return `Session ${payload.session_id}`;
-  if (payload.title) return String(payload.title);
-  if (msg.type) return String(msg.type).replace(/_/g, ' ');
-  return 'message';
-}
-
-function toIso(value) {
-  if (!value) return null;
-  if (typeof value === 'string') return value;
-  const n = Number(value);
-  return Number.isFinite(n) ? new Date(n).toISOString() : null;
-}
-
-module.exports = {
-  getInteractions,
-  readMailboxMessages,
-  readProxySnapshots,
-};

package/src/webui/observer/jsonl.js

@@ -1,75 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-
-function readJsonSafe(filePath, fallback) {
-  try {
-    if (!filePath || !fs.existsSync(filePath)) return fallback;
-    const raw = fs.readFileSync(filePath, 'utf8');
-    if (!raw.trim()) return fallback;
-    return JSON.parse(raw);
-  } catch {
-    return fallback;
-  }
-}
-
-function readJsonl(filePath, opts = {}) {
-  try {
-    if (!filePath || !fs.existsSync(filePath)) return [];
-    const raw = fs.readFileSync(filePath, 'utf8');
-    const lines = raw.split('\n').map((line) => line.trim()).filter(Boolean);
-    const selected = opts.last ? lines.slice(-toPositiveInt(opts.last, lines.length)) : lines;
-    const rows = [];
-    for (const line of selected) {
-      try {
-        rows.push(JSON.parse(line));
-      } catch {
-        // Corrupt JSONL rows should not break the dashboard.
-      }
-    }
-    return rows;
-  } catch {
-    return [];
-  }
-}
-
-function tailText(filePath, maxLines) {
-  try {
-    if (!filePath || !fs.existsSync(filePath)) return '';
-    const raw = fs.readFileSync(filePath, 'utf8');
-    const lines = raw.split('\n');
-    return lines.slice(-toPositiveInt(maxLines, 200)).join('\n');
-  } catch {
-    return '';
-  }
-}
-
-function paginate(items, query = {}) {
-  const limit = Math.min(toPositiveInt(query.limit, 50), 200);
-  const offset = Math.max(0, toPositiveInt(query.offset || query.cursor, 0));
-  const data = items.slice(offset, offset + limit);
-  const nextOffset = offset + data.length;
-  return {
-    data,
-    pagination: {
-      limit,
-      offset,
-      totalItems: items.length,
-      nextCursor: nextOffset < items.length ? String(nextOffset) : null,
-    },
-  };
-}
-
-function toPositiveInt(value, fallback) {
-  const n = Number(value);
-  if (!Number.isFinite(n) || n < 0) return fallback;
-  return Math.floor(n);
-}
-
-module.exports = {
-  readJsonSafe,
-  readJsonl,
-  tailText,
-  paginate,
-  toPositiveInt,
-};

package/src/webui/observer/paths.js

@@ -1,46 +0,0 @@
-'use strict';
-
-const path = require('path');
-const {
-  getRepoRoot,
-  getWorkspaceRoot,
-  getEvolutionDir,
-  getGepAssetsDir,
-  getSkillsDir,
-  getEvolverLogPath,
-  getAgentSessionsDir,
-  getNarrativePath,
-  getReflectionLogPath,
-} = require('../../gep/paths');
-
-function getObserverPaths() {
-  const evolutionDir = getEvolutionDir();
-  const gepAssetsDir = getGepAssetsDir();
-  return {
-    repoRoot: getRepoRoot(),
-    workspaceRoot: getWorkspaceRoot(),
-    evolutionDir,
-    gepAssetsDir,
-    skillsDir: getSkillsDir(),
-    agentSessionsDir: getAgentSessionsDir(),
-    evolverLogPath: getEvolverLogPath(),
-    cycleProgressPath: path.join(evolutionDir, 'cycle_progress.json'),
-    solidifyStatePath: path.join(evolutionDir, 'evolution_solidify_state.json'),
-    evolutionStatePath: path.join(evolutionDir, 'evolution_state.json'),
-    pipelineEventsPath: path.join(evolutionDir, 'pipeline_events.jsonl'),
-    assetCallLogPath: path.join(evolutionDir, 'asset_call_log.jsonl'),
-    reflectionLogPath: getReflectionLogPath(),
-    narrativePath: getNarrativePath(),
-    memoryGraphPath: path.join(evolutionDir, 'memory_graph.jsonl'),
-    genesPath: path.join(gepAssetsDir, 'genes.json'),
-    genesJsonlPath: path.join(gepAssetsDir, 'genes.jsonl'),
-    capsulesPath: path.join(gepAssetsDir, 'capsules.json'),
-    capsulesJsonlPath: path.join(gepAssetsDir, 'capsules.jsonl'),
-    eventsPath: path.join(gepAssetsDir, 'events.jsonl'),
-    candidatesPath: path.join(gepAssetsDir, 'candidates.jsonl'),
-    externalCandidatesPath: path.join(gepAssetsDir, 'external_candidates.jsonl'),
-    failedCapsulesPath: path.join(gepAssetsDir, 'failed_capsules.json'),
-  };
-}
-
-module.exports = { getObserverPaths };

package/src/webui/observer/personality.js

@@ -1,43 +0,0 @@
-'use strict';
-
-const path = require('path');
-const fs = require('fs');
-const { getMemoryDir, getEvolutionDir } = require('../../gep/paths');
-const { readJsonSafe, readJsonl } = require('./jsonl');
-const { redactValue } = require('./redact');
-
-function getPersonality() {
-  const evoDir = getEvolutionDir();
-  const memDir = getMemoryDir();
-  const candidates = [
-    path.join(evoDir, 'personality_state.json'),
-    path.join(memDir, 'personality_state.json'),
-  ];
-  for (const candidate of candidates) {
-    const data = readJsonSafe(candidate, null);
-    if (data) return { exists: true, ...redactValue(data) };
-  }
-  return { exists: false, current: null, history: [] };
-}
-
-function getMemoryGraph(query = {}) {
-  const limit = Math.min(Number(query.limit) || 50, 500);
-  const graphPath = path.join(getEvolutionDir(), 'memory_graph.jsonl');
-  if (!fs.existsSync(graphPath)) return { exists: false, items: [] };
-  const rows = readJsonl(graphPath, { last: limit }).map(redactValue);
-  return {
-    exists: true,
-    total: countLines(graphPath),
-    items: rows.reverse(),
-  };
-}
-
-function countLines(filePath) {
-  try {
-    return fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean).length;
-  } catch {
-    return 0;
-  }
-}
-
-module.exports = { getPersonality, getMemoryGraph };

package/src/webui/observer/pipelineEvents.js

@@ -1,58 +0,0 @@
-'use strict';
-
-const fs = require('fs');
-const path = require('path');
-const { getObserverPaths } = require('./paths');
-const { readJsonl } = require('./jsonl');
-const { redactValue } = require('./redact');
-
-const VALID_PHASE_STATUSES = new Set(['pending', 'running', 'success', 'failed', 'skipped', 'blocked']);
-
-function readPipelineEvents(opts = {}) {
-  return readJsonl(getObserverPaths().pipelineEventsPath, opts).map(normalizePipelineEvent);
-}
-
-function logPipelineEvent(event) {
-  const normalized = normalizePipelineEvent(event);
-  const target = getObserverPaths().pipelineEventsPath;
-  fs.mkdirSync(path.dirname(target), { recursive: true });
-  fs.appendFileSync(target, JSON.stringify(normalized) + '\n', 'utf8');
-  return normalized;
-}
-
-function normalizePipelineEvent(event) {
-  const input = event && typeof event === 'object' ? event : {};
-  const status = VALID_PHASE_STATUSES.has(input.status) ? input.status : 'running';
-  return redactValue({
-    run_id: stringOrNull(input.run_id),
-    cycle_id: stringOrNull(input.cycle_id),
-    phase: stringOrDefault(input.phase, 'unknown'),
-    status,
-    started_at: stringOrNull(input.started_at),
-    finished_at: stringOrNull(input.finished_at),
-    summary: stringOrDefault(input.summary, ''),
-    evidence_refs: arrayOfObjects(input.evidence_refs),
-    asset_refs: arrayOfObjects(input.asset_refs),
-    validation_refs: arrayOfObjects(input.validation_refs),
-    requires_confirmation: Boolean(input.requires_confirmation),
-    timestamp: input.timestamp || new Date().toISOString(),
-  });
-}
-
-function stringOrNull(value) {
-  return typeof value === 'string' && value ? value : null;
-}
-
-function stringOrDefault(value, fallback) {
-  return typeof value === 'string' ? value : fallback;
-}
-
-function arrayOfObjects(value) {
-  return Array.isArray(value) ? value.filter((entry) => entry && typeof entry === 'object') : [];
-}
-
-module.exports = {
-  readPipelineEvents,
-  logPipelineEvent,
-  normalizePipelineEvent,
-};

package/src/webui/observer/redact.js

@@ -1,63 +0,0 @@
-'use strict';
-
-const SECRET_KEY_RE = /(secret|token|api[_-]?key|authorization|cookie|oauth|password|private[_-]?key|node_secret)/i;
-// Three alternation arms, each with one capturing group so the replace()
-// callback can keep the prefix and only mask the value:
-//   1) HTTP Bearer auth header
-//   2) Env/INI/CLI style NAME=value (or NAME: value) where NAME ends in
-//      SECRET / TOKEN / KEY / PASSWORD / PASSWD / PWD. The original
-//      regex only matched SECRET / TOKEN; without KEY / PASSWORD a line
-//      like `OPENAI_API_KEY=sk-...` or `DB_PASSWORD=hunter2` would slip
-//      through to the /webui/logs/evolver tail, leaking live credentials.
-//   3) Bare OpenAI/Anthropic-shape API keys (`sk-...` / `sk-ant-...`) for
-//      cases where they appear as standalone tokens in log lines without
-//      a NAME= prefix.
-// Flagged /gi so lower-case variants (`password=...`, `bearer ...`,
-// `sk-...`) are also caught; over-redacting harmless prose tokens like
-// "monkey=" is acceptable, leaking a real key is not.
-const SECRET_TEXT_RE = new RegExp([
-  '(Bearer\\s+)[A-Za-z0-9._~+/-]+=*',
-  '([A-Za-z0-9_]*(?:SECRET|TOKEN|KEY|PASSWORD|PASSWD|PWD)[A-Za-z0-9_]*\\s*[:=]\\s*)["\']?[^\\s,"\'`]+',
-  '(sk-(?:ant-)?[A-Za-z0-9_-]{20,})',
-].join('|'), 'gi');
-
-function redactValue(value, depth = 0) {
-  if (depth > 8) return '[REDACTED_DEPTH]';
-  if (Array.isArray(value)) return value.map((entry) => redactValue(entry, depth + 1));
-  if (!value || typeof value !== 'object') {
-    return typeof value === 'string' ? redactText(value) : value;
-  }
-
-  const out = {};
-  for (const [key, child] of Object.entries(value)) {
-    if (SECRET_KEY_RE.test(key)) {
-      out[key] = '[REDACTED]';
-    } else if (isLargeSensitiveField(key, child)) {
-      out[key] = '[REDACTED_SENSITIVE_DETAIL]';
-    } else {
-      out[key] = redactValue(child, depth + 1);
-    }
-  }
-  return out;
-}
-
-function redactText(text) {
-  if (typeof text !== 'string') return text;
-  return text.replace(SECRET_TEXT_RE, (match, bearer, namedPrefix, bareKey) => {
-    if (bearer) return `${bearer}[REDACTED]`;
-    if (namedPrefix) return `${namedPrefix}[REDACTED]`;
-    if (bareKey) return '[REDACTED]';
-    return '[REDACTED]';
-  });
-}
-
-function isLargeSensitiveField(key, value) {
-  if (typeof value !== 'string') return false;
-  if (!/(prompt|diff|content|transcript|message|raw)/i.test(key)) return false;
-  return value.length > 500;
-}
-
-module.exports = {
-  redactValue,
-  redactText,
-};