@evomap/evolver 1.89.15 → 1.89.18

package/src/proxy/mailbox/store.js

@@ -3,28 +3,20 @@
 const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
+const {
+  PRIVATE_FILE_MODE,
+  bestEffortChmod,
+  ensurePrivateDir,
+  writePrivateFile,
+  readMailboxStateFile,
+  writeMergedMailboxStateFile,
+} = require('./state');
 
 const DEFAULT_CHANNEL = 'evomap-hub';
 const SCHEMA_VERSION = 1;
 const PROXY_PROTOCOL_VERSION = '0.1.0';
-const PRIVATE_DIR_MODE = 0o700;
-const PRIVATE_FILE_MODE = 0o600;
-
-function bestEffortChmod(filePath, mode) {
-  try { fs.chmodSync(filePath, mode); } catch { /* best effort; no-op on Windows */ }
-}
-
-function ensurePrivateDir(dir) {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true, mode: PRIVATE_DIR_MODE });
-  }
-  bestEffortChmod(dir, PRIVATE_DIR_MODE);
-}
-
-function writePrivateFile(filePath, content) {
-  fs.writeFileSync(filePath, content, { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
-  bestEffortChmod(filePath, PRIVATE_FILE_MODE);
-}
+const DEFAULT_MAX_JSONL_LINE_BYTES = 16 * 1024 * 1024;
+const JSONL_READ_CHUNK_BYTES = 64 * 1024;
 
 // Merge `fields` into `target` while stripping keys that can mutate the
 // prototype chain. Mailbox rows are persisted as JSONL and rebuilt on
@@ -107,22 +99,117 @@ function safeParse(payload) {
 // Windows NTFS provides the same atomicity guarantee for O_APPEND writes to
 // regular files that POSIX local filesystems do, so the removal above is
 // equally valid on Windows. No platform-specific code is needed here.
+function jsonlLineForWrite(obj, opts = {}) {
+  const line = JSON.stringify(obj) + '\n';
+  const maxLineBytes = Math.max(1, Math.floor(Number(opts.maxLineBytes) || resolveMaxJsonlLineBytes()));
+  const lineBytes = Buffer.byteLength(line, 'utf8');
+  if (lineBytes > maxLineBytes) {
+    const err = new Error(`mailbox JSONL line is ${lineBytes} bytes; maximum is ${maxLineBytes} bytes`);
+    err.code = 'MAILBOX_JSONL_LINE_TOO_LARGE';
+    err.lineBytes = lineBytes;
+    err.maxLineBytes = maxLineBytes;
+    throw err;
+  }
+  return line;
+}
+
 function appendLine(filePath, obj) {
-  fs.appendFileSync(filePath, JSON.stringify(obj) + '\n', { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
+  const line = jsonlLineForWrite(obj);
+  fs.appendFileSync(filePath, line, { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
   bestEffortChmod(filePath, PRIVATE_FILE_MODE);
 }
 
-function readLines(filePath) {
-  if (!fs.existsSync(filePath)) return [];
-  const content = fs.readFileSync(filePath, 'utf8');
-  const lines = content.split('\n');
-  const results = [];
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try { results.push(JSON.parse(trimmed)); } catch { /* skip corrupt lines */ }
+function resolveMaxJsonlLineBytes(env = process.env) {
+  const raw = Number(env.EVOMAP_MAILBOX_MAX_LINE_BYTES);
+  if (Number.isFinite(raw) && raw > 0) return Math.floor(raw);
+  return DEFAULT_MAX_JSONL_LINE_BYTES;
+}
+
+function forEachJsonLine(filePath, onRow, opts = {}) {
+  const stats = { parsed: 0, corrupt: 0, overlong: 0 };
+  if (!fs.existsSync(filePath)) return stats;
+
+  const maxLineBytes = Math.max(1, Math.floor(Number(opts.maxLineBytes) || resolveMaxJsonlLineBytes()));
+  const chunk = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES);
+  let fd = null;
+  let parts = [];
+  let lineBytes = 0;
+  let dropping = false;
+
+  const resetLine = () => {
+    parts = [];
+    lineBytes = 0;
+    dropping = false;
+  };
+
+  const appendSegment = (segment) => {
+    if (!segment || segment.length === 0 || dropping) return;
+    if (lineBytes + segment.length > maxLineBytes) {
+      parts = [];
+      lineBytes = 0;
+      dropping = true;
+      return;
+    }
+    parts.push(Buffer.from(segment));
+    lineBytes += segment.length;
+  };
+
+  const finishLine = (hasNewline = false) => {
+    const totalLineBytes = lineBytes + (hasNewline ? 1 : 0);
+    if (dropping) {
+      stats.overlong += 1;
+      resetLine();
+      return;
+    }
+    if (totalLineBytes > maxLineBytes) {
+      stats.overlong += 1;
+      resetLine();
+      return;
+    }
+    if (lineBytes === 0) {
+      resetLine();
+      return;
+    }
+    const trimmed = Buffer.concat(parts, lineBytes).toString('utf8').trim();
+    resetLine();
+    if (!trimmed) return;
+    try {
+      onRow(JSON.parse(trimmed));
+      stats.parsed += 1;
+    } catch {
+      stats.corrupt += 1;
+    }
+  };
+
+  try {
+    fd = fs.openSync(filePath, 'r');
+    while (true) {
+      const bytesRead = fs.readSync(fd, chunk, 0, chunk.length, null);
+      if (bytesRead <= 0) break;
+      let start = 0;
+      for (let i = 0; i < bytesRead; i++) {
+        if (chunk[i] !== 0x0a) continue;
+        appendSegment(chunk.subarray(start, i));
+        finishLine(true);
+        start = i + 1;
+      }
+      if (start < bytesRead) appendSegment(chunk.subarray(start, bytesRead));
+    }
+    finishLine();
+  } catch (err) {
+    stats.read_failed = 1;
+    const detail = err && err.message ? err.message : String(err);
+    const wrapped = new Error(`failed to read mailbox JSONL ${filePath}: ${detail}`);
+    wrapped.code = 'MAILBOX_JSONL_READ_FAILED';
+    wrapped.stats = stats;
+    wrapped.cause = err;
+    throw wrapped;
+  } finally {
+    if (fd !== null) {
+      try { fs.closeSync(fd); } catch { /* ignore close errors */ }
+    }
   }
-  return results;
+  return stats;
 }
 
 // --- In-memory index that backs JSONL persistence ---
@@ -150,19 +237,21 @@ class MailboxStore {
   }
 
   _loadState() {
-    try {
-      if (fs.existsSync(this._stateFile)) {
-        this._state = JSON.parse(fs.readFileSync(this._stateFile, 'utf8'));
-      }
-    } catch {
-      this._state = {};
-    }
+    this._state = readMailboxStateFile(this._stateFile) || {};
     const existingVersion = this._state._schema_version || 0;
+    const beforeMigrationState = { ...this._state };
     if (existingVersion < SCHEMA_VERSION) {
       this._runMigrations(existingVersion, SCHEMA_VERSION);
     }
     this._state._schema_version = SCHEMA_VERSION;
-    this._persistState();
+    const updatedKeys = new Set(['_schema_version']);
+    for (const key of Object.keys(this._state)) {
+      if (this._state[key] !== beforeMigrationState[key]) updatedKeys.add(key);
+    }
+    for (const key of Object.keys(beforeMigrationState)) {
+      if (!Object.prototype.hasOwnProperty.call(this._state, key)) updatedKeys.add(key);
+    }
+    this._persistState(updatedKeys);
   }
 
   _runMigrations(fromVersion, toVersion) {
@@ -174,9 +263,7 @@ class MailboxStore {
     }
   }
 
-  _persistState() {
-    const dir = path.dirname(this._stateFile);
-    ensurePrivateDir(dir);
+  _persistState(updatedKeys) {
     // Round-7 (§20.5): per-PID tmp path. Two evolver processes (daemon +
     // ad-hoc CLI / proxy + loop) writing to the same `${stateFile}.tmp`
     // would otherwise interleave: process B's writeFileSync truncates
@@ -186,20 +273,11 @@ class MailboxStore {
     // load-bearing trigger for the "401-loop -> reauth backoff -> dead
     // for 30 min..4 h" symptom this branch targets. Matches the
     // precedent set by _persistNodeSecret in src/gep/a2aProtocol.js.
-    const tmp = `${this._stateFile}.${process.pid}.tmp`;
-    writePrivateFile(tmp, JSON.stringify(this._state, null, 2) + '\n');
-    // Windows: fs.renameSync throws EPERM when the destination file already
-    // exists, unlike POSIX where rename(2) atomically replaces the target.
-    // Remove the destination first so the rename succeeds on all platforms.
-    // The window between unlink and rename is intentionally tiny; a crash in
-    // that window leaves the tmp file behind (recovered on next _persistState).
-    if (process.platform === 'win32') {
-      try { fs.unlinkSync(this._stateFile); } catch (e) {
-        if (e.code !== 'ENOENT') throw e;
-      }
-    }
-    fs.renameSync(tmp, this._stateFile);
-    bestEffortChmod(this._stateFile, PRIVATE_FILE_MODE);
+    //
+    // Use the shared merge helper so stale long-running stores cannot
+    // resurrect an older node_secret tuple after another path rotates or clears
+    // it directly on disk.
+    this._state = writeMergedMailboxStateFile(this._stateFile, this._state, updatedKeys);
   }
 
   _rebuildIndex() {
@@ -208,17 +286,20 @@ class MailboxStore {
     this._inbound = [];
 
     const TERMINAL = new Set(['synced', 'delivered', 'failed', 'rejected']);
-    const rows = readLines(this._messagesFile);
-    for (const rawRow of rows) {
+    const stats = forEachJsonLine(this._messagesFile, (rawRow) => {
       const row = sanitizeRow(rawRow);
+      if (!row || typeof row !== 'object') return;
       if (row._op === 'update') {
         const existing = this._messages.get(row.id);
         if (existing) safeAssign(existing, row.fields);
-        continue;
+        return;
       }
+      if (!row.id) return;
       this._messages.set(row.id, row);
-    }
+    });
+    this._lastRebuildStats = stats;
     for (const [id, msg] of this._messages) {
+      if (!msg || typeof msg !== 'object') continue;
       if (TERMINAL.has(msg.status)) continue;
       if (msg.direction === 'outbound') this._outbound.push(id);
       else if (msg.direction === 'inbound') this._inbound.push(id);
@@ -430,8 +511,9 @@ class MailboxStore {
   }
 
   setCursor(key, value) {
-    this._state[`cursor:${key}`] = value;
-    this._persistState();
+    const stateKey = `cursor:${key}`;
+    this._state[stateKey] = value;
+    this._persistState([stateKey]);
   }
 
   getState(key) {
@@ -441,7 +523,20 @@ class MailboxStore {
 
   setState(key, value) {
     this._state[key] = typeof value === 'string' ? value : JSON.stringify(value);
-    this._persistState();
+    this._persistState([key]);
+  }
+
+  setNodeSecretState({ secret = '', version = '', source = '', envSuppressed = '' } = {}) {
+    this._state.node_secret = secret;
+    this._state.node_secret_version = version ? String(version) : '';
+    this._state.node_secret_source = source;
+    this._state.node_secret_env_suppressed = envSuppressed;
+    this._persistState([
+      'node_secret',
+      'node_secret_version',
+      'node_secret_source',
+      'node_secret_env_suppressed',
+    ]);
   }
 
   // --- Compaction (reduces JSONL file size by rewriting only current state) ---
@@ -457,11 +552,16 @@ class MailboxStore {
     }
     entries.sort((a, b) => a.created_at - b.created_at);
 
+    const lines = entries.map((msg) => jsonlLineForWrite(msg));
+
     const fd = fs.openSync(tmpFile, 'w', PRIVATE_FILE_MODE);
-    for (const msg of entries) {
-      fs.writeSync(fd, JSON.stringify(msg) + '\n');
+    try {
+      for (const line of lines) {
+        fs.writeSync(fd, line);
+      }
+    } finally {
+      fs.closeSync(fd);
     }
-    fs.closeSync(fd);
     bestEffortChmod(tmpFile, PRIVATE_FILE_MODE);
     // Windows: renameSync throws EPERM when the destination already exists.
     // Remove it first so the swap succeeds on all platforms.

package/src/proxy/router/messages_route.js

@@ -144,8 +144,8 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
   return async ({ body, headers }) => {
     const inboundHeaders = headers || {};
     // x-api-key is satisfied by either the inbound header OR a proxy-side
-    // ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN env (token mediation, see
-    // _proxyAnthropic). The proxy server itself has already auth-checked
+    // EVOMAP_ANTHROPIC_API_KEY / ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN env
+    // (token mediation, see _proxyAnthropic). The proxy server itself has already auth-checked
     // `Authorization: Bearer <proxy_token>` before reaching this handler.
     //
     // Bedrock upstream uses SigV4 with AWS_* env, so neither inbound
@@ -161,7 +161,8 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
     if (upstreamMode !== 'bedrock') {
       const hasInboundKey = !!inboundHeaders['x-api-key'];
       const hasProxyEnvCreds = !!(
-        process.env.ANTHROPIC_API_KEY
+        process.env.EVOMAP_ANTHROPIC_API_KEY
+        || process.env.ANTHROPIC_API_KEY
         || process.env.EVOMAP_ANTHROPIC_AUTH_TOKEN
         || (process.env.EVOMAP_PROXY_AUTO_INJECTED === '1' ? '' : process.env.ANTHROPIC_AUTH_TOKEN)
       );

package/src/proxy/server/http.js

@@ -3,6 +3,11 @@
 const crypto = require('crypto');
 const http = require('http');
 const { writeSettings, readSettings, clearSettings, clearIfStale } = require('./settings');
+const {
+  isValidReusableProxyToken,
+  readReusableClientProxyToken,
+  syncClaudeProxySettings,
+} = require('../clientSettings');
 
 const MAX_PORT_ATTEMPTS = 100;
 const DEFAULT_PORT = 19820;
@@ -93,13 +98,14 @@ function tryListen(server, port) {
 }
 
 class ProxyHttpServer {
-  constructor(routes, { port, logger } = {}) {
+  constructor(routes, { port, logger, clientSettings } = {}) {
     this.routes = routes;
     this.basePort = port || Number(process.env.EVOMAP_PROXY_PORT) || DEFAULT_PORT;
     this.actualPort = null;
     this.logger = logger || console;
     this.server = null;
     this.token = null;
+    this.clientSettings = clientSettings || null;
   }
 
   async start() {
@@ -109,16 +115,22 @@ class ProxyHttpServer {
     // already exported into long-lived shells (the .bashrc auto-source only
     // runs once per terminal).
     const priorProxy = readSettings().proxy || {};
-    const previous = typeof priorProxy.token === 'string' ? priorProxy.token : null;
+    const previous = isValidReusableProxyToken(priorProxy.token) ? priorProxy.token.trim() : null;
     // settings.json is operator-edited; previous_tokens may contain non-strings
     // (numbers, booleans, objects) that would later crash Buffer.from(cand, 'utf8')
     // in _handleRequest as ERR_INVALID_ARG_TYPE — an unhandled rejection that
     // takes the daemon down under default --unhandled-rejections=throw.
     const priorPreviousTokens = Array.isArray(priorProxy.previous_tokens)
-      ? priorProxy.previous_tokens.filter((t) => typeof t === 'string' && t.length > 0)
+      ? priorProxy.previous_tokens
+        .map((t) => (typeof t === 'string' ? t.trim() : ''))
+        .filter((t) => isValidReusableProxyToken(t))
       : [];
     clearIfStale();
-    this.token = previous || crypto.randomBytes(32).toString('hex');
+    const clientSettingsOpts = this.clientSettings || {};
+    const clientToken = this.clientSettings
+      ? readReusableClientProxyToken({ ...clientSettingsOpts, port: this.basePort })
+      : null;
+    this.token = previous || clientToken || crypto.randomBytes(32).toString('hex');
     this._priorPreviousTokens = priorPreviousTokens;
     this.server = http.createServer((req, res) => this._handleRequest(req, res));
 
@@ -138,6 +150,24 @@ class ProxyHttpServer {
           proxyBlock.previous_tokens = this._priorPreviousTokens;
         }
         writeSettings({ proxy: proxyBlock });
+        if (this.clientSettings) {
+          try {
+            const syncResult = syncClaudeProxySettings({
+              ...clientSettingsOpts,
+              url,
+              port: this.basePort,
+              token: this.token,
+              runtimeEnv: process.env,
+            });
+            if (syncResult.synced && syncResult.changed) {
+              this.logger.log(`[proxy] Synced Claude client settings at ${syncResult.file}`);
+            } else if (syncResult.reason === 'invalid_settings_json') {
+              this.logger.warn?.(`[proxy] Skipped Claude client settings sync because ${syncResult.file} is not valid JSON`);
+            }
+          } catch (err) {
+            this.logger.warn?.('[proxy] Claude client settings sync failed: ' + (err && err.message ? err.message : err));
+          }
+        }
         this.logger.log(`[proxy] HTTP server listening on ${url}`);
         return { port, url, token: this.token };
       }
@@ -171,9 +201,13 @@ class ProxyHttpServer {
     // and unhandled-reject through the auth path.
     const previous = readSettings().proxy?.previous_tokens;
     const extras = Array.isArray(previous)
-      ? previous.filter((t) => typeof t === 'string' && t.length > 0)
+      ? previous
+        .map((t) => (typeof t === 'string' ? t.trim() : ''))
+        .filter((t) => isValidReusableProxyToken(t))
       : [];
-    const candidates = [this.token, ...extras].filter((t) => typeof t === 'string' && t.length > 0);
+    const candidates = [this.token, ...extras]
+      .filter((t) => isValidReusableProxyToken(t))
+      .map((t) => t.trim());
     let valid = false;
     for (const cand of candidates) {
       const expBuf = Buffer.from(cand, 'utf8');

package/src/proxy/sync/inbound.js

@@ -16,6 +16,18 @@ const {
 const DEFAULT_POLL_INTERVAL_ACTIVE = 10_000;
 const DEFAULT_POLL_INTERVAL_IDLE = 60_000;
 
+function toInboundStoreMessage(m, channel) {
+  return {
+    id: m.id,
+    type: m.type,
+    payload: m.payload,
+    channel,
+    priority: m.priority || 'normal',
+    refId: m.ref_id,
+    expiresAt: m.expires_at,
+  };
+}
+
 class InboundSync {
   constructor({ store, hubUrl, getHeaders, logger }) {
     this.store = store;
@@ -86,26 +98,33 @@ class InboundSync {
 
       const data = await readHubResponseJson(res);
       const messages = data.messages || [];
+      let stored = 0;
+      let dropped = 0;
 
       if (messages.length > 0) {
-        this.store.writeInboundBatch(
-          messages.map(m => ({
-            id: m.id,
-            type: m.type,
-            payload: m.payload,
-            channel,
-            priority: m.priority || 'normal',
-            refId: m.ref_id,
-            expiresAt: m.expires_at,
-          }))
-        );
+        for (const m of messages) {
+          try {
+            this.store.writeInbound(toInboundStoreMessage(m, channel));
+            stored++;
+          } catch (err) {
+            if (err && err.code === 'MAILBOX_JSONL_LINE_TOO_LARGE') {
+              dropped++;
+              const msgId = m && m.id ? String(m.id) : '(missing id)';
+              this.logger.warn?.(`[inbound] dropped oversized inbound message ${msgId}: ${err.message}`);
+              continue;
+            }
+            throw err;
+          }
+        }
       }
 
       if (data.next_cursor) {
         this.store.setCursor(cursorKey, data.next_cursor);
       }
 
-      return { received: messages.length, cursor: data.next_cursor || cursor };
+      const result = { received: stored, cursor: data.next_cursor || cursor };
+      if (dropped > 0) result.dropped = dropped;
+      return result;
     } catch (err) {
       if (err instanceof AuthError) throw err;
       if (isHubUnreachableError(err)) {