mcp-pacemaker 2.0.1

package/bin/pooling-trace.mjs

@@ -0,0 +1,218 @@
+import fs from 'node:fs';
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { threadId } from 'node:worker_threads';
+
+export const TRACE_MAX_EVENTS = 192;
+export const TRACE_MAX_BYTES = 65536;
+export const TRACE_MAX_LINE_BYTES = 1024;
+const methods = ['apply', 'undo', 'stageApply', 'stageUndo', 'rejectStage', 'commitBatch'];
+const codes = ['NONE', 'UNKNOWN', 'ETIMEDOUT', 'EACCES', 'EPERM', 'ENOENT', 'EIO',
+  'WRITER_DEADLINE', 'WRITER_CANCELLED', 'WRITER_FAILED', 'WRITER_OUTCOME_UNKNOWN',
+  'WRITER_DEADLINE_COMMITTED', 'WRITER_BUSY', 'WRITER_CLOSED', 'REVISION_CONFLICT',
+  'IO_ERROR', 'ACCESS_DENIED', 'FILE_BUSY', 'RECOVERY_REQUIRED', 'STORE_BUSY'];
+const integer = (value) => Number.isSafeInteger(value) && value >= 0 && value <= 0xffffffff;
+const state = (value) => Number.isInteger(value) && value >= 0 && value <= 6;
+const duration = (value) => Number.isFinite(value) && Math.abs(value) <= 120000;
+const boolean = (value) => typeof value === 'boolean';
+const method = (value) => methods.includes(value);
+const code = (value) => codes.includes(value);
+const action = (value) => ['inspect-access', 'stage', 'move-no-replace'].includes(value);
+const fields = {
+  'request-start': {},
+  'body-end': { bytes: integer },
+  'response': { status: (value) => Number.isInteger(value) && value >= 100 && value <= 599 },
+  'queue-enter': { queued: integer, applying: boolean },
+  'queue-leave': {},
+  'writer-send': { method, queued: integer, cold: boolean },
+  'worker-ready': {},
+  'worker-enter': { method, state },
+  'worker-result': { state },
+  'worker-error': { code, state },
+  'writer-expire': { expired: boolean, state },
+  'writer-result': { code, state },
+  'store-start': {},
+  'store-ready': {},
+  'helper-start': { action, call: integer, timeoutMs: duration },
+  'helper-end': { action, call: integer, helperPid: integer,
+    exit: (value) => value === null || Number.isInteger(value) && Math.abs(value) <= 0xffffffff, code },
+  'publish-start': {},
+  'publish-end': {},
+  'capture-truncated': {},
+  'capture-failed': {},
+  'capture-invalid': {},
+};
+const base = ['version', 'event', 'operation', 'pid', 'thread', 'sequence', 'elapsedMs', 'remainingMs'];
+const uuid = (value) => typeof value === 'string' &&
+  /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value);
+
+export function validateTraceRecord(record) {
+  if (!record ||
+      Object.getPrototypeOf(record) !== Object.prototype ||
+      !Object.hasOwn(fields, record.event)) throw new Error('Invalid pooling trace record.');
+  const schema = fields[record.event];
+  const keys = [...base, ...Object.keys(schema)];
+  const valid = record.version === 1 && uuid(record.operation) &&
+    integer(record.pid) && integer(record.thread) && integer(record.sequence) &&
+    record.sequence > 0 && record.sequence <= TRACE_MAX_EVENTS &&
+    duration(record.elapsedMs) && duration(record.remainingMs) &&
+    Object.keys(record).length === keys.length &&
+    keys.every((key) => Object.hasOwn(record, key)) &&
+    Object.entries(schema).every(([key, validate]) => validate(record[key]));
+  if (!valid ||
+      Buffer.byteLength(JSON.stringify(record) + '\n') > TRACE_MAX_LINE_BYTES) {
+    throw new Error('Invalid pooling trace record.');
+  }
+  return record;
+}
+
+export function traceErrorCode(error) {
+  try {
+    return error == null ? 'NONE' : codes.includes(error.code) ? error.code : 'UNKNOWN';
+  } catch {
+    return 'UNKNOWN';
+  }
+}
+
+class PoolingTraceOperation {
+  constructor(sink, deadline, context) {
+    this.sink = sink;
+    this.deadline = deadline;
+    this.context = context;
+    this.calls = 0;
+  }
+
+  helper() {
+    return ++this.calls;
+  }
+
+  record(event, extra = {}) {
+    this.sink.record(this, event, extra);
+  }
+}
+
+export class PoolingTrace {
+  #descriptor;
+  #path;
+  #identity;
+  #written = 0;
+  #owned = false;
+  #closed = false;
+
+  constructor(directory) {
+    this.directory = directory;
+    this.events = 0;
+    this.bytes = 0;
+    this.stopped = false;
+    this.failed = false;
+  }
+
+  operation(deadline, context) {
+    if (!this.directory ||
+        this.#closed) return undefined;
+    try {
+      context ??= { operation: randomUUID(), started: String(process.hrtime.bigint()), deadline: String(deadline) };
+      const valid = uuid(context.operation) &&
+        typeof context.started === 'string' && /^\d{1,24}$/.test(context.started) &&
+        typeof context.deadline === 'string' && /^\d{1,24}$/.test(context.deadline);
+      if (!valid) return undefined;
+      return new PoolingTraceOperation(this, deadline ?? BigInt(context.deadline), {
+        operation: context.operation, started: context.started, deadline: context.deadline,
+      });
+    } catch {
+      this.failed = true;
+      return undefined;
+    }
+  }
+
+  record(operation, event, extra) {
+    if (this.stopped ||
+        this.#closed) return;
+    try {
+      const now = process.hrtime.bigint();
+      const milliseconds = (value) => Math.round(Math.max(-120000, Math.min(120000, Number(value) / 1e6)) * 1000) / 1000;
+      const record = {
+        version: 1, event, operation: operation.context.operation,
+        pid: process.pid, thread: threadId, sequence: this.events + 1,
+        elapsedMs: milliseconds(now - BigInt(operation.context.started)),
+        remainingMs: milliseconds(operation.deadline - now),
+      };
+      try {
+        if (!Object.hasOwn(fields, event) ||
+            Object.keys(extra).some((key) => !Object.hasOwn(fields[event], key))) {
+          throw new Error('Invalid pooling trace fields.');
+        }
+        Object.assign(record, extra);
+        validateTraceRecord(record);
+      } catch {
+        record.event = 'capture-invalid';
+        for (const key of Object.keys(record)) {
+          if (!base.includes(key)) delete record[key];
+        }
+        this.stopped = true;
+      }
+      const full = this.events >= TRACE_MAX_EVENTS - 1 ||
+        this.bytes + Buffer.byteLength(JSON.stringify(record) + '\n') > TRACE_MAX_BYTES - TRACE_MAX_LINE_BYTES;
+      if (full ||
+          this.failed) {
+        record.event = this.failed ? 'capture-failed' : 'capture-truncated';
+        for (const key of Object.keys(record)) {
+          if (!base.includes(key)) delete record[key];
+        }
+        this.stopped = true;
+      }
+      validateTraceRecord(record);
+      const line = JSON.stringify(record) + '\n';
+      this.#append(line);
+      this.events++;
+      this.bytes += Buffer.byteLength(line);
+      if (this.stopped) this.close();
+    } catch {
+      // Diagnostics must never replace the operation's result or error.
+      this.failed = true;
+      if (this.#owned) this.close();
+    }
+  }
+
+  #append(line) {
+    if (this.#descriptor === undefined) {
+      this.#path = join(this.directory, `${process.pid}-${threadId}.jsonl`);
+      this.#descriptor = fs.openSync(this.#path, 'wx', 0o600);
+      this.#owned = true;
+      this.#identity = fs.fstatSync(this.#descriptor, { bigint: true });
+    }
+    this.#checkOwned(this.#written);
+    // A replacement after the check cannot redirect this write to another file.
+    fs.appendFileSync(this.#descriptor, line);
+    const written = this.#written + Buffer.byteLength(line);
+    this.#checkOwned(written);
+    this.#written = written;
+  }
+
+  #checkOwned(bytes) {
+    const open = fs.fstatSync(this.#descriptor, { bigint: true });
+    const named = fs.lstatSync(this.#path, { bigint: true });
+    const owned = [open, named].every((stat) =>
+      stat.isFile() && !stat.isSymbolicLink() && stat.nlink === 1n &&
+      stat.dev === this.#identity.dev && stat.ino === this.#identity.ino &&
+      stat.size === BigInt(bytes));
+    if (!owned) throw new Error('Pooling trace file ownership changed.');
+  }
+
+  close() {
+    if (this.#closed) return;
+    this.#closed = true;
+    this.stopped = true;
+    const descriptor = this.#descriptor;
+    this.#descriptor = undefined;
+    if (descriptor === undefined) return;
+    try {
+      fs.closeSync(descriptor);
+    } catch {
+      this.failed = true;
+    }
+  }
+}
+
+export const poolingTrace = new PoolingTrace(process.env.MCP_POOLING_TRACE_DIR);
+if (poolingTrace.directory) process.once('exit', () => poolingTrace.close());

package/bin/pooling-writer.mjs

@@ -0,0 +1,242 @@
+import { Worker, isMainThread, parentPort, workerData } from 'node:worker_threads';
+import { PoolingConfigStore, PoolingConfigError } from './pooling-config.mjs';
+import { PoolingExecution, POOLING_BUDGET_MS } from './pooling-execution.mjs';
+import { poolingTrace, traceErrorCode } from './pooling-trace.mjs';
+
+if (!isMainThread &&
+    workerData?.kind === 'mcp-pooling-writer') {
+  poolingTrace.operation(undefined, workerData.traceContext)?.record('worker-ready');
+  const store = new PoolingConfigStore(workerData.configPath);
+  parentPort.on('message', ({ id, method, request, deadline, buffer, traceContext }) => {
+    if (method === 'close') {
+      store.close();
+      parentPort.close();
+      return;
+    }
+    const execution = new PoolingExecution(deadline, buffer);
+    execution.trace = poolingTrace.operation(deadline, traceContext);
+    execution.trace?.record('worker-enter', { method, state: Atomics.load(execution.state, 0) });
+    let cancelledBatchId;
+    try {
+      execution.check();
+      if (!['apply', 'undo', 'stageApply', 'stageUndo', 'rejectStage', 'commitBatch'].includes(method)) {
+        throw new PoolingConfigError(400, 'INVALID_REQUEST', 'Unknown pooling operation.');
+      }
+      const result = store[method](request, execution);
+      if (result.cancelled) cancelledBatchId = result.batchId;
+      let completedLate;
+      try {
+        completedLate = execution.complete();
+      } catch (error) {
+        if (result.pending &&
+            result.accepted !== false) {
+          store.rejectStage(result, new PoolingExecution(
+            process.hrtime.bigint() + BigInt(POOLING_BUDGET_MS) * 1000000n));
+        }
+        throw error;
+      }
+      execution.trace?.record('worker-result', { state: Atomics.load(execution.state, 0) });
+      parentPort.postMessage({ id, result, completedLate });
+    } catch (error) {
+      execution.trace?.record('worker-error', { code: traceErrorCode(error), state: Atomics.load(execution.state, 0) });
+      execution.fail(error.commitState);
+      const known = error instanceof PoolingConfigError;
+      parentPort.postMessage({ id, cancelledBatchId, error: {
+        statusCode: known ? error.statusCode : 500,
+        code: known ? error.code : 'WRITER_FAILED',
+        message: known ? error.message : 'Pooling writer failed. Reread the config before retrying.',
+        commitState: error.commitState,
+      } });
+    }
+  });
+}
+
+export class PoolingConfigWriter {
+  #configPath;
+  #worker;
+  #pending = new Map();
+  #sequence = 0;
+  #closed = false;
+  #closing;
+  #resolveClose;
+  #failure;
+
+  constructor(configPath) {
+    this.#configPath = configPath;
+  }
+
+  apply(request, options = {}) {
+    return this.#send('apply', request, options);
+  }
+
+  undo(request, options = {}) {
+    return this.#send('undo', request, options);
+  }
+
+  stageApply(request, options = {}) {
+    return this.#send('stageApply', request, options);
+  }
+
+  stageUndo(request, options = {}) {
+    return this.#send('stageUndo', request, options);
+  }
+
+  commitBatch(request, options = {}) {
+    return this.#send('commitBatch', request, options);
+  }
+
+  rejectStage(request) {
+    return this.#send('rejectStage', request, {});
+  }
+
+  #send(method, request, options) {
+    if (this.#closed) {
+      return Promise.reject(new PoolingConfigError(503, 'WRITER_CLOSED', 'Pooling writer is shutting down.'));
+    }
+    if (this.#failure) return Promise.reject(this.#failure);
+    const defaultDeadline = process.hrtime.bigint() + BigInt(POOLING_BUDGET_MS) * 1000000n;
+    const deadline = options.deadline === undefined ? defaultDeadline
+      : options.deadline < defaultDeadline ? options.deadline : defaultDeadline;
+    const execution = new PoolingExecution(deadline);
+    execution.trace = options.trace ?? poolingTrace.operation(deadline);
+    if (options.signal?.aborted) return Promise.reject(execution.cancel());
+    try {
+      execution.check();
+    } catch (error) {
+      return Promise.reject(error);
+    }
+    if (this.#pending.size >= 16) {
+      return Promise.reject(new PoolingConfigError(503, 'WRITER_BUSY', 'Too many pending pooling changes.'));
+    }
+    execution.trace?.record('writer-send', { method, queued: this.#pending.size, cold: !this.#worker });
+    if (!this.#worker) this.#start(execution.trace);
+    const id = ++this.#sequence;
+    return new Promise((resolve, reject) => {
+      const pending = { resolve, reject, execution, options, method, settled: false };
+      const cancel = (expired) => {
+        if (pending.settled) return;
+        execution.trace?.record('writer-expire', { expired, state: Atomics.load(execution.state, 0) });
+        const error = execution.cancel(expired);
+        if (!error) return;
+        this.#settle(pending, error);
+      };
+      pending.abort = () => cancel(false);
+      this.#pending.set(id, pending);
+      try {
+        options.signal?.addEventListener('abort', pending.abort, { once: true });
+        pending.timer = setTimeout(() => cancel(true), execution.remainingMs());
+        this.#worker.postMessage({ id, method, request, deadline, buffer: execution.state.buffer,
+          traceContext: execution.trace?.context });
+      } catch (error) {
+        this.#pending.delete(id);
+        this.#settle(pending, error);
+      }
+    });
+  }
+
+  #start(trace) {
+    this.#worker = new Worker(new URL(import.meta.url), {
+      workerData: { kind: 'mcp-pooling-writer', configPath: this.#configPath, traceContext: trace?.context },
+    });
+    this.#worker.on('message', async ({ id, result, error, completedLate, cancelledBatchId }) => {
+      const pending = this.#pending.get(id);
+      if (!pending) return;
+      pending.execution.trace?.record('writer-result', {
+        code: traceErrorCode(error), state: Atomics.load(pending.execution.state, 0),
+      });
+      this.#pending.delete(id);
+      let failure = error
+        ? new PoolingConfigError(error.statusCode, error.code, error.message)
+        : completedLate ? pending.execution.committedLate() : undefined;
+      cancelledBatchId ??= result?.cancelled ? result.batchId : undefined;
+      if (!failure &&
+          cancelledBatchId &&
+          pending.options.signal?.aborted) {
+        failure = new PoolingConfigError(408, 'WRITER_CANCELLED',
+          'Pooling change was cancelled before config commit.');
+      }
+      if (failure) {
+        failure.commitState = error?.commitState ?? failure.commitState ??
+          (pending.method === 'commitBatch' ? 'not-committed' : undefined);
+        if (cancelledBatchId) failure.cancelledBatchId = cancelledBatchId;
+      }
+      if (!failure &&
+          result?.pending &&
+          result.accepted !== false &&
+          pending.options.signal?.aborted) {
+        try {
+          await this.rejectStage({ generation: result.generation });
+          this.#settle(pending, new PoolingConfigError(408, 'WRITER_CANCELLED',
+            'Pooling change was cancelled before config commit.'));
+        } catch (cleanupError) {
+          this.#settle(pending, cleanupError);
+        }
+        return;
+      }
+      if (pending.settled ||
+          completedLate) this.#lateSettlement(pending, failure, result);
+      this.#settle(pending, failure, result);
+    });
+    this.#worker.on('error', () => this.#failed());
+    this.#worker.on('exit', (code) => {
+      if (code !== 0 ||
+          this.#pending.size > 0 ||
+          !this.#closed) {
+        this.#failed();
+      }
+      this.#worker = null;
+      this.#resolveClose?.();
+    });
+  }
+
+  #failed() {
+    this.#failure ??= new PoolingConfigError(500, 'WRITER_FAILED',
+      'Pooling writer stopped. Reread the config and restart the bridge before retrying.');
+    for (const pending of this.#pending.values()) {
+      const commitFailure = pending.execution.workerFailure();
+      const failure = commitFailure ?? this.#failure;
+      if (pending.settled ||
+          commitFailure) this.#lateSettlement(pending, failure, undefined, Boolean(commitFailure));
+      this.#settle(pending, failure);
+    }
+    this.#pending.clear();
+  }
+
+  #settle(pending, error, result) {
+    if (pending.settled) return;
+    pending.settled = true;
+    clearTimeout(pending.timer);
+    pending.options.signal?.removeEventListener('abort', pending.abort);
+    if (error) pending.reject(error);
+    else pending.resolve(result);
+  }
+
+  #lateSettlement(pending, error, result, commitStarted = false) {
+    if (pending.options.onLateSettlement) {
+      try {
+        pending.options.onLateSettlement({
+          error, result,
+          committed: Boolean(result && !result.pending && !result.cancelled) ||
+            error?.commitState === 'committed',
+          cancelledBatchId: error?.cancelledBatchId ?? (result?.cancelled ? result.batchId : undefined),
+          commitStarted,
+        });
+      } catch {
+        process.emitWarning('Unable to reconcile a pooling operation without a normal result.');
+      }
+    } else if (error &&
+        error.code !== 'WRITER_CANCELLED' &&
+        error.code !== 'WRITER_DEADLINE') {
+      process.emitWarning(`Pooling operation settled without a normal result (${error.code}). Reread the config.`);
+    }
+  }
+
+  close() {
+    this.#closed = true;
+    if (this.#closing) return this.#closing;
+    if (!this.#worker) return Promise.resolve();
+    this.#closing = new Promise((resolve) => { this.#resolveClose = resolve; });
+    this.#worker.postMessage({ method: 'close' });
+    return this.#closing;
+  }
+}