@yemi33/minions 0.1.2177 → 0.1.2179

package/engine/diagnostics-memory.js

@@ -0,0 +1,190 @@
+/**
+ * engine/diagnostics-memory.js — In-process memory + event-loop + GC sampler.
+ *
+ * P-a1b2c3d4 (memory + perf audit plan). Importable from both engine.js and
+ * dashboard.js — each Node process gets its own module-scope state (event-loop
+ * histogram, GC counters, ring buffer). No I/O; persistence is the caller's
+ * responsibility (see P-b2c3d4e5).
+ *
+ * Exports:
+ *   - sampleSelf({ label }) → snapshot object with RSS / heap / event-loop /
+ *     GC counters / pid / uptime / capturedAt.
+ *   - getHistory({ limit }) → up-to-`limit` samples from the ring buffer
+ *     (oldest first), defaults to all 1440.
+ *   - recordSample(sample) → append to the ring buffer; rotates at the cap.
+ *   - startPeriodicSampling({ intervalMs, onSample }) → setInterval-driven
+ *     sampler; returns a stop() function. Dashboard.js uses this on boot;
+ *     engine.js drives sampling from its own tick instead.
+ *
+ * Zero deps — all Node built-ins. The event-loop histogram is a
+ * `monitorEventLoopDelay({ resolution: 20 })` singleton lazily enabled on the
+ * first `sampleSelf()` call. The PerformanceObserver subscribes to
+ * `entryTypes: ['gc']` and accumulates pause times into module-scope counters.
+ */
+
+const v8 = require('v8');
+const { monitorEventLoopDelay, PerformanceObserver, constants } = require('perf_hooks');
+
+const RING_BUFFER_CAP = 1440;
+const HISTOGRAM_RESOLUTION_MS = 20;
+const NS_PER_MS = 1e6;
+
+const ringBuffer = [];
+
+let eventLoopHistogram = null;
+let gcObserver = null;
+let gcPausesTotalMs = 0;
+let gcCount = 0;
+let lastGcPauseMs = 0;
+let lastGcKind = null;
+
+const GC_KIND_NAMES = {
+  [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major',
+  [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor',
+  [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental',
+  [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb',
+};
+
+function _ensureHistogramStarted() {
+  if (eventLoopHistogram) return eventLoopHistogram;
+  eventLoopHistogram = monitorEventLoopDelay({ resolution: HISTOGRAM_RESOLUTION_MS });
+  eventLoopHistogram.enable();
+  return eventLoopHistogram;
+}
+
+function _ensureGcObserverStarted() {
+  if (gcObserver) return gcObserver;
+  gcObserver = new PerformanceObserver((list) => {
+    for (const entry of list.getEntries()) {
+      // entry.duration is in ms (fractional). entry.kind / entry.detail.kind
+      // is the numeric NODE_PERFORMANCE_GC_* constant.
+      const pauseMs = Number(entry.duration) || 0;
+      gcPausesTotalMs += pauseMs;
+      gcCount += 1;
+      lastGcPauseMs = pauseMs;
+      const kindNum = (entry.detail && entry.detail.kind) || entry.kind || null;
+      lastGcKind = (kindNum != null && GC_KIND_NAMES[kindNum]) || (kindNum != null ? String(kindNum) : null);
+    }
+  });
+  gcObserver.observe({ entryTypes: ['gc'], buffered: false });
+  // Don't keep the event loop alive just to observe GC.
+  if (typeof gcObserver.unref === 'function') gcObserver.unref();
+  return gcObserver;
+}
+
+function _nsToMs(ns) {
+  if (!Number.isFinite(ns) || ns <= 0) return 0;
+  return ns / NS_PER_MS;
+}
+
+function _readEventLoopLag(hist) {
+  // hist.percentile(p) throws when the histogram is empty (no recorded
+  // samples yet). Guard so the first sampleSelf() call after process start
+  // returns zeros instead of throwing.
+  let p50 = 0, p99 = 0, max = 0;
+  try {
+    if (hist && typeof hist.percentile === 'function') {
+      const c = typeof hist.count === 'number' ? hist.count : (hist.totalCount || 0);
+      if (c > 0) {
+        p50 = _nsToMs(hist.percentile(50));
+        p99 = _nsToMs(hist.percentile(99));
+        max = _nsToMs(hist.max);
+      }
+    }
+  } catch {
+    // Empty / not-yet-populated histogram. Leave zeros.
+  }
+  return { p50, p99, max };
+}
+
+function sampleSelf({ label = null } = {}) {
+  const hist = _ensureHistogramStarted();
+  _ensureGcObserverStarted();
+
+  const mem = process.memoryUsage();
+  const heap = v8.getHeapStatistics();
+  const lag = _readEventLoopLag(hist);
+
+  return {
+    rss: mem.rss,
+    heapUsed: mem.heapUsed,
+    heapTotal: mem.heapTotal,
+    external: mem.external,
+    arrayBuffers: mem.arrayBuffers || 0,
+    heapSizeLimit: heap.heap_size_limit,
+    eventLoopLagP50: lag.p50,
+    eventLoopLagP99: lag.p99,
+    eventLoopLagMax: lag.max,
+    lastGcPauseMs,
+    lastGcKind,
+    gcPausesTotalMs,
+    gcCount,
+    uptime: process.uptime(),
+    pid: process.pid,
+    label,
+    capturedAt: Date.now(),
+  };
+}
+
+function recordSample(sample) {
+  if (!sample || typeof sample !== 'object') return;
+  ringBuffer.push(sample);
+  while (ringBuffer.length > RING_BUFFER_CAP) ringBuffer.shift();
+}
+
+function getHistory({ limit } = {}) {
+  const n = ringBuffer.length;
+  if (!Number.isInteger(limit) || limit <= 0 || limit >= n) {
+    return ringBuffer.slice();
+  }
+  return ringBuffer.slice(n - limit);
+}
+
+function startPeriodicSampling({ intervalMs = 60000, onSample = null } = {}) {
+  if (!Number.isFinite(intervalMs) || intervalMs <= 0) {
+    throw new Error('startPeriodicSampling: intervalMs must be a positive number');
+  }
+  const handle = setInterval(() => {
+    let sample;
+    try {
+      sample = sampleSelf({ label: 'periodic' });
+    } catch {
+      return;
+    }
+    recordSample(sample);
+    if (typeof onSample === 'function') {
+      try { onSample(sample); } catch { /* swallow — sampler must not crash on callback errors */ }
+    }
+  }, intervalMs);
+  if (typeof handle.unref === 'function') handle.unref();
+  return function stop() {
+    clearInterval(handle);
+  };
+}
+
+function _resetForTest() {
+  ringBuffer.length = 0;
+  if (eventLoopHistogram && typeof eventLoopHistogram.disable === 'function') {
+    try { eventLoopHistogram.disable(); } catch {}
+  }
+  eventLoopHistogram = null;
+  if (gcObserver && typeof gcObserver.disconnect === 'function') {
+    try { gcObserver.disconnect(); } catch {}
+  }
+  gcObserver = null;
+  gcPausesTotalMs = 0;
+  gcCount = 0;
+  lastGcPauseMs = 0;
+  lastGcKind = null;
+}
+
+module.exports = {
+  sampleSelf,
+  getHistory,
+  recordSample,
+  startPeriodicSampling,
+  // Constants for callers that need to know the cap up front.
+  RING_BUFFER_CAP,
+  // exported for testing
+  _resetForTest,
+};

package/engine/dispatch.js

@@ -295,6 +295,33 @@ function _routeToReviewQueue(item, evaluation) {
   });
 }
 
+/**
+ * Resolve whether the work item's parent PRD is in a "vetted" status —
+ * `approved` or `active` — for the PRD-sourced bypass (W-mq9acoo800177bcb).
+ * Returns true when the bypass should fire: PRD exists on disk and its
+ * status is one of the vetted set. Returns false on missing sourcePlan,
+ * missing/unreadable PRD file, or non-vetted status (rejected, paused,
+ * revision-requested, completed, awaiting-approval). Errors fail closed —
+ * if we can't confirm the PRD is approved/active, we let the normal
+ * validator run.
+ *
+ * @param {object} wi - the work item (item.meta.item)
+ * @returns {boolean}
+ */
+function _isPrdSourcedAndVetted(wi) {
+  const sourcePlan = wi && wi.sourcePlan;
+  if (!sourcePlan || typeof sourcePlan !== 'string') return false;
+  try {
+    const prdPath = path.join(queries.PRD_DIR, sourcePlan);
+    const plan = shared.safeJsonNoRestore(prdPath);
+    if (!plan || !plan.missing_features) return false;
+    const status = plan.status || shared.PLAN_STATUS.ACTIVE;
+    return status === shared.PLAN_STATUS.APPROVED || status === shared.PLAN_STATUS.ACTIVE;
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Async wrapper around addToDispatch that consults the pre-dispatch validator
  * when ENGINE_DEFAULTS.enablePreDispatchEval is true. Validator failures are
@@ -305,6 +332,8 @@ function _routeToReviewQueue(item, evaluation) {
  * @param {object} [opts.config] - engine config; defaults to queries.getConfig().
  * @param {Function} [opts.validate] - injection point for tests; defaults to
  *   require('./pre-dispatch-eval').validateAcceptanceCriteria.
+ * @param {Function} [opts.isPrdVetted] - injection point for tests; defaults
+ *   to the on-disk PRD status read via _isPrdSourcedAndVetted.
  * @returns {Promise<string|null>} dispatch id when queued; null when routed to review.
  */
 async function addToDispatchWithValidation(item, opts = {}) {
@@ -328,6 +357,29 @@ async function addToDispatchWithValidation(item, opts = {}) {
     return addToDispatch(item);
   }
 
+  // W-mq9acoo800177bcb — PRD-sourced bypass. plan-to-prd already LLM-vets
+  // every item it lands in a PRD, so re-validating each materialized item
+  // per discovery tick is the dominant cost of the workflow this
+  // optimization targets (user-observed: 2 PRDs / 11 items → ~5 min
+  // queueing). Gate behind preDispatchEvalSkipPrdSourced (default true) and
+  // only fire when the parent PRD is approved/active — rejected / paused /
+  // revision-requested / completed PRDs still flow through the normal
+  // validator path so a stale PRD reopen can't smuggle in unvetted items.
+  const skipPrdSourced = config?.engine?.preDispatchEvalSkipPrdSourced
+    ?? ENGINE_DEFAULTS.preDispatchEvalSkipPrdSourced;
+  if (skipPrdSourced) {
+    const isPrdVetted = typeof opts.isPrdVetted === 'function'
+      ? opts.isPrdVetted
+      : _isPrdSourcedAndVetted;
+    if (isPrdVetted(wi)) {
+      // Stamp the skip marker on the dispatch item so the entry that lands
+      // in dispatch.json carries observability — operators can spot which
+      // items the validator skipped vs which it ran.
+      item._preDispatchEvalSkipped = 'prd-sourced';
+      return addToDispatch(item);
+    }
+  }
+
   const validate = typeof opts.validate === 'function'
     ? opts.validate
     : require('./pre-dispatch-eval').validateAcceptanceCriteria;
@@ -1073,5 +1125,6 @@ module.exports = {
   findActivePrOrBranchLock,
   normalizeRetryableDecision,
   isCompletedWorkItemForFailure,
+  _isPrdSourcedAndVetted,
   NON_MUTATING_DISPATCH_TYPES,
 };