mandrel 1.57.0

package/.agents/scripts/lib/signals/read.js

@@ -0,0 +1,268 @@
+/**
+ * Streaming signals reader (Epic #1181 / Story #1438 / Task #1459).
+ *
+ * Provides one async-iterator entry point — `read({ epic, story?, kind?,
+ * config? })` — that consumers use instead of opening the NDJSON file
+ * themselves. The reader streams line-by-line so callers never load the
+ * full file into memory (50MB+ traces stay under file size in RSS).
+ *
+ * On-disk layout (resolved via `lib/config/temp-paths.js`):
+ *
+ *   <tempRoot>/epic-<epic>/story-<story>/signals.ndjson
+ *
+ * When `story` is omitted, the reader fans out across every
+ * `story-<id>/signals.ndjson` under `<tempRoot>/epic-<epic>/`. When the
+ * Epic directory is missing, the iterator yields nothing (consumers
+ * treat absence as "no signals yet").
+ *
+ * ## Filter semantics
+ *
+ *   - `kind` — when provided, only events whose `kind` matches the
+ *     argument are yielded. Filtering happens after the per-line JSON
+ *     parse + envelope guard (see `lib/signals/schema.js`).
+ *   - `story` — narrows to a single Story's stream; otherwise we walk
+ *     every Story directory under the Epic.
+ *
+ * ## Warn-once policy (AC #3)
+ *
+ *   Malformed JSON lines are common during a partial write race; we
+ *   warn **once per process** (module-level latch) rather than per line
+ *   so a corrupted tail doesn't drown the operator log. The latch
+ *   carries the first offending path/line and a count for follow-up.
+ *
+ * ## Robustness
+ *
+ *   The reader never throws on a missing file, a permission error, or
+ *   malformed JSON — every failure path resolves to "no more events"
+ *   so the analyzer can keep walking. Errors from the supplied
+ *   `tempRoot` resolver propagate (those are programmer errors, not
+ *   I/O conditions).
+ *
+ * @module lib/signals/read
+ */
+
+import { createReadStream } from 'node:fs';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { createInterface } from 'node:readline';
+import {
+  epicArtifactPath,
+  epicTempDir,
+  signalsFile,
+  storyTempDir,
+} from '../config/temp-paths.js';
+import { parseStoryBranch } from '../git-utils.js';
+import { Logger } from '../Logger.js';
+
+import { isPositiveInt } from './detectors/common.js';
+import { EVENT_KIND_VALUES, hasCommonEnvelope } from './schema.js';
+
+// Module-level latch. Carries `{ path, lineNumber, totalCount }` for the
+// first malformed line we see in this process; subsequent malformed
+// lines bump `totalCount` without emitting. Reset is intentionally
+// not exposed — the warn-once contract is process-wide.
+const _malformedLatch = { fired: false, totalCount: 0 };
+
+/**
+ * Test-only helper to reset the warn-once latch. Not part of the public
+ * surface — exported so the unit test can exercise per-test isolation
+ * without spawning a new Node process.
+ *
+ * @returns {void}
+ */
+export function __resetMalformedLatchForTests() {
+  _malformedLatch.fired = false;
+  _malformedLatch.totalCount = 0;
+}
+
+/**
+ * Snapshot of the current warn-once state. Test-only — surfaces the
+ * count of malformed lines observed so the unit suite can assert that
+ * "10 bad lines → 1 warn but count=10".
+ *
+ * @returns {{ fired: boolean, totalCount: number }}
+ */
+export function __getMalformedLatchForTests() {
+  return { ..._malformedLatch };
+}
+
+function warnOnceMalformed(targetPath, lineNumber, parseErr) {
+  _malformedLatch.totalCount += 1;
+  if (_malformedLatch.fired) return;
+  _malformedLatch.fired = true;
+  Logger.warn(
+    `signals/read: malformed JSON encountered (first at ${targetPath}:${lineNumber}: ${
+      parseErr instanceof Error ? parseErr.message : String(parseErr)
+    }); further malformed lines suppressed for this process.`,
+  );
+}
+
+/**
+ * Stream one signals.ndjson file as an async iterable of parsed records.
+ * Internal helper; consumers go through `read()`.
+ *
+ * @param {string} target — absolute on-disk path
+ * @param {string | null} kindFilter — kind to filter on (null = no filter)
+ * @returns {AsyncGenerator<object>}
+ */
+async function* streamFile(target, kindFilter) {
+  // Existence check before opening the stream — `createReadStream`
+  // defers ENOENT until the first `read()`, which leaves the async
+  // iterator with an uncatchable error mid-flight on some Node
+  // versions. `fs.access` is cheap and gives us a clean early return.
+  try {
+    await fs.access(target);
+  } catch {
+    return;
+  }
+
+  const stream = createReadStream(target, { encoding: 'utf8' });
+  const rl = createInterface({ input: stream, crlfDelay: Infinity });
+
+  let lineNumber = 0;
+  try {
+    for await (const rawLine of rl) {
+      lineNumber += 1;
+      if (rawLine.length === 0) continue;
+      let parsed;
+      try {
+        parsed = JSON.parse(rawLine);
+      } catch (err) {
+        warnOnceMalformed(target, lineNumber, err);
+        continue;
+      }
+      if (!hasCommonEnvelope(parsed)) continue;
+      if (kindFilter != null && parsed.kind !== kindFilter) continue;
+      yield parsed;
+    }
+  } finally {
+    rl.close();
+    // `stream.destroy()` is idempotent — `rl.close()` already pulls
+    // the underlying stream down on its own, but we drop it explicitly
+    // here to satisfy the Windows file-handle leak check.
+    if (!stream.destroyed) stream.destroy();
+  }
+}
+
+/**
+ * List every `stories/story-<id>/signals.ndjson` path under the Epic's
+ * `<tempRoot>/epic-<epic>/` directory. Returns an empty array when the
+ * Epic directory is missing.
+ *
+ * The returned paths are sorted by Story ID ascending so the iterator's
+ * output is stable across runs.
+ *
+ * Story #2940 nested per-Story directories under a `stories/` segment.
+ * Epic-level signals continue to live at the Epic root.
+ *
+ * @param {number} epic
+ * @param {object | undefined} config
+ * @returns {Promise<string[]>}
+ */
+async function listEpicStorySignalsFiles(epic, config) {
+  const epicDir = epicTempDir(epic, config);
+  let epicEntries;
+  try {
+    epicEntries = await fs.readdir(epicDir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  let hasEpicLevelSignals = false;
+  let hasStoriesDir = false;
+  for (const ent of epicEntries) {
+    if (ent.isDirectory() && ent.name === 'stories') {
+      hasStoriesDir = true;
+    } else if (ent.isFile() && ent.name === 'signals.ndjson') {
+      // Story #1430 — wave-runner lifecycle signals land here.
+      hasEpicLevelSignals = true;
+    }
+  }
+  const storyIds = [];
+  if (hasStoriesDir) {
+    let storyEntries;
+    try {
+      storyEntries = await fs.readdir(path.join(epicDir, 'stories'), {
+        withFileTypes: true,
+      });
+    } catch {
+      storyEntries = [];
+    }
+    for (const ent of storyEntries) {
+      if (!ent.isDirectory()) continue;
+      const sid = parseStoryBranch(ent.name);
+      if (sid === null || !isPositiveInt(sid)) continue;
+      storyIds.push(sid);
+    }
+  }
+  storyIds.sort((a, b) => a - b);
+  // Yield epic-level signals first (wave-start precedes per-Story friction),
+  // then walk the per-Story streams in ascending ID order.
+  const targets = hasEpicLevelSignals
+    ? [epicArtifactPath(epic, 'signals.ndjson', config)]
+    : [];
+  for (const sid of storyIds) {
+    targets.push(path.join(storyTempDir(epic, sid, config), 'signals.ndjson'));
+  }
+  return targets;
+}
+
+/**
+ * Stream every event matching `{ epic, story?, kind? }` from the
+ * configured `tempRoot`'s `epic-<epic>/[story-<story>/]signals.ndjson`
+ * file(s).
+ *
+ * Returns an async iterable so callers can `for await` over it without
+ * buffering. The reader is **streaming** — peak memory stays below the
+ * file size for any input.
+ *
+ * @param {{ epic: number, story?: number, kind?: string, config?: object }} args
+ * @returns {AsyncGenerator<object>}
+ *
+ * @example
+ *   for await (const evt of read({ epic: 1181 })) { ... }
+ *   for await (const evt of read({ epic: 1181, story: 1438, kind: 'friction' })) { ... }
+ */
+export async function* read(args) {
+  if (args == null || typeof args !== 'object') {
+    throw new TypeError(
+      `signals/read: args must be an object with at minimum { epic }; got ${args}`,
+    );
+  }
+  const { epic, story, kind, config } = args;
+  if (!isPositiveInt(epic)) {
+    throw new RangeError(
+      `signals/read: epic must be a positive integer (got ${epic})`,
+    );
+  }
+  if (story !== undefined && !isPositiveInt(story)) {
+    throw new RangeError(
+      `signals/read: story must be a positive integer when provided (got ${story})`,
+    );
+  }
+  if (kind !== undefined && kind !== null) {
+    if (typeof kind !== 'string' || kind.length === 0) {
+      throw new TypeError(
+        `signals/read: kind must be a non-empty string when provided (got ${kind})`,
+      );
+    }
+    if (!EVENT_KIND_VALUES.has(kind)) {
+      // Unknown kinds are not technically invalid — a future detector
+      // might emit a new kind that we don't yet know about. We warn
+      // (best effort) but still let the iterator run so the consumer
+      // can adopt the new kind without code change here.
+      Logger.warn(
+        `signals/read: kind '${kind}' is not in the current EVENT_KINDS enumeration; iterating anyway.`,
+      );
+    }
+  }
+
+  const kindFilter = kind ?? null;
+  const targets =
+    story !== undefined
+      ? [signalsFile(epic, story, config)]
+      : await listEpicStorySignalsFiles(epic, config);
+
+  for (const target of targets) {
+    yield* streamFile(target, kindFilter);
+  }
+}

package/.agents/scripts/lib/signals/schema.js

@@ -0,0 +1,225 @@
+/**
+ * Signals schema — single source of truth for event-kind constants and
+ * shape guards (Epic #1181 / Story #1438 / Task #1458).
+ *
+ * Both writers (`lib/observability/signals-writer.js` callers) and
+ * readers (`lib/signals/read.js`, `lib/observability/perf-aggregator.js`)
+ * import their field names from this module so the on-disk NDJSON shape
+ * stays under one schema.
+ *
+ * Pure module: importing has **no I/O side effects**. The exported
+ * `EVENT_KINDS` object is `Object.freeze`'d so consumers can use it as a
+ * compile-time-ish constant without worrying about mutation.
+ *
+ * ## Event-kind enumeration
+ *
+ * The set below is the audited union of `kind:` literals emitted via
+ * `signals-writer.appendSignal` and `signals-writer.appendTrace` across
+ * the codebase as of Epic #1181 (audit-snapshot 2026-05-11):
+ *
+ * Signals-writer `appendSignal` call sites:
+ *   - `friction`       — quality-gate / runtime friction (check-crap.js,
+ *                        check-maintainability.js, diagnose-friction.js,
+ *                        post-merge-pipeline.js, progress-reporter.js,
+ *                        auto-refresh-runner.js)
+ *   - `dispatched`     — (no live emitter; retained pending Story #3908 sweep)
+ *   - `wave-start` / `wave-complete` — `lib/wave-runner/tick.js` (read by
+ *                        perf-aggregator's `waveParallelism` report;
+ *                        `wave-start` also anchors span-tree Story spans)
+ *   - `wave-end`       — span-tree pairing anchor (no live emitter after the
+ *                        Epic #2646 listener deletion; retained for span-tree)
+ *   - `state-transition` — orchestration/ticketing.js
+ *
+ * Story #3909 retired the write-only wave kinds with no consumer (`wave-tick`,
+ * `epic-complete`) — they duplicated the checkpoint + `epic-run-progress`
+ * rollup and nothing read them back.
+ *
+ * Signals-writer `appendTrace` call sites (traces.ndjson sibling, but
+ * sharing the same envelope shape — `tool-trace-hook.js`):
+ *   - `trace`          — per-tool-call timing record
+ *
+ * Aggregator-consumed kinds (perf-aggregator.js scans these — emitters
+ * for `hotspot`, `rework`, `churn`, `idle`, `retry` are future Epic #1030
+ * detector Stories that the aggregator was built to receive; the schema
+ * pins the names so emitters land on a known shape):
+ *   - `hotspot`, `rework`, `churn`, `idle`, `retry`
+ *
+ * ## Common envelope
+ *
+ * Every signal MUST carry at minimum:
+ *   - `ts`     — ISO-8601 timestamp string (writers historically used
+ *                `timestamp:` instead; both keys are accepted by the
+ *                guards below for backward-compat — the migration to
+ *                `ts:` lands in a follow-on Story).
+ *   - `epic`   — integer Epic ID (writers historically used `epicId:`;
+ *                same backward-compat note).
+ *   - `kind`   — one of `EVENT_KINDS`.
+ *
+ * Optional but commonly carried:
+ *   - `story` / `storyId`  — integer Story ID
+ *   - `task` / `taskId`    — integer Task ID (nullable)
+ *   - `source`             — `{ tool: string }`
+ *   - `details`            — kind-specific payload (object or string)
+ *
+ * @module lib/signals/schema
+ */
+
+import { isObject } from '../json-utils.js';
+import { isPositiveInt } from './detectors/common.js';
+
+/**
+ * Frozen enumeration of every event kind currently emitted by
+ * `signals-writer.appendSignal` / `appendTrace`, plus the
+ * aggregator-consumed kinds whose detectors are future Stories.
+ *
+ * Field names are kebab-cased to match the on-disk literals.
+ */
+export const EVENT_KINDS = Object.freeze({
+  FRICTION: 'friction',
+  TRACE: 'trace',
+  DISPATCHED: 'dispatched',
+  // Wave-window forensics signals: `wave-start` / `wave-end` anchor the
+  // span-tree's Story spans, and the perf-aggregator brackets each wave's
+  // wall-clock from `wave-start` → `wave-complete` (the `waveParallelism`
+  // report). These are READ — they survive.
+  WAVE_START: 'wave-start',
+  WAVE_END: 'wave-end',
+  WAVE_COMPLETE: 'wave-complete',
+  // Story #3909 — the write-only wave-lifecycle kinds with no consumer
+  // (`wave-tick`, `epic-complete`) were retired. They duplicated the
+  // `epic-run-state` checkpoint and the `epic-run-progress` rollup and nothing
+  // read them back.
+  STATE_TRANSITION: 'state-transition',
+  HOTSPOT: 'hotspot',
+  REWORK: 'rework',
+  CHURN: 'churn',
+  IDLE: 'idle',
+  RETRY: 'retry',
+  // Story #3819 — per-criterion acceptance self-eval signal emitted by
+  // acceptance-eval.js. One record per Story per eval-loop terminus,
+  // carrying which acceptance items needed rework and the round count, so
+  // the retro and /epic-plan Phase 0 feedback fetch can see acceptance
+  // churn alongside friction/hotspot data.
+  ACCEPTANCE_EVAL: 'acceptance-eval',
+});
+
+/**
+ * The set of event-kind string values (for fast membership checks).
+ *
+ * @type {ReadonlySet<string>}
+ */
+export const EVENT_KIND_VALUES = Object.freeze(
+  new Set(Object.values(EVENT_KINDS)),
+);
+
+/**
+ * Field-name constants exported so reader/aggregator code does not
+ * spread string literals across the module graph.
+ */
+export const FIELDS = Object.freeze({
+  // Envelope (canonical names — migration to these is in flight)
+  TS: 'ts',
+  EPIC: 'epic',
+  STORY: 'story',
+  TASK: 'task',
+  KIND: 'kind',
+
+  // Legacy envelope aliases still emitted by some writers
+  TIMESTAMP: 'timestamp',
+  EPIC_ID: 'epicId',
+  STORY_ID: 'storyId',
+  TASK_ID: 'taskId',
+
+  // Common payload fields
+  SOURCE: 'source',
+  DETAILS: 'details',
+  CATEGORY: 'category',
+  PHASE: 'phase',
+});
+
+/**
+ * Return true when `v` is a non-empty ISO-8601-ish timestamp string.
+ * We don't fully validate format here — the schema's contract is "looks
+ * like a timestamp string", not "round-trips through Date.parse exactly".
+ *
+ * @param {unknown} v
+ * @returns {boolean}
+ */
+function isTimestamp(v) {
+  return typeof v === 'string' && v.length > 0;
+}
+
+/**
+ * Common envelope guard: every signal MUST carry `ts` (or legacy
+ * `timestamp`), `epic` (or legacy `epicId`), and a recognised `kind`.
+ *
+ * Returns true when the envelope is well-formed. Used by `lib/signals/read`
+ * to discard malformed lines before yielding them to the consumer.
+ *
+ * @param {unknown} evt
+ * @returns {boolean}
+ */
+export function hasCommonEnvelope(evt) {
+  if (!isObject(evt)) return false;
+  if (typeof evt.kind !== 'string' || !EVENT_KIND_VALUES.has(evt.kind)) {
+    return false;
+  }
+  const ts = evt.ts ?? evt.timestamp;
+  if (!isTimestamp(ts)) return false;
+  const epic = evt.epic ?? evt.epicId;
+  if (!isPositiveInt(epic)) return false;
+  return true;
+}
+
+/**
+ * Generic per-kind guard. Returns true when the envelope is well-formed
+ * AND (when `kind` is supplied) the event's `kind` matches.
+ *
+ * @param {unknown} evt
+ * @param {string} [kind] — optional kind to match (one of `EVENT_KINDS`).
+ * @returns {boolean}
+ */
+export function isValidSignal(evt, kind) {
+  if (!hasCommonEnvelope(evt)) return false;
+  if (kind != null && evt.kind !== kind) return false;
+  return true;
+}
+
+/**
+ * Per-kind shape guards. Each entry asserts the envelope plus any
+ * required per-kind fields. Unknown kinds fall back to the envelope
+ * check.
+ *
+ * The guards are intentionally lax — they reject records that are
+ * obviously malformed (missing `kind`, missing `ts`, missing `epic`),
+ * not records with extra fields or future schema extensions. The
+ * aggregator (perf-aggregator.js) carries its own per-kind narrowing.
+ *
+ * @type {Readonly<Record<string, (evt: unknown) => boolean>>}
+ */
+export const GUARDS = Object.freeze({
+  [EVENT_KINDS.FRICTION]: (evt) => {
+    if (!isValidSignal(evt, EVENT_KINDS.FRICTION)) return false;
+    // friction signals commonly carry a `category` field, but some
+    // writers (early in the migration) omit it. We accept both.
+    return true;
+  },
+  [EVENT_KINDS.TRACE]: (evt) => isValidSignal(evt, EVENT_KINDS.TRACE),
+  [EVENT_KINDS.DISPATCHED]: (evt) => isValidSignal(evt, EVENT_KINDS.DISPATCHED),
+  [EVENT_KINDS.WAVE_START]: (evt) => isValidSignal(evt, EVENT_KINDS.WAVE_START),
+  [EVENT_KINDS.WAVE_END]: (evt) => isValidSignal(evt, EVENT_KINDS.WAVE_END),
+  [EVENT_KINDS.WAVE_TICK]: (evt) => isValidSignal(evt, EVENT_KINDS.WAVE_TICK),
+  [EVENT_KINDS.WAVE_COMPLETE]: (evt) =>
+    isValidSignal(evt, EVENT_KINDS.WAVE_COMPLETE),
+  [EVENT_KINDS.EPIC_COMPLETE]: (evt) =>
+    isValidSignal(evt, EVENT_KINDS.EPIC_COMPLETE),
+  [EVENT_KINDS.STATE_TRANSITION]: (evt) =>
+    isValidSignal(evt, EVENT_KINDS.STATE_TRANSITION),
+  [EVENT_KINDS.HOTSPOT]: (evt) => isValidSignal(evt, EVENT_KINDS.HOTSPOT),
+  [EVENT_KINDS.REWORK]: (evt) => isValidSignal(evt, EVENT_KINDS.REWORK),
+  [EVENT_KINDS.CHURN]: (evt) => isValidSignal(evt, EVENT_KINDS.CHURN),
+  [EVENT_KINDS.IDLE]: (evt) => isValidSignal(evt, EVENT_KINDS.IDLE),
+  [EVENT_KINDS.RETRY]: (evt) => isValidSignal(evt, EVENT_KINDS.RETRY),
+  [EVENT_KINDS.ACCEPTANCE_EVAL]: (evt) =>
+    isValidSignal(evt, EVENT_KINDS.ACCEPTANCE_EVAL),
+});