@surlymochan/evalid 0.5.19 → 0.5.21

package/src/core/execution-log.js

@@ -1,8 +1,16 @@
-import { appendFile, mkdir, readFile } from 'node:fs/promises';
-import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
+import { appendFile, mkdir, open, readFile, stat } from 'node:fs/promises';
+import { appendFileSync, closeSync, mkdirSync, openSync, readSync, statSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { buildLogEntryHash, validateExecutionLogChain } from './workflow-integrity.js';
 
+const TAIL_WINDOW_BYTES = 16_384;
+
+// Tail anchors are cached per log file: an append only needs the previous entry_hash, and
+// re-reading a multi-megabyte log for every event made appends O(n^2). The cache is keyed by
+// (size, mtimeMs) so an external writer (another xflow process, a rotated or truncated log that
+// lands on the same size) invalidates it instead of serving a stale anchor.
+const tailStateCache = new Map();
+
 function nowIso() {
   return new Date().toISOString();
 }
@@ -25,28 +33,34 @@ export async function appendExecutionLog(projectRoot, entry = {}) {
 async function appendExecutionLogUnlocked(projectRoot, entry = {}) {
   const file = resolveExecutionLogPath(projectRoot);
   await mkdir(dirname(file), { recursive: true });
-  const previousHash = await readLastEntryHash(file);
+  const tail = await readTailState(file);
   const payload = {
     timestamp: nowIso(),
     ...entry,
-    prev_hash: previousHash,
+    prev_hash: tail.hash,
   };
   payload.entry_hash = buildLogEntryHash(payload);
-  await appendFile(file, `${JSON.stringify(payload)}\n`, 'utf8');
+  // A crash can leave a partial trailing line; start a fresh line so the new entry stays
+  // parseable instead of being glued onto the fragment.
+  const line = `${tail.size > 0 && !tail.endsWithNewline ? '\n' : ''}${JSON.stringify(payload)}\n`;
+  await appendFile(file, line, 'utf8');
+  await rememberAppendedTail(file, payload.entry_hash);
   return file;
 }
 
 export function appendExecutionLogSync(projectRoot, entry = {}) {
   const file = resolveExecutionLogPath(projectRoot);
   mkdirSync(dirname(file), { recursive: true });
-  const previousHash = readLastEntryHashSync(file);
+  const tail = readTailStateSync(file);
   const payload = {
     timestamp: nowIso(),
     ...entry,
-    prev_hash: previousHash,
+    prev_hash: tail.hash,
   };
   payload.entry_hash = buildLogEntryHash(payload);
-  appendFileSync(file, `${JSON.stringify(payload)}\n`, 'utf8');
+  const line = `${tail.size > 0 && !tail.endsWithNewline ? '\n' : ''}${JSON.stringify(payload)}\n`;
+  appendFileSync(file, line, 'utf8');
+  rememberAppendedTailSync(file, payload.entry_hash);
   return file;
 }
 
@@ -76,32 +90,178 @@ export async function readExecutionLog(projectRoot, options = {}) {
   }
 }
 
-async function readLastEntryHash(file) {
+export async function readLastEntryHash(file) {
+  const tail = await readTailState(file);
+  return tail.hash;
+}
+
+async function readTailState(file) {
   try {
-    const raw = await readFile(file, 'utf8');
-    return lastEntryHashFromRaw(raw);
+    const info = await stat(file);
+    const cached = tailStateCache.get(file);
+    if (cached && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
+      return cached;
+    }
+    const state = {
+      size: info.size,
+      mtimeMs: info.mtimeMs,
+      hash: await readTailHash(file, info.size),
+      endsWithNewline: await tailEndsWithNewline(file, info.size),
+    };
+    tailStateCache.set(file, state);
+    return state;
+  } catch {
+    return { size: 0, mtimeMs: null, hash: null, endsWithNewline: true };
+  }
+}
+
+function readTailStateSync(file) {
+  try {
+    const info = statSync(file);
+    const cached = tailStateCache.get(file);
+    if (cached && cached.size === info.size && cached.mtimeMs === info.mtimeMs) {
+      return cached;
+    }
+    const state = {
+      size: info.size,
+      mtimeMs: info.mtimeMs,
+      hash: readTailHashSync(file, info.size),
+      endsWithNewline: tailEndsWithNewlineSync(file, info.size),
+    };
+    tailStateCache.set(file, state);
+    return state;
+  } catch {
+    return { size: 0, mtimeMs: null, hash: null, endsWithNewline: true };
+  }
+}
+
+async function readTailHash(file, size) {
+  if (!size) return null;
+  // The window must contain at least one complete line to anchor the chain. A single entry
+  // larger than the window (an error or detail field is unbounded) leaves no complete line, so
+  // the window grows until the previous entry comes into view. Returning null instead made the
+  // next append pick up prev_hash: null, which permanently breaks the chain after a restart or
+  // from a second process.
+  let windowBytes = TAIL_WINDOW_BYTES;
+  for (;;) {
+    const start = Math.max(0, size - windowBytes);
+    const hash = await readHashWindow(file, start, size);
+    if (hash !== null || start === 0) return hash;
+    windowBytes *= 4;
+  }
+}
+
+function readTailHashSync(file, size) {
+  if (!size) return null;
+  let windowBytes = TAIL_WINDOW_BYTES;
+  for (;;) {
+    const start = Math.max(0, size - windowBytes);
+    const hash = readHashWindowSync(file, start, size);
+    if (hash !== null || start === 0) return hash;
+    windowBytes *= 4;
+  }
+}
+
+async function readHashWindow(file, start, size) {
+  const handle = await open(file, 'r');
+  try {
+    const buffer = Buffer.alloc(size - start);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
+    return lastEntryHashFromRaw(buffer.subarray(0, bytesRead).toString('utf8'), start > 0);
   } catch {
     return null;
+  } finally {
+    await handle.close();
   }
 }
 
-function readLastEntryHashSync(file) {
+function readHashWindowSync(file, start, size) {
+  const fd = openSync(file, 'r');
   try {
-    const raw = readFileSync(file, 'utf8');
-    return lastEntryHashFromRaw(raw);
+    const buffer = Buffer.alloc(size - start);
+    const bytesRead = readSync(fd, buffer, 0, buffer.length, start);
+    return lastEntryHashFromRaw(buffer.subarray(0, bytesRead).toString('utf8'), start > 0);
   } catch {
     return null;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * Record the anchor that was just written. One cheap stat keeps the cache key (size, mtimeMs)
+ * honest, so a later writer that rewrites the file at the same size invalidates the entry.
+ */
+async function rememberAppendedTail(file, entryHash) {
+  try {
+    const info = await stat(file);
+    tailStateCache.set(file, {
+      size: info.size,
+      mtimeMs: info.mtimeMs,
+      hash: entryHash,
+      endsWithNewline: true,
+    });
+  } catch {
+    tailStateCache.delete(file);
+  }
+}
+
+function rememberAppendedTailSync(file, entryHash) {
+  try {
+    const info = statSync(file);
+    tailStateCache.set(file, {
+      size: info.size,
+      mtimeMs: info.mtimeMs,
+      hash: entryHash,
+      endsWithNewline: true,
+    });
+  } catch {
+    tailStateCache.delete(file);
   }
 }
 
-function lastEntryHashFromRaw(raw) {
-  const lines = raw.split('\n').map((line) => line.trim()).filter(Boolean);
+async function tailEndsWithNewline(file, size) {
+  if (!size) return true;
+  const handle = await open(file, 'r');
+  try {
+    const buffer = Buffer.alloc(1);
+    const { bytesRead } = await handle.read(buffer, 0, 1, size - 1);
+    return bytesRead === 0 || buffer[0] === 0x0a;
+  } catch {
+    return true;
+  } finally {
+    await handle.close();
+  }
+}
+
+function tailEndsWithNewlineSync(file, size) {
+  if (!size) return true;
+  const fd = openSync(file, 'r');
+  try {
+    const buffer = Buffer.alloc(1);
+    const bytesRead = readSync(fd, buffer, 0, 1, size - 1);
+    return bytesRead === 0 || buffer[0] === 0x0a;
+  } catch {
+    return true;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+function lastEntryHashFromRaw(raw, startedMidLine = false) {
+  const lines = raw.split('\n');
+  if (startedMidLine) {
+    // The first line of a tail window is usually a fragment; it cannot anchor the chain.
+    lines.shift();
+  }
   for (let index = lines.length - 1; index >= 0; index -= 1) {
+    const line = lines[index].trim();
+    if (!line) continue;
     try {
-      const entry = JSON.parse(lines[index]);
+      const entry = JSON.parse(line);
       if (entry.entry_hash) return entry.entry_hash;
     } catch {
-      return null;
+      // A corrupt or truncated trailing line must not reset the chain; keep scanning upwards.
     }
   }
   return null;

package/src/core/lite-track-auto.js

@@ -0,0 +1,126 @@
+/**
+ * M4 lite fast track, slice 3 (evalid #101, parent #97): auto-select the
+ * track for `xflow workflow run`.
+ *
+ * With no explicit flag the run classifies the change first (slice 1's
+ * `xflow/atoms/lite_track.py`, working-tree diff against the merge-base,
+ * `--uncommitted`): a small diff walks the lite track, a big diff stays
+ * full. Explicit `--lite` / `--full` always wins; both flags together
+ * resolve to full (conservative). Any classifier failure falls back to
+ * full and records the failure -- a run must never silently go lite.
+ *
+ * The returned decision object is JSON-serialisable on purpose: the CLI
+ * attaches it to the workflow input as `lite_track_decision` and the
+ * executor records it in the `workflow_started` / `workflow_completed`
+ * events, next to slice 1's `lite_fast_track` field.
+ */
+import { spawnSync } from 'node:child_process';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+export const DEFAULT_CLASSIFIER_PATH = resolve(__dirname, '..', '..', 'xflow', 'atoms', 'lite_track.py');
+const DEFAULT_TIMEOUT_MS = 60_000;
+
+/**
+ * Run the classifier against `projectRoot` and return the auto decision.
+ *
+ * `runClassifier` is injectable (defaults to `spawnSync`) so tests can
+ * stub the classifier invocation without touching git or python.
+ */
+export function resolveLiteTrackAuto({
+  projectRoot,
+  classifierPath = DEFAULT_CLASSIFIER_PATH,
+  python = process.env.PYTHON || 'python3',
+  runClassifier = spawnSync,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+} = {}) {
+  const fallback = (detail) => ({
+    source: 'auto-fallback-full',
+    track: 'full',
+    reason: `classifier failed (${detail}); falling back to full track`,
+    files: null,
+    added: null,
+    removed: null,
+    total: null,
+    base: null,
+    force_full: [],
+  });
+
+  let result;
+  try {
+    result = runClassifier(
+      python,
+      [classifierPath, '--repo', projectRoot, '--uncommitted', '--json'],
+      { encoding: 'utf8', timeout: timeoutMs },
+    );
+  } catch (err) {
+    return fallback(`spawn error: ${err?.message || String(err)}`);
+  }
+  if (!result || result.status !== 0) {
+    const detail = String(
+      result?.stderr?.trim() || result?.error?.message || `exit ${result?.status ?? 'unknown'}`,
+    ).slice(0, 200);
+    return fallback(detail);
+  }
+  let verdict;
+  try {
+    verdict = JSON.parse(result.stdout);
+  } catch {
+    return fallback('unparseable classifier output');
+  }
+  if (verdict?.track !== 'lite' && verdict?.track !== 'full') {
+    return fallback(`unexpected verdict: ${JSON.stringify(verdict).slice(0, 200)}`);
+  }
+  return {
+    source: 'auto',
+    track: verdict.track,
+    reason: verdict.reason || '',
+    files: verdict.files ?? null,
+    added: verdict.added ?? null,
+    removed: verdict.removed ?? null,
+    total: verdict.total ?? null,
+    base: verdict.base ?? null,
+    force_full: verdict.force_full || [],
+  };
+}
+
+/**
+ * Resolve the effective track from explicit flags and (when neither flag
+ * is given) the auto decision. Returns `{ lite, decision }`.
+ */
+export function decideLiteTrack({ lite, full, autoDecision }) {
+  if (lite === true && full === true) {
+    return {
+      lite: false,
+      decision: {
+        source: 'explicit-full',
+        track: 'full',
+        reason: 'both --lite and --full given; --full wins (conservative)',
+      },
+    };
+  }
+  if (lite === true) {
+    return {
+      lite: true,
+      decision: { source: 'explicit-lite', track: 'lite', reason: 'explicit --lite flag' },
+    };
+  }
+  if (full === true) {
+    return {
+      lite: false,
+      decision: { source: 'explicit-full', track: 'full', reason: 'explicit --full flag' },
+    };
+  }
+  if (!autoDecision) {
+    return {
+      lite: false,
+      decision: {
+        source: 'auto-fallback-full',
+        track: 'full',
+        reason: 'no classifier verdict available; falling back to full track',
+      },
+    };
+  }
+  return { lite: autoDecision.track === 'lite', decision: autoDecision };
+}

package/src/core/log-tail.js

@@ -0,0 +1,176 @@
+/**
+ * log-tail — byte-offset reader for the hash-chained execution log.
+ *
+ * The execution log (.as-xflow/logs/execution.ndjson) is the single source of truth for
+ * workflow progress: the CLI writes it, `xflow serve` reads it. Live consumers therefore tail
+ * it by byte offset instead of re-reading it, and resume from the `entry_hash` they last saw
+ * (SSE Last-Event-ID). Every offset returned here sits on a line boundary, so a consumer never
+ * observes half an entry.
+ */
+import { open, stat } from 'node:fs/promises';
+import { resolveExecutionLogPath } from './execution-log.js';
+
+export const DEFAULT_TAIL_LIMIT = 200;
+export const MAX_TAIL_READ_BYTES = 4 * 1024 * 1024;
+const OVERSCAN_BYTES = 64 * 1024;
+const MAX_OVERSCAN_BYTES = 64 * 1024 * 1024;
+
+export async function readExecutionLogSize(projectRoot = process.cwd()) {
+  try {
+    return (await stat(resolveExecutionLogPath(projectRoot))).size;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Read the entries written after `offset`.
+ *
+ * Returns { entries, nextOffset, size, hasMore, partial, eof }. `nextOffset` only ever advances
+ * past a complete line, so an entry that is still being written is re-read (not skipped) on the
+ * next call. `limit` caps one call so a large backlog is drained over several calls instead of
+ * one unbounded read.
+ */
+export async function readEntriesFromOffset(projectRoot = process.cwd(), offset = 0, options = {}) {
+  const file = resolveExecutionLogPath(projectRoot);
+  const limit = Math.max(1, Math.floor(Number(options.limit) || DEFAULT_TAIL_LIMIT));
+  const size = await readExecutionLogSize(projectRoot);
+  const requested = Math.floor(Number(offset) || 0);
+  const start = Math.min(Math.max(0, requested), size);
+  if (start >= size) {
+    return { file, entries: [], nextOffset: size, size, hasMore: false, partial: false, eof: true };
+  }
+
+  const end = Math.min(size, start + MAX_TAIL_READ_BYTES);
+  const window = await readRange(file, start, end);
+  const entries = [];
+  let cursor = 0;
+  let nextOffset = start;
+  let sawNewline = false;
+
+  while (entries.length < limit) {
+    const newlineAt = window.indexOf(0x0a, cursor);
+    if (newlineAt === -1) break;
+    sawNewline = true;
+    const line = window.subarray(cursor, newlineAt).toString('utf8').trim();
+    cursor = newlineAt + 1;
+    nextOffset = start + cursor;
+    if (!line) continue;
+    entries.push(parseEntryLine(line));
+  }
+
+  if (!sawNewline && end < size) {
+    // One entry larger than the read window would stall the tail forever (the same offset
+    // would be returned on every poll). Skip to the next line boundary and tell the caller.
+    const skippedTo = await skipOversizedLine(file, end, size);
+    entries.push({
+      kind: 'invalid_log_line',
+      raw: 'line_exceeds_read_window',
+      reason: 'entry_larger_than_read_window',
+      truncated: true,
+    });
+    return {
+      file,
+      entries,
+      nextOffset: skippedTo,
+      size,
+      hasMore: skippedTo < size,
+      partial: false,
+      eof: skippedTo >= size,
+    };
+  }
+
+  return {
+    file,
+    entries,
+    nextOffset,
+    size,
+    hasMore: nextOffset < size,
+    // Anything left in the window sits after the last newline, i.e. an entry still being written.
+    partial: cursor < window.length,
+    eof: nextOffset >= size,
+  };
+}
+
+/**
+ * Map an entry_hash to the byte offset just past that entry.
+ *
+ * Returns null when the hash is unknown (rotated or deleted log), which callers treat as
+ * "replay is impossible, fall back to a snapshot" rather than as an error.
+ */
+export async function resolveEntryOffset(projectRoot = process.cwd(), entryHash) {
+  const wanted = String(entryHash || '').trim();
+  if (!wanted) return null;
+  const file = resolveExecutionLogPath(projectRoot);
+  let size;
+  try {
+    size = (await stat(file)).size;
+  } catch {
+    return null;
+  }
+  if (!size) return null;
+
+  // A reconnect resolves a single hash, and the entry the client last saw is almost always
+  // among the most recent ones, so scan the tail first and widen the window until the log is
+  // covered. Every window ends at EOF on purpose: a window that stopped at an arbitrary offset
+  // would split an entry across two reads and never see it whole.
+  let windowBytes = OVERSCAN_BYTES;
+  for (;;) {
+    const start = Math.max(0, size - windowBytes);
+    const window = await readRange(file, start, size);
+    const offset = matchEntryOffset(window, start, wanted);
+    if (offset !== null) return offset;
+    if (start === 0) return null;
+    windowBytes *= 4;
+  }
+}
+
+function matchEntryOffset(window, base, wanted) {
+  let cursor = 0;
+  while (cursor < window.length) {
+    const newlineAt = window.indexOf(0x0a, cursor);
+    if (newlineAt === -1) break;
+    const line = window.subarray(cursor, newlineAt).toString('utf8').trim();
+    cursor = newlineAt + 1;
+    if (!line) continue;
+    try {
+      if (JSON.parse(line).entry_hash === wanted) return base + cursor;
+    } catch {
+      // Corrupt lines cannot match a hash; keep scanning.
+    }
+  }
+  return null;
+}
+
+function parseEntryLine(line) {
+  try {
+    const entry = JSON.parse(line);
+    return entry && typeof entry === 'object' ? entry : { kind: 'invalid_log_line', raw: line.slice(0, 512) };
+  } catch {
+    return { kind: 'invalid_log_line', raw: line.slice(0, 512), reason: 'invalid_json' };
+  }
+}
+
+async function readRange(file, start, end) {
+  const handle = await open(file, 'r');
+  try {
+    const buffer = Buffer.alloc(end - start);
+    const { bytesRead } = await handle.read(buffer, 0, buffer.length, start);
+    return buffer.subarray(0, bytesRead);
+  } finally {
+    await handle.close();
+  }
+}
+
+async function skipOversizedLine(file, from, size) {
+  let position = from;
+  while (position < size) {
+    const chunkEnd = Math.min(size, position + OVERSCAN_BYTES);
+    const chunk = await readRange(file, position, chunkEnd);
+    const newlineAt = chunk.indexOf(0x0a);
+    if (newlineAt !== -1) return position + newlineAt + 1;
+    position = chunkEnd;
+    if (position - from > MAX_OVERSCAN_BYTES) return position;
+  }
+  return size;
+}

package/src/core/workflow-executor.js

@@ -75,9 +75,13 @@ export class WorkflowExecutor {
 
   // ─── Main execution loop ─────────────────────────────────────────────────
 
-  // M4 lite fast track: explicit `--lite` input only (the classifier in
-  // xflow/atoms/lite_track.py is the decision procedure; this flag is the
-  // enforcement switch). No silent auto-switching.
+  // M4 lite fast track: `input.lite === true` engages it. Slice 1 set this
+  // only from the explicit `--lite` flag; slice 3 (evalid #101) lets the
+  // CLI resolve it first -- `decideLiteTrack` in lite-track-auto.js runs
+  // the classifier when no explicit flag is given and sets `input.lite`
+  // when the verdict is lite, so the executor itself never decides. The
+  // decision itself arrives as `input.lite_track_decision` and is recorded
+  // in the workflow_started / workflow_completed events.
   isLiteFastTrack() {
     return this.input?.lite === true;
   }
@@ -137,6 +141,10 @@ export class WorkflowExecutor {
       workflow: this.workflow.name,
       track: this.workflow.track,
       lite_fast_track: this.isLiteFastTrack(),
+      // M4 slice 3: how the track was chosen (auto / explicit-lite /
+      // explicit-full / auto-fallback-full); null when the run was started
+      // without the CLI decision step (e.g. direct executor use).
+      lite_track_decision: this.input?.lite_track_decision ?? null,
       workflow_integrity: this.workflowManifest,
       max_parallel_agents: this.maxParallelAgents,
       parallel_policy: this.concurrencyPolicy.mode,
@@ -351,6 +359,7 @@ export class WorkflowExecutor {
       workflow: this.workflow.name,
       track: this.workflow.track,
       lite_fast_track: this.isLiteFastTrack(),
+      lite_track_decision: this.input?.lite_track_decision ?? null,
       workflow_integrity: this.workflowManifest,
       change_id: this.changeId || null,
       mission_id: this.runtime.missionId || null,