@khoavu882/doflow 0.0.0-stage → 2.0.0-beta.4

package/src/runtime/task-state.js

@@ -0,0 +1,176 @@
+'use strict';
+
+// TypeScript migration (agent-docs/BRIEF_GOAL_DOFLOW.md): run directly by Node native type-stripping.
+
+// ── Durable per-task state: locked, merged, versioned ────────────────────────────────────────────
+//
+// EvidenceLedger.save and ClaimsManager.save used to overwrite their whole JSON file with
+// writeFileSync — no writer exclusion, no revision check. Two sessions whose read-modify-write
+// operations overlapped both reported success and the final file held only the later writer's
+// items (architecture review R4, reproduced with two ledger instances on one task). Atomic rename
+// alone does not fix that: it prevents a torn file, not a lost update.
+//
+// This module is the one seam both stores persist through:
+//
+//   - a per-file lock (`<file>.lock` directory — mkdirSync is atomic on every platform) makes the
+//     read-merge-write below a critical section across processes on this machine;
+//   - inside the lock, the writer re-reads the file and MERGES what is on disk into its own view
+//     before writing, so an overlapping writer's items survive;
+//   - the payload carries a monotonically increasing `revision`, so an overlapped-and-merged
+//     history is visible in the record rather than silent;
+//   - the write itself is tmp-file + rename, so an interrupted writer leaves the previous state
+//     intact plus an ignorable `*.tmp` — never a half-written file;
+//   - `readTaskState` validates the schema version and refuses a file from a future schema with
+//     the version named, instead of guessing at its shape.
+
+const path = require('node:path');
+const { isDeepStrictEqual } = require('node:util');
+
+/** Three-way merge against the reader's snapshot. Unchanged local records never overwrite a
+ * newer disk version; concurrent edits to the same record must be retried after an explicit read. */
+function mergeRecords(local       , disk       , baseline               )        {
+  const merged = new Map(disk.map((item     ) => [item.id, item]));
+  for (const item of local) {
+    const previous = baseline.get(item.id);
+    const current = merged.get(item.id);
+    if (isDeepStrictEqual(item, previous)) continue;
+    if (current && !isDeepStrictEqual(current, previous) && !isDeepStrictEqual(current, item)) {
+      throw new Error(`Concurrent update to '${item.id}'; reload the task and reapply the change. Nothing was written.`);
+    }
+    merged.set(item.id, item);
+  }
+  return [...merged.values()];
+}
+
+const SCHEMA_VERSION = 1;
+const LOCK_STALE_MS = 10_000;
+const LOCK_RETRIES = 250;
+const LOCK_WAIT_MS = 20;
+
+/** Synchronous sleep without spinning: Atomics.wait on a throwaway buffer. */
+function sleep(ms        )       {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * Makes one non-waiting attempt to take the cross-process lock for a state file. A lock older than
+ * LOCK_STALE_MS may be broken; a live holder is never waited on. Returns a release function or null.
+ * @param {Object} fsImpl
+ * @param {string} file
+ * @returns {(() => void)|null}
+ */
+function tryAcquireLock(fsImpl     , file        )                      {
+  const lockDir = `${file}.lock`;
+  const release = () => { try { fsImpl.rmdirSync(lockDir); } catch { /* released is released */ } };
+  try {
+    fsImpl.mkdirSync(lockDir);
+    return release;
+  } catch (error     ) {
+    if (error.code !== 'EEXIST') throw error;
+  }
+  let lock     ;
+  try { lock = fsImpl.statSync(lockDir); } catch (error     ) {
+    if (error.code !== 'ENOENT') return null;
+    try { fsImpl.mkdirSync(lockDir); return release; } catch (retryError     ) {
+      if (retryError.code === 'EEXIST') return null;
+      throw retryError;
+    }
+  }
+  if (Date.now() - lock.mtimeMs <= LOCK_STALE_MS) return null;
+  try { fsImpl.rmdirSync(lockDir); } catch (error     ) {
+    if (error.code !== 'ENOENT') return null;
+  }
+  try { fsImpl.mkdirSync(lockDir); return release; } catch (error     ) {
+    if (error.code === 'EEXIST') return null;
+    throw error;
+  }
+}
+
+/**
+ * Takes the cross-process lock for one state file, retrying for the existing bounded wait interval.
+ * A stale lock is broken because its holder died mid-write; the rename-based write keeps the file whole.
+ * @param {Object} fsImpl
+ * @param {string} file
+ * @returns {() => void}
+ */
+function acquireLock(fsImpl     , file        )             {
+  const lockDir = `${file}.lock`;
+  // The stale rule below is wall-clock, so the wait must be too: on a host where each attempt costs
+  // real time, counting attempts alone would outlive a live holder's staleness window and then
+  // break (and later release) its lock.
+  const deadline = Date.now() + LOCK_RETRIES * LOCK_WAIT_MS;
+  for (let attempt = 0; attempt < LOCK_RETRIES && (attempt === 0 || Date.now() < deadline); attempt += 1) {
+    try {
+      fsImpl.mkdirSync(lockDir);
+      return () => { try { fsImpl.rmdirSync(lockDir); } catch { /* released is released */ } };
+    } catch {
+      try {
+        if (Date.now() - fsImpl.statSync(lockDir).mtimeMs > LOCK_STALE_MS) {
+          fsImpl.rmdirSync(lockDir);
+          continue;
+        }
+      } catch { /* the holder released between our mkdir and stat — retry immediately */ }
+      sleep(LOCK_WAIT_MS);
+    }
+  }
+  throw new Error(`Could not lock '${file}' after ${(LOCK_RETRIES * LOCK_WAIT_MS) / 1000}s — another writer holds '${file}.lock'. `
+    + 'If no other session is running, the lock is leftover from a crash younger than its stale timeout; retry shortly.');
+}
+
+/**
+ * Reads and validates one task-state file. Returns null when the file does not exist.
+ * @param {Object} fsImpl
+ * @param {string} file
+ * @returns {Object|null} the parsed payload
+ */
+function readTaskState(fsImpl     , file        )      {
+  if (!fsImpl.existsSync(file)) return null;
+  let data     ;
+  try {
+    data = JSON.parse(fsImpl.readFileSync(file, 'utf8'));
+  } catch (error     ) {
+    throw new Error(`Failed to load state file '${file}': ${error.message}. The write path is atomic `
+      + '(tmp + rename), so a torn file means outside interference; recover it from version control '
+      + 'or move it aside and re-record the task\'s items.');
+  }
+  if (data && typeof data === 'object' && data.version !== undefined && data.version !== SCHEMA_VERSION) {
+    throw new Error(`State file '${file}' has schema version ${data.version}; this runtime reads version ${SCHEMA_VERSION}. `
+      + 'Refusing to guess at its shape — upgrade the runtime or migrate the file.');
+  }
+  return data;
+}
+
+/**
+ * The one durable-update path: lock, re-read, merge, bump revision, atomic write.
+ *
+ * @param {Object} options
+ * @param {Object} options.fsImpl fs implementation (tests substitute one)
+ * @param {string} options.file absolute path of the state file
+ * @param {(diskPayload: Object|null) => Object} options.build called INSIDE the lock with what is
+ *   currently on disk; must fold any disk items unknown to the caller into the caller's view and
+ *   return the complete new payload (without `version`/`revision`, which are stamped here)
+ * @param {() => void} [options.prepare] called INSIDE the lock before the file is read, for a store
+ *   that must inspect or set aside what is there under the same exclusion as the write
+ * @returns {string} the file path written
+ */
+function updateTaskState({ fsImpl, file, build, prepare }                                                                                )         {
+  fsImpl.mkdirSync(path.dirname(file), { recursive: true });
+  const release = acquireLock(fsImpl, file);
+  try {
+    if (prepare) prepare();
+    const disk = readTaskState(fsImpl, file);
+    const payload = {
+      ...build(disk),
+      version: SCHEMA_VERSION,
+      revision: (disk && Number.isInteger(disk.revision) ? disk.revision : 0) + 1,
+    };
+    const tmp = `${file}.${process.pid}.tmp`;
+    fsImpl.writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8');
+    fsImpl.renameSync(tmp, file);
+    return file;
+  } finally {
+    release();
+  }
+}
+
+module.exports = { acquireLock, tryAcquireLock, updateTaskState, readTaskState, mergeRecords, SCHEMA_VERSION };

package/src/runtime/trace/indicators.js

@@ -0,0 +1,235 @@
+'use strict';
+
+// TypeScript migration (agent-docs/BRIEF_GOAL_DOFLOW.md): run directly by Node native type-stripping.
+
+/**
+ * Workflow indicators over the orchestration record (feature 033).
+ *
+ * Why this is not part of `stats`: a run-ledger record carries exactly `timestamp`, `verb`,
+ * `exit_code`, `duration_ms` and `arg_count`. It names no task, no stage and no class, so no amount of
+ * extending `stats` can answer "how long from discovery to plan" — the data is not in that file. The
+ * orchestrator writes a different one, per task, and that is what this reads.
+ *
+ * What it deliberately does not do:
+ *
+ * - No score, grade, percentage or confidence. The shipped guidance forbids expressing evidence that
+ *   way, and one figure over stages that do unlike work would hide exactly what the report is for.
+ * - No git, pull-request or CI indicators. The record holds none of that data, and inventing a proxy
+ *   would be worse than the gap.
+ * - No writes. The correctness of every figure here rests on the record being untouched by reading it.
+ *
+ * One honest limitation, stated in the output as well as here: the record marks when a stage
+ * *completed*, never when it began. A stage's figure is therefore the interval since the previous
+ * recorded event, which includes whatever time passed before that stage started. Fixing that needs the
+ * orchestrator to record starts, which this feature deliberately does not change.
+ */
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { formatMs, finish } = require('./render.js');
+
+const ORCHESTRATION_DIRNAME = 'orchestration';
+
+/** Mirrors resolveRunsLocation's shape, for the sibling state directory this reads. */
+function orchestrationDir(root        )         {
+  return path.join(root, '.doflow', 'state', ORCHESTRATION_DIRNAME);
+}
+
+function parseAt(value     )                {
+  const ms = Date.parse(value);
+  return Number.isNaN(ms) ? null : ms;
+}
+
+/** Every record in a directory, plus the names of the ones that did not parse. A file that cannot be
+ * read is counted and named rather than skipped: a report that quietly drops what it could not read
+ * overstates its own coverage, which is the defect `malformedLines` already guards against for runs. */
+function readRecords(dir        , fsImpl      = fs)      {
+  if (!fsImpl.existsSync(dir)) return { records: [], unreadable: [], exists: false };
+  const records        = [];
+  const unreadable        = [];
+  for (const name of fsImpl.readdirSync(dir).sort()) {
+    if (!name.endsWith('.json')) continue;
+    try {
+      const parsed = JSON.parse(fsImpl.readFileSync(path.join(dir, name), 'utf8'));
+      if (!parsed || typeof parsed !== 'object' || !parsed.taskId) {
+        unreadable.push({ file: name, reason: 'not an orchestration record' });
+        continue;
+      }
+      records.push(parsed);
+    } catch (err     ) {
+      unreadable.push({ file: name, reason: err.message });
+    }
+  }
+  return { records, unreadable, exists: true };
+}
+
+/** A stage row is "not work done" when the orchestrator backfilled it or imported it rather than
+ * executing it. Its duration measures when the record was written, so it is listed but never averaged. */
+function isSynthetic(entry     )          {
+  return Boolean(entry.backfilled) || (entry.executionStatus && entry.executionStatus !== 'completed');
+}
+
+function median(values          )                {
+  if (!values.length) return null;
+  const sorted = [...values].sort((a, b) => a - b);
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 ? sorted[mid] : Math.round((sorted[mid - 1] + sorted[mid]) / 2);
+}
+
+/** One run's rows, walking history in recorded order with a cursor so no interval is counted twice. */
+function runRows(record     )      {
+  const history = Array.isArray(record.history) ? record.history : [];
+  let cursor = parseAt(record.startedAt);
+  const stages        = [];
+  const gates        = [];
+  let reruns = 0;
+
+  for (const entry of history) {
+    const at = parseAt(entry.at);
+    // A negative elapsed can only mean out-of-order history. It is reported as the negative number it
+    // is rather than clamped, because hiding it would hide a corrupt record.
+    const elapsedMs = at !== null && cursor !== null ? at - cursor : null;
+    if (entry.action === 'complete-stage') {
+      stages.push({
+        stage: entry.node ?? null,
+        elapsedMs,
+        outcome: entry.outcome ?? null,
+        synthetic: isSynthetic(entry),
+      });
+      cursor = at ?? cursor;
+    } else if (entry.action === 'decide-gate') {
+      gates.push({
+        gate: entry.node ?? null,
+        elapsedMs,
+        decision: entry.detail ?? null,
+        forced: Boolean(entry.forced),
+      });
+      cursor = at ?? cursor;
+    } else if (entry.action === 'annotate') {
+      reruns += 1;
+      cursor = at ?? cursor;
+    } else {
+      cursor = at ?? cursor;
+    }
+  }
+
+  const started = parseAt(record.startedAt);
+  const updated = parseAt(record.updatedAt);
+  return {
+    taskId: record.taskId,
+    taskClass: record.taskClass ?? null,
+    state: record.state ?? null,
+    startedAt: record.startedAt ?? null,
+    updatedAt: record.updatedAt ?? null,
+    totalMs: started !== null && updated !== null ? updated - started : null,
+    reruns,
+    stages,
+    gates,
+  };
+}
+
+/** Median elapsed per stage id across runs of one class. Synthetic rows are excluded here and still
+ * listed in their run: averaging the moment a file was written into the figure most likely to be
+ * quoted would corrupt it. */
+function aggregateStages(runs       )        {
+  const perStage = new Map                  ();
+  for (const run of runs) {
+    for (const row of run.stages) {
+      if (row.synthetic || row.elapsedMs === null || !row.stage) continue;
+      if (!perStage.has(row.stage)) perStage.set(row.stage, []);
+      perStage.get(row.stage) .push(row.elapsedMs);
+    }
+  }
+  return [...perStage.entries()].map(([stage, values]) => ({
+    stage, runs: values.length, medianMs: median(values),
+  }));
+}
+
+/** Pure. Reads nothing and writes nothing; takes already-parsed records. */
+function buildIndicators({ records = [], unreadable = [], exists = true } = {})      {
+  const runs = (records         ).map(runRows);
+  const byClass = new Map             ();
+
+  for (const run of runs) {
+    const key = run.taskClass || '(unclassified)';
+    if (!byClass.has(key)) byClass.set(key, { taskClass: key, runs: [], stageAggregate: [] });
+    byClass.get(key) .runs.push(run);
+  }
+
+  for (const group of byClass.values()) group.stageAggregate = aggregateStages(group.runs);
+
+  return {
+    view: 'indicators',
+    source: `.doflow/state/${ORCHESTRATION_DIRNAME}`,
+    exists,
+    runCount: runs.length,
+    unreadable,
+    byClass: [...byClass.values()],
+    // Printed as well as returned: a limitation a reader has to look up is one they will not read.
+    limits: [
+      'A stage figure is the interval since the previous recorded event, not pure execution time: the record marks completions, never starts.',
+      'A gate figure is a human deciding. It is charged to no stage and never aggregated.',
+      'Backfilled and imported stages are listed per run but excluded from every aggregate.',
+      'No git, pull-request or continuous-integration indicator is computable here: the orchestration record holds none of that data.',
+    ],
+  };
+}
+
+/** One class block: its runs, each run's stage and gate rows, then the class median. */
+function printClass(group     )       {
+  console.log('─'.repeat(78));
+  console.log(`CLASS: ${group.taskClass}   (${group.runs.length} run(s))`);
+  for (const run of group.runs) {
+    console.log(`  ${run.taskId}  state=${run.state}  total ${formatMs(run.totalMs)}  reruns ${run.reruns}`);
+    for (const row of run.stages) {
+      console.log(`     stage ${String(row.stage).padEnd(24)} ${formatMs(row.elapsedMs).padStart(9)}  outcome=${row.outcome ?? 'not recorded'}${row.synthetic ? '  [backfilled/imported]' : ''}`);
+    }
+    for (const row of run.gates) {
+      console.log(`     wait  ${String(row.gate).padEnd(24)} ${formatMs(row.elapsedMs).padStart(9)}  decision=${row.decision ?? 'not recorded'}${row.forced ? '  [forced]' : ''}`);
+    }
+  }
+  if (!group.stageAggregate.length) return;
+  console.log(`  median per stage (${group.taskClass}, backfilled excluded):`);
+  for (const agg of group.stageAggregate) {
+    console.log(`     ${agg.stage.padEnd(24)} ${formatMs(agg.medianMs).padStart(9)}  over ${agg.runs} run(s)`);
+  }
+}
+
+function handleIndicatorsCommand({ json = false, projectRoot = process.cwd() } = {})         {
+  const dir = orchestrationDir(projectRoot);
+  const read = readRecords(dir);
+  const view = buildIndicators(read);
+
+  if (json) {
+    console.log(JSON.stringify(view, null, 2));
+    return finish(0);
+  }
+
+  console.log('\nDoFlow Workflow Indicators');
+  console.log('═'.repeat(78));
+  if (!view.runCount) {
+    console.log(read.exists
+      ? `No readable orchestration record under ${view.source}.`
+      : `No orchestration record has been written yet at ${view.source}.`);
+    if (view.unreadable.length) {
+      for (const u of view.unreadable) console.log(`  unreadable: ${u.file} — ${u.reason}`);
+    }
+    console.log('═'.repeat(78) + '\n');
+    return finish(0);
+  }
+
+  console.log(`Runs:      ${view.runCount} across ${view.byClass.length} task class(es)`);
+  if (view.unreadable.length) {
+    console.log(`Unreadable: ${view.unreadable.length} record(s) — ${view.unreadable.map((u     ) => u.file).join(', ')}`);
+  }
+
+  for (const group of view.byClass) printClass(group);
+
+  console.log('─'.repeat(78));
+  console.log('What this does not measure:');
+  for (const limit of view.limits) console.log(`  · ${limit}`);
+  console.log('═'.repeat(78) + '\n');
+  return finish(0);
+}
+
+module.exports = { orchestrationDir, readRecords, buildIndicators, handleIndicatorsCommand };