@worca/ui 0.41.0 → 0.43.0

package/server/graph-query-aggregator.js

@@ -0,0 +1,165 @@
+/**
+ * Graph-query aggregator — reads pipeline.hook.graph_query events from a run's
+ * events.jsonl and turns them into live per-iteration query counts
+ * (graphify_invocations / crg_invocations / crg_tool_counts), assigned to the
+ * matching iteration in status.json by timestamp range.
+ *
+ * This is the live counterpart to the runner's completion-time tally: the
+ * runner writes the authoritative graphify_invocations / crg_invocations onto
+ * each iteration when the stage *completes*. For a still-running iteration that
+ * count is absent, so the graphify/CRG badges would otherwise stay blank until
+ * completion. This aggregator fills the gap by counting the live hook events —
+ * exactly like dispatch-events-aggregator does for skills/subagents — and only
+ * for iterations that don't already carry the runner's number (so the
+ * authoritative completion count is never clobbered).
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+
+const GRAPH_QUERY_EVENT_TYPE = 'pipeline.hook.graph_query';
+
+/**
+ * Classify a single parsed events.jsonl line. Returns the normalised graph-query
+ * entry if the line is a graph-query event, or null otherwise. Extracted so a
+ * combined single-pass reader (events-jsonl-reader.js) can share the exact same
+ * normalisation as the standalone reader below.
+ *
+ * @param {object} e — a parsed events.jsonl object
+ * @returns {{engine, op, timestamp}|null}
+ */
+export function parseGraphQueryEventLine(e) {
+  if (!e || e.event_type !== GRAPH_QUERY_EVENT_TYPE) return null;
+  const payload = e.payload || {};
+  const engine = payload.engine;
+  if (engine !== 'graphify' && engine !== 'crg') return null;
+  return { engine, op: payload.op || '', timestamp: e.timestamp };
+}
+
+/**
+ * Parse events.jsonl and return only the graph-query events.
+ * Malformed lines are skipped so a corrupt event doesn't break the run view.
+ *
+ * @param {string} eventsPath — absolute path to events.jsonl
+ * @returns {Array<{engine, op, timestamp}>}
+ */
+export function readGraphQueryEventsFromJsonl(eventsPath) {
+  if (!eventsPath || !existsSync(eventsPath)) return [];
+  let content;
+  try {
+    content = readFileSync(eventsPath, 'utf8');
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const line of content.split('\n')) {
+    if (!line.trim()) continue;
+    let e;
+    try {
+      e = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const entry = parseGraphQueryEventLine(e);
+    if (entry) out.push(entry);
+  }
+  return out;
+}
+
+/**
+ * Given a list of graph-query events and a stages map from status.json, return
+ * a new stages map where each iteration that overlaps an event's timestamp gets
+ * live query counts — but ONLY when the iteration does not already carry the
+ * runner's authoritative count (i.e. the still-running iteration). Completed
+ * iterations are left exactly as the runner wrote them.
+ *
+ * Counts added per matching running iteration:
+ *   graphify_invocations: <number>   (graphify-engine events)
+ *   crg_invocations:      <number>   (crg-engine events)
+ *   crg_tool_counts:      { <op>: <number> }   (crg-engine, by tool — tooltip)
+ *
+ * Non-destructive: input stages object is shallow-copied; touched iterations
+ * get new objects. Existing iteration fields are preserved.
+ *
+ * @param {Array<{engine, op, timestamp}>} events
+ * @param {object} stages — status.stages
+ * @returns {object} enriched stages
+ */
+export function assignGraphQueryCountsToIterations(events, stages) {
+  if (!stages || typeof stages !== 'object') return stages;
+  if (!events || events.length === 0) return stages;
+
+  // Bucket events into iterations by timestamp. Bucket key: `${stageKey}|${num}`.
+  const buckets = new Map();
+  for (const ev of events) {
+    if (!ev.timestamp) continue;
+    const eventTime = Date.parse(ev.timestamp);
+    if (Number.isNaN(eventTime)) continue;
+
+    let matched = false;
+    for (const [stageKey, stage] of Object.entries(stages)) {
+      const iterations = stage?.iterations;
+      if (!Array.isArray(iterations)) continue;
+      for (const iter of iterations) {
+        // Only fill counts for iterations the runner hasn't tallied yet —
+        // a present number means the stage completed and is authoritative.
+        if (typeof iter.graphify_invocations === 'number') continue;
+        if (typeof iter.crg_invocations === 'number') continue;
+        const start = iter.started_at ? Date.parse(iter.started_at) : NaN;
+        if (Number.isNaN(start)) continue;
+        const end = iter.completed_at
+          ? Date.parse(iter.completed_at)
+          : Number.POSITIVE_INFINITY;
+        if (eventTime >= start && eventTime <= end) {
+          const key = `${stageKey}|${iter.number}`;
+          if (!buckets.has(key)) buckets.set(key, []);
+          buckets.get(key).push(ev);
+          matched = true;
+          break;
+        }
+      }
+      if (matched) break;
+    }
+  }
+
+  if (buckets.size === 0) return stages;
+
+  const enrichedStages = { ...stages };
+  for (const [key, bucketEvents] of buckets) {
+    const [stageKey, iterNumStr] = key.split('|');
+    const iterNum = Number(iterNumStr);
+    const stage = enrichedStages[stageKey];
+    if (!stage) continue;
+    const counts = tally(bucketEvents);
+    const newIterations = stage.iterations.map((iter) =>
+      iter.number === iterNum ? { ...iter, ...counts } : iter,
+    );
+    enrichedStages[stageKey] = { ...stage, iterations: newIterations };
+  }
+  return enrichedStages;
+}
+
+/**
+ * Tally graph-query events into the count shape the badges read.
+ *
+ * @param {Array<{engine, op}>} events
+ * @returns {{graphify_invocations, crg_invocations, crg_tool_counts}}
+ */
+function tally(events) {
+  let graphify = 0;
+  let crg = 0;
+  const crgToolCounts = {};
+  for (const ev of events) {
+    if (ev.engine === 'graphify') {
+      graphify += 1;
+    } else if (ev.engine === 'crg') {
+      crg += 1;
+      const op = ev.op || 'unknown';
+      crgToolCounts[op] = (crgToolCounts[op] || 0) + 1;
+    }
+  }
+  return {
+    graphify_invocations: graphify,
+    crg_invocations: crg,
+    crg_tool_counts: crgToolCounts,
+  };
+}

package/server/integrations/renderers.js

@@ -134,6 +134,16 @@ function renderGitPrCreated(envelope) {
   return mdMsg(parts.join('\n'), 'info');
 }
 
+function renderGitPrDeferred(envelope) {
+  const p = envelope.payload;
+  const parts = [`\u{1F7E1} **Run:** \`${runId(envelope)}\``];
+  parts.push(
+    `   **PR deferred:** branch \`${p.head_branch}\` pushed — open the run and click **Create PR** to publish.`,
+  );
+  if (p.pr_title) parts.push(`   **Title:** ${p.pr_title}`);
+  return mdMsg(parts.join('\n'), 'warning');
+}
+
 function renderGitPrMerged(envelope) {
   const p = envelope.payload;
   const parts = [`\u2705 **PR merged:** [#${p.pr_number}](${p.pr_url})`];
@@ -560,6 +570,7 @@ const EVENT_RENDERERS = {
   'pipeline.stage.completed': renderStageCompleted,
   'pipeline.stage.interrupted': renderStageInterrupted,
   'pipeline.git.pr_created': renderGitPrCreated,
+  'pipeline.git.pr_deferred': renderGitPrDeferred,
   'pipeline.git.pr_merged': renderGitPrMerged,
   'pipeline.circuit_breaker.tripped': renderCbTripped,
   'pipeline.cost.budget_warning': renderCostBudgetWarning,

package/server/process-manager.js

@@ -541,6 +541,9 @@ export class ProcessManager {
     if (opts.mloops && opts.mloops > 1) {
       args.push('--mloops', String(opts.mloops));
     }
+    if (opts.maxBeads != null) {
+      args.push('--max-beads', String(opts.maxBeads));
+    }
     if (opts.planFile) {
       args.push('--plan', opts.planFile);
     }
@@ -554,6 +557,89 @@ export class ProcessManager {
     const env = { ...process.env };
     delete env.CLAUDECODE;
 
+    // run_worktree.py is a *launcher*: it performs all setup (PR-metadata
+    // fetch, worktree checkout, registry write) and only exits 0 after the
+    // real pipeline wrote its status.json (_await_pipeline_startup), printing
+    // diagnostics to stderr and exiting non-zero on any failure. Its detached
+    // grandchild redirects its own stdio to a log file, so it never holds our
+    // pipes open. That makes the launcher's exit code an authoritative
+    // success/failure signal — wait for it instead of guessing with a fixed
+    // timer. The old 2s timer resolved "started" before slow failures (e.g. a
+    // PR fetch followed by a worktree collision) surfaced, so the UI reported
+    // success while nothing ran. run_pipeline.py (in-place / resume) is NOT a
+    // launcher — it *is* the long-lived pipeline — so it keeps the timer path.
+    const isFireAndForget = scriptRel === worktreeScriptRel;
+
+    if (isFireAndForget) {
+      return new Promise((resolve, reject) => {
+        const child = spawn('python3', args, {
+          detached: true,
+          // Capture stderr to surface the launcher's error; ignore stdin/stdout
+          // (stdout carries only run_id+path and is not needed).
+          stdio: ['ignore', 'ignore', 'pipe'],
+          cwd,
+          env,
+        });
+
+        let settled = false;
+        let stderr = '';
+        const STDERR_CAP = 8192;
+        // Generous safety net: the launcher normally exits within seconds, but
+        // a hung gh/network call shouldn't block the launch request forever.
+        const hardCap = setTimeout(() => {
+          if (settled) return;
+          settled = true;
+          child.removeAllListeners('error');
+          child.removeAllListeners('exit');
+          cleanupPromptFile(promptFilePath);
+          const err = new Error(
+            'Pipeline launcher did not finish within 180s — aborting launch',
+          );
+          err.code = 'spawn_timeout';
+          reject(err);
+        }, 180000);
+        hardCap.unref?.();
+
+        if (child.stderr) {
+          child.stderr.on('data', (d) => {
+            if (stderr.length < STDERR_CAP) stderr += d.toString();
+          });
+        }
+
+        child.on('error', (spawnErr) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(hardCap);
+          cleanupPromptFile(promptFilePath);
+          const err = new Error(
+            `Failed to start pipeline: ${spawnErr.message}`,
+          );
+          err.code = 'spawn_error';
+          reject(err);
+        });
+
+        child.on('exit', (code, signal) => {
+          if (settled) return;
+          settled = true;
+          clearTimeout(hardCap);
+          cleanupPromptFile(promptFilePath);
+          if (code === 0) {
+            child.unref();
+            resolve({ pid: child.pid });
+            return;
+          }
+          const detail = stderr.trim().split('\n').slice(-6).join('\n').trim();
+          const reason =
+            code !== null ? `exit code ${code}` : `signal ${signal}`;
+          const err = new Error(
+            `Pipeline failed to start (${reason})${detail ? `:\n${detail}` : ''}`,
+          );
+          err.code = 'spawn_error';
+          reject(err);
+        });
+      });
+    }
+
     return new Promise((resolve, reject) => {
       const child = spawn('python3', args, {
         detached: true,

package/server/project-routes.js

@@ -53,7 +53,7 @@ import { validateSettingsPayload } from './settings-validator.js';
 import { createTemplatesRoutes } from './templates-routes.js';
 import { isVersionBehind } from './version-check.js';
 import { getVersionInfo } from './versions.js';
-import { discoverRuns } from './watcher.js';
+import { discoverRuns, discoverRunsAsync } from './watcher.js';
 import {
   checkWorcaInstalled,
   readProjectWorcaVersion,
@@ -365,7 +365,12 @@ export function createProjectScopedRoutes({
   // GET /api/projects/:projectId/runs — list runs for this project
   router.get('/runs', requireWorcaDir, async (req, res) => {
     try {
-      const runs = discoverRuns(req.project.worcaDir);
+      // List/sidebar path: scan off the event loop (async) and skip
+      // events.jsonl enrichment entirely — neither the run list nor the sidebar
+      // render dispatch_events / graph-query counts (issue #296).
+      const runs = await discoverRunsAsync(req.project.worcaDir, {
+        enrich: false,
+      });
       const default_branch = getDefaultBranch(req.project.projectRoot);
 
       const { getBeadsCounts } = req.app.locals;
@@ -924,6 +929,7 @@ export function createProjectScopedRoutes({
       planFile,
       msize,
       mloops,
+      maxBeads,
       branch,
       template,
     } = body;
@@ -1013,6 +1019,10 @@ export function createProjectScopedRoutes({
       mloops != null
         ? Math.max(1, Math.min(10, Math.round(Number(mloops))))
         : 1;
+    const maxBeadsVal =
+      maxBeads != null
+        ? Math.max(0, Math.min(50, Math.round(Number(maxBeads))))
+        : undefined;
 
     // Atomically check global cap and start pipeline under lock
     await launchLock.withLock(async () => {
@@ -1040,6 +1050,7 @@ export function createProjectScopedRoutes({
           prompt: hasPrompt ? prompt : undefined,
           msize: msizeVal,
           mloops: mloopsVal,
+          maxBeads: maxBeadsVal,
           planFile: hasPlan ? planFile.trim() : undefined,
           branch: branch || undefined,
           template: template || undefined,
@@ -1709,7 +1720,9 @@ export function createProjectScopedRoutes({
   // Reads per-iteration token_usage from each run's status.json.
   router.get('/costs', requireWorcaDir, (req, res) => {
     const { worcaDir } = req.project;
-    const runs = discoverRuns(worcaDir);
+    // Costs read only per-iteration token_usage from status.json — no
+    // events.jsonl enrichment needed (issue #296).
+    const runs = discoverRuns(worcaDir, { enrich: false });
     const tokenData = {};
 
     for (const run of runs) {

package/server/schemas/keys.json

@@ -33,6 +33,11 @@
       "milestones": {
         "plan_approval": true,
         "pr_approval": false
+      },
+      "telemetry": {
+        "file_access": {
+          "enabled": true
+        }
       }
     }
   }

package/server/template-prompts.js

@@ -0,0 +1,136 @@
+/**
+ * Effective prompt model for the Pipelines editor "Prompts" tab.
+ *
+ * For each stage prompt file (agent `*.md` and user-prompt `*.block.md`) this
+ * resolves what the pipeline actually runs, classifying each file as one of:
+ *
+ *   - 'builtin'  — the template has no overlay; the built-in core prompt is used
+ *                  unchanged (a fallback).
+ *   - 'pipeline' — the template overlay replaces the built-in prompt entirely
+ *                  (default mode, or an explicit `<!-- replace -->`).
+ *   - 'extends'  — the overlay is `<!-- append -->`; it merges into the built-in
+ *                  via `## Override: <Section>` blocks (each appending, or
+ *                  overwriting when the block opens with `<!-- replace -->`), or
+ *                  a raw trailing append when there are no override blocks.
+ *
+ * Mode/override parsing mirrors src/worca/orchestrator/overlay.py so the editor
+ * preview matches what the runtime actually assembles.
+ */
+
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const OVERLAY_NAME_RE = /^[a-z0-9._-]{1,64}\.(md|block\.md)$/;
+const APPEND_TAG = '<!-- append -->';
+const REPLACE_TAG = '<!-- replace -->';
+
+/**
+ * Split an `<!-- append -->` overlay body into `## Override: <Section>` blocks.
+ * Mirrors overlay.py:_parse_overrides. A block whose first non-blank line is
+ * `<!-- replace -->` overwrites the matching built-in section; otherwise it
+ * appends. Returns `[{ section, mode: 'append'|'overwrite', body }]`.
+ */
+export function parseOverrides(content) {
+  const parts = content.split(/^(## Override:\s*.+)$/m);
+  const overrides = [];
+  for (let i = 1; i < parts.length - 1; i += 2) {
+    const headingLine = parts[i];
+    const section = headingLine.replace(/^##\s*Override:\s*/, '').trim();
+    const lines = parts[i + 1].split('\n');
+    const kept = [];
+    let foundReplace = false;
+    let replace = false;
+    for (const line of lines) {
+      if (!foundReplace && line.trim() === REPLACE_TAG) {
+        replace = true;
+        foundReplace = true;
+        continue;
+      }
+      kept.push(line);
+    }
+    overrides.push({
+      section,
+      mode: replace ? 'overwrite' : 'append',
+      body: kept.join('\n').trim(),
+    });
+  }
+  return overrides;
+}
+
+/**
+ * Classify a single file given its built-in (core) and overlay contents.
+ * Either may be null. Returns the per-file model the editor renders.
+ */
+export function classifyPromptFile(name, coreContent, overlayContent) {
+  const role = name.endsWith('.block.md') ? 'block' : 'agent';
+  const base = { name, role };
+
+  if (overlayContent == null) {
+    return { ...base, source: 'builtin', content: coreContent ?? '' };
+  }
+
+  const stripped = overlayContent.replace(/^\s+/, '');
+
+  if (stripped.startsWith(APPEND_TAG)) {
+    const body = stripped.slice(APPEND_TAG.length);
+    const overrides = parseOverrides(body);
+    return {
+      ...base,
+      source: 'extends',
+      builtin: coreContent ?? '',
+      contributions: overrides,
+      rawAppend: overrides.length === 0 ? body.trim() : null,
+    };
+  }
+
+  const content = stripped.startsWith(REPLACE_TAG)
+    ? stripped.slice(REPLACE_TAG.length).trim()
+    : overlayContent.trim();
+  return { ...base, source: 'pipeline', content };
+}
+
+/**
+ * Read every prompt file under a directory (filtered to overlay-name shape),
+ * returning `{ filename: content }`. Missing dir → empty object.
+ */
+function readPromptDir(dir) {
+  const out = {};
+  if (!dir || !existsSync(dir)) return out;
+  let names;
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return out;
+  }
+  for (const f of names) {
+    if (!OVERLAY_NAME_RE.test(f)) continue;
+    try {
+      out[f] = readFileSync(join(dir, f), 'utf8');
+    } catch {
+      /* skip unreadable files */
+    }
+  }
+  return out;
+}
+
+/**
+ * Build the prompts model for a template.
+ *
+ * @param {string} coreDir     - built-in core prompts dir (.../agents/core)
+ * @param {string} overlayDir  - template overlay dir (.../<template>/agents); may not exist
+ * @returns {object} `{ filename: model }` over the union of core+overlay files
+ */
+export function buildPromptsModel(coreDir, overlayDir) {
+  const core = readPromptDir(coreDir);
+  const overlay = readPromptDir(overlayDir);
+  const names = new Set([...Object.keys(core), ...Object.keys(overlay)]);
+  const model = {};
+  for (const name of names) {
+    model[name] = classifyPromptFile(
+      name,
+      Object.hasOwn(core, name) ? core[name] : null,
+      Object.hasOwn(overlay, name) ? overlay[name] : null,
+    );
+  }
+  return model;
+}