@tagma/sdk 0.1.7 → 0.1.8

package/README.md

@@ -113,7 +113,10 @@ Executes the pipeline. Returns `{ success, runId, logPath, summary, states }`.
 Options:
 - `approvalGateway` -- custom `ApprovalGateway` instance (defaults to `InMemoryApprovalGateway`)
 - `signal` -- `AbortSignal` to cancel the run externally
-- `onEvent` -- callback for real-time `PipelineEvent` updates (task status changes, pipeline start/end)
+- `onEvent` -- callback for real-time `PipelineEvent` updates:
+  - `pipeline_start` — pipeline began; includes `states: ReadonlyMap<taskId, TaskState>` (initial snapshot of all tasks at `waiting`)
+  - `task_status_change` — a task changed status; includes `state: TaskState` (complete snapshot at the time of change, with `result` and `finishedAt` already populated for terminal statuses)
+  - `pipeline_end` — pipeline finished; includes `success: boolean`
 - `maxLogRuns` -- number of per-run log directories to keep under `<workDir>/logs/` (default: 20)
 
 ### `PipelineRunner`
@@ -133,8 +136,9 @@ runner.start(); // returns Promise<EngineResult>, idempotent
 // Cancel from IPC
 runner.abort();
 
-// After completion
-const states = runner.getStates(); // ReadonlyMap<taskId, TaskState>
+// Available from the first pipeline_start event onward (not just after completion)
+// Returns null only if the pipeline has never started
+const states = runner.getStates(); // ReadonlyMap<taskId, TaskState> | null
 ```
 
 Properties:
@@ -184,7 +188,7 @@ const yaml = serializePipeline(config);
 | `moveTrack(config, trackId, toIndex)` | Reorder a track |
 | `updateTrack(config, trackId, fields)` | Patch track fields (not tasks) |
 | `upsertTask(config, trackId, task)` | Insert or replace a task |
-| `removeTask(config, trackId, taskId)` | Remove a task |
+| `removeTask(config, trackId, taskId, cleanRefs?)` | Remove a task; pass `cleanRefs: true` to also strip dangling `depends_on` / `continue_from` references from other tasks |
 | `moveTask(config, trackId, taskId, toIndex)` | Reorder a task within its track |
 | `transferTask(config, fromTrackId, taskId, toTrackId)` | Move a task across tracks |
 
@@ -231,6 +235,20 @@ if (errors.length > 0) {
 }
 ```
 
+### `buildRawDag(config: RawPipelineConfig): RawDag`
+
+Extracts the topology of a raw (unresolved) pipeline config as a graph — no `workDir` or plugin registration required. Intended for the visual editor to render the flow graph during editing.
+
+Returns `{ nodes: ReadonlyMap<taskId, RawDagNode>, edges: { from, to }[] }` where each edge represents a dependency (from must complete before to). Template-expansion tasks (`use:` field) and unresolvable refs are silently skipped.
+
+```ts
+const { nodes, edges } = buildRawDag(draftConfig);
+// nodes — keyed by "trackId.taskId"
+// edges — [{ from: "track.taskA", to: "track.taskB" }, ...]
+```
+
+Use `buildDag` instead when you have a fully resolved `PipelineConfig` and need topological sort order.
+
 ## Related Packages
 
 | Package | Description |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tagma/sdk",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "type": "module",
   "workspaces": [
     "plugins/*"

package/src/config-ops.ts

@@ -117,19 +117,56 @@ export function upsertTask(
 
 /**
  * Remove a task from a track. No-op if either id is not found.
+ *
+ * When `cleanRefs` is true, all `depends_on` and `continue_from` references to the
+ * removed task are also removed from every other task in the pipeline. This prevents
+ * validateRaw from reporting dangling-ref errors after the deletion.
  */
 export function removeTask(
   config: RawPipelineConfig,
   trackId: string,
   taskId: string,
+  cleanRefs = false,
 ): RawPipelineConfig {
-  return {
+  const withoutTask = {
     ...config,
     tracks: config.tracks.map(t => {
       if (t.id !== trackId) return t;
       return { ...t, tasks: t.tasks.filter(tk => tk.id !== taskId) };
     }),
   };
+
+  if (!cleanRefs) return withoutTask;
+
+  // Both bare ("taskId") and fully-qualified ("trackId.taskId") forms are valid refs
+  const qualId = `${trackId}.${taskId}`;
+  const isRemoved = (ref: string) => ref === taskId || ref === qualId;
+
+  return {
+    ...withoutTask,
+    tracks: withoutTask.tracks.map(t => ({
+      ...t,
+      tasks: t.tasks.map(tk => cleanTaskRefs(tk, isRemoved)),
+    })),
+  };
+}
+
+function cleanTaskRefs(
+  task: RawTaskConfig,
+  isRemoved: (ref: string) => boolean,
+): RawTaskConfig {
+  const filteredDeps = task.depends_on?.filter(d => !isRemoved(d));
+  const dropContinueFrom = task.continue_from !== undefined && isRemoved(task.continue_from);
+
+  const depsUnchanged = filteredDeps === undefined || filteredDeps.length === task.depends_on!.length;
+  if (depsUnchanged && !dropContinueFrom) return task;
+
+  const { depends_on, continue_from, ...rest } = task;
+  return {
+    ...rest,
+    ...(filteredDeps !== undefined && filteredDeps.length > 0 ? { depends_on: filteredDeps } : {}),
+    ...(!dropContinueFrom && continue_from !== undefined ? { continue_from } : {}),
+  } as RawTaskConfig;
 }
 
 /**

package/src/dag.ts

@@ -1,4 +1,4 @@
-import type { PipelineConfig, TaskConfig, TrackConfig } from './types';
+import type { PipelineConfig, RawPipelineConfig, RawTaskConfig, TaskConfig, TrackConfig } from './types';
 
 export interface DagNode {
   readonly taskId: string;   // fully qualified: track_id.task_id or just task_id
@@ -135,3 +135,88 @@ export function buildDag(config: PipelineConfig): Dag {
 
   return { nodes, sorted };
 }
+
+// ═══ Raw DAG (for visual editor — no workDir required) ═══
+
+export interface RawDagNode {
+  readonly taskId: string;        // fully qualified: track_id.task_id
+  readonly trackId: string;
+  readonly rawTask: RawTaskConfig;
+  readonly dependsOn: readonly string[];  // fully qualified IDs, best-effort resolved
+}
+
+export interface RawDag {
+  readonly nodes: ReadonlyMap<string, RawDagNode>;
+  /** Directed edges: from → to means "from must complete before to starts" */
+  readonly edges: readonly { readonly from: string; readonly to: string }[];
+}
+
+/**
+ * Build a lightweight DAG from a raw (unresolved) pipeline config.
+ * Unlike buildDag, this function:
+ *   - Does not require a workDir or resolved PipelineConfig
+ *   - Is lenient: missing or ambiguous refs are silently skipped
+ *   - Skips template-expansion tasks (those with a `use` field)
+ *
+ * Intended for the visual editor to render the flow graph before a pipeline is run.
+ */
+export function buildRawDag(config: RawPipelineConfig): RawDag {
+  const nodes = new Map<string, RawDagNode>();
+  const bareToQualified = new Map<string, string>();
+
+  // 1. Register all concrete tasks
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      if (task.use) continue; // template-expansion tasks are not yet materialized
+      const qid = `${track.id}.${task.id}`;
+      if (nodes.has(qid)) continue; // skip duplicates silently
+
+      if (bareToQualified.has(task.id)) {
+        bareToQualified.set(task.id, '__ambiguous__');
+      } else {
+        bareToQualified.set(task.id, qid);
+      }
+      nodes.set(qid, { taskId: qid, trackId: track.id, rawTask: task, dependsOn: [] });
+    }
+  }
+
+  // 2. Resolve dependency refs leniently (missing / ambiguous refs are skipped)
+  function tryResolve(ref: string, fromTrackId: string): string | null {
+    if (ref.includes('.')) return nodes.has(ref) ? ref : null;
+    const sameTrack = `${fromTrackId}.${ref}`;
+    if (nodes.has(sameTrack)) return sameTrack;
+    const global = bareToQualified.get(ref);
+    if (global && global !== '__ambiguous__') return global;
+    return null;
+  }
+
+  const edges: { from: string; to: string }[] = [];
+
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      if (task.use) continue;
+      const qid = `${track.id}.${task.id}`;
+      const deps: string[] = [];
+
+      for (const ref of task.depends_on ?? []) {
+        const resolved = tryResolve(ref, track.id);
+        if (resolved && !deps.includes(resolved)) {
+          deps.push(resolved);
+          edges.push({ from: resolved, to: qid });
+        }
+      }
+      if (task.continue_from) {
+        const resolved = tryResolve(task.continue_from, track.id);
+        if (resolved && !deps.includes(resolved)) {
+          deps.push(resolved);
+          edges.push({ from: resolved, to: qid });
+        }
+      }
+
+      const node = nodes.get(qid)!;
+      nodes.set(qid, { ...node, dependsOn: deps });
+    }
+  }
+
+  return { nodes, edges };
+}

package/src/engine.ts

@@ -97,8 +97,8 @@ export interface EngineResult {
 // ═══ Pipeline Events ═══
 
 export type PipelineEvent =
-  | { readonly type: 'task_status_change'; readonly taskId: string; readonly status: TaskStatus; readonly prevStatus: TaskStatus; readonly runId: string }
-  | { readonly type: 'pipeline_start'; readonly runId: string }
+  | { readonly type: 'task_status_change'; readonly taskId: string; readonly status: TaskStatus; readonly prevStatus: TaskStatus; readonly runId: string; readonly state: TaskState }
+  | { readonly type: 'pipeline_start'; readonly runId: string; readonly states: ReadonlyMap<string, TaskState> }
   | { readonly type: 'pipeline_end'; readonly runId: string; readonly success: boolean };
 
 export interface RunPipelineOptions {
@@ -198,7 +198,11 @@ export async function runPipeline(
   for (const [, state] of states) {
     state.status = 'waiting';
   }
-  options.onEvent?.({ type: 'pipeline_start', runId });
+  // Include a full states snapshot so listeners can initialize their mirrors without missing events
+  const statesSnapshot: ReadonlyMap<string, TaskState> = new Map(
+    [...states.entries()].map(([id, s]) => [id, { ...s }])
+  );
+  options.onEvent?.({ type: 'pipeline_start', runId, states: statesSnapshot });
 
   const sessionMap = new Map<string, string>();
   const outputMap = new Map<string, string>();
@@ -246,7 +250,16 @@ export async function runPipeline(
     const state = states.get(taskId)!;
     const prevStatus = state.status;
     state.status = newStatus;
-    emit({ type: 'task_status_change', taskId, status: newStatus, prevStatus, runId });
+    // Snapshot state at emit time — result and finishedAt must be set before calling this for terminal statuses
+    const snapshot: TaskState = {
+      config: state.config,
+      trackConfig: state.trackConfig,
+      status: state.status,
+      result: state.result,
+      startedAt: state.startedAt,
+      finishedAt: state.finishedAt,
+    };
+    emit({ type: 'task_status_change', taskId, status: newStatus, prevStatus, runId, state: snapshot });
   }
 
   function getOnFailure(taskId: string): OnFailure {
@@ -319,8 +332,8 @@ export async function runPipeline(
       if (result === 'skip') {
         const depStatus = states.get(depId)?.status ?? 'unknown';
         log.debug(`[task:${taskId}]`, `skipped (upstream "${depId}" status=${depStatus})`);
-        setTaskStatus(taskId, 'skipped');
         state.finishedAt = nowISO();
+        setTaskStatus(taskId, 'skipped');
         return;
       }
       if (result === 'unsatisfied') return; // still waiting
@@ -343,6 +356,7 @@ export async function runPipeline(
         const msg = err instanceof Error ? err.message : String(err);
         // If pipeline was aborted while we were still waiting for the trigger,
         // this task never entered running state → skipped, not timeout.
+        state.finishedAt = nowISO();
         if (pipelineAborted) {
           setTaskStatus(taskId, 'skipped');
         } else if (msg.includes('rejected') || msg.includes('denied')) {
@@ -352,7 +366,6 @@ export async function runPipeline(
         } else {
           setTaskStatus(taskId, 'failed');        // plugin error, watcher crash, etc.
         }
-        state.finishedAt = nowISO();
         await fireHook(taskId, 'task_failure');
         return;
       }
@@ -366,8 +379,8 @@ export async function runPipeline(
         `task_start hook exit=${hookResult.exitCode} allowed=${hookResult.allowed}`);
     }
     if (!hookResult.allowed) {
-      setTaskStatus(taskId, 'blocked');
       state.finishedAt = nowISO();
+      setTaskStatus(taskId, 'blocked');
       await fireHook(taskId, 'task_failure');
       return;
     }
@@ -443,18 +456,19 @@ export async function runPipeline(
         result = await runSpawn(spec, driver, runOpts);
       }
 
-      // 5. Determine status
+      // 5. Determine terminal status (without emitting yet — result must be complete first)
+      let terminalStatus: TaskStatus;
       if (result.exitCode === -1) {
-        setTaskStatus(taskId, 'timeout');
+        terminalStatus = 'timeout';
       } else if (result.exitCode !== 0) {
-        setTaskStatus(taskId, 'failed');
+        terminalStatus = 'failed';
       } else if (task.completion) {
         const plugin = getHandler<CompletionPlugin>('completions', task.completion.type);
         const completionCtx = { workDir: task.cwd ?? workDir };
         const passed = await plugin.check(task.completion as Record<string, unknown>, result, completionCtx);
-        setTaskStatus(taskId, passed ? 'success' : 'failed');
+        terminalStatus = passed ? 'success' : 'failed';
       } else {
-        setTaskStatus(taskId, 'success');
+        terminalStatus = 'success';
       }
 
       // 6. Write output file with RAW stdout (preserves driver output format).
@@ -488,16 +502,18 @@ export async function runPipeline(
         if (!sessionMap.has(bareId)) sessionMap.set(bareId, result.sessionId);
       }
 
+      // Set result and finishedAt before emitting terminal status so listeners see complete state
       state.result = result;
       state.finishedAt = nowISO();
+      setTaskStatus(taskId, terminalStatus);
 
       // Log task outcome with relevant details
       const durSec = (result.durationMs / 1000).toFixed(1);
-      if (state.status === 'success') {
+      if (terminalStatus === 'success') {
         log.info(`[task:${taskId}]`, `success (${durSec}s)`);
       } else {
         log.error(`[task:${taskId}]`,
-          `${state.status} exit=${result.exitCode} duration=${durSec}s`);
+          `${terminalStatus} exit=${result.exitCode} duration=${durSec}s`);
         if (result.stderr) {
           const tail = tailLines(result.stderr, 10);
           log.error(`[task:${taskId}]`, `stderr tail:\n${tail}`);
@@ -524,12 +540,10 @@ export async function runPipeline(
       }
       if (task.completion) {
         log.debug(`[task:${taskId}]`,
-          `completion check: type=${task.completion.type} result=${state.status}`);
+          `completion check: type=${task.completion.type} result=${terminalStatus}`);
       }
 
     } catch (err: unknown) {
-      setTaskStatus(taskId, 'failed');
-      state.finishedAt = nowISO();
       const errMsg = err instanceof Error ? (err.stack ?? err.message) : String(err);
       log.error(`[task:${taskId}]`, `failed before execution: ${errMsg}`);
       state.result = {
@@ -539,6 +553,8 @@ export async function runPipeline(
         outputPath: null, stderrPath: null, durationMs: 0,
         sessionId: null, normalizedOutput: null,
       };
+      state.finishedAt = nowISO();
+      setTaskStatus(taskId, 'failed');
     }
 
     // 7. Fire hooks
@@ -589,8 +605,8 @@ export async function runPipeline(
       for (const [id, state] of states) {
         if (!isTerminal(state.status)) {
           // Running tasks get timeout (they were killed); waiting tasks get skipped
-          setTaskStatus(id, state.status === 'running' ? 'timeout' : 'skipped');
           state.finishedAt = nowISO();
+          setTaskStatus(id, state.status === 'running' ? 'timeout' : 'skipped');
         }
       }
     }

package/src/pipeline-runner.ts

@@ -42,6 +42,7 @@ export class PipelineRunner {
   private _abortController = new AbortController();
   private _handlers = new Set<(event: PipelineEvent) => void>();
   private _states: ReadonlyMap<string, TaskState> | null = null;
+  private _statesMirror = new Map<string, TaskState>();
 
   constructor(
     private readonly config: PipelineConfig,
@@ -67,6 +68,14 @@ export class PipelineRunner {
       onEvent: (event) => {
         if (event.type === 'pipeline_start') {
           this._runId = event.runId;
+          // Initialize the live mirror with the full initial state snapshot
+          for (const [id, state] of event.states) {
+            this._statesMirror.set(id, { ...state });
+          }
+        }
+        if (event.type === 'task_status_change') {
+          // Keep the mirror up to date so getStates() works during the run
+          this._statesMirror.set(event.taskId, event.state);
         }
         if (event.type === 'pipeline_end') {
           this._status = this._abortController.signal.aborted ? 'aborted' : 'done';
@@ -94,11 +103,14 @@ export class PipelineRunner {
   }
 
   /**
-   * Snapshot of task states. Populated after the run completes.
-   * During a run, listen to subscribe() events for incremental updates.
+   * Live snapshot of task states. Available from the first pipeline_start event onward
+   * (i.e. as soon as start() is called) and remains accessible after the run completes.
+   * Returns null only if the pipeline has never started.
    */
   getStates(): ReadonlyMap<string, TaskState> | null {
-    return this._states;
+    if (this._states) return this._states;
+    if (this._statesMirror.size > 0) return this._statesMirror;
+    return null;
   }
 
   /**

package/src/sdk.ts

@@ -33,8 +33,8 @@ export type { ValidationError } from './validate-raw';
 export { parseYaml, resolveConfig, expandTemplates, loadPipeline, serializePipeline, deresolvePipeline, validateConfig } from './schema';
 
 // ── DAG ──
-export { buildDag } from './dag';
-export type { DagNode, Dag } from './dag';
+export { buildDag, buildRawDag } from './dag';
+export type { DagNode, Dag, RawDagNode, RawDag } from './dag';
 
 // ── Plugin registry ──
 export { bootstrapBuiltins } from './bootstrap';