@tagma/sdk 0.2.4 → 0.2.6

package/src/dag.ts

@@ -1,222 +1,222 @@
-import type { PipelineConfig, RawPipelineConfig, RawTaskConfig, TaskConfig, TrackConfig } from './types';
-
-export interface DagNode {
-  readonly taskId: string;   // fully qualified: track_id.task_id or just task_id
-  readonly task: TaskConfig;
-  readonly track: TrackConfig;
-  readonly dependsOn: readonly string[];
-}
-
-export interface Dag {
-  readonly nodes: ReadonlyMap<string, DagNode>;
-  readonly sorted: readonly string[];   // topological order
-}
-
-// Build a global task ID: for cross-track refs we use "track_id.task_id"
-// Within a track, bare "task_id" is also valid
-function qualifyId(trackId: string, taskId: string): string {
-  return `${trackId}.${taskId}`;
-}
-
-export function buildDag(config: PipelineConfig): Dag {
-  const nodes = new Map<string, DagNode>();
-  // Map bare task IDs to qualified IDs (for resolving unqualified refs)
-  const bareToQualified = new Map<string, string>();
-
-  // 1. Register all nodes
-  for (const track of config.tracks) {
-    for (const task of track.tasks) {
-      const qid = qualifyId(track.id, task.id);
-
-      if (nodes.has(qid)) {
-        throw new Error(`Duplicate task ID: "${qid}"`);
-      }
-
-      // Track bare ID → qualified. If same bare ID in multiple tracks, mark ambiguous
-      if (bareToQualified.has(task.id)) {
-        bareToQualified.set(task.id, '__ambiguous__');
-      } else {
-        bareToQualified.set(task.id, qid);
-      }
-
-      nodes.set(qid, {
-        taskId: qid,
-        task,
-        track,
-        dependsOn: [],  // filled below
-      });
-    }
-  }
-
-  // Helper to resolve a dependency ref to a qualified ID
-  function resolveRef(ref: string, fromTrackId: string): string {
-    // Already qualified (contains dot)
-    if (ref.includes('.')) {
-      if (!nodes.has(ref)) {
-        throw new Error(`Task reference "${ref}" not found`);
-      }
-      return ref;
-    }
-    // Try within same track first
-    const sameTrack = qualifyId(fromTrackId, ref);
-    if (nodes.has(sameTrack)) return sameTrack;
-    // Try global bare lookup
-    const global = bareToQualified.get(ref);
-    if (global && global !== '__ambiguous__') return global;
-    if (global === '__ambiguous__') {
-      throw new Error(
-        `Ambiguous task reference "${ref}" exists in multiple tracks. ` +
-        `Use "track_id.task_id" format.`
-      );
-    }
-    throw new Error(`Task reference "${ref}" not found`);
-  }
-
-  // 2. Resolve depends_on and continue_from to qualified IDs
-  for (const track of config.tracks) {
-    for (const task of track.tasks) {
-      const qid = qualifyId(track.id, task.id);
-      const deps: string[] = [];
-
-      if (task.depends_on) {
-        for (const dep of task.depends_on) {
-          deps.push(resolveRef(dep, track.id));
-        }
-      }
-      if (task.continue_from) {
-        const resolved = resolveRef(task.continue_from, track.id);
-        if (!deps.includes(resolved)) {
-          deps.push(resolved); // continue_from implies dependency
-        }
-      }
-
-      // Replace node with resolved deps
-      const node = nodes.get(qid)!;
-      nodes.set(qid, { ...node, dependsOn: deps });
-    }
-  }
-
-  // 3. Topological sort + cycle detection (Kahn's algorithm)
-  const inDegree = new Map<string, number>();
-  const adjacency = new Map<string, string[]>(); // parent → children
-
-  for (const [id] of nodes) {
-    inDegree.set(id, 0);
-    adjacency.set(id, []);
-  }
-
-  for (const [id, node] of nodes) {
-    for (const dep of node.dependsOn) {
-      adjacency.get(dep)!.push(id);
-      inDegree.set(id, (inDegree.get(id) ?? 0) + 1);
-    }
-  }
-
-  const queue: string[] = [];
-  for (const [id, degree] of inDegree) {
-    if (degree === 0) queue.push(id);
-  }
-
-  const sorted: string[] = [];
-  while (queue.length > 0) {
-    const current = queue.shift()!;
-    sorted.push(current);
-    for (const child of adjacency.get(current)!) {
-      const newDegree = inDegree.get(child)! - 1;
-      inDegree.set(child, newDegree);
-      if (newDegree === 0) queue.push(child);
-    }
-  }
-
-  if (sorted.length !== nodes.size) {
-    const remaining = [...nodes.keys()].filter(id => !sorted.includes(id));
-    throw new Error(`Circular dependency detected involving tasks: ${remaining.join(', ')}`);
-  }
-
-  return { nodes, sorted };
-}
-
-// ═══ Raw DAG (for visual editor — no workDir required) ═══
-
-export interface RawDagNode {
-  readonly taskId: string;        // fully qualified: track_id.task_id
-  readonly trackId: string;
-  readonly rawTask: RawTaskConfig;
-  readonly dependsOn: readonly string[];  // fully qualified IDs, best-effort resolved
-}
-
-export interface RawDag {
-  readonly nodes: ReadonlyMap<string, RawDagNode>;
-  /** Directed edges: from → to means "from must complete before to starts" */
-  readonly edges: readonly { readonly from: string; readonly to: string }[];
-}
-
-/**
- * Build a lightweight DAG from a raw (unresolved) pipeline config.
- * Unlike buildDag, this function:
- *   - Does not require a workDir or resolved PipelineConfig
- *   - Is lenient: missing or ambiguous refs are silently skipped
- *   - Skips template-expansion tasks (those with a `use` field)
- *
- * Intended for the visual editor to render the flow graph before a pipeline is run.
- */
-export function buildRawDag(config: RawPipelineConfig): RawDag {
-  const nodes = new Map<string, RawDagNode>();
-  const bareToQualified = new Map<string, string>();
-
-  // 1. Register all concrete tasks
-  for (const track of config.tracks) {
-    for (const task of track.tasks) {
-      if (task.use) continue; // template-expansion tasks are not yet materialized
-      const qid = `${track.id}.${task.id}`;
-      if (nodes.has(qid)) continue; // skip duplicates silently
-
-      if (bareToQualified.has(task.id)) {
-        bareToQualified.set(task.id, '__ambiguous__');
-      } else {
-        bareToQualified.set(task.id, qid);
-      }
-      nodes.set(qid, { taskId: qid, trackId: track.id, rawTask: task, dependsOn: [] });
-    }
-  }
-
-  // 2. Resolve dependency refs leniently (missing / ambiguous refs are skipped)
-  function tryResolve(ref: string, fromTrackId: string): string | null {
-    if (ref.includes('.')) return nodes.has(ref) ? ref : null;
-    const sameTrack = `${fromTrackId}.${ref}`;
-    if (nodes.has(sameTrack)) return sameTrack;
-    const global = bareToQualified.get(ref);
-    if (global && global !== '__ambiguous__') return global;
-    return null;
-  }
-
-  const edges: { from: string; to: string }[] = [];
-
-  for (const track of config.tracks) {
-    for (const task of track.tasks) {
-      if (task.use) continue;
-      const qid = `${track.id}.${task.id}`;
-      const deps: string[] = [];
-
-      for (const ref of task.depends_on ?? []) {
-        const resolved = tryResolve(ref, track.id);
-        if (resolved && !deps.includes(resolved)) {
-          deps.push(resolved);
-          edges.push({ from: resolved, to: qid });
-        }
-      }
-      if (task.continue_from) {
-        const resolved = tryResolve(task.continue_from, track.id);
-        if (resolved && !deps.includes(resolved)) {
-          deps.push(resolved);
-          edges.push({ from: resolved, to: qid });
-        }
-      }
-
-      const node = nodes.get(qid)!;
-      nodes.set(qid, { ...node, dependsOn: deps });
-    }
-  }
-
-  return { nodes, edges };
-}
+import type { PipelineConfig, RawPipelineConfig, RawTaskConfig, TaskConfig, TrackConfig } from './types';
+
+export interface DagNode {
+  readonly taskId: string;   // fully qualified: track_id.task_id or just task_id
+  readonly task: TaskConfig;
+  readonly track: TrackConfig;
+  readonly dependsOn: readonly string[];
+}
+
+export interface Dag {
+  readonly nodes: ReadonlyMap<string, DagNode>;
+  readonly sorted: readonly string[];   // topological order
+}
+
+// Build a global task ID: for cross-track refs we use "track_id.task_id"
+// Within a track, bare "task_id" is also valid
+function qualifyId(trackId: string, taskId: string): string {
+  return `${trackId}.${taskId}`;
+}
+
+export function buildDag(config: PipelineConfig): Dag {
+  const nodes = new Map<string, DagNode>();
+  // Map bare task IDs to qualified IDs (for resolving unqualified refs)
+  const bareToQualified = new Map<string, string>();
+
+  // 1. Register all nodes
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      const qid = qualifyId(track.id, task.id);
+
+      if (nodes.has(qid)) {
+        throw new Error(`Duplicate task ID: "${qid}"`);
+      }
+
+      // Track bare ID → qualified. If same bare ID in multiple tracks, mark ambiguous
+      if (bareToQualified.has(task.id)) {
+        bareToQualified.set(task.id, '__ambiguous__');
+      } else {
+        bareToQualified.set(task.id, qid);
+      }
+
+      nodes.set(qid, {
+        taskId: qid,
+        task,
+        track,
+        dependsOn: [],  // filled below
+      });
+    }
+  }
+
+  // Helper to resolve a dependency ref to a qualified ID
+  function resolveRef(ref: string, fromTrackId: string): string {
+    // Already qualified (contains dot)
+    if (ref.includes('.')) {
+      if (!nodes.has(ref)) {
+        throw new Error(`Task reference "${ref}" not found`);
+      }
+      return ref;
+    }
+    // Try within same track first
+    const sameTrack = qualifyId(fromTrackId, ref);
+    if (nodes.has(sameTrack)) return sameTrack;
+    // Try global bare lookup
+    const global = bareToQualified.get(ref);
+    if (global && global !== '__ambiguous__') return global;
+    if (global === '__ambiguous__') {
+      throw new Error(
+        `Ambiguous task reference "${ref}" exists in multiple tracks. ` +
+        `Use "track_id.task_id" format.`
+      );
+    }
+    throw new Error(`Task reference "${ref}" not found`);
+  }
+
+  // 2. Resolve depends_on and continue_from to qualified IDs
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      const qid = qualifyId(track.id, task.id);
+      const deps: string[] = [];
+
+      if (task.depends_on) {
+        for (const dep of task.depends_on) {
+          deps.push(resolveRef(dep, track.id));
+        }
+      }
+      if (task.continue_from) {
+        const resolved = resolveRef(task.continue_from, track.id);
+        if (!deps.includes(resolved)) {
+          deps.push(resolved); // continue_from implies dependency
+        }
+      }
+
+      // Replace node with resolved deps
+      const node = nodes.get(qid)!;
+      nodes.set(qid, { ...node, dependsOn: deps });
+    }
+  }
+
+  // 3. Topological sort + cycle detection (Kahn's algorithm)
+  const inDegree = new Map<string, number>();
+  const adjacency = new Map<string, string[]>(); // parent → children
+
+  for (const [id] of nodes) {
+    inDegree.set(id, 0);
+    adjacency.set(id, []);
+  }
+
+  for (const [id, node] of nodes) {
+    for (const dep of node.dependsOn) {
+      adjacency.get(dep)!.push(id);
+      inDegree.set(id, (inDegree.get(id) ?? 0) + 1);
+    }
+  }
+
+  const queue: string[] = [];
+  for (const [id, degree] of inDegree) {
+    if (degree === 0) queue.push(id);
+  }
+
+  const sorted: string[] = [];
+  while (queue.length > 0) {
+    const current = queue.shift()!;
+    sorted.push(current);
+    for (const child of adjacency.get(current)!) {
+      const newDegree = inDegree.get(child)! - 1;
+      inDegree.set(child, newDegree);
+      if (newDegree === 0) queue.push(child);
+    }
+  }
+
+  if (sorted.length !== nodes.size) {
+    const remaining = [...nodes.keys()].filter(id => !sorted.includes(id));
+    throw new Error(`Circular dependency detected involving tasks: ${remaining.join(', ')}`);
+  }
+
+  return { nodes, sorted };
+}
+
+// ═══ Raw DAG (for visual editor — no workDir required) ═══
+
+export interface RawDagNode {
+  readonly taskId: string;        // fully qualified: track_id.task_id
+  readonly trackId: string;
+  readonly rawTask: RawTaskConfig;
+  readonly dependsOn: readonly string[];  // fully qualified IDs, best-effort resolved
+}
+
+export interface RawDag {
+  readonly nodes: ReadonlyMap<string, RawDagNode>;
+  /** Directed edges: from → to means "from must complete before to starts" */
+  readonly edges: readonly { readonly from: string; readonly to: string }[];
+}
+
+/**
+ * Build a lightweight DAG from a raw (unresolved) pipeline config.
+ * Unlike buildDag, this function:
+ *   - Does not require a workDir or resolved PipelineConfig
+ *   - Is lenient: missing or ambiguous refs are silently skipped
+ *   - Skips template-expansion tasks (those with a `use` field)
+ *
+ * Intended for the visual editor to render the flow graph before a pipeline is run.
+ */
+export function buildRawDag(config: RawPipelineConfig): RawDag {
+  const nodes = new Map<string, RawDagNode>();
+  const bareToQualified = new Map<string, string>();
+
+  // 1. Register all concrete tasks
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      if (task.use) continue; // template-expansion tasks are not yet materialized
+      const qid = `${track.id}.${task.id}`;
+      if (nodes.has(qid)) continue; // skip duplicates silently
+
+      if (bareToQualified.has(task.id)) {
+        bareToQualified.set(task.id, '__ambiguous__');
+      } else {
+        bareToQualified.set(task.id, qid);
+      }
+      nodes.set(qid, { taskId: qid, trackId: track.id, rawTask: task, dependsOn: [] });
+    }
+  }
+
+  // 2. Resolve dependency refs leniently (missing / ambiguous refs are skipped)
+  function tryResolve(ref: string, fromTrackId: string): string | null {
+    if (ref.includes('.')) return nodes.has(ref) ? ref : null;
+    const sameTrack = `${fromTrackId}.${ref}`;
+    if (nodes.has(sameTrack)) return sameTrack;
+    const global = bareToQualified.get(ref);
+    if (global && global !== '__ambiguous__') return global;
+    return null;
+  }
+
+  const edges: { from: string; to: string }[] = [];
+
+  for (const track of config.tracks) {
+    for (const task of track.tasks) {
+      if (task.use) continue;
+      const qid = `${track.id}.${task.id}`;
+      const deps: string[] = [];
+
+      for (const ref of task.depends_on ?? []) {
+        const resolved = tryResolve(ref, track.id);
+        if (resolved && !deps.includes(resolved)) {
+          deps.push(resolved);
+          edges.push({ from: resolved, to: qid });
+        }
+      }
+      if (task.continue_from) {
+        const resolved = tryResolve(task.continue_from, track.id);
+        if (resolved && !deps.includes(resolved)) {
+          deps.push(resolved);
+          edges.push({ from: resolved, to: qid });
+        }
+      }
+
+      const node = nodes.get(qid)!;
+      nodes.set(qid, { ...node, dependsOn: deps });
+    }
+  }
+
+  return { nodes, edges };
+}

package/src/engine.ts

@@ -16,7 +16,7 @@ import {
   buildPipelineCompleteContext, buildPipelineErrorContext,
   type PipelineInfo, type TrackInfo, type TaskInfo,
 } from './hooks';
-import { Logger, tailLines, clip } from './logger';
+import { Logger, tailLines, clip, type LogLevel } from './logger';
 import { InMemoryApprovalGateway, type ApprovalGateway } from './approval';
 
 // ═══ Preflight Validation ═══
@@ -115,7 +115,15 @@ export interface EngineResult {
 export type PipelineEvent =
   | { readonly type: 'task_status_change'; readonly taskId: string; readonly status: TaskStatus; readonly prevStatus: TaskStatus; readonly runId: string; readonly state: TaskState }
   | { readonly type: 'pipeline_start'; readonly runId: string; readonly states: ReadonlyMap<string, TaskState> }
-  | { readonly type: 'pipeline_end'; readonly runId: string; readonly success: boolean };
+  | { readonly type: 'pipeline_end'; readonly runId: string; readonly success: boolean }
+  /**
+   * Fine-grained log line emitted alongside every write to pipeline.log.
+   * Consumers use this to stream the full run process into UIs without
+   * tailing the log file. `taskId` is non-null for task-scoped lines and
+   * null for pipeline-wide messages (e.g. configuration dumps, DAG
+   * topology, pipeline start/end).
+   */
+  | { readonly type: 'task_log'; readonly runId: string; readonly taskId: string | null; readonly level: LogLevel; readonly timestamp: string; readonly text: string };
 
 export interface RunPipelineOptions {
   readonly approvalGateway?: ApprovalGateway;
@@ -160,7 +168,19 @@ export async function runPipeline(
 
   const startedAt = nowISO();
   const pipelineInfo: PipelineInfo = { name: config.name, run_id: runId, started_at: startedAt };
-  const log = new Logger(workDir, runId);
+  // Forward every structured log line to subscribers as task_log events.
+  // Reading options.onEvent inside the callback (vs. capturing it once) keeps
+  // the SDK behavior correct if callers pass a fresh onEvent on each run.
+  const log = new Logger(workDir, runId, (record) => {
+    options.onEvent?.({
+      type: 'task_log',
+      runId,
+      taskId: record.taskId,
+      level: record.level,
+      timestamp: record.timestamp,
+      text: record.text,
+    });
+  });
   log.info('[pipeline]', `start "${config.name}" run_id=${runId}`);
 
   // File-only: dump the resolved pipeline shape + DAG topology for post-mortem.
@@ -352,7 +372,7 @@ export async function runPipeline(
     const task = node.task;
     const track = node.track;
 
-    log.section(`Task ${taskId}`);
+    log.section(`Task ${taskId}`, taskId);
     log.debug(`[task:${taskId}]`,
       `type=${task.prompt ? 'ai' : 'cmd'} track=${track.id} deps=[${node.dependsOn.join(', ') || '(root)'}]`);
 
@@ -469,7 +489,7 @@ export async function runPipeline(
         }
         log.debug(`[task:${taskId}]`,
           `prompt: ${originalLen} chars (final: ${prompt.length} chars)`);
-        log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`);
+        log.quiet(`--- prompt (final) ---\n${clip(prompt)}\n--- end prompt ---`, taskId);
 
         const enrichedTask: TaskConfig = { ...task, prompt };
         const driverCtx: DriverContext = {
@@ -565,10 +585,10 @@ export async function runPipeline(
         log.debug(`[task:${taskId}]`, `wrote stderr: ${result.stderrPath}`);
       }
       if (result.stdout) {
-        log.quiet(`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`);
+        log.quiet(`--- stdout (${taskId}) ---\n${clip(result.stdout)}\n--- end stdout ---`, taskId);
       }
       if (result.stderr) {
-        log.quiet(`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`);
+        log.quiet(`--- stderr (${taskId}) ---\n${clip(result.stderr)}\n--- end stderr ---`, taskId);
       }
       if (task.completion) {
         log.debug(`[task:${taskId}]`,