chainlesschain 0.162.71 → 0.162.73

package/src/lib/cowork-workflow.js

@@ -625,6 +625,206 @@ export async function runLoopStep({
   };
 }
 
+// ─── Step node + no-barrier pipeline ──────────────────────────────────────────
+
+/**
+ * Run a single step to completion, resolving its when-gate / loop / forEach /
+ * plain task exactly as the batch executor does, and storing results (including
+ * a forEach parent aggregate) in `resultsById`. Returns `{ outcomes, failed }`:
+ * `outcomes` are the entries to add to the run's step list (forEach contributes
+ * its children, not the parent); `failed` is true when any non-skipped outcome
+ * is not "completed". Used by the no-barrier pipeline scheduler.
+ */
+export async function runStepNode(step, ctx) {
+  const { resultsById, cwd, llmOptions, onStepStart, onStepComplete } = ctx;
+  const recordId = step.id;
+  const single = (status, summary) => {
+    const o = { id: recordId, status, taskId: null, result: { summary } };
+    resultsById.set(recordId, o);
+    return {
+      outcomes: [o],
+      failed: status !== "completed" && status !== "skipped",
+    };
+  };
+
+  let runThis;
+  try {
+    runThis = shouldRunStep(step, resultsById);
+  } catch (err) {
+    return single("failed", err.message);
+  }
+  if (!runThis) return single("skipped", "when-condition false");
+
+  if (isLoopStep(step)) {
+    if (onStepStart) onStepStart({ stepId: recordId, message: step.message });
+    const o = await runLoopStep({
+      step,
+      recordId,
+      cwd,
+      llmOptions,
+      resultsById,
+    });
+    resultsById.set(recordId, o);
+    if (onStepComplete) onStepComplete(o);
+    return {
+      outcomes: [o],
+      failed: o.status !== "completed" && o.status !== "skipped",
+    };
+  }
+
+  if (step.forEach !== undefined) {
+    let items;
+    try {
+      items = resolveForEachItems(step.forEach, resultsById);
+    } catch (err) {
+      return single("failed", err.message);
+    }
+    if (items.length === 0) return single("skipped", "forEach items empty");
+    const children = await Promise.all(
+      items.map(async (item, k) => {
+        const childId = `${recordId}[${k}]`;
+        const withItem = substituteItem(step.message, item);
+        const msg = substitutePlaceholders(withItem, resultsById);
+        if (onStepStart) onStepStart({ stepId: childId, message: msg });
+        const r = await runStepWithRetry({
+          step,
+          message: msg,
+          cwd,
+          llmOptions,
+        });
+        const co = outcomeFromRetry(childId, r);
+        resultsById.set(childId, co);
+        if (onStepComplete) onStepComplete(co);
+        return co;
+      }),
+    );
+    const allOk = children.every((c) => c.status === "completed");
+    const anyOk = children.some((c) => c.status === "completed");
+    resultsById.set(recordId, {
+      id: recordId,
+      status: allOk ? "completed" : anyOk ? "partial" : "failed",
+      taskId: null,
+      result: {
+        summary: children.map((c) => c.result?.summary ?? "").join("\n"),
+        children: children.length,
+      },
+    });
+    return {
+      outcomes: children,
+      failed: children.some((c) => c.status !== "completed"),
+    };
+  }
+
+  const message = substitutePlaceholders(step.message, resultsById);
+  if (onStepStart) onStepStart({ stepId: recordId, message });
+  const r = await runStepWithRetry({ step, message, cwd, llmOptions });
+  const o = outcomeFromRetry(recordId, r);
+  resultsById.set(recordId, o);
+  if (onStepComplete) onStepComplete(o);
+  return { outcomes: [o], failed: o.status !== "completed" };
+}
+
+/**
+ * No-barrier pipeline scheduler: start each step the instant *its own*
+ * dependencies finish, rather than waiting for the whole dependency level.
+ * Up to `maxParallel` step nodes run concurrently. On failure with
+ * `continueOnError` off, no new steps are scheduled (in-flight ones finish) and
+ * the rest are marked skipped. Produces the same outcome set as the batch
+ * executor — only the wall-clock idle between levels is removed.
+ */
+export async function runPipeline({
+  steps,
+  resultsById,
+  maxParallel = 4,
+  continueOnError = false,
+  cwd,
+  llmOptions = {},
+  onStepStart,
+  onStepComplete,
+}) {
+  const limit = Math.max(1, Math.floor(maxParallel) || 1);
+  const remainingDeps = new Map(
+    steps.map((s) => [s.id, new Set(s.dependsOn || [])]),
+  );
+  const dependents = new Map(steps.map((s) => [s.id, []]));
+  for (const s of steps) {
+    for (const d of s.dependsOn || []) {
+      if (dependents.has(d)) dependents.get(d).push(s.id);
+    }
+  }
+  const scheduled = new Set();
+  const stepOutcomes = [];
+  let anyFailure = false;
+  let halted = false;
+  let active = 0;
+
+  await new Promise((resolve) => {
+    function pump() {
+      if (!halted) {
+        for (const s of steps) {
+          if (active >= limit) break;
+          if (scheduled.has(s.id)) continue;
+          if (remainingDeps.get(s.id).size !== 0) continue;
+          scheduled.add(s.id);
+          active++;
+          launch(s);
+        }
+      }
+      if (active === 0 && (halted || scheduled.size === steps.length)) {
+        for (const s of steps) {
+          if (scheduled.has(s.id)) continue;
+          const o = {
+            id: s.id,
+            status: "skipped",
+            taskId: null,
+            result: { summary: "skipped due to earlier failure" },
+          };
+          resultsById.set(s.id, o);
+          stepOutcomes.push(o);
+          scheduled.add(s.id);
+        }
+        resolve();
+      }
+    }
+
+    async function launch(step) {
+      let res;
+      try {
+        res = await runStepNode(step, {
+          resultsById,
+          cwd,
+          llmOptions,
+          onStepStart,
+          onStepComplete,
+        });
+      } catch (err) {
+        const o = {
+          id: step.id,
+          status: "failed",
+          taskId: null,
+          result: { summary: `Step threw: ${err.message}` },
+        };
+        resultsById.set(step.id, o);
+        res = { outcomes: [o], failed: true };
+      }
+      stepOutcomes.push(...res.outcomes);
+      active--;
+      if (res.failed) {
+        anyFailure = true;
+        if (!continueOnError) halted = true;
+      }
+      for (const depId of dependents.get(step.id) || []) {
+        remainingDeps.get(depId).delete(step.id);
+      }
+      pump();
+    }
+
+    pump();
+  });
+
+  return { stepOutcomes, anyFailure };
+}
+
 // ─── Execution ───────────────────────────────────────────────────────────────
 
 /**
@@ -646,8 +846,12 @@ export async function runLoopStep({
  * @param {object} options
  * @param {object} options.workflow - Workflow definition
  * @param {string} [options.cwd] - Working directory for history
- * @param {number} [options.maxParallel] - Max parallel steps per batch
+ * @param {number} [options.maxParallel] - Max parallel steps (per batch, or
+ *   concurrent step nodes in pipeline mode)
  * @param {boolean} [options.continueOnError] - Keep running after a failure
+ * @param {boolean} [options.pipeline] - No-barrier scheduling: start each step
+ *   as soon as its own deps finish instead of waiting for the dependency level.
+ *   Defaults to `workflow.pipeline ?? false`. Same outcomes, less idle wait.
  * @param {object} [options.llmOptions] - Forwarded to each task
  * @param {function} [options.onStepStart]
  * @param {function} [options.onStepComplete]
@@ -668,6 +872,7 @@ export async function executeWorkflow(options = {}) {
     llmOptions = {},
     onStepStart,
     onStepComplete,
+    pipeline,
   } = options;
 
   const { valid, errors } = validateWorkflow(workflow);
@@ -678,79 +883,54 @@ export async function executeWorkflow(options = {}) {
     );
   }
 
-  const batches = planBatches(workflow.steps);
+  const usePipeline = pipeline ?? workflow.pipeline ?? false;
   const resultsById = new Map();
-  const stepOutcomes = [];
   const startedAt = new Date(_deps.now()).toISOString();
-  let anyFailure = false;
+  let stepOutcomes;
+  let anyFailure;
 
-  for (const batch of batches) {
-    // Respect maxParallel by slicing batch into chunks
-    const chunks = [];
-    for (let i = 0; i < batch.length; i += maxParallel) {
-      chunks.push(batch.slice(i, i + maxParallel));
-    }
+  if (usePipeline) {
+    ({ stepOutcomes, anyFailure } = await runPipeline({
+      steps: workflow.steps,
+      resultsById,
+      maxParallel,
+      continueOnError,
+      cwd,
+      llmOptions,
+      onStepStart,
+      onStepComplete,
+    }));
+  } else {
+    stepOutcomes = [];
+    anyFailure = false;
+    const batches = planBatches(workflow.steps);
+    for (const batch of batches) {
+      // Respect maxParallel by slicing batch into chunks
+      const chunks = [];
+      for (let i = 0; i < batch.length; i += maxParallel) {
+        chunks.push(batch.slice(i, i + maxParallel));
+      }
 
-    for (const chunk of chunks) {
-      // Expand forEach / when into concrete tasks for this chunk
-      const runnable = []; // { step, message, recordId, parentId }
-      const preOutcomes = []; // outcomes produced synchronously (skipped)
-      for (const step of chunk) {
-        if (anyFailure && !continueOnError) {
-          const outcome = {
-            id: step.id,
-            status: "skipped",
-            taskId: null,
-            result: { summary: "skipped due to earlier failure" },
-          };
-          resultsById.set(step.id, outcome);
-          preOutcomes.push(outcome);
-          continue;
-        }
-        // when-gate
-        let runThis = true;
-        try {
-          runThis = shouldRunStep(step, resultsById);
-        } catch (err) {
-          anyFailure = true;
-          const outcome = {
-            id: step.id,
-            status: "failed",
-            taskId: null,
-            result: { summary: err.message },
-          };
-          resultsById.set(step.id, outcome);
-          preOutcomes.push(outcome);
-          continue;
-        }
-        if (!runThis) {
-          const outcome = {
-            id: step.id,
-            status: "skipped",
-            taskId: null,
-            result: { summary: "when-condition false" },
-          };
-          resultsById.set(step.id, outcome);
-          preOutcomes.push(outcome);
-          continue;
-        }
-        // loop node — runs its body repeatedly; per-iteration substitution
-        // happens inside runLoopStep, so push the raw template.
-        if (isLoopStep(step)) {
-          runnable.push({
-            step,
-            message: step.message,
-            recordId: step.id,
-            parentId: null,
-            isLoop: true,
-          });
-          continue;
-        }
-        // forEach-expansion
-        if (step.forEach !== undefined) {
-          let items;
+      for (const chunk of chunks) {
+        // Expand forEach / when into concrete tasks for this chunk
+        const runnable = []; // { step, message, recordId, parentId }
+        const preOutcomes = []; // outcomes produced synchronously (skipped)
+        for (const step of chunk) {
+          if (anyFailure && !continueOnError) {
+            const outcome = {
+              id: step.id,
+              status: "skipped",
+              taskId: null,
+              result: { summary: "skipped due to earlier failure" },
+            };
+            resultsById.set(step.id, outcome);
+            preOutcomes.push(outcome);
+            continue;
+          }
+          // when-gate
+          let runThis = true;
           try {
-            items = resolveForEachItems(step.forEach, resultsById);
+            runThis = shouldRunStep(step, resultsById);
           } catch (err) {
             anyFailure = true;
             const outcome = {
@@ -763,91 +943,132 @@ export async function executeWorkflow(options = {}) {
             preOutcomes.push(outcome);
             continue;
           }
-          if (items.length === 0) {
+          if (!runThis) {
             const outcome = {
               id: step.id,
               status: "skipped",
               taskId: null,
-              result: { summary: "forEach items empty" },
+              result: { summary: "when-condition false" },
             };
             resultsById.set(step.id, outcome);
             preOutcomes.push(outcome);
             continue;
           }
-          for (let k = 0; k < items.length; k++) {
-            const childId = `${step.id}[${k}]`;
-            const withItem = substituteItem(step.message, items[k]);
-            const msg = substitutePlaceholders(withItem, resultsById);
+          // loop node — runs its body repeatedly; per-iteration substitution
+          // happens inside runLoopStep, so push the raw template.
+          if (isLoopStep(step)) {
             runnable.push({
               step,
-              message: msg,
-              recordId: childId,
-              parentId: step.id,
+              message: step.message,
+              recordId: step.id,
+              parentId: null,
+              isLoop: true,
             });
+            continue;
           }
-          continue;
-        }
-        const message = substitutePlaceholders(step.message, resultsById);
-        runnable.push({ step, message, recordId: step.id, parentId: null });
-      }
-
-      const promises = runnable.map(
-        async ({ step, message, recordId, isLoop }) => {
-          if (onStepStart) onStepStart({ stepId: recordId, message });
-          let outcome;
-          if (isLoop) {
-            outcome = await runLoopStep({
-              step,
-              recordId,
-              cwd,
-              llmOptions,
-              resultsById,
-            });
-          } else {
-            const r = await runStepWithRetry({
-              step,
-              message,
-              cwd,
-              llmOptions,
-            });
-            outcome = outcomeFromRetry(recordId, r);
+          // forEach-expansion
+          if (step.forEach !== undefined) {
+            let items;
+            try {
+              items = resolveForEachItems(step.forEach, resultsById);
+            } catch (err) {
+              anyFailure = true;
+              const outcome = {
+                id: step.id,
+                status: "failed",
+                taskId: null,
+                result: { summary: err.message },
+              };
+              resultsById.set(step.id, outcome);
+              preOutcomes.push(outcome);
+              continue;
+            }
+            if (items.length === 0) {
+              const outcome = {
+                id: step.id,
+                status: "skipped",
+                taskId: null,
+                result: { summary: "forEach items empty" },
+              };
+              resultsById.set(step.id, outcome);
+              preOutcomes.push(outcome);
+              continue;
+            }
+            for (let k = 0; k < items.length; k++) {
+              const childId = `${step.id}[${k}]`;
+              const withItem = substituteItem(step.message, items[k]);
+              const msg = substitutePlaceholders(withItem, resultsById);
+              runnable.push({
+                step,
+                message: msg,
+                recordId: childId,
+                parentId: step.id,
+              });
+            }
+            continue;
           }
-          if (outcome.status !== "completed") anyFailure = true;
-          resultsById.set(recordId, outcome);
-          if (onStepComplete) onStepComplete(outcome);
-          return outcome;
-        },
-      );
+          const message = substitutePlaceholders(step.message, resultsById);
+          runnable.push({ step, message, recordId: step.id, parentId: null });
+        }
 
-      const results = await Promise.all(promises);
-      stepOutcomes.push(...preOutcomes, ...results);
-
-      // Aggregate forEach children into a parent entry so downstream
-      // `${step.<parent>.summary}` references still work.
-      const byParent = new Map();
-      for (let k = 0; k < runnable.length; k++) {
-        const r = runnable[k];
-        if (!r.parentId) continue;
-        if (!byParent.has(r.parentId)) byParent.set(r.parentId, []);
-        byParent.get(r.parentId).push(results[k]);
-      }
-      for (const [parentId, children] of byParent) {
-        const allOk = children.every((c) => c.status === "completed");
-        const anyOk = children.some((c) => c.status === "completed");
-        const status = allOk ? "completed" : anyOk ? "partial" : "failed";
-        resultsById.set(parentId, {
-          id: parentId,
-          status,
-          taskId: null,
-          result: {
-            summary: children.map((c) => c.result?.summary ?? "").join("\n"),
-            children: children.length,
+        const promises = runnable.map(
+          async ({ step, message, recordId, isLoop }) => {
+            if (onStepStart) onStepStart({ stepId: recordId, message });
+            let outcome;
+            if (isLoop) {
+              outcome = await runLoopStep({
+                step,
+                recordId,
+                cwd,
+                llmOptions,
+                resultsById,
+              });
+            } else {
+              const r = await runStepWithRetry({
+                step,
+                message,
+                cwd,
+                llmOptions,
+              });
+              outcome = outcomeFromRetry(recordId, r);
+            }
+            if (outcome.status !== "completed") anyFailure = true;
+            resultsById.set(recordId, outcome);
+            if (onStepComplete) onStepComplete(outcome);
+            return outcome;
           },
-        });
+        );
+
+        const results = await Promise.all(promises);
+        stepOutcomes.push(...preOutcomes, ...results);
+
+        // Aggregate forEach children into a parent entry so downstream
+        // `${step.<parent>.summary}` references still work.
+        const byParent = new Map();
+        for (let k = 0; k < runnable.length; k++) {
+          const r = runnable[k];
+          if (!r.parentId) continue;
+          if (!byParent.has(r.parentId)) byParent.set(r.parentId, []);
+          byParent.get(r.parentId).push(results[k]);
+        }
+        for (const [parentId, children] of byParent) {
+          const allOk = children.every((c) => c.status === "completed");
+          const anyOk = children.some((c) => c.status === "completed");
+          const status = allOk ? "completed" : anyOk ? "partial" : "failed";
+          resultsById.set(parentId, {
+            id: parentId,
+            status,
+            taskId: null,
+            result: {
+              summary: children.map((c) => c.result?.summary ?? "").join("\n"),
+              children: children.length,
+            },
+          });
+        }
       }
-    }
 
-    if (anyFailure && !continueOnError) break;
+      if (anyFailure && !continueOnError) break;
+    }
   }
 
   const finishedAt = new Date(_deps.now()).toISOString();

package/src/lib/micro-compact.js

@@ -0,0 +1,52 @@
+/**
+ * Micro-compaction (Claude-Code `/microcompact` parity). Where full compaction
+ * summarizes the whole history when it grows large, micro-compaction is
+ * surgical: it trims the bulkiest OLD items — large tool results — in place,
+ * keeping the conversation flow and the most-recent messages verbatim. Because
+ * it only SHORTENS a tool message's content (never removes the message), it can
+ * never orphan a tool_call→tool_result pair, so it's safe to run any time.
+ *
+ * Pure → unit-testable.
+ */
+
+const DEFAULT_KEEP_RECENT = 6; // last N messages kept verbatim
+const DEFAULT_MAX_TOOL_CHARS = 400; // trim older tool results longer than this
+
+/**
+ * @param {Array} messages  OpenAI-shaped messages ({role, content, …}).
+ * @param {object} [opts]    { keepRecent, maxToolChars }
+ * @returns {{ messages: Array, stats: {trimmed:number, saved:number, kept:number} }}
+ *          A NEW array (originals untouched); `saved` ≈ characters freed.
+ */
+export function microCompact(messages, opts = {}) {
+  if (!Array.isArray(messages)) {
+    return { messages, stats: { trimmed: 0, saved: 0, kept: 0 } };
+  }
+  const keepRecent = Number.isFinite(opts.keepRecent)
+    ? opts.keepRecent
+    : DEFAULT_KEEP_RECENT;
+  const maxChars = Number.isFinite(opts.maxToolChars)
+    ? opts.maxToolChars
+    : DEFAULT_MAX_TOOL_CHARS;
+  // messages at index >= cutoff are "recent" and never touched.
+  const cutoff = Math.max(0, messages.length - keepRecent);
+
+  let trimmed = 0;
+  let saved = 0;
+  const out = messages.map((m, i) => {
+    if (i >= cutoff) return m; // recent → verbatim
+    if (!m || m.role !== "tool") return m; // only old tool results
+    const content = typeof m.content === "string" ? m.content : "";
+    if (content.length <= maxChars) return m; // small enough already
+    trimmed += 1;
+    saved += content.length - maxChars;
+    const head = content.slice(0, maxChars);
+    return {
+      ...m,
+      content: `${head}\n… [tool result trimmed — ${content.length} chars; older context dropped to save space]`,
+      _microCompacted: true,
+    };
+  });
+
+  return { messages: out, stats: { trimmed, saved, kept: keepRecent } };
+}

package/src/lib/output-styles.js

@@ -17,6 +17,7 @@
 import fsDefault from "node:fs";
 import pathDefault from "node:path";
 import { homedir as homedirDefault } from "node:os";
+import { projectRootBase } from "./project-root.cjs";
 
 export const _deps = {
   fs: fsDefault,
@@ -46,7 +47,8 @@ export const BUILTIN_OUTPUT_STYLES = Object.freeze({
   },
   learning: {
     name: "learning",
-    description: "Collaborative — leaves small instructive pieces for the user.",
+    description:
+      "Collaborative — leaves small instructive pieces for the user.",
     body: [
       "## Output style: Learning",
       "Work collaboratively. When a small, well-scoped piece of the task would be",
@@ -82,11 +84,31 @@ function parseFrontmatter(content) {
 /** Directories scanned for style files (project first, then personal). */
 function styleDirs(cwd, home) {
   const { path } = _deps;
-  return [
-    { dir: path.join(cwd, ".chainlesschain", "output-styles"), scope: "project" },
+  const dirs = [
+    {
+      dir: path.join(cwd, ".chainlesschain", "output-styles"),
+      scope: "project",
+    },
     { dir: path.join(cwd, ".claude", "output-styles"), scope: "project" },
-    { dir: path.join(home, ".claude", "output-styles"), scope: "personal" },
   ];
+  // Subdirectory run: also scan the project-root `.claude` (closest cwd wins on
+  // a name clash via the reverse + last-write-wins in discoverOutputStyles).
+  const root = projectRootBase(cwd, { fs: _deps.fs, path });
+  if (root) {
+    dirs.push({
+      dir: path.join(root, ".chainlesschain", "output-styles"),
+      scope: "project",
+    });
+    dirs.push({
+      dir: path.join(root, ".claude", "output-styles"),
+      scope: "project",
+    });
+  }
+  dirs.push({
+    dir: path.join(home, ".claude", "output-styles"),
+    scope: "personal",
+  });
+  return dirs;
 }
 
 /** Discover all styles: built-ins + files (a file shadows a built-in by name). */
@@ -155,7 +177,11 @@ export function settingsDefaultOutputStyle(cwd = process.cwd(), opts = {}) {
     try {
       if (!fs.existsSync(f)) continue;
       const data = JSON.parse(fs.readFileSync(f, "utf-8"));
-      if (data && typeof data.outputStyle === "string" && data.outputStyle.trim()) {
+      if (
+        data &&
+        typeof data.outputStyle === "string" &&
+        data.outputStyle.trim()
+      ) {
         value = data.outputStyle.trim();
       }
     } catch {
@@ -170,8 +196,16 @@ export function settingsDefaultOutputStyle(cwd = process.cwd(), opts = {}) {
  * Precedence: explicit name → settings.json `outputStyle` → none.
  * Returns `{ name, body }` (body may be "" for `default`) or null if unresolved.
  */
-export function resolveOutputStyle(explicitName, cwd = process.cwd(), opts = {}) {
-  const name = (explicitName || settingsDefaultOutputStyle(cwd, opts) || "").trim();
+export function resolveOutputStyle(
+  explicitName,
+  cwd = process.cwd(),
+  opts = {},
+) {
+  const name = (
+    explicitName ||
+    settingsDefaultOutputStyle(cwd, opts) ||
+    ""
+  ).trim();
   if (!name) return null;
   const style = getOutputStyle(name, cwd, opts);
   if (!style) return { name, body: "", missing: true };