@yycholla/pi-dynamic-workflows 3.3.0

package/src/workflow-context-measurement.ts

@@ -0,0 +1,262 @@
+import { readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { createWorkflowTool } from "./workflow-tool.js";
+
+/** Package-relative generated context-measurement artifact. */
+export const WORKFLOW_CONTEXT_MEASUREMENT_PATH = "docs/workflow-context-surfaces.json";
+const ROOT = join(import.meta.dirname, "..");
+const SKILL_ROOT = "skills/workflow-authoring";
+const SKILL_PATH = `${SKILL_ROOT}/SKILL.md`;
+
+/** Canonical task profiles used for stable byte-based on-demand context measurements. */
+export const WORKFLOW_AUTHORING_PROFILES = [
+  {
+    name: "write",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/pattern-selection.md`,
+      `${SKILL_ROOT}/references/focused-recipes.md`,
+      `${SKILL_ROOT}/examples/fan-out-and-synthesize.js`,
+      `${SKILL_ROOT}/examples/structured-output.js`,
+    ],
+  },
+  {
+    name: "edit",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/lifecycle.md`,
+      `${SKILL_ROOT}/references/focused-recipes.md`,
+      `${SKILL_ROOT}/examples/phased-budgets.js`,
+      `${SKILL_ROOT}/examples/saved-nested-workflows.js`,
+    ],
+  },
+  {
+    name: "review",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/review.md`,
+      `${SKILL_ROOT}/references/quality-helpers.md`,
+      `${SKILL_ROOT}/examples/adversarial-verification.js`,
+    ],
+  },
+  {
+    name: "debug",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/debugging.md`,
+      `${SKILL_ROOT}/references/specialized-helpers.md`,
+      `${SKILL_ROOT}/examples/validated-gate.js`,
+    ],
+  },
+  {
+    name: "loop",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/pattern-selection.md`,
+      `${SKILL_ROOT}/references/lifecycle.md`,
+      `${SKILL_ROOT}/references/focused-recipes.md`,
+      `${SKILL_ROOT}/examples/loop-until-done.js`,
+      `${SKILL_ROOT}/examples/structured-output.js`,
+    ],
+  },
+  {
+    name: "retry",
+    files: [
+      SKILL_PATH,
+      `${SKILL_ROOT}/references/runtime.md`,
+      `${SKILL_ROOT}/references/retry-helper.md`,
+      `${SKILL_ROOT}/references/focused-recipes.md`,
+      `${SKILL_ROOT}/examples/bounded-semantic-retry.js`,
+      `${SKILL_ROOT}/examples/structured-output.js`,
+    ],
+  },
+] as const;
+
+interface ByteSurface {
+  serialization: string;
+  bytes: number;
+}
+
+/** One registered skill's always-on discovery-entry byte cost. */
+export interface SkillDiscoverySurface extends ByteSurface {
+  root: string;
+}
+
+/** Versioned byte measurements for always-on, discovery, corpus, and representative authoring surfaces. */
+export interface WorkflowContextMeasurement {
+  formatVersion: 3;
+  encoding: "utf8";
+  sources: ["src/workflow-tool.ts", "skills/workflow-authoring", "package.json#pi.skills"];
+  surfaces: {
+    permanentWorkflowPrompt: ByteSurface;
+    providerVisibleWorkflowToolDefinition: ByteSurface;
+    /**
+     * Every skill this package registers (package.json's `pi.skills` — read
+     * from disk, not hardcoded here) contributes an always-on discovery entry
+     * (name + description) the model sees regardless of whether the skill is
+     * ever loaded. This sums all of them, not just workflow-authoring, so a
+     * new skill's always-on cost can't silently go untracked.
+     */
+    registeredSkillsDiscovery: ByteSurface & { skills: SkillDiscoverySurface[] };
+    ordinaryWorkflowOwnedAlwaysOn: ByteSurface;
+    workflowAuthoringSkillCorpus: ByteSurface & { files: number };
+    representativeAuthoringProfiles: {
+      serialization: "sum of UTF-8 bytes for each profile's declared package-relative files";
+      medianBytes: number;
+      profiles: Array<{ name: string; files: string[]; bytes: number }>;
+    };
+  };
+}
+
+function bytes(value: string): number {
+  return Buffer.byteLength(value, "utf8");
+}
+
+function fileBytes(root: string, path: string): number {
+  return bytes(readFileSync(join(root, path), "utf8"));
+}
+
+function skillFiles(root: string): string[] {
+  const absoluteRoot = join(root, SKILL_ROOT);
+  const pending = [absoluteRoot];
+  const files: string[] = [];
+  while (pending.length > 0) {
+    const directory = pending.pop();
+    if (!directory) continue;
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      const absolute = join(directory, entry.name);
+      if (entry.isDirectory()) pending.push(absolute);
+      else if (entry.isFile()) files.push(relative(root, absolute).replaceAll("\\", "/"));
+    }
+  }
+  return files.sort();
+}
+
+/**
+ * Every skill root this package registers, read from package.json's
+ * `pi.skills` array rather than hardcoded — so a newly added skill is picked
+ * up automatically instead of silently missing from the always-on tally.
+ */
+function registeredSkillRoots(root: string): string[] {
+  const manifest = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) as {
+    pi?: { skills?: unknown };
+  };
+  const skills = manifest.pi?.skills;
+  if (!Array.isArray(skills) || skills.length === 0 || !skills.every((s) => typeof s === "string")) {
+    throw new Error("package.json must declare a non-empty pi.skills array of skill root paths");
+  }
+  return skills;
+}
+
+function skillDiscoveryEntry(root: string, skillRoot: string): string {
+  const skillPath = `${skillRoot}/SKILL.md`;
+  const skill = readFileSync(join(root, skillPath), "utf8");
+  const name = /^name:\s*(.+)$/m.exec(skill)?.[1]?.trim();
+  const description = /^description:\s*(.+)$/m.exec(skill)?.[1]?.trim();
+  if (!name || !description) throw new Error(`${skillPath} must declare name and description`);
+  return [
+    "<skill>",
+    `  <name>${name}</name>`,
+    `  <description>${description}</description>`,
+    `  <location>${skillPath}</location>`,
+    "</skill>",
+  ].join("\n");
+}
+
+function median(values: number[]): number {
+  const sorted = [...values].sort((left, right) => left - right);
+  const middle = sorted.length / 2;
+  return sorted.length % 2 === 0
+    ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2
+    : (sorted[Math.floor(middle)] ?? 0);
+}
+
+/** Measures permanent, discovery, corpus, and canonical on-demand workflow context surfaces. */
+export function measureWorkflowContextSurfaces(root: string = ROOT): WorkflowContextMeasurement {
+  const tool = createWorkflowTool();
+  const permanentWorkflowPrompt = [
+    `- workflow: ${tool.promptSnippet}`,
+    ...(tool.promptGuidelines ?? []).map((guideline) => `- ${guideline}`),
+  ].join("\n");
+  const providerVisibleWorkflowToolDefinition = JSON.stringify({
+    name: tool.name,
+    description: tool.description,
+    parameters: tool.parameters,
+  });
+  const skillRoots = registeredSkillRoots(root);
+  const skillDiscoverySurfaces: SkillDiscoverySurface[] = skillRoots.map((skillRoot) => ({
+    root: skillRoot,
+    serialization: "UTF-8 bytes of normalized Pi skill XML with package-relative location",
+    bytes: bytes(skillDiscoveryEntry(root, skillRoot)),
+  }));
+  const registeredSkillsDiscoveryBytes = skillDiscoverySurfaces.reduce((sum, surface) => sum + surface.bytes, 0);
+  const corpusFiles = skillFiles(root);
+  const corpusBytes = corpusFiles.reduce((sum, path) => sum + fileBytes(root, path), 0);
+  const profiles = WORKFLOW_AUTHORING_PROFILES.map((profile) => ({
+    name: profile.name,
+    files: [...profile.files],
+    bytes: profile.files.reduce((sum, path) => sum + fileBytes(root, path), 0),
+  }));
+  const promptBytes = bytes(permanentWorkflowPrompt);
+  const toolBytes = bytes(providerVisibleWorkflowToolDefinition);
+
+  return {
+    formatVersion: 3,
+    encoding: "utf8",
+    sources: ["src/workflow-tool.ts", "skills/workflow-authoring", "package.json#pi.skills"],
+    surfaces: {
+      permanentWorkflowPrompt: {
+        serialization: "UTF-8 bytes of LF-joined Pi prompt lines",
+        bytes: promptBytes,
+      },
+      providerVisibleWorkflowToolDefinition: {
+        serialization: "UTF-8 bytes of JSON.stringify({ name, description, parameters })",
+        bytes: toolBytes,
+      },
+      registeredSkillsDiscovery: {
+        serialization:
+          "sum of UTF-8 bytes of normalized Pi skill XML (name + description + location) across every root in package.json's pi.skills",
+        bytes: registeredSkillsDiscoveryBytes,
+        skills: skillDiscoverySurfaces,
+      },
+      ordinaryWorkflowOwnedAlwaysOn: {
+        serialization:
+          "sum of permanent prompt, provider-visible tool definition, and every registered skill's discovery bytes",
+        bytes: promptBytes + toolBytes + registeredSkillsDiscoveryBytes,
+      },
+      workflowAuthoringSkillCorpus: {
+        serialization: "sum of UTF-8 bytes for every file under skills/workflow-authoring",
+        files: corpusFiles.length,
+        bytes: corpusBytes,
+      },
+      representativeAuthoringProfiles: {
+        serialization: "sum of UTF-8 bytes for each profile's declared package-relative files",
+        medianBytes: median(profiles.map(({ bytes: profileBytes }) => profileBytes)),
+        profiles,
+      },
+    },
+  };
+}
+
+/** Render the current measurement as deterministic formatted JSON. */
+export function renderWorkflowContextMeasurement(): string {
+  return `${JSON.stringify(measureWorkflowContextSurfaces(), null, 2)}\n`;
+}
+
+/** Write the generated measurement under root and return the measured values. */
+export function writeWorkflowContextMeasurement(root: string): WorkflowContextMeasurement {
+  const measurement = measureWorkflowContextSurfaces(root);
+  writeFileSync(join(root, WORKFLOW_CONTEXT_MEASUREMENT_PATH), `${JSON.stringify(measurement, null, 2)}\n`);
+  return measurement;
+}
+
+/** Report whether committed or supplied measurement JSON matches current package bytes. */
+export function checkWorkflowContextMeasurement(root: string, actual?: string): boolean {
+  const committed = actual ?? readFileSync(join(root, WORKFLOW_CONTEXT_MEASUREMENT_PATH), "utf8");
+  return committed === `${JSON.stringify(measureWorkflowContextSurfaces(root), null, 2)}\n`;
+}

package/src/workflow-control-tool.ts

@@ -0,0 +1,238 @@
+import { defineTool, type ToolDefinition } from "@earendil-works/pi-coding-agent";
+import { type Static, Type } from "typebox";
+import { aggregateAgentUsage, tokenFigures, type WorkflowAgentSnapshot, type WorkflowSnapshot } from "./display.js";
+import type { PersistedRunState, RunStatus } from "./run-persistence.js";
+import type { WorkflowManager } from "./workflow-manager.js";
+
+// A tool's top-level parameter schema must be a JSON Schema object (`type:
+// "object"`). A discriminated Type.Union of two objects serializes to a
+// top-level `anyOf` with no `type`, which strict providers (e.g. DeepSeek)
+// reject with "schema must be type object, got type: null". So the schema is a
+// single object: `action` is the full set of verbs and `runId` is optional at
+// the schema level. The per-action requirement (runId is mandatory for every
+// action except `list`, and `list` takes no runId) is enforced at runtime in
+// normalizeInput() and guarded again in execute().
+const workflowControlSchema = Type.Object(
+  {
+    action: Type.Union(
+      [
+        Type.Literal("list"),
+        Type.Literal("status"),
+        Type.Literal("pause"),
+        Type.Literal("resume"),
+        Type.Literal("stop"),
+      ],
+      { description: "list = all runs (no runId); status/pause/resume/stop act on one run and require runId." },
+    ),
+    runId: Type.Optional(
+      Type.String({
+        minLength: 1,
+        description: "Canonical workflow run ID. Required for status, pause, resume, and stop; omit for list.",
+      }),
+    ),
+  },
+  { additionalProperties: false },
+);
+
+export type WorkflowControlInput = Static<typeof workflowControlSchema>;
+
+export interface WorkflowControlToolOptions {
+  manager: WorkflowManager;
+}
+
+export interface WorkflowControlRunDetails {
+  runId: string;
+  workflowName: string;
+  status: RunStatus;
+  phase: string | null;
+  counts: {
+    total: number;
+    done: number;
+    running: number;
+    queued: number;
+    error: number;
+    skipped: number;
+  };
+  activeLabels: string[];
+  tokenTotal: number;
+}
+
+type ControlResult = {
+  content: Array<{ type: "text"; text: string }>;
+  details: Record<string, unknown>;
+};
+
+export function createWorkflowControlTool(
+  options: WorkflowControlToolOptions,
+): ToolDefinition<typeof workflowControlSchema, Record<string, unknown>> {
+  const manager = options.manager;
+  return defineTool({
+    name: "workflow_control",
+    label: "Workflow Control",
+    description:
+      "List and inspect workflow runs, or pause, resume, and stop them without asking the user to run slash commands.",
+    promptSnippet: "Inspect and manage workflow runs directly by canonical run ID.",
+    promptGuidelines: [
+      "Use workflow_control for workflow lifecycle management; do not ask the user to type /workflows when this tool can perform the action.",
+      "Use stop to terminate or quit a run. Closing the navigator does not stop a run.",
+    ],
+    parameters: workflowControlSchema,
+    prepareArguments: normalizeInput,
+    async execute(_toolCallId, params) {
+      if (params.action === "list") {
+        const runs = manager.listRuns();
+        const summaries = runs.map((run) => summarizeRun(run, manager.getSnapshot(run.runId)));
+        return result(
+          summaries.length
+            ? `action=list result=ok runs=${summaries.length}\n${summaries.map(formatRun).join("\n")}`
+            : "action=list result=ok runs=0",
+          { action: "list", result: "ok", runs: summaries },
+        );
+      }
+
+      // runId is optional in the schema (see workflowControlSchema) but required
+      // for every non-list action; normalizeInput already enforces this, and this
+      // guard both narrows the type and returns a structured error if a model
+      // somehow calls a run action without one.
+      if (!params.runId) return controlError(params.action, "", "runId is required for this action", ["list"]);
+      const run = findRun(manager, params.runId);
+      if (!run) return controlError(params.action, params.runId, "run not found", ["list"]);
+
+      try {
+        switch (params.action) {
+          case "status": {
+            const summary = summarizeRun(run, manager.getSnapshot(run.runId));
+            return result(`action=status result=ok ${formatRun(summary)}`, {
+              action: "status",
+              result: "ok",
+              run: summary,
+            });
+          }
+          case "pause":
+            if (!manager.pause(run.runId)) return invalidTransition("pause", run);
+            return actionSuccess("pause", "paused", currentSummary(manager, run));
+          case "resume":
+            if (!(await manager.resume(run.runId))) return invalidTransition("resume", run);
+            return actionSuccess("resume", "resumed", currentSummary(manager, run));
+          case "stop":
+            if (!manager.stop(run.runId)) return invalidTransition("stop", run);
+            return actionSuccess("stop", "stopped", currentSummary(manager, run));
+        }
+      } catch (err) {
+        // A transient persistence I/O error (or any unexpected throw from the
+        // manager) shouldn't surface as a raw stack trace to the model — report
+        // it via the tool's normal structured error shape instead.
+        const message = err instanceof Error ? err.message : String(err);
+        return controlError(params.action, run.runId, message, allowedActions(run.status));
+      }
+    },
+  });
+}
+
+function normalizeInput(value: unknown): WorkflowControlInput {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("workflow_control requires an object argument");
+  }
+  const input = value as Record<string, unknown>;
+  const actions = new Set(["list", "status", "pause", "resume", "stop"]);
+  if (typeof input.action !== "string" || !actions.has(input.action)) {
+    throw new Error("workflow_control requires action: list|status|pause|resume|stop");
+  }
+
+  const allowedKeys = input.action === "list" ? new Set(["action"]) : new Set(["action", "runId"]);
+  const extraKey = Object.keys(input).find((key) => !allowedKeys.has(key));
+  if (extraKey) throw new Error(`workflow_control action "${input.action}" does not accept ${extraKey}`);
+
+  if (input.action !== "list" && (typeof input.runId !== "string" || !input.runId.trim())) {
+    throw new Error(`workflow_control action "${input.action}" requires runId`);
+  }
+  return input as WorkflowControlInput;
+}
+
+function result(text: string, details: Record<string, unknown>): ControlResult {
+  return { content: [{ type: "text", text }], details };
+}
+
+function findRun(manager: WorkflowManager, runId: string): PersistedRunState | undefined {
+  return manager.listRuns().find((candidate) => candidate.runId === runId);
+}
+
+function currentSummary(manager: WorkflowManager, fallback: PersistedRunState): WorkflowControlRunDetails {
+  const current = findRun(manager, fallback.runId) ?? fallback;
+  return summarizeRun(current, manager.getSnapshot(current.runId));
+}
+
+function actionSuccess(action: string, actionResult: string, run: WorkflowControlRunDetails): ControlResult {
+  return result(`action=${action} result=${actionResult} ${formatRun(run)}`, {
+    action,
+    result: actionResult,
+    run,
+  });
+}
+
+function invalidTransition(action: string, run: PersistedRunState): ControlResult {
+  return controlError(action, run.runId, `cannot ${action} run with status ${run.status}`, allowedActions(run.status));
+}
+
+function controlError(action: string, runId: string, message: string, allowed: string[]): ControlResult {
+  return result(
+    `action=${action} result=error runId=${runId} error=${message} allowed=${allowed.join(",") || "none"}`,
+    { action, result: "error", runId, error: message, allowedActions: allowed },
+  );
+}
+
+function allowedActions(status: RunStatus): string[] {
+  switch (status) {
+    case "running":
+      return ["status", "pause", "stop"];
+    case "paused":
+      return ["status", "resume", "stop"];
+    case "failed":
+    case "pending":
+      return ["status", "resume"];
+    case "completed":
+    case "aborted":
+      return ["status"];
+  }
+}
+
+function summarizeRun(run: PersistedRunState, live?: WorkflowSnapshot | null): WorkflowControlRunDetails {
+  const agents = live?.agents ?? run.agents;
+  const counts = countAgents(agents);
+  const liveUsage = tokenFigures(live?.tokenUsage);
+  const persistedUsage = tokenFigures(run.tokenUsage);
+  const agentUsage = aggregateAgentUsage(agents);
+  return {
+    runId: run.runId,
+    workflowName: live?.name ?? run.workflowName,
+    status: run.status,
+    phase: live?.currentPhase ?? run.currentPhase ?? null,
+    counts,
+    activeLabels: agents.filter((agent) => agent.status === "running").map((agent) => agent.label),
+    tokenTotal: Math.max(
+      liveUsage.fresh + liveUsage.cacheRead,
+      persistedUsage.fresh + persistedUsage.cacheRead,
+      agentUsage.fresh + agentUsage.cacheRead,
+    ),
+  };
+}
+
+function countAgents(agents: Array<Pick<WorkflowAgentSnapshot, "status">>): WorkflowControlRunDetails["counts"] {
+  return {
+    total: agents.length,
+    done: agents.filter((agent) => agent.status === "done").length,
+    running: agents.filter((agent) => agent.status === "running").length,
+    queued: agents.filter((agent) => agent.status === "queued").length,
+    error: agents.filter((agent) => agent.status === "error").length,
+    skipped: agents.filter((agent) => agent.status === "skipped").length,
+  };
+}
+
+function formatRun(run: WorkflowControlRunDetails): string {
+  const active = run.activeLabels.join(",") || "-";
+  return `runId=${run.runId} name=${quote(run.workflowName)} status=${run.status} phase=${quote(run.phase ?? "-")} total=${run.counts.total} done=${run.counts.done} running=${run.counts.running} queued=${run.counts.queued} error=${run.counts.error} skipped=${run.counts.skipped} active=${quote(active)} tokens=${run.tokenTotal}`;
+}
+
+function quote(value: string): string {
+  return JSON.stringify(value);
+}

package/src/workflow-delivery-choice.ts

@@ -0,0 +1,69 @@
+/** One model-facing timing decision for invoking the workflow tool. */
+export interface WorkflowDeliveryChoiceScenario {
+  id: string;
+  prompt: string;
+  expectedBackground: boolean;
+}
+
+/** Pure scoring result for one captured workflow tool invocation. */
+export interface WorkflowDeliveryChoiceEvaluation {
+  passed: boolean;
+  resolvedBackground: boolean | null;
+  assertions: Array<{ name: string; passed: boolean; details: string }>;
+}
+
+/** Focused #89 scenarios for default background delivery versus same-turn use. */
+export const WORKFLOW_DELIVERY_CHOICE_SCENARIOS: readonly WorkflowDeliveryChoiceScenario[] = [
+  {
+    id: "background-delivery",
+    prompt:
+      "Run a workflow to audit the repository from several independent angles. I do not need the result in this turn; let it be delivered back later so I can keep working.",
+    expectedBackground: true,
+  },
+  {
+    id: "inline-result",
+    prompt:
+      "Run a workflow to compare the two proposed designs, then use its result in your answer in this same turn. I am waiting for the result before you respond.",
+    expectedBackground: false,
+  },
+];
+
+/** Score captured workflow arguments without executing the submitted workflow. */
+export function evaluateWorkflowDeliveryChoice(
+  scenario: WorkflowDeliveryChoiceScenario,
+  value: unknown,
+): WorkflowDeliveryChoiceEvaluation {
+  const input = isRecord(value) ? value : null;
+  const hasScript = typeof input?.script === "string" && input.script.trim().length > 0;
+  const validBackground = input !== null && (input.background === undefined || typeof input.background === "boolean");
+  const resolvedBackground = validBackground ? (typeof input.background === "boolean" ? input.background : true) : null;
+  const timingMatches = resolvedBackground === scenario.expectedBackground;
+  const assertions = [
+    {
+      name: "workflow:called-with-script",
+      passed: hasScript,
+      details: "the model must invoke the real workflow tool shape with a nonblank script",
+    },
+    {
+      name: "background:valid-type",
+      passed: validBackground,
+      details: "background must be omitted for its true default or supplied as a boolean",
+    },
+    {
+      name: "background:matches-user-timing",
+      passed: timingMatches,
+      details: scenario.expectedBackground
+        ? "later delivery should use the default background run"
+        : "same-turn use should pass background: false",
+    },
+  ];
+  return {
+    passed: assertions.every(({ passed }) => passed),
+    resolvedBackground,
+    assertions,
+  };
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}