@elmundi/ship-cli 0.14.1 → 0.15.3

package/lib/commands/run.mjs

@@ -4,19 +4,21 @@
  * Customer's GH Actions cron fires this once per routine slot. The
  * pipeline:
  *
- *   1. Read the routine from `.ship/config.yml` (pattern_id, optional
- *      user-authored prompt).
- *   2. Fetch the pattern body+frontmatter from Ship server
- *      (`POST /fetch kind=pattern id=<pattern_id>`).
- *   3. If the pattern declares `spec.fsm_stage`, ask Ship server for
- *      the next ticket in that FSM stage
+ *   1. Read the routine from `.ship/config.yml` (specialist slug +
+ *      optional inline prompt).
+ *   2. Resolve the agent role body via the workspace endpoint
+ *      `GET /v1/workspaces/{ws}/agent-roles/{slug}/resolve` —
+ *      workspace overrides win, otherwise the Ship default. Pull
+ *      the `system` (shared base) body in parallel.
+ *   3. If the resolved role declares `fsm_stage`, ask Ship server
+ *      for the next ticket in that stage
  *      (`GET /v1/.../tracker/next?state=<stage>`). Server picks the
  *      adapter (Linear / GH Issues / etc.) — CLI doesn't care.
- *   4. Mint a `run_id` and render the prompt: pattern body + ticket
- *      details + a finish-protocol block with `SHIP_API_BASE`,
- *      `SHIP_API_TOKEN`, `SHIP_WORKSPACE_ID`, `RUN_ID`, `TICKET_REF`,
- *      `FSM_STAGE` already substituted so the agent can call the
- *      finish endpoint directly.
+ *   4. Mint a `run_id` and render the prompt: system body + role
+ *      body + routine prompt + ticket details + a finish-protocol
+ *      block with `SHIP_API_BASE`, `SHIP_API_TOKEN`, `SHIP_WORKSPACE_ID`,
+ *      `RUN_ID`, `TICKET_REF`, `FSM_STAGE` already substituted so the
+ *      agent can call the finish endpoint directly.
  *   5. Launch the configured agent runtime (`cli/lib/agents/`) — Cursor
  *      Cloud today. Block until the runtime terminates.
  *   6. The agent itself calls
@@ -46,15 +48,12 @@
 import crypto from "node:crypto";
 import path from "node:path";
 
-import yaml from "yaml";
-
 import { readConfig, findShipRoot } from "../config/io.mjs";
 import { resolveExecutable } from "../runtime/routines.mjs";
-import { fetchArtifact } from "../http.mjs";
-import { readArtifactFile } from "../artifacts/fs-index.mjs";
-import { resolveShipRepoRootForCatalog } from "../find-ship-root.mjs";
 import { resolveProvider, runAgent } from "../agents/index.mjs";
 
+const SYSTEM_ROLE_SLUG = "system";
+
 
 const EXIT_OK = 0;
 const EXIT_USAGE = 1;
@@ -73,8 +72,14 @@ export async function runCommand(ctx, rest) {
     printHelp();
     process.exit(EXIT_OK);
   }
-  if (!args.routine) {
-    die(EXIT_USAGE, "`--routine <id>` is required.\nRun: shipctl run --help");
+  if (!args.routine && !args.specialist) {
+    die(
+      EXIT_USAGE,
+      "either `--routine <id>` or `--specialist <slug>` is required.\nRun: shipctl run --help",
+    );
+  }
+  if (args.routine && args.specialist) {
+    die(EXIT_USAGE, "`--routine` and `--specialist` are mutually exclusive.");
   }
 
   const cwd = args.cwd || process.cwd();
@@ -84,33 +89,80 @@ export async function runCommand(ctx, rest) {
   }
 
   const { config } = readConfig(cwd);
-  const resolved = resolveExecutable(config, args.routine);
-  if (!resolved) {
-    die(EXIT_USAGE, `unknown routine '${args.routine}' in .ship/config.yml`);
+  // Routine mode: resolve from ``.ship/config.yml``. Specialist mode
+  // (used by the pipeline-pick fallback in the trigger workflow):
+  // synthesize a minimal executable so the rest of the pipeline can
+  // stay routine-shaped without inventing a ``pipeline:<slug>``
+  // routine in the YAML.
+  let resolved;
+  if (args.specialist) {
+    resolved = {
+      kind: "specialist",
+      id: args.specialist,
+      source: { specialist: args.specialist },
+      executable: {
+        id: args.specialist,
+        type: "specialist",
+        kind: "pipeline_pick",
+        specialist: args.specialist,
+        prompt: null,
+      },
+    };
+  } else {
+    resolved = resolveExecutable(config, args.routine);
+    if (!resolved) {
+      die(EXIT_USAGE, `unknown routine '${args.routine}' in .ship/config.yml`);
+    }
   }
 
+  // ``runId`` for emit/branch naming carries either the routine id or
+  // the specialist slug. Logging downstream uses ``runHandle``.
+  const runHandle = args.routine || `pipeline:${args.specialist}`;
+
   const env = readEnv();
   const { apiBase, apiToken, workspaceId, githubRepo } = env;
 
-  // 1) Resolve pattern
-  const patternId = resolved.executable.pattern;
-  if (!patternId) {
-    die(EXIT_USAGE, `routine '${args.routine}' has no pattern set`);
+  // 1) Resolve specialist slug.
+  // ``routine.specialist`` is the canonical Phase-2.4 form; the legacy
+  // ``routine.pattern`` is mapped to a slug in
+  // ``cli/lib/runtime/routines.mjs`` (drops the ``role-`` prefix).
+  const specialistSlug = resolved.executable.specialist || args.specialist;
+  if (!specialistSlug) {
+    die(
+      EXIT_USAGE,
+      `routine '${args.routine}' has no 'specialist:' (or legacy 'pattern:') set`,
+    );
+  }
+  if (!apiBase || !apiToken || !workspaceId) {
+    die(
+      EXIT_USAGE,
+      "agent-role resolve requires SHIP_API_BASE + SHIP_API_TOKEN + SHIP_WORKSPACE_ID",
+    );
   }
 
-  const fetchBase = methodologyBase(env, config);
-  const rawPatternBody = await loadPattern({ id: patternId, fetchBase });
-  const { frontmatter, frontmatterRaw, body: patternBody } = splitFrontmatter(rawPatternBody);
-  const fsmStage = pickFsmStage(frontmatter, frontmatterRaw);
-
-  // Patterns reference ``{{BASE}}`` to splice in common-base. Fetch it
-  // up-front so renderPrompt can do the substitution. ``{{SKILLS_CONTEXT}}``
-  // inside common-base is left as "(no skills directory)" for the MVP —
-  // skills bundling lands in a follow-up.
-  const baseRaw = await loadPattern({ id: "common-base", fetchBase, optional: true });
-  const baseBody = baseRaw ? splitFrontmatter(baseRaw).body : "";
+  // 2) Pull the resolved role + the shared system prompt in parallel.
+  // ``resolveAgentRole`` returns the workspace override when present,
+  // otherwise falls back to the Ship default. ``system`` is fetched
+  // optional — older deployments may not have it; we render without
+  // a system header in that case.
+  const [roleResolved, systemResolved] = await Promise.all([
+    resolveAgentRole({ apiBase, apiToken, workspaceId, slug: specialistSlug }),
+    resolveAgentRole({
+      apiBase,
+      apiToken,
+      workspaceId,
+      slug: SYSTEM_ROLE_SLUG,
+      optional: true,
+    }),
+  ]);
+  if (!roleResolved) {
+    die(EXIT_USAGE, `unknown agent role '${specialistSlug}' for this workspace`);
+  }
+  const fsmStage = roleResolved.fsm_stage || null;
+  const roleBody = roleResolved.prompt || "";
+  const systemBody = systemResolved?.prompt || "";
 
-  // 2) Resolve task. ``--dry-run`` skips the server call and uses a
+  // 3) Resolve task. ``--dry-run`` skips the server call and uses a
   // synthetic task so the operator can see the prompt shape without
   // needing the new endpoints deployed.
   let task = null;
@@ -134,20 +186,40 @@ export async function runCommand(ctx, rest) {
         state: fsmStage,
       });
       if (!task) {
-        emit(args, { status: "noop", routine: args.routine, pattern: patternId, fsm_stage: fsmStage, reason: "no_eligible_ticket" });
+        emit(args, {
+          status: "noop",
+          routine: args.routine,
+          specialist: specialistSlug,
+          fsm_stage: fsmStage,
+          reason: "no_eligible_ticket",
+          run_handle: runHandle,
+        });
         process.exit(EXIT_NO_TASK);
       }
     }
   }
 
-  // 3) Mint a run_id + render prompt with finish-protocol values
+  // 4) Fetch the workspace policy preamble so the agent prompt
+  // carries the same standing rules the Navigator chat does. Best-
+  // effort: a missing token, missing API base, or a network failure
+  // quietly skips the prepend — local / offline runs still work.
+  // ``role`` is now the specialist slug directly (no more
+  // ``spec.role`` indirection).
+  const policiesPreamble = await fetchPoliciesPreamble({
+    apiBase,
+    apiToken,
+    workspaceId,
+    role: specialistSlug,
+  });
+
+  // 5) Mint a run_id + render prompt with finish-protocol values
   // already substituted so the agent can call /agent-runs/finish from
   // inside Cursor without holding any extra config.
   const runId = `run_${crypto.randomBytes(8).toString("hex")}`;
-  const prompt = renderPrompt({
-    patternBody,
-    baseBody,
-    role: patternId,
+  const renderedPrompt = renderPrompt({
+    patternBody: roleBody,
+    baseBody: systemBody,
+    role: specialistSlug,
     routineSpec: resolved.executable,
     task,
     fsmStage,
@@ -156,27 +228,31 @@ export async function runCommand(ctx, rest) {
       apiToken,
       workspaceId,
       runId,
-      role: patternId,
+      role: specialistSlug,
       ticketRef: task?.ticket_ref || null,
       fsmStage: fsmStage || null,
     },
   });
+  const prompt = policiesPreamble
+    ? `${policiesPreamble.trim()}\n\n---\n\n${renderedPrompt}`
+    : renderedPrompt;
 
   // ``--dry-run`` exits here so the operator can eyeball the rendered
   // prompt + resolved task without launching an agent or touching any
-  // tracker. Useful when iterating on pattern bodies.
+  // tracker. Useful when iterating on prompts.
   if (args.dryRun || ctx.dryRun) {
     if (args.json) {
       console.log(JSON.stringify({
         status: "dry-run",
         routine: args.routine,
-        pattern: patternId,
+        specialist: specialistSlug,
+        specialist_source: roleResolved.source,
         fsm_stage: fsmStage,
         task,
         prompt,
       }, null, 2));
     } else {
-      console.error(`# ship: dry-run routine=${args.routine} pattern=${patternId} fsm_stage=${fsmStage || "(context-free)"}`);
+      console.error(`# ship: dry-run handle=${runHandle} specialist=${specialistSlug} (${roleResolved.source}) fsm_stage=${fsmStage || "(context-free)"}`);
       if (task) {
         console.error(`# ship: task ticket_ref=${task.ticket_ref} title=${JSON.stringify(task.title || "")}`);
       } else {
@@ -189,8 +265,8 @@ export async function runCommand(ctx, rest) {
   }
 
   // 4) Launch agent runtime
-  const provider = resolveProvider(config, args.routine);
-  const branchName = makeBranchName(args.routine, task?.ticket_ref);
+  const provider = resolveProvider(config, args.routine || specialistSlug);
+  const branchName = makeBranchName(runHandle, task?.ticket_ref);
   const repoUrl = githubRepo ? `https://github.com/${githubRepo}` : null;
   if (!repoUrl) die(EXIT_USAGE, "GITHUB_REPOSITORY env var is required to launch agent");
 
@@ -207,8 +283,9 @@ export async function runCommand(ctx, rest) {
     emit(args, {
       status: "error",
       routine: args.routine,
-      pattern: patternId,
+      specialist: specialistSlug,
       run_id: runId,
+      run_handle: runHandle,
       stage: "launch_agent",
       error: err instanceof Error ? err.message : String(err),
     });
@@ -223,13 +300,14 @@ export async function runCommand(ctx, rest) {
   emit(args, {
     status: "completed",
     routine: args.routine,
-    pattern: patternId,
+    specialist: specialistSlug,
     fsm_stage: fsmStage,
     ticket_ref: task?.ticket_ref || null,
     agent_id: runtime.agentId,
     branch: runtime.branchName,
     cursor_status: runtime.status,
     run_id: runId,
+    run_handle: runHandle,
   });
   process.exit(EXIT_OK);
 }
@@ -256,49 +334,107 @@ function stripSlash(s) {
 }
 
 
-function methodologyBase(env, config) {
-  // Server's POST /fetch lives next to /v1, not under /api/methodology.
-  if (env.apiBase) return env.apiBase;
-  const fromConfig = config?.api?.base_url;
-  if (typeof fromConfig === "string" && fromConfig.trim()) {
-    return stripSlash(fromConfig);
+/**
+ * Resolve an agent role through the workspace endpoint.
+ *
+ * Returns ``{slug, name, prompt, fsm_stage, source}`` (workspace row
+ * or Ship default), ``null`` for 404, throws on auth/network errors
+ * unless ``optional`` is set (then 404 *and* errors return ``null``
+ * with a one-line warning).
+ */
+async function resolveAgentRole({
+  apiBase,
+  apiToken,
+  workspaceId,
+  slug,
+  optional = false,
+}) {
+  const url = `${apiBase}/v1/workspaces/${encodeURIComponent(workspaceId)}/agent-roles/${encodeURIComponent(slug)}/resolve`;
+  let res;
+  try {
+    res = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${apiToken}`,
+      },
+    });
+  } catch (err) {
+    if (optional) {
+      console.error(
+        `warn: agent-role '${slug}' resolve failed (network): ${err instanceof Error ? err.message : err}`,
+      );
+      return null;
+    }
+    throw err;
   }
-  throw new Error("SHIP_API_BASE not set and no api.base_url in .ship/config.yml");
+  if (res.status === 404) return null;
+  if (!res.ok) {
+    if (optional) {
+      console.error(
+        `warn: agent-role '${slug}' resolve returned ${res.status}; running without it`,
+      );
+      return null;
+    }
+    throw new Error(
+      `agent-roles resolve ${res.status}: ${(await res.text()).slice(0, 300)}`,
+    );
+  }
+  return res.json();
 }
 
 
-function splitFrontmatter(raw) {
-  if (!raw.startsWith("---")) return { frontmatter: {}, frontmatterRaw: "", body: raw };
-  const end = raw.indexOf("\n---\n", 4);
-  if (end < 0) return { frontmatter: {}, frontmatterRaw: "", body: raw };
-  const headRaw = raw.slice(3, end + 1);
-  const body = raw.slice(end + 5);
-  let parsed = {};
+/**
+ * Best-effort fetch of the workspace's policy preamble. Returns the
+ * markdown block to prepend, or ``null`` when there's nothing to
+ * inject (no policies, missing token / API base, network error,
+ * non-200 response). Never throws — a broken policies path mustn't
+ * break a routine run.
+ *
+ * Auth: workspace-membership token (``SHIP_API_TOKEN`` — same one
+ * the rest of ``run.mjs`` uses). The companion run-token endpoint
+ * at ``/v1/pipelines/runs/{run_id}/policies-preamble`` doesn't fit
+ * this flow because the CLI mints ``run_id`` locally; the
+ * workspace-scoped variant takes membership instead.
+ */
+async function fetchPoliciesPreamble({ apiBase, apiToken, workspaceId, role }) {
+  if (!apiBase || !apiToken || !workspaceId) return null;
+  const qs = role ? `?role=${encodeURIComponent(role)}` : "";
+  const url = `${apiBase}/v1/workspaces/${encodeURIComponent(workspaceId)}/policies/preamble${qs}`;
+  let res;
   try {
-    parsed = yaml.parse(headRaw) || {};
-  } catch {
-    // Some Ship patterns have unquoted ``@elmundi/ship-core`` in
-    // ``authors`` which strict YAML rejects. The CLI doesn't need the
-    // full document — ``pickFsmStage`` falls back to a regex on the
-    // raw frontmatter text in that case.
-    parsed = {};
+    res = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${apiToken}`,
+      },
+    });
+  } catch (err) {
+    console.error(
+      `warn: policies preamble fetch failed (network): ${err instanceof Error ? err.message : err}`,
+    );
+    return null;
   }
-  return { frontmatter: parsed, frontmatterRaw: headRaw, body };
-}
-
-
-function pickFsmStage(frontmatter, frontmatterRaw) {
-  const spec = frontmatter?.spec || {};
-  const v = spec.fsm_stage ?? spec.fsmStage;
-  if (typeof v === "string" && v.trim()) return v.trim();
-  if (frontmatterRaw) {
-    // Strict-YAML-fallback: regex on the raw frontmatter. Matches
-    // ``fsm_stage: triage`` (with optional surrounding whitespace, with
-    // or without quotes) anywhere in the spec block.
-    const m = frontmatterRaw.match(/^\s*fsm_stage:\s*['"]?([\w.-]+)['"]?/m);
-    if (m && m[1]) return m[1].trim();
+  if (!res.ok) {
+    // 404 happens against older backends that don't have the
+    // workspace-scoped endpoint yet — silent skip there. Other
+    // statuses (401, 403, 500) get a one-line warning so operators
+    // notice misconfigurations without aborting the run.
+    if (res.status !== 404) {
+      console.error(
+        `warn: policies preamble fetch returned ${res.status}; running without preamble`,
+      );
+    }
+    return null;
   }
-  return null;
+  let body;
+  try {
+    body = await res.json();
+  } catch {
+    return null;
+  }
+  return typeof body?.preamble === "string" && body.preamble.trim()
+    ? body.preamble
+    : null;
 }
 
 
@@ -345,6 +481,8 @@ function renderPrompt({ patternBody, baseBody, role, routineSpec, task, fsmStage
     out.push("");
   }
   out.push(expanded.trim());
+  out.push("");
+  out.push(renderLifecycleHooks());
   if (task) {
     out.push("");
     out.push("## Task");
@@ -367,6 +505,33 @@ function renderPrompt({ patternBody, baseBody, role, routineSpec, task, fsmStage
 }
 
 
+function renderLifecycleHooks() {
+  // Phase 4: every run is bracketed by two auditable lifecycle hooks
+  // — knowledge-fetch first, knowledge-feedback last. We render them
+  // as explicit prompt instructions so they show up in the agent's
+  // tool-use log (the runner audits the log to flag runs that
+  // skipped them). Soft for now (no run-blocking enforcement) but
+  // the prompt is the canonical contract until the runner grows
+  // tool-stream interception.
+  return [
+    "## Lifecycle hooks (Phase 4)",
+    "",
+    "**First call — knowledge fetch.** Before any other tool call,",
+    `run \`shipctl knowledge fetch <bucket>\` for at least one bucket`,
+    "relevant to your role. The audit log flags runs whose first",
+    "tool call wasn't a knowledge fetch; do not skip this to save",
+    "tokens.",
+    "",
+    "**Last call — knowledge feedback.** Before calling the finish",
+    "endpoint, leave one-line learnings via the Ship knowledge",
+    "feedback channel (`shipctl feedback draft` → `feedback submit`)",
+    "if you discovered something a future run on this codebase would",
+    "want to know. Empty findings are a valid outcome — better no",
+    "feedback than fabricated polish.",
+  ].join("\n");
+}
+
+
 function renderExitProtocol(ctx) {
   // Substitute the run-time values directly into the example so the
   // agent doesn't have to figure out env var hookup. The token is
@@ -412,7 +577,8 @@ curl -fsS -X POST '${apiBase}/v1/workspaces/${workspaceId}/agent-runs/finish' \\
   ${ticketLine}
   ${fsmLine}
   "stage_next": "<next FSM stage, e.g. ba_requirements>",
-  "comment": "Markdown summary of what you did. End with [Ship SDLC:${ctx?.role || "{{ROLE}}"}].",
+  "description": "<Full rewritten ticket body in Markdown — Problem / Goal / Acceptance criteria / Scope / Non-goals / Risks / etc. — when your role's job is to shape the ticket itself (intake, BA, planner). Omit (null) when your role is not supposed to rewrite the body.>",
+  "comment": "<One-paragraph audit narration of what you changed and why, ending with [Ship SDLC:${ctx?.role || "{{ROLE}}"}]. Do NOT paste the new description here — that's what the description field is for.>",
   "summary": null,
   "payload": {}
 }
@@ -424,8 +590,15 @@ JSON
 - **\`ready_next_step\`** — your role finished cleanly. Two shapes:
 
   1. **You worked on a ticket.** Set \`ticket_ref\` and \`stage_next\`
-     to the next FSM stage; server moves the ticket and posts
-     \`comment\` if provided.
+     to the next FSM stage; server moves the ticket. If your role's
+     job is to shape the ticket (intake / BA / planner), set
+     \`description\` to the **full rewritten body** — the server
+     replaces the tracker description (Linear keeps the prior body
+     in the activity feed, so nothing is lost). Use \`comment\` for
+     a short audit narration of what changed and why; **do not put
+     the new spec text in a comment**, otherwise the ticket
+     description rots while comments accumulate. Pure-narration
+     roles (security-officer, retro) skip \`description\` entirely.
   2. **There was nothing to do.** Pass \`ticket_ref: null\` and omit
      \`stage_next\`. The server records the run in the audit log and
      does **nothing** else — no inbox row, no tracker mutation. This
@@ -462,33 +635,6 @@ as a one-shot credential for this run.
 }
 
 
-/**
- * Load a pattern body. Resolution order:
- *   1) when running inside the Ship monorepo, read from
- *      ``artifacts/patterns/<id>/ARTIFACT.md`` on disk — fast and
- *      always reflects the working tree (good for dry-runs / local
- *      smoke tests before the server is rebuilt).
- *   2) otherwise hit the server's ``POST /fetch``.
- */
-async function loadPattern({ id, fetchBase, optional = false }) {
-  const shipRepo = resolveShipRepoRootForCatalog();
-  if (shipRepo) {
-    const file = readArtifactFile(shipRepo, "pattern", id);
-    if (file && typeof file.content === "string") return file.content;
-  }
-  try {
-    const { content } = await fetchArtifact(fetchBase, "pattern", id);
-    return content;
-  } catch (err) {
-    if (optional) {
-      console.error(`warn: failed to fetch pattern '${id}': ${err.message}`);
-      return "";
-    }
-    throw err;
-  }
-}
-
-
 function makeBranchName(routine, ticketRef) {
   const stamp = Date.now().toString(36);
   if (ticketRef) {
@@ -505,7 +651,14 @@ function makeBranchName(routine, ticketRef) {
 
 
 function parseArgs(rest) {
-  const out = { routine: null, cwd: null, json: false, help: false, dryRun: false };
+  const out = {
+    routine: null,
+    specialist: null,
+    cwd: null,
+    json: false,
+    help: false,
+    dryRun: false,
+  };
   const copy = [...rest];
   while (copy.length) {
     const a = copy[0];
@@ -513,6 +666,7 @@ function parseArgs(rest) {
     if (a === "--json") { out.json = true; copy.shift(); continue; }
     if (a === "--dry-run") { out.dryRun = true; copy.shift(); continue; }
     if (a === "--routine" && copy[1] !== undefined) { out.routine = copy[1]; copy.splice(0, 2); continue; }
+    if (a === "--specialist" && copy[1] !== undefined) { out.specialist = copy[1]; copy.splice(0, 2); continue; }
     if (a === "--cwd" && copy[1] !== undefined) { out.cwd = path.resolve(copy[1]); copy.splice(0, 2); continue; }
     // Soft-ignore legacy flags that older trigger workflows still pass —
     // the new pipeline doesn't need them and refusing would break repos
@@ -527,10 +681,14 @@ function parseArgs(rest) {
 
 
 function printHelp() {
-  console.log(`shipctl run — execute one E14 routine end-to-end.
+  console.log(`shipctl run — execute one E14 routine or pipeline-pick specialist end-to-end.
 
 Run
   shipctl run --routine <id> [--json] [--cwd <dir>] [--dry-run]
+  shipctl run --specialist <slug> [--json] [--cwd <dir>] [--dry-run]
+
+  --routine     id from process.routines in .ship/config.yml (cron-driven)
+  --specialist  agent-role slug from the Ship registry (pipeline-pick fallback)
 
 ENV
   SHIP_API_BASE        Ship server base URL (e.g. https://api.ship.elmundi.com)