@juicesharp/rpiv-workflow 1.14.0

package/sessions/extraction.ts

@@ -0,0 +1,297 @@
+/**
+ * Output production + validation retry loop. Sits between the
+ * post-session classifier (which decides "stage finished cleanly?") and
+ * the persistence helpers ("record this stage").
+ *
+ * Public entry: `produceAndValidateOutput`. Returns a tagged outcome
+ * — `ok` with the output, `fatal` (halt with a wording the
+ * collector/parser supplied), or `validation-exhausted` (halt after the
+ * retry budget tripped without a passing schema).
+ *
+ * The two-step contract:
+ *   1. `outcome.collector.collect(ctx)` — enumerate artifacts.
+ *   2. `outcome.parser?.parse(ctx)`     — shape the typed data channel
+ *                                        (default: data = artifacts,
+ *                                        kind = "artifacts").
+ */
+
+import type { StageDef, StageSchema } from "../api.js";
+import { nowIso } from "../audit.js";
+import type { Artifact } from "../handle.js";
+import { assertNever, withTimeout } from "../internal-utils.js";
+import { buildLifecycleContext, skillStageRef } from "../lifecycle.js";
+import { ERR_SCHEMA_TIMEOUT, MSG_VALIDATION_RETRY, MSG_VALIDATION_RETRY_PROMPT } from "../messages.js";
+import { sideEffectOutcome } from "../outcomes/index.js";
+import { finalizeOutput, type Output } from "../output.js";
+import type { CollectCtx, OutputSpec } from "../output-spec.js";
+import { type BranchEntry, readBranch } from "../transcript.js";
+import type { RunnerCtx, StageSession } from "../types.js";
+import {
+	DEFAULT_VALIDATION_RETRIES,
+	DEFAULT_VALIDATION_RETRY_TIMEOUT_MS,
+	MAX_VALIDATION_RETRIES,
+	MAX_VALIDATION_RETRY_TIMEOUT_MS,
+	MIN_VALIDATION_RETRIES,
+	MIN_VALIDATION_RETRY_TIMEOUT_MS,
+	type SchemaValidationFailure,
+	type ValidationResult,
+	validateOutputData,
+} from "../validate-output.js";
+import { handlerFor } from "./spawn.js";
+
+export type OutputProduction =
+	| { kind: "ok"; output: Output }
+	| { kind: "fatal"; message: string }
+	| { kind: "validation-exhausted"; failureSummary: string };
+
+/** Retry loop re-produces against the latest branch after each fix request. */
+export async function produceAndValidateOutput(
+	ctx: RunnerCtx,
+	s: StageSession,
+	branch: BranchEntry[],
+	branchOffset: number | undefined,
+): Promise<OutputProduction> {
+	const outcome = resolveOutcome(s.stage, s.skill);
+	const collectCtx = buildCollectCtx(s, branch, branchOffset);
+	const finalize = (parts: { kind: string; artifacts: readonly Artifact[]; data: unknown }) => wrapOutput(s, parts);
+
+	const first = await runOutcome(outcome, collectCtx, finalize);
+	if (first.kind === "fatal") return first;
+	const initialOutput = enforceCompletionContract(s.stage, s.skill, first.output);
+	if (initialOutput.kind === "fatal") return initialOutput;
+
+	if (!shouldValidateOutput(s.stage, initialOutput.output)) return initialOutput;
+
+	return retryUntilValid(ctx, s, { outcome, collectCtx, finalize }, initialOutput.output);
+}
+
+/**
+ * Explicit `stage.outcome` wins. Defaults:
+ *  - `side-effect` → `sideEffectOutcome` (universal — emits empty artifacts).
+ *  - `produces`    → throws. There is no framework-wide default; the
+ *    `.rpiv/artifacts/<bucket>/<file>.md` layout is an rpiv-pi convention
+ *    and lives in that package. `validate-workflow.ts` rejects this at
+ *    load time; the runtime throw is defense-in-depth for programmatic
+ *    embedders that bypassed validation.
+ */
+function resolveOutcome(stage: StageDef, skill: string): OutputSpec {
+	if (stage.outcome) return stage.outcome;
+	switch (stage.kind) {
+		case "side-effect":
+			return sideEffectOutcome;
+		case "produces":
+			throw new Error(
+				`runStage: stage "${skill}" has kind "produces" but no \`outcome\` — ` +
+					"there is no framework default for produces stages (the `.rpiv/artifacts/` layout is " +
+					"an rpiv-pi convention). Either wire `outcome: rpivArtifactMdOutcome` (from @juicesharp/rpiv-pi) " +
+					"or supply your own `{ collector, parser? }`.",
+			);
+		default:
+			return assertNever(stage.kind);
+	}
+}
+
+/**
+ * Contract: `branch` is always the FULL unsliced branch and
+ * `branchOffset` is always the policy-derived offset (continue → the
+ * stage's captured offset; fresh → undefined). Collectors slice on
+ * demand via the `branchOffset` field. Initial production and retry
+ * production use the same offset value.
+ */
+function buildCollectCtx(s: StageSession, branch: BranchEntry[], branchOffset: number | undefined): CollectCtx {
+	return {
+		cwd: s.cwd,
+		runId: s.runId,
+		stageIndex: s.stageIndex,
+		state: s.state,
+		branch,
+		branchOffset,
+		snapshot: s.snapshot,
+		skill: s.skill,
+	};
+}
+
+function wrapOutput(s: StageSession, parts: { kind: string; artifacts: readonly Artifact[]; data: unknown }): Output {
+	return finalizeOutput(parts, {
+		stage: s.stageName,
+		skill: s.skill,
+		stageNumber: s.state.lastAllocatedStageNumber + 1,
+		ts: nowIso(),
+		runId: s.runId,
+	});
+}
+
+type RunOutcomeResult = { kind: "ok"; output: Output } | { kind: "fatal"; message: string };
+
+/**
+ * The collector → parser pipeline. When `parser` is omitted, the
+ * output emits `kind: "artifacts"` with `data = artifacts` — a stage
+ * that only needs to enumerate doesn't have to write a parser.
+ */
+async function runOutcome(
+	outcome: OutputSpec,
+	ctx: CollectCtx,
+	finalize: (parts: { kind: string; artifacts: readonly Artifact[]; data: unknown }) => Output,
+): Promise<RunOutcomeResult> {
+	const collected = await outcome.collector.collect(ctx);
+	if (collected.kind === "fatal") return collected;
+
+	if (!outcome.parser) {
+		return {
+			kind: "ok",
+			output: finalize({ kind: "artifacts", artifacts: collected.artifacts, data: collected.artifacts }),
+		};
+	}
+
+	const parsed = await outcome.parser.parse({ ...ctx, artifacts: collected.artifacts });
+	if (parsed.kind === "fatal") return parsed;
+	return {
+		kind: "ok",
+		output: finalize({
+			kind: parsed.payload.kind,
+			artifacts: collected.artifacts,
+			data: parsed.payload.data,
+		}),
+	};
+}
+
+/**
+ * Contract check: `produces` stages MUST emit at least one
+ * artifact. The collector/parser pair can succeed structurally
+ * (kind: "ok") with zero artifacts — that's a chain halt for
+ * `produces` (the stage promised an output and didn't deliver)
+ * but a normal pass-through for `side-effect`.
+ */
+function enforceCompletionContract(
+	stage: StageDef,
+	skill: string,
+	output: Output,
+): { kind: "ok"; output: Output } | { kind: "fatal"; message: string } {
+	if (stage.kind === "produces" && output.artifacts.length === 0) {
+		return {
+			kind: "fatal",
+			message: `${skill} finished without producing any artifact (collector returned an empty list)`,
+		};
+	}
+	return { kind: "ok", output };
+}
+
+function shouldValidateOutput(stage: StageDef, output: Output): boolean {
+	return !!(stage.outputSchema && output.data !== undefined);
+}
+
+interface RetryDeps {
+	outcome: OutputSpec;
+	collectCtx: CollectCtx;
+	finalize: (parts: { kind: string; artifacts: readonly Artifact[]; data: unknown }) => Output;
+}
+
+async function retryUntilValid(
+	ctx: RunnerCtx,
+	s: StageSession,
+	deps: RetryDeps,
+	initial: Output,
+): Promise<OutputProduction> {
+	const schema = s.stage.outputSchema!;
+	const maxRetries = Math.max(
+		MIN_VALIDATION_RETRIES,
+		Math.min(s.stage.maxRetries ?? DEFAULT_VALIDATION_RETRIES, MAX_VALIDATION_RETRIES),
+	);
+	const timeoutMs = Math.max(
+		MIN_VALIDATION_RETRY_TIMEOUT_MS,
+		Math.min(s.stage.validateTimeoutMs ?? DEFAULT_VALIDATION_RETRY_TIMEOUT_MS, MAX_VALIDATION_RETRY_TIMEOUT_MS),
+	);
+
+	let output = initial;
+	const initialValidation = await validateOrFatal(schema, output.data, s.skill, timeoutMs);
+	if (initialValidation.kind === "fatal") return initialValidation;
+	let result = initialValidation.result;
+	let attempts = 0;
+
+	while (!result.valid && attempts < maxRetries && s.stage.onInvalid !== "halt") {
+		attempts++;
+		// onStageRetry fires before the agent is re-prompted; `attempt` is 1-based.
+		await s.lifecycle.fire(
+			ctx,
+			"onStageRetry",
+			skillStageRef(s.stageName, s.stageIndex + 1, s.skill),
+			attempts,
+			buildLifecycleContext({
+				cwd: s.cwd,
+				runId: s.runId,
+				workflow: s.runIdentity.workflow,
+				totalStages: s.runIdentity.totalStages,
+				trigger: s.runIdentity.trigger,
+				state: s.state,
+			}),
+		);
+		try {
+			await askAgentToFix(ctx, s, attempts, result.failures, timeoutMs);
+		} catch (e) {
+			const msg = e instanceof Error ? e.message : String(e);
+			return { kind: "fatal", message: msg };
+		}
+
+		const retryBranch = readBranch(ctx);
+		const retryCtx: CollectCtx = { ...deps.collectCtx, branch: retryBranch };
+		const reRun = await runOutcome(deps.outcome, retryCtx, deps.finalize);
+		if (reRun.kind === "fatal") return reRun;
+		const contract = enforceCompletionContract(s.stage, s.skill, reRun.output);
+		if (contract.kind === "fatal") return contract;
+
+		output = contract.output;
+		const reValidation = await validateOrFatal(schema, output.data, s.skill, timeoutMs);
+		if (reValidation.kind === "fatal") return reValidation;
+		result = reValidation.result;
+	}
+
+	if (!result.valid) return validationExhausted(result.failures);
+	return { kind: "ok", output };
+}
+
+/**
+ * Translate a thrown `validateOutputData` (user-authored schemas may throw
+ * synchronously or reject their Promise) into the canonical fatal-extraction
+ * outcome. Async schemas are guarded by `timeoutMs` — the same
+ * `validateTimeoutMs` budget that bounds the agent-settle step on a
+ * retry.
+ */
+async function validateOrFatal(
+	schema: StageSchema,
+	data: unknown,
+	skill: string,
+	timeoutMs: number,
+): Promise<{ kind: "ok"; result: ValidationResult } | { kind: "fatal"; message: string }> {
+	try {
+		const result = await withTimeout(
+			Promise.resolve(validateOutputData(schema, data)),
+			timeoutMs,
+			ERR_SCHEMA_TIMEOUT("outputSchema", timeoutMs),
+		);
+		return { kind: "ok", result };
+	} catch (e) {
+		const reason = e instanceof Error ? e.message : String(e);
+		return { kind: "fatal", message: `${skill}: ${reason}` };
+	}
+}
+
+async function askAgentToFix(
+	ctx: RunnerCtx,
+	s: StageSession,
+	attempt: number,
+	failures: SchemaValidationFailure[],
+	timeoutMs: number,
+): Promise<void> {
+	ctx.ui.notify(MSG_VALIDATION_RETRY(s.skill, attempt), "warning");
+	const errorLines = failures.map((f) => ` • ${f.path} — ${f.message}`).join("\n");
+	await withTimeout(
+		handlerFor(s.stage.sessionPolicy).send(ctx, MSG_VALIDATION_RETRY_PROMPT(s.skill, errorLines), s.continueHost),
+		timeoutMs,
+		`${s.skill}: validation retry attempt ${attempt} exceeded ${timeoutMs}ms — agent did not settle`,
+	);
+}
+
+function validationExhausted(failures: SchemaValidationFailure[]): OutputProduction {
+	const failureSummary = failures.map((f) => `${f.path}: ${f.message}`).join("; ");
+	return { kind: "validation-exhausted", failureSummary };
+}

package/sessions/index.ts

@@ -0,0 +1,7 @@
+/**
+ * Session execution public surface. The runner is internally split into
+ * three files (see `sessions.ts`'s header for the module map); this
+ * barrel re-exports only the symbols the rest of the package consumes.
+ */
+
+export { runFanoutSession, runStageSession } from "./sessions.js";

package/sessions/sessions.ts

@@ -0,0 +1,269 @@
+/**
+ * Session execution — one Pi session per workflow stage / fanout unit.
+ * `runStageSession` and `runFanoutSession` are the two public entries.
+ *
+ * The fresh-vs-continue policy split is owned by `SessionPolicyHandler`
+ * (see `spawn.ts`): `FRESH_HANDLER` and `CONTINUE_HANDLER` implement
+ * the three policy-specific decisions. Everything in this file —
+ * post-processing, halt routing, success persistence, outcome reading
+ * — is policy-agnostic.
+ *
+ * Companion modules:
+ *   - extraction.ts — produceAndValidateOutput + retry loop +
+ *                     outcome helpers (collector → parser pipeline).
+ *   - spawn.ts      — SessionPolicyHandler + FRESH/CONTINUE handlers +
+ *                     handlerFor.
+ */
+
+import {
+	type AuditCtx,
+	fanoutRowStage,
+	nowIso,
+	recordCancellation,
+	recordStage,
+	recordStopFailure,
+	recordTerminalFailure,
+} from "../audit.js";
+import { resolvePublishName } from "../internal-utils.js";
+import { buildLifecycleContext, skillStageRef } from "../lifecycle.js";
+import {
+	ERR_AUDIT_WRITE_FAILED,
+	ERR_VALIDATION_FAILED,
+	MSG_AUDIT_WRITE_FAILED,
+	MSG_STAGE_COMPLETE,
+	MSG_STAGE_FAILED,
+	MSG_VALIDATION_EXHAUSTED,
+} from "../messages.js";
+import type { Output } from "../output.js";
+import { type BranchEntry, classifyStop, readBranch, type StopSignal } from "../transcript.js";
+import type { FanoutSession, RunnerCtx, SessionContext, StageSession } from "../types.js";
+import { produceAndValidateOutput } from "./extraction.js";
+import { FRESH_HANDLER, handlerFor } from "./spawn.js";
+
+// ===========================================================================
+// PUBLIC ENTRIES — what the orchestrator calls
+// ===========================================================================
+
+/** Execute one DAG stage in its own session. */
+export async function runStageSession(ctx: RunnerCtx, s: StageSession): Promise<void> {
+	const handler = handlerFor(s.stage.sessionPolicy);
+	const { cancelled } = await handler.spawn(ctx, s.prompt, (sessionCtx) => postStage(sessionCtx, s), s.continueHost);
+	if (cancelled) await recordCancellation(ctx, auditFor(s));
+}
+
+/** Execute one fanout-unit iteration. Always fresh. */
+export async function runFanoutSession(ctx: RunnerCtx, s: FanoutSession): Promise<void> {
+	const { cancelled } = await FRESH_HANDLER.spawn(ctx, s.prompt, (sessionCtx) => postFanout(sessionCtx, s));
+	if (cancelled) await recordCancellation(ctx, auditFor(s));
+}
+
+// ===========================================================================
+// POST-PROCESSING — runs after the agent loop settles
+// ===========================================================================
+
+/** Stage post-processing: classify outcome → produce & validate output → persist → chain. */
+async function postStage(ctx: RunnerCtx, s: StageSession): Promise<void> {
+	const handler = handlerFor(s.stage.sessionPolicy);
+	const offset = handler.branchOffset(s.branchOffset);
+	const outcome = readSessionOutcome(ctx, offset);
+	if (outcome.stop !== "stop") return haltStage(ctx, s, outcome.stop);
+
+	const result = await produceAndValidateOutput(ctx, s, outcome.branch, offset);
+	if (result.kind === "fatal") return haltStageWithExtractionError(ctx, s, result.message);
+	if (result.kind === "validation-exhausted") return haltStageWithValidationFailure(ctx, s, result.failureSummary);
+
+	if (!(await recordStageSuccess(ctx, s, result.output))) return;
+	await s.onSuccess(ctx, result.output.artifacts[0]);
+}
+
+/** Fanout-unit post-processing: classify outcome → persist bare row → chain. */
+async function postFanout(ctx: RunnerCtx, s: FanoutSession): Promise<void> {
+	const outcome = readSessionOutcome(ctx, undefined);
+	if (outcome.stop !== "stop") return haltFanout(ctx, s, outcome.stop);
+
+	if (!(await recordFanoutSuccess(ctx, s))) return;
+	await s.onSuccess(ctx);
+}
+
+// ===========================================================================
+// HALT HELPERS — turn a halt reason into the right audit-layer call
+// ===========================================================================
+
+async function haltStage(ctx: RunnerCtx, s: StageSession, stop: Exclude<StopSignal, "stop">): Promise<void> {
+	await recordStopFailure(ctx, auditFor(s), stop, `${s.skill} failed`, s.onFailure);
+}
+
+async function haltStageWithExtractionError(ctx: RunnerCtx, s: StageSession, message: string): Promise<void> {
+	await recordTerminalFailure(
+		ctx,
+		auditFor(s),
+		{ status: "failed", notifyMsg: MSG_STAGE_FAILED(s.skill), notifyLevel: "error", errMsg: message },
+		s.onFailure,
+	);
+}
+
+async function haltStageWithValidationFailure(ctx: RunnerCtx, s: StageSession, failureSummary: string): Promise<void> {
+	await recordTerminalFailure(
+		ctx,
+		auditFor(s),
+		{
+			status: "failed",
+			notifyMsg: MSG_VALIDATION_EXHAUSTED(s.skill),
+			notifyLevel: "error",
+			errMsg: ERR_VALIDATION_FAILED(s.skill, failureSummary),
+		},
+		s.onFailure,
+	);
+}
+
+async function haltFanout(ctx: RunnerCtx, s: FanoutSession, stop: Exclude<StopSignal, "stop">): Promise<void> {
+	await recordStopFailure(ctx, auditFor(s), stop, `${s.skill} unit ${s.unitIndex} (${s.label}) failed`);
+}
+
+// ===========================================================================
+// SUCCESS-PERSISTENCE HELPERS
+// ===========================================================================
+
+/**
+ * Write + counter-increment guard shared by `recordStageSuccess` and
+ * `recordFanoutSuccess`. Returns `true` iff the JSONL row landed.
+ * Output assignment lives here so callers get the same "output is
+ * set iff the row that carried it landed" invariant.
+ */
+function tryRecordStage(s: SessionContext, row: { stage: string; skill?: string; output?: Output }): boolean {
+	const assigned = recordStage(
+		s.cwd,
+		s.runId,
+		{
+			stage: row.stage,
+			skill: row.skill,
+			status: "completed",
+			ts: nowIso(),
+			output: row.output,
+		},
+		s.state,
+	);
+	if (assigned === undefined) return false;
+	if (row.output) s.state.output = row.output;
+	s.state.stagesCompleted++;
+	return true;
+}
+
+/**
+ * Update the rolling chain-input slot. Three cases:
+ *   1. `produces` stages whose collector returned at least one artifact
+ *      advance the primary (first artifact wins; `role` is user-facing
+ *      metadata, not a framework gate).
+ *   2. `side-effect` stages with `inheritsArtifacts: false` (authored via
+ *      `terminal()`) CLEAR the slot — they explicitly break the chain
+ *      so anything after also starts without an inherited artifact.
+ *   3. Other `side-effect` stages (commit, implement) leave it in place
+ *      so a stage after them inherits the upstream chain input.
+ */
+function maybeAdvancePrimary(s: StageSession, output: Output): void {
+	if (s.stage.kind === "produces") {
+		const next = output.artifacts[0];
+		if (next) s.state.primaryArtifact = next;
+		const key = resolvePublishName(s.stage, s.stageName);
+		const slot = s.state.named[key];
+		if (slot) slot.push(output);
+		else s.state.named[key] = [output];
+		return;
+	}
+	if (s.stage.inheritsArtifacts === false) {
+		s.state.primaryArtifact = undefined;
+	}
+}
+
+/**
+ * Returns true on successful write — caller gates `onSuccess` on this so the
+ * chain advances only when the audit row landed. On failure, leaves
+ * `state.output` / `state.primaryArtifact` at their prior values and sets
+ * `state.termination.error` to halt the run.
+ */
+async function recordStageSuccess(ctx: RunnerCtx, s: StageSession, output: Output): Promise<boolean> {
+	if (tryRecordStage(s, { stage: s.stageName, skill: s.skill, output })) {
+		maybeAdvancePrimary(s, output);
+		ctx.ui.notify(MSG_STAGE_COMPLETE(s.skill), "info");
+		await s.lifecycle.fire(
+			ctx,
+			"onStageEnd",
+			skillStageRef(s.stageName, s.state.lastAllocatedStageNumber, s.skill),
+			output,
+			lifecycleCtxFromSession(s),
+		);
+		return true;
+	}
+	ctx.ui.notify(MSG_AUDIT_WRITE_FAILED(s.skill), "error");
+	s.state.termination.error = ERR_AUDIT_WRITE_FAILED(s.skill);
+	return false;
+}
+
+async function recordFanoutSuccess(ctx: RunnerCtx, s: FanoutSession): Promise<boolean> {
+	const stageLabel = fanoutRowStage(s);
+	if (tryRecordStage(s, { stage: stageLabel, skill: s.skill })) {
+		await s.lifecycle.fire(
+			ctx,
+			"onFanoutUnitEnd",
+			skillStageRef(s.stageName, s.stageIndex + 1, s.skill),
+			{ prompt: s.prompt, label: s.label, ...(s.id !== undefined && { id: s.id }) },
+			s.unitIndex,
+			lifecycleCtxFromSession(s),
+		);
+		return true;
+	}
+	s.state.termination.error = ERR_AUDIT_WRITE_FAILED(stageLabel);
+	return false;
+}
+
+/** Build a `LifecycleContext` from any SessionContext-shaped object. */
+function lifecycleCtxFromSession(s: SessionContext) {
+	return buildLifecycleContext({
+		cwd: s.cwd,
+		runId: s.runId,
+		workflow: s.runIdentity.workflow,
+		totalStages: s.runIdentity.totalStages,
+		trigger: s.runIdentity.trigger,
+		state: s.state,
+	});
+}
+
+// ===========================================================================
+// OUTCOME READER
+// ===========================================================================
+
+interface SessionOutcome {
+	branch: BranchEntry[];
+	stop: StopSignal;
+}
+
+/**
+ * Always reads the full unsliced branch + applies the policy-derived
+ * `branchOffset` to `classifyStop` so the prior-stage prefix is
+ * skipped in place. The same offset value flows through to
+ * `produceAndValidateOutput` (initial == retry).
+ *
+ * No longer scans the transcript for an artifact path — discovery is
+ * the collector's job, not the runner's.
+ */
+function readSessionOutcome(ctx: RunnerCtx, branchOffset: number | undefined): SessionOutcome {
+	const branch = readBranch(ctx);
+	return {
+		branch,
+		stop: classifyStop(branch, branchOffset),
+	};
+}
+
+// ===========================================================================
+// Helpers
+// ===========================================================================
+
+const auditFor = (s: StageSession | FanoutSession): AuditCtx => ({
+	cwd: s.cwd,
+	runId: s.runId,
+	state: s.state,
+	stageName: "unitIndex" in s ? fanoutRowStage(s) : s.stageName,
+	skill: s.skill,
+	lifecycle: s.lifecycle,
+	runIdentity: s.runIdentity,
+});