@agwab/pi-workflow 0.2.1 → 0.4.0

package/src/workflow-partial-output.ts

@@ -0,0 +1,299 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { hashDynamicRequest } from "./dynamic-events.js";
+import { readJson, writeJsonAtomic } from "./store.js";
+
+export const WORKFLOW_PARTIAL_OUTPUT_PROTOCOL =
+	"workflow-partial-output-v1" as const;
+export const WORKFLOW_PARTIAL_OUTPUT_LEDGER_SCHEMA =
+	"workflow-partial-output-ledger-v1" as const;
+export const WORKFLOW_PARTIAL_OUTPUT_LEDGER_FILE = "partial-control.json";
+
+export type WorkflowPartialOutputIssueCode =
+	| "invalid_json"
+	| "invalid_type"
+	| "invalid_schema"
+	| "invalid_path"
+	| "disallowed_path"
+	| "missing_items"
+	| "missing_item_id"
+	| "duplicate_item_id";
+
+export interface WorkflowPartialOutputIssue {
+	code: WorkflowPartialOutputIssueCode;
+	message: string;
+	sectionIndex?: number;
+	path?: string;
+	itemId?: string;
+}
+
+export interface WorkflowPartialOutputItem {
+	path: string;
+	itemId: string;
+	itemHash: string;
+	item: unknown;
+	ordinal: number;
+	sectionIndex: number;
+	sectionItemIndex: number;
+	itemRef: string;
+}
+
+export interface WorkflowPartialOutputLedger {
+	schema: typeof WORKFLOW_PARTIAL_OUTPUT_LEDGER_SCHEMA;
+	protocol: typeof WORKFLOW_PARTIAL_OUTPUT_PROTOCOL;
+	items: WorkflowPartialOutputItem[];
+	issues: WorkflowPartialOutputIssue[];
+}
+
+export interface ParseWorkflowPartialOutputOptions {
+	allowedPaths?: readonly string[];
+}
+
+interface PartialSectionMatch {
+	content: string;
+	start: number;
+	end: number;
+	index: number;
+}
+
+const PARTIAL_CONTROL_OPEN = "partial-control";
+
+export function partialOutputLedgerPath(taskDir: string): string {
+	return join(taskDir, WORKFLOW_PARTIAL_OUTPUT_LEDGER_FILE);
+}
+
+export async function readWorkflowPartialOutputLedger(
+	taskDir: string,
+): Promise<WorkflowPartialOutputLedger | undefined> {
+	return await readJson<WorkflowPartialOutputLedger>(
+		partialOutputLedgerPath(taskDir),
+	);
+}
+
+export async function writeWorkflowPartialOutputLedger(options: {
+	taskDir: string;
+	rawOutput: string;
+	allowedPaths?: readonly string[];
+}): Promise<WorkflowPartialOutputLedger> {
+	const ledger = parseWorkflowPartialOutput(options.rawOutput, {
+		allowedPaths: options.allowedPaths,
+	});
+	await writeJsonAtomic(partialOutputLedgerPath(options.taskDir), ledger);
+	return ledger;
+}
+
+export async function writeWorkflowPartialOutputLedgerFromFile(options: {
+	taskDir: string;
+	outputFile: string;
+	allowedPaths?: readonly string[];
+}): Promise<WorkflowPartialOutputLedger | undefined> {
+	const rawOutput = await readFile(options.outputFile, "utf8").catch(
+		() => undefined,
+	);
+	if (rawOutput === undefined) return undefined;
+	return await writeWorkflowPartialOutputLedger({
+		taskDir: options.taskDir,
+		rawOutput,
+		allowedPaths: options.allowedPaths,
+	});
+}
+
+export function stripWorkflowPartialOutputSections(raw: string): string {
+	if (!raw.includes(PARTIAL_CONTROL_OPEN)) return raw;
+	return raw.replace(partialControlSectionRegExp(), "");
+}
+
+export function parseWorkflowPartialOutput(
+	raw: string,
+	options: ParseWorkflowPartialOutputOptions = {},
+): WorkflowPartialOutputLedger {
+	const allowedPaths = options.allowedPaths
+		? new Set(options.allowedPaths)
+		: undefined;
+	const items: WorkflowPartialOutputItem[] = [];
+	const issues: WorkflowPartialOutputIssue[] = [];
+	const byPathAndId = new Map<string, WorkflowPartialOutputItem>();
+
+	for (const section of collectPartialControlSections(raw)) {
+		const parsed = parsePartialSectionJson(section, issues);
+		if (!parsed) continue;
+		const path = parsePartialSectionPath(parsed, section, allowedPaths, issues);
+		if (!path) continue;
+		const rawItems = parsePartialSectionItems(parsed, section, path, issues);
+		if (!rawItems) continue;
+		for (const [sectionItemIndex, item] of rawItems.entries()) {
+			const itemId = stablePartialItemId(item);
+			if (!itemId) {
+				issues.push({
+					code: "missing_item_id",
+					sectionIndex: section.index,
+					path,
+					message:
+						"partial output items must be objects with a stable non-empty string id",
+				});
+				continue;
+			}
+			const itemHash = hashDynamicRequest(item);
+			const key = `${path}\0${itemId}`;
+			const existing = byPathAndId.get(key);
+			if (existing) {
+				if (existing.itemHash !== itemHash) {
+					issues.push({
+						code: "duplicate_item_id",
+						sectionIndex: section.index,
+						path,
+						itemId,
+						message: `partial output item ${itemId} at ${path} changed after it was published`,
+					});
+				}
+				continue;
+			}
+			const ordinal = items.length;
+			const partialItem: WorkflowPartialOutputItem = {
+				path,
+				itemId,
+				itemHash,
+				item,
+				ordinal,
+				sectionIndex: section.index,
+				sectionItemIndex,
+				itemRef: `${WORKFLOW_PARTIAL_OUTPUT_LEDGER_FILE}#/items/${ordinal}`,
+			};
+			items.push(partialItem);
+			byPathAndId.set(key, partialItem);
+		}
+	}
+
+	return {
+		schema: WORKFLOW_PARTIAL_OUTPUT_LEDGER_SCHEMA,
+		protocol: WORKFLOW_PARTIAL_OUTPUT_PROTOCOL,
+		items,
+		issues,
+	};
+}
+
+export function hasFatalPartialOutputIssue(
+	ledger: Pick<WorkflowPartialOutputLedger, "issues"> | undefined,
+): WorkflowPartialOutputIssue | undefined {
+	return ledger?.issues.find((issue) => issue.code === "duplicate_item_id");
+}
+
+function collectPartialControlSections(raw: string): PartialSectionMatch[] {
+	if (!raw.includes(PARTIAL_CONTROL_OPEN)) return [];
+	const matches: PartialSectionMatch[] = [];
+	const re = partialControlSectionRegExp();
+	let match: RegExpExecArray | null;
+	while ((match = re.exec(raw)) !== null) {
+		matches.push({
+			content: match[1] ?? "",
+			start: match.index,
+			end: re.lastIndex,
+			index: matches.length,
+		});
+	}
+	return matches;
+}
+
+function partialControlSectionRegExp(): RegExp {
+	return /[ \t]*<partial-control\s*>([\s\S]*?)<\/partial-control>[ \t]*(?:\r?\n)?/gi;
+}
+
+function parsePartialSectionJson(
+	section: PartialSectionMatch,
+	issues: WorkflowPartialOutputIssue[],
+): Record<string, unknown> | undefined {
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(section.content.trim());
+	} catch (error) {
+		issues.push({
+			code: "invalid_json",
+			sectionIndex: section.index,
+			message: error instanceof Error ? error.message : String(error),
+		});
+		return undefined;
+	}
+	if (!isRecord(parsed)) {
+		issues.push({
+			code: "invalid_type",
+			sectionIndex: section.index,
+			message: "partial-control section must contain a JSON object",
+		});
+		return undefined;
+	}
+	if (parsed.schema !== WORKFLOW_PARTIAL_OUTPUT_PROTOCOL) {
+		issues.push({
+			code: "invalid_schema",
+			sectionIndex: section.index,
+			message: `partial-control schema must be ${WORKFLOW_PARTIAL_OUTPUT_PROTOCOL}`,
+		});
+		return undefined;
+	}
+	return parsed;
+}
+
+function parsePartialSectionPath(
+	section: Record<string, unknown>,
+	match: PartialSectionMatch,
+	allowedPaths: Set<string> | undefined,
+	issues: WorkflowPartialOutputIssue[],
+): string | undefined {
+	const path = section.path;
+	if (typeof path !== "string" || !path.startsWith("$.")) {
+		issues.push({
+			code: "invalid_path",
+			sectionIndex: match.index,
+			message: "partial-control path must be a control JSONPath starting with $.",
+		});
+		return undefined;
+	}
+	if (allowedPaths && !allowedPaths.has(path)) {
+		issues.push({
+			code: "disallowed_path",
+			sectionIndex: match.index,
+			path,
+			message: `partial-control path ${path} is not declared for this stage`,
+		});
+		return undefined;
+	}
+	return path;
+}
+
+function parsePartialSectionItems(
+	section: Record<string, unknown>,
+	match: PartialSectionMatch,
+	path: string,
+	issues: WorkflowPartialOutputIssue[],
+): unknown[] | undefined {
+	const items = section.items;
+	if (!Array.isArray(items)) {
+		issues.push({
+			code: "missing_items",
+			sectionIndex: match.index,
+			path,
+			message: "partial-control items must be an array",
+		});
+		return undefined;
+	}
+	return items;
+}
+
+function stablePartialItemId(item: unknown): string | undefined {
+	if (!isRecord(item) || typeof item.id !== "string") return undefined;
+	const sanitized = sanitizePartialItemId(item.id);
+	return sanitized || undefined;
+}
+
+function sanitizePartialItemId(value: string): string {
+	return value
+		.trim()
+		.toLowerCase()
+		.replace(/[^a-z0-9_.-]+/g, "-")
+		.replace(/^-+|-+$/g, "")
+		.slice(0, 64);
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/workflow-progress-health.ts

@@ -104,16 +104,16 @@ export function diagnoseWorkflowRunHealth(
 	options: WorkflowHealthOptions = {},
 ): WorkflowProgressHealth {
 	const nowMs = options.nowMs ?? Date.now();
-	const runningTask = currentRunningTask(run.tasks ?? []);
-	if (runningTask)
-		return diagnoseWorkflowTaskHealth(runningTask, run, { nowMs });
-
 	const problem = (run.tasks ?? []).find((task) =>
 		isProblemStatus(task.status),
 	);
 	if (problem) return problemRunHealth(problem, nowMs);
 	if (isProblemStatus(run.status)) return problemWorkflowHealth(run.status);
 	if (run.status === "completed") return completedWorkflowHealth();
+
+	const runningTask = currentRunningTask(run.tasks ?? []);
+	if (runningTask)
+		return diagnoseWorkflowTaskHealth(runningTask, run, { nowMs });
 	return waitingWorkflowHealth(run, nowMs);
 }
 
@@ -143,19 +143,20 @@ export function classifyWorkflowTaskDuration(
 		.filter(Boolean)
 		.join(" ")
 		.toLowerCase();
-	if (/\b(render|helper|support|schema|partition|gate)\b/.test(text))
-		return "short";
-	if (
-		/\b(research|audit|synthesis|review|verify|verifier|normalize|plan|impact|spec)\b/.test(
-			text,
-		)
-	)
-		return "long";
 	const maxRuntimeMs = task.runtime?.maxRuntimeMs;
 	if (maxRuntimeMs !== undefined && Number.isFinite(maxRuntimeMs)) {
 		if (maxRuntimeMs <= 5 * 60_000) return "short";
 		if (maxRuntimeMs >= 60 * 60_000) return "long";
 	}
+	if (task.kind === "support") return "short";
+	if (
+		/\b(research|audit|synthesi[sz]e?r?|review(?:er|ers|ing|s)?|verif(?:y|ier|iers|ication)|normaliz(?:e|er|ing|ation)?|plan(?:ning)?|impact|spec)\b/.test(
+			text,
+		)
+	)
+		return "long";
+	if (/\b(render|helper|support|schema|partition|gate)\b/.test(text))
+		return "short";
 	return "medium";
 }
 
@@ -215,6 +216,33 @@ function waitingWorkflowHealth(
 	nowMs: number,
 ): WorkflowProgressHealth {
 	const hasPending = run.taskSummary.pending > 0;
+	const lastActivityAgeMs = ageMs(run.updatedAt, nowMs);
+	if (hasPending && lastActivityAgeMs !== undefined) {
+		if (lastActivityAgeMs >= STUCK_BY_DURATION.medium) {
+			return {
+				state: "likely-stuck",
+				label: "scheduler stuck",
+				summary: "pending tasks have not scheduled",
+				tone: "error",
+				suggestion: "resume",
+				reason: "no task is running and run activity is stale",
+				lastActivityAt: run.updatedAt,
+				lastActivityAgeMs,
+			};
+		}
+		if (lastActivityAgeMs >= STALL_BY_DURATION.medium) {
+			return {
+				state: "stalled",
+				label: "scheduler quiet",
+				summary: "pending tasks are waiting without recent activity",
+				tone: "warning",
+				suggestion: "inspect",
+				reason: "no task is running and run activity is stale",
+				lastActivityAt: run.updatedAt,
+				lastActivityAgeMs,
+			};
+		}
+	}
 	return {
 		state: hasPending ? "pending" : "active",
 		label: hasPending ? "pending" : "active",
@@ -227,7 +255,7 @@ function waitingWorkflowHealth(
 			? "no task is currently running"
 			: "workflow is still in progress",
 		lastActivityAt: run.updatedAt,
-		lastActivityAgeMs: ageMs(run.updatedAt, nowMs),
+		lastActivityAgeMs,
 	};
 }
 
@@ -335,8 +363,12 @@ function runningContext(
 	const durationClass = classifyWorkflowTaskDuration(task);
 	const startedAtMs = parseTime(task.startedAt);
 	const heartbeatAt = parseHeartbeatAt(task.lastMessage);
+	const heartbeatAgeMs = ageMs(heartbeatAt, nowMs);
 	const activityAt = latestIso([heartbeatAt, run?.updatedAt, task.startedAt]);
 	const lastActivityAgeMs = ageMs(activityAt, nowMs);
+	const hasFreshHeartbeat =
+		heartbeatAgeMs !== undefined &&
+		heartbeatAgeMs <= STALL_BY_DURATION[durationClass];
 	return {
 		task,
 		nowMs,
@@ -346,8 +378,8 @@ function runningContext(
 		activityAt,
 		lastActivityAgeMs,
 		heartbeatAt,
-		heartbeatAgeMs: ageMs(heartbeatAt, nowMs),
-		hasBackendSignal: Boolean(task.backendHandle || task.pid || heartbeatAt),
+		heartbeatAgeMs,
+		hasBackendSignal: Boolean(task.pid || hasFreshHeartbeat),
 		staleMs: lastActivityAgeMs ?? Number.POSITIVE_INFINITY,
 	};
 }

package/src/workflow-runtime.ts

@@ -345,9 +345,25 @@ export function readSimpleJsonPath(value: unknown, path: string): unknown {
 	const parts = path.slice(2).split(".").filter(Boolean);
 	let current = value as any;
 	for (const part of parts) {
-		if (current === null || typeof current !== "object" || !(part in current))
-			return undefined;
+		if (!canReadJsonPathPart(current, part)) return undefined;
 		current = current[part];
 	}
 	return current;
 }
+
+function canReadJsonPathPart(
+	value: unknown,
+	part: string,
+): value is Record<string, unknown> {
+	return (
+		isSafeJsonPathPart(part) && isRecord(value) && Object.hasOwn(value, part)
+	);
+}
+
+function isSafeJsonPathPart(part: string): boolean {
+	return part !== "__proto__" && part !== "prototype" && part !== "constructor";
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null;
+}

package/src/workflow-view.ts

@@ -1370,10 +1370,11 @@ function statusForSummary(
 ): WorkflowRunStatus | TaskRunStatus {
 	if (summary.running > 0) return "running";
 	if (summary.blocked > 0) return "blocked";
-	if (summary.failed > 0 || summary.interrupted > 0) return "failed";
+	if (summary.failed > 0) return "failed";
 	if (summary.pending > 0) return "pending";
 	if (summary.total > 0 && summary.completed === summary.total)
 		return "completed";
+	if (summary.interrupted > 0) return "interrupted";
 	return "interrupted";
 }