@hotmeshio/long-tail 0.5.6 → 0.5.9

package/build/services/diagnostics/patterns.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.matchPatterns = matchPatterns;
+/**
+ * Match event stream + stream messages against known failure patterns.
+ * Returns findings ordered by severity (critical first).
+ */
+function matchPatterns(execution, workerMessages, engineMessages, escalationRow) {
+    const findings = [];
+    const events = execution.events ?? [];
+    const now = Date.now();
+    // ── Derive open signals ──────────────────────────────────────────────────
+    const signalCompletions = new Set(events
+        .filter(e => e.event_type === 'workflow_execution_signaled')
+        .map(e => e.attributes.timeline_key)
+        .filter(Boolean));
+    const openSignals = events.filter(e => e.event_type === 'signal_wait_started' &&
+        !signalCompletions.has(e.attributes.timeline_key));
+    // ── Dead-lettered messages ────────────────────────────────────────────────
+    const deadWorker = workerMessages.filter(m => m.dead_lettered_at);
+    const deadEngine = engineMessages.filter(m => m.dead_lettered_at);
+    // ── Reservation leaks: claimed but not ACK'd for > 30s ───────────────────
+    const leaks = workerMessages.filter(m => m.reserved_at && !m.expired_at && !m.dead_lettered_at &&
+        now - new Date(m.reserved_at).getTime() > 30_000);
+    // ── Pattern: orphaned_signal ──────────────────────────────────────────────
+    if (openSignals.length > 0 && !escalationRow) {
+        const workerMsg = workerMessages.find(m => m.expired_at && m.aid?.endsWith('/worker'));
+        const payload = safeParseJson(workerMsg?.message);
+        const queueConfig = payload?.data?.queueConfig;
+        const signalId = payload?.data?.signalId;
+        findings.push({
+            condition: 'orphaned_signal',
+            confidence: 0.98,
+            severity: 'critical',
+            evidence: [
+                `${openSignals.length} open signal(s) with no matching resolution`,
+                `No escalation row in hmsh_escalations for this workflow`,
+                queueConfig == null
+                    ? `Worker result missing queueConfig — escalation INSERT was skipped (pre-0.22 SDK)`
+                    : `Worker result has queueConfig but escalation INSERT did not complete`,
+                ...(signalId ? [`Signal key: ${signalId}`] : []),
+            ],
+            treatment: [
+                { action: 'create_escalation_row', note: 'INSERT missing row into public.hmsh_escalations' },
+                ...(signalId ? [{ action: 'resolve_by_signal_key', signal_key: signalId, note: 'POST /api/escalations/resolve-by-signal-key after inserting escalation row' }] : []),
+            ],
+        });
+    }
+    // ── Pattern: normal_wait ──────────────────────────────────────────────────
+    // Two paths reach here:
+    // 1. Open signal in event stream (signal_wait_started) + pending escalation
+    // 2. No terminal event + pending escalation (conditionLT path — no signal_wait_started in export)
+    const isRunning = !events.some(e => e.event_type === 'workflow_execution_completed' || e.event_type === 'workflow_execution_failed');
+    const isNormalWait = (openSignals.length > 0 && escalationRow?.status === 'pending') ||
+        (isRunning && events.length > 0 && escalationRow?.status === 'pending');
+    if (isNormalWait) {
+        findings.push({
+            condition: 'normal_wait',
+            confidence: 0.99,
+            severity: 'info',
+            evidence: [
+                `Workflow suspended — escalation exists (status: pending)`,
+                ...(escalationRow.role ? [`Role: ${escalationRow.role}, Type: ${escalationRow.type}`] : []),
+                ...(escalationRow.id ? [`Escalation id: ${escalationRow.id}`] : []),
+            ],
+            treatment: [{ action: 'none', note: 'Waiting for human to claim and resolve escalation' }],
+        });
+    }
+    // ── Pattern: dead_lettered_activity ──────────────────────────────────────
+    if (deadWorker.length > 0 || deadEngine.length > 0) {
+        const samples = [...deadWorker, ...deadEngine].slice(0, 3);
+        findings.push({
+            condition: 'dead_lettered_activity',
+            confidence: 0.99,
+            severity: 'critical',
+            evidence: [
+                `${deadWorker.length} worker message(s) dead-lettered, ${deadEngine.length} engine message(s) dead-lettered`,
+                ...samples.map(m => `aid: ${m.aid ?? 'engine'}, retries: ${m.retry_attempt}/${m.max_retry_attempts}, stream: ${m.stream_name}`),
+            ],
+            treatment: [{
+                    action: 'investigate_dead_letter',
+                    count: deadWorker.length + deadEngine.length,
+                    note: 'Message exhausted retries — check worker health and DB connectivity',
+                }],
+        });
+    }
+    // ── Pattern: reservation_leak ─────────────────────────────────────────────
+    if (leaks.length > 0) {
+        findings.push({
+            condition: 'reservation_leak',
+            confidence: 0.85,
+            severity: 'warning',
+            evidence: leaks.map(m => `aid: ${m.aid}, claimed by: ${m.reserved_by ?? 'unknown'}, open for ${Math.round((now - new Date(m.reserved_at).getTime()) / 1000)}s`),
+            treatment: [{ action: 'check_worker_health', note: 'Worker claimed message but never ACKd — check for crashed workers' }],
+        });
+    }
+    // ── Pattern: terminal_failure ─────────────────────────────────────────────
+    const lastEvent = events.at(-1);
+    if (lastEvent?.event_type === 'workflow_execution_failed') {
+        const attrs = lastEvent.attributes;
+        findings.push({
+            condition: 'terminal_failure',
+            confidence: 0.99,
+            severity: 'critical',
+            evidence: [
+                `Workflow terminated with failure`,
+                ...(attrs?.error ? [`Error: ${attrs.error}`] : []),
+            ],
+            treatment: [{ action: 'none', note: 'Workflow terminated — review error and restart if needed' }],
+        });
+    }
+    // ── Pattern: never_started ────────────────────────────────────────────────
+    const nonStartEvents = events.filter(e => e.event_type !== 'workflow_execution_started');
+    if (events.length > 0 && nonStartEvents.length === 0) {
+        findings.push({
+            condition: 'never_started',
+            confidence: 0.90,
+            severity: 'critical',
+            evidence: [`Workflow created but no activities have run — engine may not be consuming this stream`],
+            treatment: [{ action: 'check_engine_health', note: 'Check engine health and stream backlog for this workflow type' }],
+        });
+    }
+    // ── Default: healthy ──────────────────────────────────────────────────────
+    if (findings.length === 0) {
+        const isCompleted = events.some(e => e.event_type === 'workflow_execution_completed' || e.event_type === 'workflow_execution_failed');
+        findings.push({
+            condition: isCompleted ? 'completed' : 'running',
+            confidence: 0.95,
+            severity: 'info',
+            evidence: [isCompleted ? 'Workflow completed normally' : 'Workflow is running — no anomalies detected'],
+            treatment: [{ action: 'none', note: 'No intervention required' }],
+        });
+    }
+    // Critical first
+    const order = { critical: 0, warning: 1, info: 2 };
+    return findings.sort((a, b) => order[a.severity] - order[b.severity]);
+}
+function safeParseJson(raw) {
+    if (!raw)
+        return null;
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}

package/build/services/escalation/bulk.d.ts

@@ -20,6 +20,15 @@ export declare function bulkAssignEscalations(ids: string[], targetUserId: strin
  * Clears assignment on all affected rows.
  */
 export declare function bulkEscalateToRole(ids: string[], targetRole: string): Promise<number>;
+/**
+ * Bulk cancel escalations. Each row is cancelled individually; rows already in
+ * a terminal state are silently skipped by the SDK. Returns the count
+ * successfully cancelled.
+ */
+export declare function bulkCancelEscalations(ids: string[]): Promise<{
+    cancelled: number;
+    skipped: number;
+}>;
 /**
  * Bulk resolve escalations for AI triage.
  * Returns full records so the caller can start triage workflows. No signal is

package/build/services/escalation/bulk.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.bulkClaimEscalations = bulkClaimEscalations;
 exports.bulkAssignEscalations = bulkAssignEscalations;
 exports.bulkEscalateToRole = bulkEscalateToRole;
+exports.bulkCancelEscalations = bulkCancelEscalations;
 exports.bulkResolveForTriage = bulkResolveForTriage;
 const client_1 = require("./client");
 const map_1 = require("./map");
@@ -41,6 +42,26 @@ async function bulkEscalateToRole(ids, targetRole) {
     const client = await (0, client_1.escalations)();
     return client.escalateManyToRole({ ids, targetRole });
 }
+/**
+ * Bulk cancel escalations. Each row is cancelled individually; rows already in
+ * a terminal state are silently skipped by the SDK. Returns the count
+ * successfully cancelled.
+ */
+async function bulkCancelEscalations(ids) {
+    if (ids.length === 0)
+        return { cancelled: 0, skipped: 0 };
+    const client = await (0, client_1.escalations)();
+    let cancelled = 0;
+    let skipped = 0;
+    await Promise.all(ids.map(async (id) => {
+        const result = await client.cancel(id);
+        if (result.ok)
+            cancelled++;
+        else
+            skipped++;
+    }));
+    return { cancelled, skipped };
+}
 /**
  * Bulk resolve escalations for AI triage.
  * Returns full records so the caller can start triage workflows. No signal is

package/build/services/escalation/crud.d.ts

@@ -32,6 +32,13 @@ export declare function getEscalationBySignalKey(signalKey: string): Promise<LTE
  * pre-check (that would be a TOCTOU window) — the atomic resolve is the arbiter.
  */
 export declare function resolveEscalationBySignalKey(signalKey: string, resolverPayload: Record<string, any>): Promise<LTEscalationRecord | null>;
+/**
+ * Mark an escalation cancelled — used when the tied workflow has terminated and
+ * can never receive the resolution signal. The escalation is removed from the
+ * active queue but preserved for audit. Returns null when the row is missing or
+ * already terminal.
+ */
+export declare function cancelEscalation(id: string): Promise<LTEscalationRecord | null>;
 /**
  * Bulk update priority for a set of escalations. Only pending escalations are
  * updated.
@@ -62,6 +69,16 @@ export declare function escalateToRole(id: string, targetRole: string): Promise<
 export declare function getEscalation(id: string): Promise<LTEscalationRecord | null>;
 export declare function getEscalationsByTaskId(taskId: string): Promise<LTEscalationRecord[]>;
 export declare function getEscalationsByWorkflowId(workflowId: string): Promise<LTEscalationRecord[]>;
+/**
+ * Cancel all pending escalations tied to a workflow in one atomic UPDATE.
+ * Called after workflow termination so escalations don't remain in the queue.
+ *
+ * HotMesh's interrupt() is supposed to do this atomically, but it filters by
+ * `app_id = <engine-appId>`. Long-tail's Durable client uses app_id='durable'
+ * while escalations are stored with app_id='hmsh' — the WHERE never matches.
+ * This single UPDATE bypasses the SDK entirely and closes that gap.
+ */
+export declare function cancelEscalationsByWorkflowId(workflowId: string): Promise<number>;
 export declare function updateEscalationMetadata(id: string, patch: Record<string, any>): Promise<LTEscalationRecord | null>;
 export declare function enrichEscalationRouting(id: string, metadataPatch: Record<string, any>, workflowFields: {
     workflowType?: string;

package/build/services/escalation/crud.js

@@ -5,6 +5,7 @@ exports.claimEscalation = claimEscalation;
 exports.resolveEscalation = resolveEscalation;
 exports.getEscalationBySignalKey = getEscalationBySignalKey;
 exports.resolveEscalationBySignalKey = resolveEscalationBySignalKey;
+exports.cancelEscalation = cancelEscalation;
 exports.updateEscalationsPriority = updateEscalationsPriority;
 exports.getEscalationRoles = getEscalationRoles;
 exports.releaseEscalation = releaseEscalation;
@@ -13,6 +14,7 @@ exports.escalateToRole = escalateToRole;
 exports.getEscalation = getEscalation;
 exports.getEscalationsByTaskId = getEscalationsByTaskId;
 exports.getEscalationsByWorkflowId = getEscalationsByWorkflowId;
+exports.cancelEscalationsByWorkflowId = cancelEscalationsByWorkflowId;
 exports.updateEscalationMetadata = updateEscalationMetadata;
 exports.enrichEscalationRouting = enrichEscalationRouting;
 exports.getEscalationsByOriginId = getEscalationsByOriginId;
@@ -138,6 +140,30 @@ async function resolveEscalationBySignalKey(signalKey, resolverPayload) {
         return null;
     return resolveEscalation(escalation.id, resolverPayload);
 }
+/**
+ * Mark an escalation cancelled — used when the tied workflow has terminated and
+ * can never receive the resolution signal. The escalation is removed from the
+ * active queue but preserved for audit. Returns null when the row is missing or
+ * already terminal.
+ */
+async function cancelEscalation(id) {
+    const client = await (0, client_1.escalations)();
+    const result = await client.cancel(id);
+    if (!result.ok)
+        return null;
+    const cancelled = (0, map_1.toEscalationRecord)(result.entry);
+    (0, publish_1.publishEscalationEvent)({
+        type: 'escalation.cancelled',
+        source: 'service',
+        workflowId: cancelled.workflow_id || '',
+        workflowName: cancelled.workflow_type || '',
+        taskQueue: cancelled.task_queue || '',
+        escalationId: cancelled.id,
+        status: 'cancelled',
+        data: { reason: 'workflow_terminated' },
+    });
+    return cancelled;
+}
 /**
  * Bulk update priority for a set of escalations. Only pending escalations are
  * updated.
@@ -227,6 +253,37 @@ async function getEscalationsByWorkflowId(workflowId) {
     });
     return (0, map_1.toEscalationRecords)(rows);
 }
+/**
+ * Cancel all pending escalations tied to a workflow in one atomic UPDATE.
+ * Called after workflow termination so escalations don't remain in the queue.
+ *
+ * HotMesh's interrupt() is supposed to do this atomically, but it filters by
+ * `app_id = <engine-appId>`. Long-tail's Durable client uses app_id='durable'
+ * while escalations are stored with app_id='hmsh' — the WHERE never matches.
+ * This single UPDATE bypasses the SDK entirely and closes that gap.
+ */
+async function cancelEscalationsByWorkflowId(workflowId) {
+    await (0, client_1.ensureEscalationCompatView)();
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(`UPDATE public.hmsh_escalations
+        SET status = 'cancelled', updated_at = NOW()
+      WHERE workflow_id = $1
+        AND status = 'pending'
+      RETURNING id, workflow_id, workflow_type, task_queue`, [workflowId]);
+    for (const row of rows) {
+        (0, publish_1.publishEscalationEvent)({
+            type: 'escalation.cancelled',
+            source: 'service',
+            workflowId: row.workflow_id || '',
+            workflowName: row.workflow_type || '',
+            taskQueue: row.task_queue || '',
+            escalationId: row.id,
+            status: 'cancelled',
+            data: { reason: 'workflow_terminated' },
+        });
+    }
+    return rows.length;
+}
 async function updateEscalationMetadata(id, patch) {
     const client = await (0, client_1.escalations)();
     const entry = await client.update({ id, metadata: patch });

package/build/services/orchestrator/condition.d.ts

@@ -36,4 +36,4 @@ import type { Types } from '@hotmeshio/hotmesh';
  * const decision = await conditionLT<{ approved: boolean }>(signalId);
  * ```
  */
-export declare function conditionLT<T = Record<string, any>>(signalId: string, escalation?: Types.ConditionQueueConfig): Promise<T>;
+export declare function conditionLT<T = Record<string, any>>(signalId: string, escalation?: Types.ConditionQueueConfig): Promise<T | false | null>;

package/build/services/orchestrator/condition.js

@@ -76,6 +76,9 @@ const LT_ACTIVITY_QUEUE = 'lt-interceptor';
  */
 async function conditionLT(signalId, escalation) {
     const raw = await hotmesh_1.Durable.workflow.condition(signalId, escalation);
+    // false = timeout, null = escalation was cancelled — propagate both as-is
+    if (raw === null || raw === false)
+        return raw;
     const escalationId = raw.$escalation_id;
     if (escalationId) {
         // Resolve the escalation as a durable activity (crash-safe)

package/build/services/yaml-workflow/deployer.js

@@ -49,6 +49,7 @@ const logger_1 = require("../../lib/logger");
 const system_events_1 = require("../../lib/events/system-events");
 const namespaceService = __importStar(require("../namespace"));
 const deployer_helpers_1 = require("./deployer-helpers");
+const pipe_validator_1 = require("./pipe-validator");
 // Re-export helpers so existing `import * from './deployer'` consumers keep working
 var deployer_helpers_2 = require("./deployer-helpers");
 Object.defineProperty(exports, "buildMergedYaml", { enumerable: true, get: function () { return deployer_helpers_2.buildMergedYaml; } });
@@ -90,6 +91,10 @@ async function deployAppId(appId, version) {
     await namespaceService.registerNamespace(appId);
     const mergedYaml = await (0, deployer_helpers_1.buildMergedYaml)(appId, version);
     logger_1.loggerRegistry.debug(`[yaml-workflow] merged YAML for ${appId} v${version}:\n${mergedYaml}`);
+    // Structural guard: reject YAML with malformed @pipe rows before they reach
+    // the HotMesh engine. A bad pipe passes deploy() silently but causes
+    // collation errors on every execution, poisoning the stream.
+    (0, pipe_validator_1.validatePipeStructure)(mergedYaml);
     const engine = await getEngine(appId);
     try {
         const manifest = await engine.deploy(mergedYaml);

package/build/services/yaml-workflow/pipe-validator.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Structural validator for HotMesh YAML pipe syntax.
+ *
+ * HotMesh collation errors are silent at deploy time but catastrophic at
+ * runtime — a single malformed @pipe in an activated workflow poisons the
+ * engine's stream, causing every execution attempt to fail with a
+ * collation-error until the database is wiped.
+ *
+ * This module provides:
+ *   validatePipeStructure  — throws on any structurally invalid @pipe
+ *   repairPipeStructure    — auto-repairs safe-to-fix patterns, then validates
+ *
+ * Call repairPipeStructure before storing or deploying any YAML.
+ *
+ * ## What constitutes a valid @pipe row
+ *
+ * An @pipe value is a YAML sequence. Each element (row) must be one of:
+ *   1. A YAML sequence (array)          — operands row OR function call row
+ *   2. A YAML mapping with '@pipe' key  — nested sub-pipe
+ *
+ * Invalid row types that cause collation errors:
+ *   - A bare string:  '{@object.create}'     → must be ['{@object.create}']
+ *   - A number/bool:  42, true               → must be wrapped in an array
+ *   - A double-nested sequence: [[...]]      → row itself is an array of arrays
+ */
+export interface PipeViolation {
+    path: string;
+    rowIndex: number;
+    rowValue: unknown;
+    message: string;
+    repairable: boolean;
+}
+/**
+ * Parse the YAML and check every @pipe for structural violations.
+ * Throws a descriptive error listing all violations if any are found.
+ */
+export declare function validatePipeStructure(yamlContent: string): void;
+/**
+ * Auto-repair safe violations (bare strings/scalars → array rows), then
+ * validate that no unfixable violations remain.
+ *
+ * Returns the repaired YAML string. Throws if unfixable violations exist.
+ */
+export declare function repairPipeStructure(yamlContent: string): string;

package/build/services/yaml-workflow/pipe-validator.js

@@ -0,0 +1,215 @@
+"use strict";
+/**
+ * Structural validator for HotMesh YAML pipe syntax.
+ *
+ * HotMesh collation errors are silent at deploy time but catastrophic at
+ * runtime — a single malformed @pipe in an activated workflow poisons the
+ * engine's stream, causing every execution attempt to fail with a
+ * collation-error until the database is wiped.
+ *
+ * This module provides:
+ *   validatePipeStructure  — throws on any structurally invalid @pipe
+ *   repairPipeStructure    — auto-repairs safe-to-fix patterns, then validates
+ *
+ * Call repairPipeStructure before storing or deploying any YAML.
+ *
+ * ## What constitutes a valid @pipe row
+ *
+ * An @pipe value is a YAML sequence. Each element (row) must be one of:
+ *   1. A YAML sequence (array)          — operands row OR function call row
+ *   2. A YAML mapping with '@pipe' key  — nested sub-pipe
+ *
+ * Invalid row types that cause collation errors:
+ *   - A bare string:  '{@object.create}'     → must be ['{@object.create}']
+ *   - A number/bool:  42, true               → must be wrapped in an array
+ *   - A double-nested sequence: [[...]]      → row itself is an array of arrays
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validatePipeStructure = validatePipeStructure;
+exports.repairPipeStructure = repairPipeStructure;
+const jsYaml = __importStar(require("js-yaml"));
+// ── internal traversal ────────────────────────────────────────────────────────
+function isPlainObject(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+function collectPipeViolations(node, path, violations) {
+    if (Array.isArray(node)) {
+        node.forEach((item, i) => collectPipeViolations(item, `${path}[${i}]`, violations));
+        return;
+    }
+    if (!isPlainObject(node))
+        return;
+    for (const [key, value] of Object.entries(node)) {
+        const childPath = `${path}.${key}`;
+        if (key === '@pipe') {
+            if (!Array.isArray(value)) {
+                violations.push({
+                    path: childPath,
+                    rowIndex: -1,
+                    rowValue: value,
+                    message: `@pipe value must be a sequence (array), got ${typeof value}`,
+                    repairable: false,
+                });
+                continue;
+            }
+            value.forEach((row, i) => {
+                if (Array.isArray(row)) {
+                    // Valid row — but check for double-nested arrays: [[...]]
+                    if (row.length > 0 && row.every((el) => Array.isArray(el))) {
+                        violations.push({
+                            path: childPath,
+                            rowIndex: i,
+                            rowValue: row,
+                            message: `row ${i} is a double-nested array (array of arrays) — each @pipe row must be a flat array`,
+                            repairable: false,
+                        });
+                    }
+                }
+                else if (isPlainObject(row) && '@pipe' in row) {
+                    // Valid nested sub-pipe — recurse into it
+                    collectPipeViolations(row, `${childPath}[${i}]`, violations);
+                }
+                else if (typeof row === 'string') {
+                    // '{@fn}' as bare string — repairable: wrap in array
+                    violations.push({
+                        path: childPath,
+                        rowIndex: i,
+                        rowValue: row,
+                        message: `row ${i} is a bare string "${row}" — must be an array row like ['${row}']`,
+                        repairable: true,
+                    });
+                }
+                else if (typeof row === 'number' || typeof row === 'boolean') {
+                    violations.push({
+                        path: childPath,
+                        rowIndex: i,
+                        rowValue: row,
+                        message: `row ${i} is a bare scalar ${JSON.stringify(row)} — must be wrapped in an array`,
+                        repairable: true,
+                    });
+                }
+                else {
+                    violations.push({
+                        path: childPath,
+                        rowIndex: i,
+                        rowValue: row,
+                        message: `row ${i} has unexpected type ${typeof row}`,
+                        repairable: false,
+                    });
+                }
+            });
+        }
+        else {
+            collectPipeViolations(value, childPath, violations);
+        }
+    }
+}
+// ── public API ────────────────────────────────────────────────────────────────
+/**
+ * Parse the YAML and check every @pipe for structural violations.
+ * Throws a descriptive error listing all violations if any are found.
+ */
+function validatePipeStructure(yamlContent) {
+    let parsed;
+    try {
+        parsed = jsYaml.load(yamlContent);
+    }
+    catch (err) {
+        throw new Error(`YAML parse error: ${err.message}`);
+    }
+    const violations = [];
+    collectPipeViolations(parsed, 'root', violations);
+    if (violations.length > 0) {
+        const detail = violations
+            .map((v) => `  • ${v.path}[${v.rowIndex}]: ${v.message}`)
+            .join('\n');
+        throw new Error(`Malformed @pipe structure — ${violations.length} violation(s) detected. ` +
+            `This YAML will cause HotMesh collation errors at runtime and must not be deployed.\n${detail}`);
+    }
+}
+/**
+ * Auto-repair safe violations (bare strings/scalars → array rows), then
+ * validate that no unfixable violations remain.
+ *
+ * Returns the repaired YAML string. Throws if unfixable violations exist.
+ */
+function repairPipeStructure(yamlContent) {
+    let parsed;
+    try {
+        parsed = jsYaml.load(yamlContent);
+    }
+    catch (err) {
+        throw new Error(`YAML parse error: ${err.message}`);
+    }
+    const repairCount = { n: 0 };
+    repairNode(parsed, repairCount);
+    if (repairCount.n > 0) {
+        // Re-serialize — use lineWidth: -1 to prevent wrapping
+        yamlContent = jsYaml.dump(parsed, { lineWidth: -1, noRefs: true, quotingType: "'" });
+    }
+    // Final validation pass — throws if unfixable violations remain
+    validatePipeStructure(yamlContent);
+    return yamlContent;
+}
+function repairNode(node, count) {
+    if (Array.isArray(node)) {
+        node.forEach((item) => repairNode(item, count));
+        return;
+    }
+    if (!isPlainObject(node))
+        return;
+    for (const [key, value] of Object.entries(node)) {
+        if (key === '@pipe' && Array.isArray(value)) {
+            const rows = value;
+            for (let i = 0; i < rows.length; i++) {
+                const row = rows[i];
+                if (typeof row === 'string' || typeof row === 'number' || typeof row === 'boolean') {
+                    rows[i] = [row];
+                    count.n++;
+                }
+                else if (isPlainObject(row) && '@pipe' in row) {
+                    repairNode(row, count);
+                }
+                else if (Array.isArray(row)) {
+                    repairNode(row, count);
+                }
+            }
+        }
+        else {
+            repairNode(value, count);
+        }
+    }
+}

package/build/system/mcp-servers/admin/diagnostics.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Diagnostics tools — mirrors routes/diagnostics.ts
+ *
+ * Three tools:
+ *   diagnose_job            — full per-job diagnosis: events + stream messages + findings
+ *   find_stalled_jobs       — fleet-level: running jobs with no recent progress
+ *   find_orphaned_signals   — fleet-level: suspended jobs with no escalation row
+ */
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+export declare function registerDiagnosticsTools(server: McpServer): void;