@hotmeshio/long-tail 0.5.6 → 0.5.9

package/build/api/diagnostics.d.ts

@@ -0,0 +1,15 @@
+import type { LTApiResult } from '../types/sdk';
+export declare function diagnose(input: {
+    workflowId: string;
+    appId?: string;
+}): Promise<LTApiResult>;
+export declare function stalledJobs(input: {
+    appId?: string;
+    stalledMinutes?: number;
+    workflowType?: string;
+    limit?: number;
+}): Promise<LTApiResult>;
+export declare function orphanedSignals(input: {
+    appId?: string;
+    limit?: number;
+}): Promise<LTApiResult>;

package/build/api/diagnostics.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diagnose = diagnose;
+exports.stalledJobs = stalledJobs;
+exports.orphanedSignals = orphanedSignals;
+const diagnostics_1 = require("../services/diagnostics");
+async function diagnose(input) {
+    try {
+        const diagnosis = await (0, diagnostics_1.diagnoseJob)(input.workflowId, input.appId);
+        return { status: 200, data: diagnosis };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function stalledJobs(input) {
+    try {
+        const result = await (0, diagnostics_1.findStalledJobs)(input);
+        return { status: 200, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+async function orphanedSignals(input) {
+    try {
+        const result = await (0, diagnostics_1.findOrphanedSignals)(input);
+        return { status: 200, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}

package/build/api/escalations/cancel.d.ts

@@ -0,0 +1,20 @@
+import type { LTApiResult, LTApiAuth } from '../../types/sdk';
+/**
+ * Cancel a single escalation. The caller must hold global escalation access
+ * or the escalation's role.
+ *
+ * @returns `{ status: 200, data: { cancelled: true } }` or 403/404/409/500
+ */
+export declare function cancelSingleEscalation(input: {
+    id: string;
+}, auth: LTApiAuth): Promise<LTApiResult>;
+/**
+ * Cancel multiple escalations at once.
+ *
+ * @param input.ids — array of escalation UUIDs
+ * @param auth — authenticated user context (admin required)
+ * @returns `{ status: 200, data: { cancelled, skipped } }`
+ */
+export declare function bulkCancel(input: {
+    ids: string[];
+}, auth: LTApiAuth): Promise<LTApiResult>;

package/build/api/escalations/cancel.js

@@ -0,0 +1,93 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.cancelSingleEscalation = cancelSingleEscalation;
+exports.bulkCancel = bulkCancel;
+const escalationService = __importStar(require("../../services/escalation"));
+const helpers_1 = require("./helpers");
+const userService = __importStar(require("../../services/user"));
+/**
+ * Cancel a single escalation. The caller must hold global escalation access
+ * or the escalation's role.
+ *
+ * @returns `{ status: 200, data: { cancelled: true } }` or 403/404/409/500
+ */
+async function cancelSingleEscalation(input, auth) {
+    try {
+        const { id } = input;
+        if (!id)
+            return { status: 400, error: 'id is required' };
+        const escalation = await escalationService.getEscalation(id);
+        if (!escalation)
+            return { status: 404, error: 'Escalation not found' };
+        const hasGlobal = await (0, helpers_1.hasGlobalEscalationAccess)(auth.userId);
+        if (!hasGlobal) {
+            const hasRole = await userService.hasRole(auth.userId, escalation.role);
+            if (!hasRole)
+                return { status: 403, error: 'Forbidden' };
+        }
+        const cancelled = await escalationService.cancelEscalation(id);
+        if (!cancelled) {
+            return { status: 409, error: 'Escalation is not cancellable (already terminal)' };
+        }
+        return { status: 200, data: { cancelled: true, escalationId: id } };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}
+/**
+ * Cancel multiple escalations at once.
+ *
+ * @param input.ids — array of escalation UUIDs
+ * @param auth — authenticated user context (admin required)
+ * @returns `{ status: 200, data: { cancelled, skipped } }`
+ */
+async function bulkCancel(input, auth) {
+    try {
+        const { ids } = input;
+        if (!(0, helpers_1.validateIds)(ids)) {
+            return { status: 400, error: 'ids must be a non-empty array' };
+        }
+        const perm = await (0, helpers_1.checkBulkPermission)(auth.userId, ids);
+        if (!perm.allowed)
+            return perm;
+        const result = await escalationService.bulkCancelEscalations(ids);
+        return { status: 200, data: result };
+    }
+    catch (err) {
+        return { status: 500, error: err.message };
+    }
+}

package/build/api/escalations/index.d.ts

@@ -5,3 +5,4 @@ export { claimEscalation, releaseEscalation } from './claim';
 export { releaseExpiredClaims, updatePriority, bulkClaim, bulkAssign, bulkEscalate, bulkTriage } from './bulk';
 export { resolveEscalation, resolveBySignalKey } from './resolve';
 export { findByMetadata, claimByMetadata, resolveByMetadata } from './metadata';
+export { cancelSingleEscalation, bulkCancel } from './cancel';

package/build/api/escalations/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.resolveByMetadata = exports.claimByMetadata = exports.findByMetadata = exports.resolveBySignalKey = exports.resolveEscalation = exports.bulkTriage = exports.bulkEscalate = exports.bulkAssign = exports.bulkClaim = exports.updatePriority = exports.releaseExpiredClaims = exports.releaseEscalation = exports.claimEscalation = exports.escalateToRole = exports.getEscalationsByWorkflowId = exports.getEscalation = exports.getEscalationStats = exports.listDistinctTypes = exports.listAvailableEscalations = exports.listEscalations = exports.createEscalation = void 0;
+exports.bulkCancel = exports.cancelSingleEscalation = exports.resolveByMetadata = exports.claimByMetadata = exports.findByMetadata = exports.resolveBySignalKey = exports.resolveEscalation = exports.bulkTriage = exports.bulkEscalate = exports.bulkAssign = exports.bulkClaim = exports.updatePriority = exports.releaseExpiredClaims = exports.releaseEscalation = exports.claimEscalation = exports.escalateToRole = exports.getEscalationsByWorkflowId = exports.getEscalation = exports.getEscalationStats = exports.listDistinctTypes = exports.listAvailableEscalations = exports.listEscalations = exports.createEscalation = void 0;
 var create_1 = require("./create");
 Object.defineProperty(exports, "createEscalation", { enumerable: true, get: function () { return create_1.createEscalation; } });
 var list_1 = require("./list");
@@ -29,3 +29,6 @@ var metadata_1 = require("./metadata");
 Object.defineProperty(exports, "findByMetadata", { enumerable: true, get: function () { return metadata_1.findByMetadata; } });
 Object.defineProperty(exports, "claimByMetadata", { enumerable: true, get: function () { return metadata_1.claimByMetadata; } });
 Object.defineProperty(exports, "resolveByMetadata", { enumerable: true, get: function () { return metadata_1.resolveByMetadata; } });
+var cancel_1 = require("./cancel");
+Object.defineProperty(exports, "cancelSingleEscalation", { enumerable: true, get: function () { return cancel_1.cancelSingleEscalation; } });
+Object.defineProperty(exports, "bulkCancel", { enumerable: true, get: function () { return cancel_1.bulkCancel; } });

package/build/api/escalations/resolve.js

@@ -66,6 +66,8 @@ async function resolveEscalation(input, _auth) {
         const escalation = await escalationService.getEscalation(id);
         if (!escalation)
             return { status: 404, error: 'Escalation not found' };
+        if (escalation.status === 'cancelled')
+            return { status: 409, error: 'Escalation is cancelled' };
         if (escalation.status !== 'pending')
             return { status: 409, error: 'Escalation not available for resolution' };
         // Path A: conditionLT signal

package/build/api/workflows/invocation.js

@@ -42,6 +42,7 @@ const workers_1 = require("../../workers");
 const exportService = __importStar(require("../../services/export"));
 const task_1 = require("../../services/task");
 const workflow_invocation_1 = require("../../services/workflow-invocation");
+const crud_1 = require("../../services/escalation/crud");
 function isResolveError(err) {
     return err?.message?.includes('Cannot resolve workflow');
 }
@@ -154,6 +155,11 @@ async function terminateWorkflow(input) {
         const client = (0, workers_1.createClient)();
         const handle = await client.workflow.getHandle(resolved.taskQueue, resolved.workflowName, input.workflowId);
         await handle.terminate();
+        // HotMesh's interrupt() cancels escalations in the same Postgres transaction,
+        // but only when the engine app_id matches the escalation row's app_id. The
+        // Durable client uses app_id='durable' while escalations are stored with
+        // app_id='hmsh', so the automatic cancel is a no-op. Cancel explicitly here.
+        await (0, crud_1.cancelEscalationsByWorkflowId)(input.workflowId);
         return {
             status: 200,
             data: { terminated: true, workflowId: input.workflowId },

package/build/lib/events/publish.d.ts

@@ -35,7 +35,7 @@ export declare function publishTaskEvent(params: {
  * Subject: system.escalation.{escalationId}.{action}
  */
 export declare function publishEscalationEvent(params: {
-    type: 'escalation.created' | 'escalation.resolved' | 'escalation.claimed' | 'escalation.released';
+    type: 'escalation.created' | 'escalation.resolved' | 'escalation.claimed' | 'escalation.released' | 'escalation.cancelled';
     source: string;
     workflowId: string;
     workflowName: string;

package/build/routes/diagnostics.d.ts

@@ -0,0 +1,2 @@
+declare const router: import("express-serve-static-core").Router;
+export default router;

package/build/routes/diagnostics.js

@@ -0,0 +1,88 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const express_1 = require("express");
+const api = __importStar(require("../api/diagnostics"));
+const auth_1 = require("../modules/auth");
+const router = (0, express_1.Router)();
+/**
+ * GET /api/diagnostics/jobs/:workflowId
+ * Full diagnosis for a single workflow job.
+ *
+ * Query params:
+ *   app_id — HotMesh namespace (default: durable)
+ */
+router.get('/jobs/:workflowId', auth_1.requireAdmin, async (req, res) => {
+    const result = await api.diagnose({
+        workflowId: req.params.workflowId,
+        appId: typeof req.query.app_id === 'string' ? req.query.app_id : undefined,
+    });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * GET /api/diagnostics/stalled
+ * Running jobs with no recent progress.
+ *
+ * Query params:
+ *   app_id          — HotMesh namespace (default: durable)
+ *   stalled_minutes — minimum minutes since last event (default: 5)
+ *   workflow_type   — filter by workflow function name
+ *   limit           — max results (default: 50, max: 200)
+ */
+router.get('/stalled', auth_1.requireAdmin, async (req, res) => {
+    const result = await api.stalledJobs({
+        appId: typeof req.query.app_id === 'string' ? req.query.app_id : undefined,
+        stalledMinutes: req.query.stalled_minutes ? Number(req.query.stalled_minutes) : undefined,
+        workflowType: typeof req.query.workflow_type === 'string' ? req.query.workflow_type : undefined,
+        limit: req.query.limit ? Number(req.query.limit) : undefined,
+    });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+/**
+ * GET /api/diagnostics/orphaned-signals
+ * Suspended jobs with no escalation row.
+ *
+ * Query params:
+ *   app_id — HotMesh namespace (default: durable)
+ *   limit  — max results (default: 100, max: 500)
+ */
+router.get('/orphaned-signals', auth_1.requireAdmin, async (req, res) => {
+    const result = await api.orphanedSignals({
+        appId: typeof req.query.app_id === 'string' ? req.query.app_id : undefined,
+        limit: req.query.limit ? Number(req.query.limit) : undefined,
+    });
+    res.status(result.status).json(result.data ?? { error: result.error });
+});
+exports.default = router;

package/build/routes/escalations/bulk.js

@@ -64,4 +64,8 @@ function registerBulkRoutes(router) {
         const result = await api.bulkTriage({ ids: req.body?.ids, hint: req.body?.hint }, req.auth);
         res.status(result.status).json(result.data ?? { error: result.error });
     });
+    router.post('/bulk-cancel', async (req, res) => {
+        const result = await api.bulkCancel({ ids: req.body?.ids }, req.auth);
+        res.status(result.status).json(result.data ?? { error: result.error });
+    });
 }

package/build/routes/escalations/single.js

@@ -83,4 +83,13 @@ function registerSingleRoutes(router) {
         const result = await api.releaseEscalation({ id: req.params.id }, req.auth);
         res.status(result.status).json(result.data ?? { error: result.error });
     });
+    /**
+     * POST /api/escalations/:id/cancel
+     * Permanently cancel an escalation. Caller must hold the escalation's role
+     * or have global escalation access. Terminal escalations return 409.
+     */
+    router.post('/:id/cancel', async (req, res) => {
+        const result = await api.cancelSingleEscalation({ id: req.params.id }, req.auth);
+        res.status(result.status).json(result.data ?? { error: result.error });
+    });
 }

package/build/routes/index.js

@@ -70,4 +70,6 @@ router.use('/capabilities', capabilities_1.default);
 router.use('/topics', topics_1.default);
 const overview_1 = __importDefault(require("./overview"));
 router.use('/overview', overview_1.default);
+const diagnostics_1 = __importDefault(require("./diagnostics"));
+router.use('/diagnostics', diagnostics_1.default);
 exports.default = router;

package/build/services/diagnostics/index.d.ts

@@ -0,0 +1,52 @@
+import type { Finding } from './patterns';
+import type { StreamMessage } from '../controlplane/types';
+export type { Finding };
+export interface EscalationSummary {
+    exists: boolean;
+    id: string | null;
+    status: string | null;
+    signal_key: string | null;
+    role: string | null;
+    type: string | null;
+    created_at: string | null;
+}
+export interface StreamSummary {
+    worker_total: number;
+    engine_total: number;
+    dead_lettered: number;
+    pending: number;
+    in_flight: number;
+}
+export interface JobDiagnosis {
+    workflow_id: string;
+    app_id: string;
+    status: 'running' | 'completed' | 'failed' | 'not_found';
+    stalled_for_ms: number | null;
+    workflow_type: string | null;
+    last_event_at: string | null;
+    stream_summary: StreamSummary;
+    escalation: EscalationSummary;
+    findings: Finding[];
+    execution_events: unknown[];
+    stream_messages: {
+        worker: StreamMessage[];
+        engine: StreamMessage[];
+    };
+}
+export declare function diagnoseJob(workflowId: string, appId?: string): Promise<JobDiagnosis>;
+export declare function findStalledJobs(params: {
+    appId?: string;
+    stalledMinutes?: number;
+    workflowType?: string | null;
+    limit?: number;
+}): Promise<{
+    jobs: unknown[];
+    total: number;
+}>;
+export declare function findOrphanedSignals(params: {
+    appId?: string;
+    limit?: number;
+}): Promise<{
+    orphans: unknown[];
+    total: number;
+}>;

package/build/services/diagnostics/index.js

@@ -0,0 +1,169 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.diagnoseJob = diagnoseJob;
+exports.findStalledJobs = findStalledJobs;
+exports.findOrphanedSignals = findOrphanedSignals;
+const db_1 = require("../../lib/db");
+const export_1 = require("../export");
+const controlplane_1 = require("../controlplane");
+const resolve_1 = require("../task/resolve");
+const patterns_1 = require("./patterns");
+// ── SQL ──────────────────────────────────────────────────────────────────────
+const ESCALATION_BY_JOB = `
+  SELECT id, status, signal_key, role, type, assigned_to, created_at
+  FROM public.hmsh_escalations
+  WHERE workflow_id = $1
+  ORDER BY created_at DESC LIMIT 5
+`;
+const STALLED_JOBS = (appId) => `
+  SELECT
+    REPLACE(j.key, $1, '') AS workflow_id,
+    j.entity               AS workflow_type,
+    j.status,
+    j.created_at,
+    j.updated_at,
+    EXTRACT(EPOCH FROM (NOW() - j.updated_at)) * 1000 AS stalled_ms
+  FROM "${appId}".jobs j
+  WHERE j.status > 0
+    AND j.is_live = TRUE
+    AND j.updated_at < NOW() - ($2 || ' minutes')::INTERVAL
+    AND ($3::text IS NULL OR j.entity = $3)
+  ORDER BY j.updated_at ASC
+  LIMIT $4
+`;
+const ORPHANED_SIGNALS = (appId) => `
+  WITH committed_waiters AS (
+    SELECT
+      ws.jid                                AS job_id,
+      ws.message->'data'->>'signalId'       AS signal_id,
+      ws.message->'data'->>'queueConfig'    AS queue_config,
+      ws.workflow_name,
+      ws.created_at                         AS suspended_at
+    FROM "${appId}".worker_streams ws
+    WHERE ws.aid LIKE '%/waiter'
+      AND ws.expired_at IS NOT NULL
+      AND ws.dead_lettered_at IS NULL
+  ),
+  live_jobs AS (
+    SELECT REPLACE(key, 'hmsh:${appId}:j:', '') AS wid
+    FROM "${appId}".jobs
+    WHERE status > 0 AND is_live = TRUE
+  )
+  SELECT
+    cw.job_id,
+    cw.signal_id,
+    cw.workflow_name,
+    cw.suspended_at,
+    EXTRACT(EPOCH FROM (NOW() - cw.suspended_at)) * 1000 AS stalled_ms,
+    cw.queue_config IS NULL AS missing_queue_config
+  FROM committed_waiters cw
+  JOIN live_jobs lj ON lj.wid = cw.job_id
+  WHERE NOT EXISTS (
+    SELECT 1 FROM public.hmsh_escalations e WHERE e.workflow_id = cw.job_id
+  )
+  ORDER BY cw.suspended_at ASC
+  LIMIT $1
+`;
+// ── diagnoseJob ──────────────────────────────────────────────────────────────
+async function diagnoseJob(workflowId, appId = 'durable') {
+    const pool = (0, db_1.getPool)();
+    let resolved;
+    try {
+        resolved = await (0, resolve_1.resolveWorkflowHandle)(workflowId);
+    }
+    catch {
+        return notFound(workflowId, appId);
+    }
+    const [execution, workerResult, engineResult, escalationResult] = await Promise.all([
+        (0, export_1.exportWorkflowExecution)(workflowId, resolved.taskQueue, resolved.workflowName)
+            .catch(() => null),
+        (0, controlplane_1.getStreamMessages)(appId, { source: 'worker', jid: workflowId, limit: 100, order: 'asc' }),
+        (0, controlplane_1.getStreamMessages)(appId, { source: 'engine', jid: workflowId, limit: 100, order: 'asc' }),
+        pool.query(ESCALATION_BY_JOB, [workflowId]),
+    ]);
+    const escalationRow = escalationResult.rows[0] ?? null;
+    const workerMessages = workerResult.messages;
+    const engineMessages = engineResult.messages;
+    const events = execution?.events ?? [];
+    const lastEvent = events.at(-1);
+    const now = Date.now();
+    const isCompleted = events.some(e => e.event_type === 'workflow_execution_completed');
+    const isFailed = events.some(e => e.event_type === 'workflow_execution_failed');
+    const status = isCompleted ? 'completed' : isFailed ? 'failed' : 'running';
+    const lastEventAt = lastEvent?.event_time ?? null;
+    const stalledMs = status === 'running' && lastEventAt
+        ? now - new Date(lastEventAt).getTime()
+        : null;
+    const allMessages = [...workerMessages, ...engineMessages];
+    const streamSummary = {
+        worker_total: workerMessages.length,
+        engine_total: engineMessages.length,
+        dead_lettered: allMessages.filter(m => m.dead_lettered_at).length,
+        pending: allMessages.filter(m => !m.reserved_at && !m.expired_at && !m.dead_lettered_at).length,
+        in_flight: allMessages.filter(m => m.reserved_at && !m.expired_at && !m.dead_lettered_at).length,
+    };
+    const findings = execution
+        ? (0, patterns_1.matchPatterns)(execution, workerMessages, engineMessages, escalationRow)
+        : [{ condition: 'export_unavailable', confidence: 0.9, severity: 'warning', evidence: ['Execution export unavailable — job may have expired or HotMesh engine is not running'], treatment: [] }];
+    return {
+        workflow_id: workflowId,
+        app_id: appId,
+        status,
+        stalled_for_ms: stalledMs,
+        workflow_type: resolved.workflowName,
+        last_event_at: lastEventAt,
+        stream_summary: streamSummary,
+        escalation: {
+            exists: !!escalationRow,
+            id: escalationRow?.id ?? null,
+            status: escalationRow?.status ?? null,
+            signal_key: escalationRow?.signal_key ?? null,
+            role: escalationRow?.role ?? null,
+            type: escalationRow?.type ?? null,
+            created_at: escalationRow?.created_at ?? null,
+        },
+        findings,
+        execution_events: events,
+        stream_messages: { worker: workerMessages, engine: engineMessages },
+    };
+}
+// ── findStalledJobs ──────────────────────────────────────────────────────────
+async function findStalledJobs(params) {
+    const appId = params.appId ?? 'durable';
+    const stalledMinutes = params.stalledMinutes ?? 5;
+    const limit = Math.min(params.limit ?? 50, 200);
+    const keyPrefix = `hmsh:${appId}:j:`;
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(STALLED_JOBS(appId), [
+        keyPrefix,
+        String(stalledMinutes),
+        params.workflowType ?? null,
+        limit,
+    ]).catch(() => ({ rows: [] }));
+    return { jobs: rows, total: rows.length };
+}
+// ── findOrphanedSignals ──────────────────────────────────────────────────────
+async function findOrphanedSignals(params) {
+    const appId = params.appId ?? 'durable';
+    const limit = Math.min(params.limit ?? 100, 500);
+    const pool = (0, db_1.getPool)();
+    const { rows } = await pool.query(ORPHANED_SIGNALS(appId), [limit])
+        .catch(() => ({ rows: [] }));
+    return { orphans: rows, total: rows.length };
+}
+// ── helpers ──────────────────────────────────────────────────────────────────
+function notFound(workflowId, appId) {
+    return {
+        workflow_id: workflowId,
+        app_id: appId,
+        status: 'not_found',
+        stalled_for_ms: null,
+        workflow_type: null,
+        last_event_at: null,
+        stream_summary: { worker_total: 0, engine_total: 0, dead_lettered: 0, pending: 0, in_flight: 0 },
+        escalation: { exists: false, id: null, status: null, signal_key: null, role: null, type: null, created_at: null },
+        findings: [{ condition: 'not_found', confidence: 1, severity: 'critical', evidence: ['Workflow ID not found — no task record or job entity'], treatment: [] }],
+        execution_events: [],
+        stream_messages: { worker: [], engine: [] },
+    };
+}

package/build/services/diagnostics/patterns.d.ts

@@ -0,0 +1,19 @@
+import type { WorkflowExecution } from '@hotmeshio/hotmesh/build/types/exporter';
+import type { StreamMessage } from '../controlplane/types';
+export interface Finding {
+    condition: string;
+    confidence: number;
+    severity: 'critical' | 'warning' | 'info';
+    evidence: string[];
+    treatment: RecoveryOption[];
+}
+export interface RecoveryOption {
+    action: string;
+    note: string;
+    [key: string]: unknown;
+}
+/**
+ * Match event stream + stream messages against known failure patterns.
+ * Returns findings ordered by severity (critical first).
+ */
+export declare function matchPatterns(execution: WorkflowExecution, workerMessages: StreamMessage[], engineMessages: StreamMessage[], escalationRow: Record<string, unknown> | null): Finding[];