@hotmeshio/hotmesh 0.26.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/package.json +1 -1
- package/build/services/durable/handle.js +45 -6
- package/build/services/escalations/client.d.ts +21 -1
- package/build/services/escalations/client.js +68 -0
- package/build/services/store/providers/postgres/postgres.d.ts +22 -0
- package/build/services/store/providers/postgres/postgres.js +111 -0
- package/build/types/hmsh_escalations.d.ts +41 -0
- package/build/types/index.d.ts +1 -1
- package/build/types/system_events.d.ts +2 -2
- package/package.json +1 -1
- package/OPTIMIZATIONS.md +0 -128
- package/RELEASE-0.26.0.md +0 -111
package/build/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WorkflowHandleService = void 0;
|
|
4
|
+
const utils_1 = require("../../modules/utils");
|
|
4
5
|
const exporter_1 = require("./exporter");
|
|
5
6
|
/**
|
|
6
7
|
* Handle to a running or completed workflow execution. Returned by
|
|
@@ -253,8 +254,13 @@ class WorkflowHandleService {
|
|
|
253
254
|
return complete(null, JSON.parse(state.metadata.err));
|
|
254
255
|
}
|
|
255
256
|
}
|
|
256
|
-
//subscribe to 'done' topic
|
|
257
|
-
|
|
257
|
+
//subscribe to 'done' topic. Await the registration: the getStatus
|
|
258
|
+
//fallback below only covers completions that land BEFORE it reads.
|
|
259
|
+
//Unawaited, a job that completes after the status read but before
|
|
260
|
+
//the subscription goes active is seen by neither arm and result()
|
|
261
|
+
//never settles — hit reliably when a bulk resolve completes many
|
|
262
|
+
//workflows in the same instant.
|
|
263
|
+
await this.hotMesh.sub(topic, async (_topic, state) => {
|
|
258
264
|
this.hotMesh.unsub(topic);
|
|
259
265
|
if (state.data.done && !state.data?.$error) {
|
|
260
266
|
await complete(state.data?.response);
|
|
@@ -267,10 +273,43 @@ class WorkflowHandleService {
|
|
|
267
273
|
return await complete(null, error);
|
|
268
274
|
}
|
|
269
275
|
});
|
|
270
|
-
//check state in case completed during wiring
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
276
|
+
//check state in case completed during wiring. One read is not
|
|
277
|
+
//enough: the executed event commits before the status decrement,
|
|
278
|
+
//so a job caught mid-close shows status > 0 with its event already
|
|
279
|
+
//published — a subscriber wiring up in that window misses both
|
|
280
|
+
//arms. Escalating re-checks close the window (the status settles
|
|
281
|
+
//milliseconds after the publish), and the slow steady poll keeps
|
|
282
|
+
//result() live even if the notification itself is lost (NOTIFY
|
|
283
|
+
//carries no durability across connection drops).
|
|
284
|
+
const settleIfComplete = async () => {
|
|
285
|
+
if (isResolved)
|
|
286
|
+
return true;
|
|
287
|
+
const status = await this.hotMesh.getStatus(this.workflowId);
|
|
288
|
+
if (isResolved)
|
|
289
|
+
return true;
|
|
290
|
+
if (status <= 0) {
|
|
291
|
+
await complete();
|
|
292
|
+
this.hotMesh.unsub(topic);
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
return false;
|
|
296
|
+
};
|
|
297
|
+
if (!(await settleIfComplete())) {
|
|
298
|
+
void (async () => {
|
|
299
|
+
const RECHECK_DELAYS_MS = [250, 1000, 5000];
|
|
300
|
+
for (const delay of RECHECK_DELAYS_MS) {
|
|
301
|
+
await (0, utils_1.sleepFor)(delay);
|
|
302
|
+
if (await settleIfComplete())
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
while (!isResolved) {
|
|
306
|
+
await (0, utils_1.sleepFor)(30000);
|
|
307
|
+
if (await settleIfComplete())
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
})().catch(() => {
|
|
311
|
+
//the subscription remains the primary completion arm
|
|
312
|
+
});
|
|
274
313
|
}
|
|
275
314
|
});
|
|
276
315
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HotMesh } from '../hotmesh';
|
|
2
2
|
import { EventsConfig } from '../../types/system_events';
|
|
3
3
|
import { Connection } from '../../types/durable';
|
|
4
|
-
import { EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, StatsEscalationsParams, EscalationStats, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, ClaimManyParams, EscalateManyToRoleParams, UpdateManyPriorityParams, ResolveManyParams, PruneEscalationsParams, PruneEscalationsResult } from '../../types/hmsh_escalations';
|
|
4
|
+
import { EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, StatsEscalationsParams, EscalationStats, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, ClaimManyParams, EscalateManyToRoleParams, UpdateManyPriorityParams, ResolveManyParams, ResolveAllOrNoneParams, ResolveAllOrNoneResult, PruneEscalationsParams, PruneEscalationsResult } from '../../types/hmsh_escalations';
|
|
5
5
|
export type GetHotMeshFn = (topic: string | null, namespace?: string) => Promise<HotMesh>;
|
|
6
6
|
export interface EscalationClientConfig {
|
|
7
7
|
/** Postgres connection options — used when creating a standalone EscalationClient. */
|
|
@@ -226,6 +226,26 @@ export declare class EscalationClientService {
|
|
|
226
226
|
* bulk resolution carries no wake and would strand the workflow.
|
|
227
227
|
*/
|
|
228
228
|
resolveMany(params: ResolveManyParams): Promise<EscalationEntry[]>;
|
|
229
|
+
/**
|
|
230
|
+
* All-or-none bulk resolve with per-row payloads. Every item's escalation
|
|
231
|
+
* must be pending (and, when `assertAssignee` is set, currently assigned to
|
|
232
|
+
* that assignee) or NOTHING resolves — one atomic SQL statement locks the
|
|
233
|
+
* rows in deterministic order, applies each row's own `resolverPayload`,
|
|
234
|
+
* and commits each waiter's wake WITH its resolve (same durability contract
|
|
235
|
+
* as `resolve()`). Unlike `resolveMany()`, rows backing a live `condition()`
|
|
236
|
+
* waiter are first-class here: each receives its own payload as
|
|
237
|
+
* `condition()`'s return value.
|
|
238
|
+
*
|
|
239
|
+
* On `ok: false` no row was written and `failed` lists exactly the blocking
|
|
240
|
+
* rows with reasons (`not-found`, `already-resolved`, `already-cancelled`,
|
|
241
|
+
* `already-expired`, `assignee-mismatch`) — rows that were themselves
|
|
242
|
+
* resolvable stay pending and are not listed.
|
|
243
|
+
*
|
|
244
|
+
* Pass `params.metadata` to merge one shared outcome patch into every row's
|
|
245
|
+
* GIN-indexed `metadata` in the same statement. Ids must be unique; an empty
|
|
246
|
+
* `items` array returns `ok: true` with no entries.
|
|
247
|
+
*/
|
|
248
|
+
resolveAllOrNone(params: ResolveAllOrNoneParams, namespace?: string): Promise<ResolveAllOrNoneResult>;
|
|
229
249
|
/**
|
|
230
250
|
* Returns dashboard-ready escalation counts. `period` controls the window
|
|
231
251
|
* used for `created` and `resolved` counts (default `'24h'`). When `roles`
|
|
@@ -476,6 +476,74 @@ class EscalationClientService {
|
|
|
476
476
|
this._emitMany('resolved', entries);
|
|
477
477
|
return entries;
|
|
478
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* All-or-none bulk resolve with per-row payloads. Every item's escalation
|
|
481
|
+
* must be pending (and, when `assertAssignee` is set, currently assigned to
|
|
482
|
+
* that assignee) or NOTHING resolves — one atomic SQL statement locks the
|
|
483
|
+
* rows in deterministic order, applies each row's own `resolverPayload`,
|
|
484
|
+
* and commits each waiter's wake WITH its resolve (same durability contract
|
|
485
|
+
* as `resolve()`). Unlike `resolveMany()`, rows backing a live `condition()`
|
|
486
|
+
* waiter are first-class here: each receives its own payload as
|
|
487
|
+
* `condition()`'s return value.
|
|
488
|
+
*
|
|
489
|
+
* On `ok: false` no row was written and `failed` lists exactly the blocking
|
|
490
|
+
* rows with reasons (`not-found`, `already-resolved`, `already-cancelled`,
|
|
491
|
+
* `already-expired`, `assignee-mismatch`) — rows that were themselves
|
|
492
|
+
* resolvable stay pending and are not listed.
|
|
493
|
+
*
|
|
494
|
+
* Pass `params.metadata` to merge one shared outcome patch into every row's
|
|
495
|
+
* GIN-indexed `metadata` in the same statement. Ids must be unique; an empty
|
|
496
|
+
* `items` array returns `ok: true` with no entries.
|
|
497
|
+
*/
|
|
498
|
+
async resolveAllOrNone(params, namespace) {
|
|
499
|
+
const ns = (params.namespace ?? namespace) ?? factory_1.APP_ID;
|
|
500
|
+
const items = params.items ?? [];
|
|
501
|
+
if (!items.length)
|
|
502
|
+
return { ok: true, entries: [] };
|
|
503
|
+
const ids = items.map((i) => i.id);
|
|
504
|
+
if (new Set(ids).size !== ids.length) {
|
|
505
|
+
throw new Error('resolveAllOrNone: duplicate escalation ids in items');
|
|
506
|
+
}
|
|
507
|
+
const hm = await this._engine(null, ns);
|
|
508
|
+
const store = hm.engine.store;
|
|
509
|
+
//pre-build each waiter's wake so it commits INSIDE the resolve
|
|
510
|
+
//statement; signal routing (signal_key, topic) is immutable after
|
|
511
|
+
//creation, so the preview rows are authoritative for wake targeting
|
|
512
|
+
const payloadById = new Map(items.map((i) => [i.id, i.resolverPayload ?? {}]));
|
|
513
|
+
const previews = await store.listEscalations({
|
|
514
|
+
ids,
|
|
515
|
+
namespace: params.namespace,
|
|
516
|
+
});
|
|
517
|
+
const wakeCommands = [];
|
|
518
|
+
for (const row of previews) {
|
|
519
|
+
if (!row.signal_key)
|
|
520
|
+
continue;
|
|
521
|
+
const cmd = await this._buildWakeCommand(ns, row.topic, row.signal_key, payloadById.get(row.id) ?? {});
|
|
522
|
+
if (cmd)
|
|
523
|
+
wakeCommands.push(cmd);
|
|
524
|
+
}
|
|
525
|
+
const dbResult = await store.resolveAllOrNoneEscalations({
|
|
526
|
+
items,
|
|
527
|
+
namespace: params.namespace,
|
|
528
|
+
metadata: params.metadata,
|
|
529
|
+
assertAssignee: params.assertAssignee,
|
|
530
|
+
}, wakeCommands);
|
|
531
|
+
if (!dbResult.ok)
|
|
532
|
+
return { ok: false, failed: dbResult.failed };
|
|
533
|
+
//rows whose wake was not part of the commit (no hook rule deployed, or
|
|
534
|
+
//signal routing enriched after the preview) — deliver post-commit
|
|
535
|
+
const enqueuedKeys = new Set(wakeCommands.map((w) => w.forSignalKey));
|
|
536
|
+
for (const entry of dbResult.entries) {
|
|
537
|
+
if (entry.signal_key && !enqueuedKeys.has(entry.signal_key)) {
|
|
538
|
+
await this._deliverEscalationSignal(ns, entry.topic, {
|
|
539
|
+
id: entry.signal_key,
|
|
540
|
+
data: payloadById.get(entry.id) ?? {},
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
this._emitMany('resolved', dbResult.entries);
|
|
545
|
+
return { ok: true, entries: dbResult.entries };
|
|
546
|
+
}
|
|
479
547
|
// ─── Aggregates ─────────────────────────────────────────────────────────────
|
|
480
548
|
/**
|
|
481
549
|
* Returns dashboard-ready escalation counts. `period` controls the window
|
|
@@ -328,6 +328,28 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
|
|
|
328
328
|
escalateManyEscalationsToRole(params: import('../../../../types/hmsh_escalations').EscalateManyToRoleParams): Promise<number>;
|
|
329
329
|
updateManyEscalationsPriority(params: import('../../../../types/hmsh_escalations').UpdateManyPriorityParams): Promise<number>;
|
|
330
330
|
resolveManyEscalations(params: import('../../../../types/hmsh_escalations').ResolveManyParams): Promise<import('../../../../types/hmsh_escalations').EscalationEntry[]>;
|
|
331
|
+
/**
|
|
332
|
+
* All-or-none bulk resolve with per-row payloads. One SQL statement:
|
|
333
|
+
* the FOR UPDATE lock (ordered by id — deterministic acquisition, no
|
|
334
|
+
* deadlock between overlapping batches), the count-gated UPDATE, and
|
|
335
|
+
* the per-row wake INSERTs are one atomic unit. When ANY row is
|
|
336
|
+
* missing, non-pending, or fails the assignee assertion, the gate
|
|
337
|
+
* count mismatches, the UPDATE matches zero rows, no wake is written,
|
|
338
|
+
* and the statement degrades to a pure read — the returned snapshot
|
|
339
|
+
* names each blocking row and why. Same durability contract as
|
|
340
|
+
* `resolveEscalation`: each wake commits WITH its row's resolve.
|
|
341
|
+
*/
|
|
342
|
+
resolveAllOrNoneEscalations(params: import('../../../../types/hmsh_escalations').ResolveAllOrNoneParams, wakeCommands?: import('../../../../types/hmsh_escalations').EscalationWakeCommand[]): Promise<{
|
|
343
|
+
ok: true;
|
|
344
|
+
entries: import('../../../../types/hmsh_escalations').EscalationEntry[];
|
|
345
|
+
wakeCount: number;
|
|
346
|
+
} | {
|
|
347
|
+
ok: false;
|
|
348
|
+
failed: Array<{
|
|
349
|
+
id: string;
|
|
350
|
+
reason: import('../../../../types/hmsh_escalations').ResolveAllOrNoneBlockReason;
|
|
351
|
+
}>;
|
|
352
|
+
}>;
|
|
331
353
|
escalationStats(params?: import('../../../../types/hmsh_escalations').StatsEscalationsParams): Promise<import('../../../../types/hmsh_escalations').EscalationStats>;
|
|
332
354
|
listDistinctEscalationTypes(namespace?: string): Promise<string[]>;
|
|
333
355
|
releaseExpiredEscalations(_namespace?: string): Promise<number>;
|
|
@@ -2222,6 +2222,117 @@ class PostgresStoreService extends __1.StoreService {
|
|
|
2222
2222
|
RETURNING *`, namespace ? [payloadJson, ids, metaJson, namespace] : [payloadJson, ids, metaJson]);
|
|
2223
2223
|
return result.rows;
|
|
2224
2224
|
}
|
|
2225
|
+
/**
|
|
2226
|
+
* All-or-none bulk resolve with per-row payloads. One SQL statement:
|
|
2227
|
+
* the FOR UPDATE lock (ordered by id — deterministic acquisition, no
|
|
2228
|
+
* deadlock between overlapping batches), the count-gated UPDATE, and
|
|
2229
|
+
* the per-row wake INSERTs are one atomic unit. When ANY row is
|
|
2230
|
+
* missing, non-pending, or fails the assignee assertion, the gate
|
|
2231
|
+
* count mismatches, the UPDATE matches zero rows, no wake is written,
|
|
2232
|
+
* and the statement degrades to a pure read — the returned snapshot
|
|
2233
|
+
* names each blocking row and why. Same durability contract as
|
|
2234
|
+
* `resolveEscalation`: each wake commits WITH its row's resolve.
|
|
2235
|
+
*/
|
|
2236
|
+
async resolveAllOrNoneEscalations(params, wakeCommands) {
|
|
2237
|
+
const { items, namespace, metadata, assertAssignee } = params;
|
|
2238
|
+
if (!items?.length)
|
|
2239
|
+
return { ok: true, entries: [], wakeCount: 0 };
|
|
2240
|
+
const ids = items.map((i) => i.id);
|
|
2241
|
+
if (new Set(ids).size !== ids.length) {
|
|
2242
|
+
throw new Error('resolveAllOrNoneEscalations: duplicate escalation ids in items');
|
|
2243
|
+
}
|
|
2244
|
+
const payloads = items.map((i) => i.resolverPayload ? JSON.stringify(i.resolverPayload) : null);
|
|
2245
|
+
const metaJson = metadata ? JSON.stringify(metadata) : null;
|
|
2246
|
+
const sqlParams = [ids, payloads, metaJson, assertAssignee ?? null];
|
|
2247
|
+
let nsClause = '';
|
|
2248
|
+
if (namespace) {
|
|
2249
|
+
sqlParams.push(namespace);
|
|
2250
|
+
nsClause = `AND e.namespace = $${sqlParams.length}`;
|
|
2251
|
+
}
|
|
2252
|
+
//per-row wakes: one engine_streams INSERT per resolved row whose
|
|
2253
|
+
//signal_key matches its pre-built command; zero rows when the gate fails
|
|
2254
|
+
let wakeCTE = '';
|
|
2255
|
+
let wakeCount = '0::int AS wake_count';
|
|
2256
|
+
if (wakeCommands?.length) {
|
|
2257
|
+
const schemaName = this.kvsql().safeName(this.appId);
|
|
2258
|
+
const base = sqlParams.length;
|
|
2259
|
+
sqlParams.push(this.appId, wakeCommands.map((w) => w.forSignalKey), wakeCommands.map((w) => w.message));
|
|
2260
|
+
wakeCTE = `,
|
|
2261
|
+
wake AS (
|
|
2262
|
+
INSERT INTO ${schemaName}.engine_streams (stream_name, message, priority)
|
|
2263
|
+
SELECT $${base + 1}, w.message, 5
|
|
2264
|
+
FROM unnest($${base + 2}::text[], $${base + 3}::text[]) AS w(signal_key, message)
|
|
2265
|
+
JOIN resolved r ON r.signal_key = w.signal_key
|
|
2266
|
+
RETURNING id
|
|
2267
|
+
)`;
|
|
2268
|
+
wakeCount = '(SELECT COUNT(*) FROM wake)::int AS wake_count';
|
|
2269
|
+
}
|
|
2270
|
+
const result = await this.pgClient.query(`
|
|
2271
|
+
WITH input AS MATERIALIZED (
|
|
2272
|
+
SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, payload_text)
|
|
2273
|
+
),
|
|
2274
|
+
target AS MATERIALIZED (
|
|
2275
|
+
SELECT e.id, e.signal_key, e.topic, e.status, e.assigned_to
|
|
2276
|
+
FROM public.hmsh_escalations e
|
|
2277
|
+
WHERE e.id = ANY($1::uuid[]) ${nsClause}
|
|
2278
|
+
ORDER BY e.id
|
|
2279
|
+
FOR UPDATE
|
|
2280
|
+
),
|
|
2281
|
+
eligible AS (
|
|
2282
|
+
SELECT t.id FROM target t
|
|
2283
|
+
WHERE t.status = 'pending'
|
|
2284
|
+
AND ($4::text IS NULL OR t.assigned_to = $4::text)
|
|
2285
|
+
),
|
|
2286
|
+
resolved AS (
|
|
2287
|
+
UPDATE public.hmsh_escalations e
|
|
2288
|
+
SET status = 'resolved', resolved_at = NOW(),
|
|
2289
|
+
resolver_payload = i.payload_text::jsonb,
|
|
2290
|
+
metadata = CASE WHEN $3::jsonb IS NOT NULL
|
|
2291
|
+
THEN COALESCE(e.metadata, '{}'::jsonb) || $3::jsonb
|
|
2292
|
+
ELSE e.metadata END,
|
|
2293
|
+
updated_at = NOW()
|
|
2294
|
+
FROM input i
|
|
2295
|
+
WHERE e.id = i.id
|
|
2296
|
+
AND e.status = 'pending'
|
|
2297
|
+
AND (SELECT COUNT(*) FROM eligible) = (SELECT COUNT(*) FROM input)
|
|
2298
|
+
RETURNING e.*
|
|
2299
|
+
)${wakeCTE}
|
|
2300
|
+
SELECT
|
|
2301
|
+
i.id AS input_id,
|
|
2302
|
+
t.status AS prior_status,
|
|
2303
|
+
CASE
|
|
2304
|
+
WHEN r.id IS NOT NULL THEN 'resolved'
|
|
2305
|
+
WHEN t.id IS NULL THEN 'not-found'
|
|
2306
|
+
WHEN t.status = 'cancelled' THEN 'already-cancelled'
|
|
2307
|
+
WHEN t.status = 'expired' THEN 'already-expired'
|
|
2308
|
+
WHEN t.status <> 'pending' THEN 'already-resolved'
|
|
2309
|
+
WHEN $4::text IS NOT NULL AND t.assigned_to IS DISTINCT FROM $4::text THEN 'assignee-mismatch'
|
|
2310
|
+
ELSE 'blocked-by-batch'
|
|
2311
|
+
END AS outcome,
|
|
2312
|
+
row_to_json(r.*) AS entry_json,
|
|
2313
|
+
${wakeCount}
|
|
2314
|
+
FROM input i
|
|
2315
|
+
LEFT JOIN target t ON t.id = i.id
|
|
2316
|
+
LEFT JOIN resolved r ON r.id = i.id
|
|
2317
|
+
ORDER BY i.id
|
|
2318
|
+
`, sqlParams);
|
|
2319
|
+
const rows = result.rows;
|
|
2320
|
+
if (rows.length && rows.every((r) => r.outcome === 'resolved')) {
|
|
2321
|
+
return {
|
|
2322
|
+
ok: true,
|
|
2323
|
+
entries: rows.map((r) => r.entry_json),
|
|
2324
|
+
wakeCount: rows[0].wake_count,
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
return {
|
|
2328
|
+
ok: false,
|
|
2329
|
+
//'blocked-by-batch' rows were themselves resolvable — only the true
|
|
2330
|
+
//blockers are reported; the caller re-gangs around exactly these
|
|
2331
|
+
failed: rows
|
|
2332
|
+
.filter((r) => r.outcome !== 'resolved' && r.outcome !== 'blocked-by-batch')
|
|
2333
|
+
.map((r) => ({ id: r.input_id, reason: r.outcome })),
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2225
2336
|
async escalationStats(params = {}) {
|
|
2226
2337
|
const { namespace, roles, period = '24h' } = params;
|
|
2227
2338
|
if (roles !== undefined && roles.length === 0) {
|
|
@@ -352,6 +352,47 @@ export interface ResolveManyParams {
|
|
|
352
352
|
*/
|
|
353
353
|
metadata?: Record<string, unknown>;
|
|
354
354
|
}
|
|
355
|
+
/** One member of a `resolveAllOrNone()` batch — its own `resolverPayload` is
|
|
356
|
+
* stored as that row's `resolver_payload` and delivered to that row's waiting
|
|
357
|
+
* workflow as `condition()`'s return value. */
|
|
358
|
+
export interface ResolveAllOrNoneItem {
|
|
359
|
+
id: string;
|
|
360
|
+
resolverPayload?: Record<string, unknown>;
|
|
361
|
+
}
|
|
362
|
+
export interface ResolveAllOrNoneParams {
|
|
363
|
+
/** The batch. Ids must be unique; each item carries its own payload. */
|
|
364
|
+
items: ResolveAllOrNoneItem[];
|
|
365
|
+
namespace?: string;
|
|
366
|
+
/**
|
|
367
|
+
* Shared outcome patch merged (not replaced) into EVERY row's GIN-indexed
|
|
368
|
+
* `metadata` in the single atomic statement. See {@link ResolveEscalationParams.metadata}.
|
|
369
|
+
*/
|
|
370
|
+
metadata?: Record<string, unknown>;
|
|
371
|
+
/**
|
|
372
|
+
* When provided, every row must currently be assigned to this assignee
|
|
373
|
+
* (`assigned_to` equality, asserted inside the same statement). Closes the
|
|
374
|
+
* claim-race window for claim-then-resolve flows: a row re-claimed by another
|
|
375
|
+
* principal between the caller's claim and this resolve blocks the batch.
|
|
376
|
+
*/
|
|
377
|
+
assertAssignee?: string;
|
|
378
|
+
}
|
|
379
|
+
/** Why a specific row blocked a `resolveAllOrNone()` batch. */
|
|
380
|
+
export type ResolveAllOrNoneBlockReason = 'not-found' | 'already-resolved' | 'already-cancelled' | 'already-expired' | 'assignee-mismatch';
|
|
381
|
+
/**
|
|
382
|
+
* Result of `resolveAllOrNone()`. On `ok: false` NOTHING was written; `failed`
|
|
383
|
+
* lists only the rows that blocked the batch (rows that were themselves
|
|
384
|
+
* resolvable are not listed — they remain pending, untouched).
|
|
385
|
+
*/
|
|
386
|
+
export type ResolveAllOrNoneResult = {
|
|
387
|
+
ok: true;
|
|
388
|
+
entries: EscalationEntry[];
|
|
389
|
+
} | {
|
|
390
|
+
ok: false;
|
|
391
|
+
failed: Array<{
|
|
392
|
+
id: string;
|
|
393
|
+
reason: ResolveAllOrNoneBlockReason;
|
|
394
|
+
}>;
|
|
395
|
+
};
|
|
355
396
|
/**
|
|
356
397
|
* Full-fidelity migration params. Extends `CreateEscalationParams` with:
|
|
357
398
|
* - `id` (required) — preserves the original UUID; no auto-generation
|
package/build/types/index.d.ts
CHANGED
|
@@ -25,5 +25,5 @@ export { ReclaimedMessageType, RetryPolicy, RouterConfig, StreamCode, StreamConf
|
|
|
25
25
|
export { context, Context, Counter, Meter, metrics, propagation, SpanContext, Span, SpanStatus, SpanStatusCode, SpanKind, trace, Tracer, ValueType, } from './telemetry';
|
|
26
26
|
export { WorkListTaskType } from './task';
|
|
27
27
|
export { TransitionMatch, TransitionRule, Transitions } from './transition';
|
|
28
|
-
export { ConditionQueueConfig, EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, } from './hmsh_escalations';
|
|
28
|
+
export { ConditionQueueConfig, EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, ResolveAllOrNoneItem, ResolveAllOrNoneParams, ResolveAllOrNoneBlockReason, ResolveAllOrNoneResult, EscalateToRoleParams, MigrateEscalationParams, } from './hmsh_escalations';
|
|
29
29
|
export { EscalationVerb, EngineVerb, WorkerVerb, SystemEvent, EventsConfig } from './system_events';
|
|
@@ -143,8 +143,8 @@ export interface SystemEvent {
|
|
|
143
143
|
span_id?: string;
|
|
144
144
|
/**
|
|
145
145
|
* Full committed row for escalation events; lifecycle metadata for
|
|
146
|
-
* engine/worker events.
|
|
147
|
-
*
|
|
146
|
+
* engine/worker events. Each dependent cherry-picks fields for its
|
|
147
|
+
* own event shape.
|
|
148
148
|
*/
|
|
149
149
|
data: Record<string, unknown>;
|
|
150
150
|
}
|
package/package.json
CHANGED
package/OPTIMIZATIONS.md
DELETED
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
# HotMesh needs — from long-tail
|
|
2
|
-
|
|
3
|
-
A needs document from the long-tail project: the usage patterns we run at
|
|
4
|
-
production scale and the engine support that would let us run them well. Each item
|
|
5
|
-
states what we do, the concrete access pattern, and the engine capability or
|
|
6
|
-
guarantee we need. Ordered by leverage.
|
|
7
|
-
|
|
8
|
-
Context: long-tail is a durable HITL escalation platform on Postgres. Escalations
|
|
9
|
-
live in `public.hmsh_escalations` (engine-created); we layer a `public.lt_escalations`
|
|
10
|
-
view and a few app-owned indexes on top. Our dominant runtime shape is
|
|
11
|
-
**single-role** work-queue traffic — claim / available / count / settle scoped to
|
|
12
|
-
one `role` — plus a per-role escalation form the resolve UI renders.
|
|
13
|
-
|
|
14
|
-
---
|
|
15
|
-
|
|
16
|
-
## 1. Safe retention for terminal escalation rows
|
|
17
|
-
|
|
18
|
-
**Usage.** An escalation is written once, resolved once, then read for audit. The
|
|
19
|
-
pending working set stays small — a role-leading pending partial index keeps
|
|
20
|
-
single-role claim/available queries `O(pending)` regardless of table size:
|
|
21
|
-
|
|
22
|
-
```sql
|
|
23
|
-
WHERE role = $1 AND status = 'pending'
|
|
24
|
-
AND (assigned_to IS NULL OR assigned_until <= NOW())
|
|
25
|
-
AND metadata @> $facets
|
|
26
|
-
ORDER BY priority ASC, created_at ASC
|
|
27
|
-
FOR UPDATE SKIP LOCKED
|
|
28
|
-
```
|
|
29
|
-
|
|
30
|
-
The terminal set (resolved / expired / cancelled) grows for the life of the
|
|
31
|
-
deployment — the source of long-run table bloat, autovacuum load, and index growth.
|
|
32
|
-
|
|
33
|
-
**Need.** A safe, engine-blessed way to age out terminal rows. Tell us what state
|
|
34
|
-
the engine still reads once an escalation is terminal (replay, completion, audit),
|
|
35
|
-
so we know what is inert, then one of:
|
|
36
|
-
- a supported retention/archive call the engine owns; or
|
|
37
|
-
- a documented contract that a row is safe to delete once its status is terminal
|
|
38
|
-
and older than a horizon; or
|
|
39
|
-
- time sub-partitioning of the terminal set the engine can `DETACH`/`DROP`.
|
|
40
|
-
|
|
41
|
-
`hmsh_escalations` stays a single physical table keyed by `id`/`signal_key` for
|
|
42
|
-
the engine's lookups; retention touches only rows the engine confirms are inert. We
|
|
43
|
-
can supply the horizon and exact DDL.
|
|
44
|
-
|
|
45
|
-
---
|
|
46
|
-
|
|
47
|
-
## 2. Stable ownership of app-added indexes on `hmsh_escalations`
|
|
48
|
-
|
|
49
|
-
**Usage.** We add indexes to the engine table via advisory-locked
|
|
50
|
-
`CREATE INDEX CONCURRENTLY IF NOT EXISTS`:
|
|
51
|
-
- a role-leading pending index `(role, priority, created_at) WHERE status='pending'`
|
|
52
|
-
for the claim/available order;
|
|
53
|
-
- a **pending-scoped GIN** on `metadata` (`jsonb_path_ops`) `WHERE status='pending'`
|
|
54
|
-
— keeps the GIN sized to the working set and cuts write amplification on the
|
|
55
|
-
high-churn insert→resolve path;
|
|
56
|
-
- covering partial indexes for resolved-row reads.
|
|
57
|
-
|
|
58
|
-
**Need.** A stable contract that app-added indexes on `hmsh_escalations` survive
|
|
59
|
-
engine migrations — the engine keeps the table in place across upgrades — or a
|
|
60
|
-
documented post-migration hook we can register to re-ensure them. Shipping these
|
|
61
|
-
index shapes in the engine itself also works; the requirement is that they persist.
|
|
62
|
-
|
|
63
|
-
---
|
|
64
|
-
|
|
65
|
-
## 3. Atomic metadata write in the `condition()` Leg1 checkpoint
|
|
66
|
-
|
|
67
|
-
**Usage.** A workflow surfaces a HITL wait with `condition(signalId, config)`. We
|
|
68
|
-
pass the target `role`, claim/routing facets, and `metadata.schema_version` (the
|
|
69
|
-
form version the resolve UI renders) in `config.metadata`. Claim/resolve-by-metadata
|
|
70
|
-
and the dashboard read those facets via GIN `@>`; the resolve UI reads
|
|
71
|
-
`schema_version` to render the exact form.
|
|
72
|
-
|
|
73
|
-
**Need.** Guarantee — as a documented public contract — that `config.metadata` is
|
|
74
|
-
written into the escalation row **atomically in the same Leg1 checkpoint** that
|
|
75
|
-
creates the row (one commit, crash-safe). This is the foundation our JIT-versioned
|
|
76
|
-
form pinning and metadata-driven routing stand on.
|
|
77
|
-
|
|
78
|
-
---
|
|
79
|
-
|
|
80
|
-
## 4. Honored long-activity timeout + reclaim safety
|
|
81
|
-
|
|
82
|
-
**Usage.** We run a batch pattern: a single proxyActivity loops internally for
|
|
83
|
-
~60s (poll → reconcile → act), returning one durable checkpoint per call instead of
|
|
84
|
-
one per step, so replay stays compact.
|
|
85
|
-
|
|
86
|
-
**Need.** A documented guarantee that:
|
|
87
|
-
- `proxyActivities({ startToCloseTimeout })` is honored for long (30–120s) activities;
|
|
88
|
-
- the stream reclaim window (`HMSH_XCLAIM_DELAY_MS`) holds an in-flight activity for
|
|
89
|
-
the duration of its `startToCloseTimeout`, so a long activity is not reclaimed and
|
|
90
|
-
re-dispatched (duplicate execution) before it completes. Deriving the reclaim
|
|
91
|
-
window from `startToCloseTimeout` automatically makes a 60s+ activity safe by
|
|
92
|
-
default.
|
|
93
|
-
|
|
94
|
-
---
|
|
95
|
-
|
|
96
|
-
## 5. Documented concurrent-condition bound for harvest fan-out
|
|
97
|
-
|
|
98
|
-
**Usage.** A market-maker workflow opens N `condition()` waits and harvests them
|
|
99
|
-
with `Promise.all`, chunked to stay within the safe concurrent-condition bound (a
|
|
100
|
-
fast signaler can arrive before a later condition registers).
|
|
101
|
-
|
|
102
|
-
**Need.** Publish the safe concurrent-condition bound as a stable number with the
|
|
103
|
-
recommended chunking pattern, so we size harvest fan-out to spec rather than
|
|
104
|
-
empirically — and raise it where the engine can.
|
|
105
|
-
|
|
106
|
-
---
|
|
107
|
-
|
|
108
|
-
## 6. Signal buffering + startChild leader-keeps-seat as public contract
|
|
109
|
-
|
|
110
|
-
**Usage.** Two behaviors our crash-safe patterns build on:
|
|
111
|
-
- a signal delivered to a workflow **not yet parked** on that signalId is buffered
|
|
112
|
-
and delivered on the next `condition()` await;
|
|
113
|
-
- `startChild` with a **duplicate workflowId** lets the sitting execution keep its
|
|
114
|
-
seat (leader-keeps-seat), enabling self-continuing link chains (`x-l1`, `x-l2`, …)
|
|
115
|
-
as an alternative to `continueAsNew`.
|
|
116
|
-
|
|
117
|
-
**Need.** Elevate both to documented, stable public guarantees (including the signal
|
|
118
|
-
buffering TTL), versioned as contract.
|
|
119
|
-
|
|
120
|
-
---
|
|
121
|
-
|
|
122
|
-
### How we consume these
|
|
123
|
-
|
|
124
|
-
Items 1–2 are the scale needs — retention for terminal rows, and durable app-owned
|
|
125
|
-
indexes that keep the hot path `O(pending)`. Item 3 is the atomicity guarantee our
|
|
126
|
-
versioned forms and metadata routing depend on. Items 4–6 are behaviors we already
|
|
127
|
-
build on and want made explicit. We can provide `EXPLAIN` plans, row-count/latency
|
|
128
|
-
profiles, and the exact DDL for any of these on request.
|
package/RELEASE-0.26.0.md
DELETED
|
@@ -1,111 +0,0 @@
|
|
|
1
|
-
# HotMesh v0.26.0 — for the long-tail project
|
|
2
|
-
|
|
3
|
-
This release answers the needs document. Items 1–5 and the signal-buffering
|
|
4
|
-
half of item 6 ship as published contracts with proving tests; the startChild
|
|
5
|
-
contract (item 6b) is scheduled for its own follow-up release. Numbering below
|
|
6
|
-
matches your document.
|
|
7
|
-
|
|
8
|
-
## 1. Retention for terminal escalation rows — new API
|
|
9
|
-
|
|
10
|
-
Terminal rows (`resolved`, `cancelled`, `expired`) are inert: every engine
|
|
11
|
-
state transition guards on `status = 'pending'`, and the engine reads terminal
|
|
12
|
-
rows only through `list()` / `get()` / `stats()`. Age them out with the new
|
|
13
|
-
engine-owned call:
|
|
14
|
-
|
|
15
|
-
```typescript
|
|
16
|
-
let deleted: number;
|
|
17
|
-
do {
|
|
18
|
-
({ deleted } = await client.escalations.prune({ olderThan: '90 days' }));
|
|
19
|
-
} while (deleted > 0);
|
|
20
|
-
```
|
|
21
|
-
|
|
22
|
-
- `olderThan` is a Postgres interval; rows qualify when
|
|
23
|
-
`updated_at < NOW() - olderThan`.
|
|
24
|
-
- `statuses` narrows which terminal states are pruned (defaults to all three);
|
|
25
|
-
`namespace` scopes to one app; `limit` bounds rows per call (default 10,000,
|
|
26
|
-
cap 100,000) so vacuum pressure stays flat.
|
|
27
|
-
- Each call is one atomic statement — candidate selection
|
|
28
|
-
(`FOR UPDATE SKIP LOCKED`) and the DELETE commit together, so concurrent
|
|
29
|
-
pruners cooperate and live waiters, claims, and signal delivery proceed
|
|
30
|
-
untouched. The proving test backdates a parked waiter 120 days, prunes, and
|
|
31
|
-
resolves it to completion.
|
|
32
|
-
- If you run retention with your own tooling instead, the documented
|
|
33
|
-
safe-to-delete predicate is exactly the one `prune()` uses:
|
|
34
|
-
`status IN ('resolved','cancelled','expired') AND updated_at` older than
|
|
35
|
-
your horizon.
|
|
36
|
-
- Choose a horizon at least as long as your reporting window: `stats()`
|
|
37
|
-
created/resolved counts and `get()` by id reflect the rows retained.
|
|
38
|
-
|
|
39
|
-
## 2. App-added indexes survive engine migrations — published contract
|
|
40
|
-
|
|
41
|
-
Engine migrations on `public.hmsh_escalations` are strictly additive
|
|
42
|
-
(`CREATE TABLE / INDEX IF NOT EXISTS`, `ADD COLUMN IF NOT EXISTS`, advisory-
|
|
43
|
-
locked), and the table persists in place across upgrades. The engine manages
|
|
44
|
-
only indexes with the `idx_hmsh_esc_*` prefix; indexes under any other name
|
|
45
|
-
are yours and persist. Your advisory-locked
|
|
46
|
-
`CREATE INDEX CONCURRENTLY IF NOT EXISTS` pattern is the recommended one.
|
|
47
|
-
|
|
48
|
-
Two notes on your shapes: the engine's `idx_hmsh_esc_available`
|
|
49
|
-
(`(namespace, app_id, role, priority ASC, created_at ASC) WHERE
|
|
50
|
-
status = 'pending'`) already serves the single-role claim/available order at
|
|
51
|
-
`O(pending)`. Your pending-scoped GIN stays app-owned: the engine keeps its
|
|
52
|
-
full-table GIN because resolver-merged metadata (v0.24.0) is read on resolved
|
|
53
|
-
rows.
|
|
54
|
-
|
|
55
|
-
## 3. Atomic metadata in the Leg1 checkpoint — published contract
|
|
56
|
-
|
|
57
|
-
Every field of the `condition()` config — `metadata` facets included — is one
|
|
58
|
-
INSERT committed inside the Leg1 checkpoint transaction: one commit,
|
|
59
|
-
crash-safe. From the row's first visible moment it carries its complete
|
|
60
|
-
routing context and metadata, so claim-by-metadata routing and
|
|
61
|
-
`schema_version`-pinned forms can trust every row they read. The proving test
|
|
62
|
-
tight-polls from workflow start and asserts the first sighting is complete.
|
|
63
|
-
|
|
64
|
-
## 4. Long activities execute exactly once — published contract
|
|
65
|
-
|
|
66
|
-
- `startToCloseTimeout` is honored for long activities (deadline enforcement
|
|
67
|
-
is covered by positive and negative tests).
|
|
68
|
-
- While an activity runs, the consumer heartbeats its stream reservation at
|
|
69
|
-
half the base window, so the message stays leased for the full run —
|
|
70
|
-
your 60s batch loop (poll → reconcile → act, one checkpoint per call) is a
|
|
71
|
-
supported shape at default settings, for any duration. Redelivery happens
|
|
72
|
-
when a consumer crashes and stops heartbeating, and the collation ledger
|
|
73
|
-
settles a redelivered message as a duplicate before the activity function
|
|
74
|
-
re-executes.
|
|
75
|
-
- The proving test runs a 45s activity against a 30s reservation window with
|
|
76
|
-
a second worker actively polling the same queue: exactly one execution.
|
|
77
|
-
- Deployment note: standard Postgres connections have this lease extension.
|
|
78
|
-
A `securedWorker` connection (SECURITY DEFINER stored-proc mode) enforces
|
|
79
|
-
the adaptive reservation window instead — 30s base, scaling to 30 minutes
|
|
80
|
-
under load — so size activities to that window in secured mode.
|
|
81
|
-
|
|
82
|
-
## 5. Concurrent-condition bound — the published number is a TTL
|
|
83
|
-
|
|
84
|
-
Fan-out is sized by time, and by count it scales freely. A signal that races
|
|
85
|
-
ahead of its `condition()` registration is buffered as a pending signal and
|
|
86
|
-
delivered — crash-safe, in a single transaction — when the wait registers.
|
|
87
|
-
The buffer holds a signal for **10 minutes** by default; pass `expire` to
|
|
88
|
-
`signal()` (e.g. `'1h'`, `'30d'`) when signaling early on purpose.
|
|
89
|
-
|
|
90
|
-
Chunk your harvest so every condition in a chunk registers within the TTL of
|
|
91
|
-
its fastest signaler. The proving test opens 25 waits in one `Promise.all`
|
|
92
|
-
and fires all 25 signals before any wait has registered: every signal is
|
|
93
|
-
delivered to its own wait, in order.
|
|
94
|
-
|
|
95
|
-
## 6. Contracts
|
|
96
|
-
|
|
97
|
-
- **Signal buffering** is now a documented public guarantee (item 5 above,
|
|
98
|
-
TTL included).
|
|
99
|
-
- **startChild leader-keeps-seat**: the sitting execution keeps its seat
|
|
100
|
-
today, and fire-and-forget `startChild` link chains (`x-l1`, `x-l2`, …)
|
|
101
|
-
work on that behavior. The *awaited* duplicate path (`executeChild` on an
|
|
102
|
-
id that already exists) gets defined semantics — a typed result rather
|
|
103
|
-
than an open wait — in a dedicated follow-up release; until then, treat
|
|
104
|
-
awaited duplicates as unspecified and build chains on the fire-and-forget
|
|
105
|
-
form.
|
|
106
|
-
|
|
107
|
-
## Upgrading
|
|
108
|
-
|
|
109
|
-
`npm install @hotmeshio/hotmesh@0.26.0`. Migrations are additive and run
|
|
110
|
-
automatically under the existing advisory lock; your indexes and views
|
|
111
|
-
persist. Full durable suite: 433 tests green.
|