@hotmeshio/hotmesh 0.25.6 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/package.json +1 -1
- package/build/services/durable/handle.js +45 -6
- package/build/services/durable/workflow/condition.d.ts +20 -3
- package/build/services/durable/workflow/condition.js +20 -3
- package/build/services/durable/workflow/proxyActivities.d.ts +14 -0
- package/build/services/durable/workflow/proxyActivities.js +14 -0
- package/build/services/escalations/client.d.ts +33 -1
- package/build/services/escalations/client.js +83 -0
- package/build/services/store/providers/postgres/postgres.d.ts +35 -0
- package/build/services/store/providers/postgres/postgres.js +150 -0
- package/build/types/hmsh_escalations.d.ts +64 -0
- package/build/types/index.d.ts +1 -1
- package/build/types/system_events.d.ts +2 -2
- package/package.json +1 -1
- package/ISSUE.md +0 -64
- package/RESPONSE-0.25.6.md +0 -77
- package/wedge-capture-20260706.csv +0 -19
package/build/package.json
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.WorkflowHandleService = void 0;
|
|
4
|
+
const utils_1 = require("../../modules/utils");
|
|
4
5
|
const exporter_1 = require("./exporter");
|
|
5
6
|
/**
|
|
6
7
|
* Handle to a running or completed workflow execution. Returned by
|
|
@@ -253,8 +254,13 @@ class WorkflowHandleService {
|
|
|
253
254
|
return complete(null, JSON.parse(state.metadata.err));
|
|
254
255
|
}
|
|
255
256
|
}
|
|
256
|
-
//subscribe to 'done' topic
|
|
257
|
-
|
|
257
|
+
//subscribe to 'done' topic. Await the registration: the getStatus
|
|
258
|
+
//fallback below only covers completions that land BEFORE it reads.
|
|
259
|
+
//Unawaited, a job that completes after the status read but before
|
|
260
|
+
//the subscription goes active is seen by neither arm and result()
|
|
261
|
+
//never settles — hit reliably when a bulk resolve completes many
|
|
262
|
+
//workflows in the same instant.
|
|
263
|
+
await this.hotMesh.sub(topic, async (_topic, state) => {
|
|
258
264
|
this.hotMesh.unsub(topic);
|
|
259
265
|
if (state.data.done && !state.data?.$error) {
|
|
260
266
|
await complete(state.data?.response);
|
|
@@ -267,10 +273,43 @@ class WorkflowHandleService {
|
|
|
267
273
|
return await complete(null, error);
|
|
268
274
|
}
|
|
269
275
|
});
|
|
270
|
-
//check state in case completed during wiring
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
276
|
+
//check state in case completed during wiring. One read is not
|
|
277
|
+
//enough: the executed event commits before the status decrement,
|
|
278
|
+
//so a job caught mid-close shows status > 0 with its event already
|
|
279
|
+
//published — a subscriber wiring up in that window misses both
|
|
280
|
+
//arms. Escalating re-checks close the window (the status settles
|
|
281
|
+
//milliseconds after the publish), and the slow steady poll keeps
|
|
282
|
+
//result() live even if the notification itself is lost (NOTIFY
|
|
283
|
+
//carries no durability across connection drops).
|
|
284
|
+
const settleIfComplete = async () => {
|
|
285
|
+
if (isResolved)
|
|
286
|
+
return true;
|
|
287
|
+
const status = await this.hotMesh.getStatus(this.workflowId);
|
|
288
|
+
if (isResolved)
|
|
289
|
+
return true;
|
|
290
|
+
if (status <= 0) {
|
|
291
|
+
await complete();
|
|
292
|
+
this.hotMesh.unsub(topic);
|
|
293
|
+
return true;
|
|
294
|
+
}
|
|
295
|
+
return false;
|
|
296
|
+
};
|
|
297
|
+
if (!(await settleIfComplete())) {
|
|
298
|
+
void (async () => {
|
|
299
|
+
const RECHECK_DELAYS_MS = [250, 1000, 5000];
|
|
300
|
+
for (const delay of RECHECK_DELAYS_MS) {
|
|
301
|
+
await (0, utils_1.sleepFor)(delay);
|
|
302
|
+
if (await settleIfComplete())
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
while (!isResolved) {
|
|
306
|
+
await (0, utils_1.sleepFor)(30000);
|
|
307
|
+
if (await settleIfComplete())
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
})().catch(() => {
|
|
311
|
+
//the subscription remains the primary completion arm
|
|
312
|
+
});
|
|
274
313
|
}
|
|
275
314
|
});
|
|
276
315
|
}
|
|
@@ -46,9 +46,12 @@ import { ConditionQueueConfig } from '../../../types/hmsh_escalations';
|
|
|
46
46
|
* ## With escalation queue config
|
|
47
47
|
*
|
|
48
48
|
* Pass a {@link ConditionQueueConfig} as the second argument to surface the
|
|
49
|
-
* pause as a claimable row in `public.hmsh_escalations`. The INSERT
|
|
50
|
-
*
|
|
51
|
-
*
|
|
49
|
+
* pause as a claimable row in `public.hmsh_escalations`. The INSERT — every
|
|
50
|
+
* field of the config, including `metadata` facets — is committed atomically
|
|
51
|
+
* with the workflow checkpoint: one write, one commit, crash-safe. From the
|
|
52
|
+
* row's first visible moment it carries its complete routing context and
|
|
53
|
+
* metadata, so claim-by-metadata routing and version-pinned facets (e.g. a
|
|
54
|
+
* `schema_version` the resolver UI renders) can trust every row they read.
|
|
52
55
|
*
|
|
53
56
|
* ```typescript
|
|
54
57
|
* const decision = await Durable.workflow.condition<{ approved: boolean }>(
|
|
@@ -81,6 +84,14 @@ import { ConditionQueueConfig } from '../../../types/hmsh_escalations';
|
|
|
81
84
|
* SLA-gated human waits in the workflow body and let hook functions report
|
|
82
85
|
* back via `signal()`.
|
|
83
86
|
*
|
|
87
|
+
* ## Early signals are buffered
|
|
88
|
+
*
|
|
89
|
+
* A signal delivered before its `condition()` registers — a fast signaler,
|
|
90
|
+
* or a payload deposited before the workflow starts — is buffered as a
|
|
91
|
+
* pending signal and delivered when the wait registers. The buffer holds a
|
|
92
|
+
* signal for 10 minutes by default; pass `expire` to `signal()` (e.g.
|
|
93
|
+
* `'1h'`, `'30d'`) to hold it longer when signaling early on purpose.
|
|
94
|
+
*
|
|
84
95
|
* ## Fan-in: wait for multiple signals in parallel
|
|
85
96
|
*
|
|
86
97
|
* ```typescript
|
|
@@ -90,6 +101,12 @@ import { ConditionQueueConfig } from '../../../types/hmsh_escalations';
|
|
|
90
101
|
* ]);
|
|
91
102
|
* ```
|
|
92
103
|
*
|
|
104
|
+
* Harvest fan-out scales the same way: open N waits with `Promise.all` and
|
|
105
|
+
* signal all of them at once. Buffering covers every signal that races
|
|
106
|
+
* ahead of registration, so size fan-out by the pending-signal TTL — how
|
|
107
|
+
* long a racing signal may wait for its condition to register — with no
|
|
108
|
+
* separate bound on the number of concurrent waits.
|
|
109
|
+
*
|
|
93
110
|
* ## Paired with hook: spawn work, wait for its signal
|
|
94
111
|
*
|
|
95
112
|
* ```typescript
|
|
@@ -51,9 +51,12 @@ const didRun_1 = require("./didRun");
|
|
|
51
51
|
* ## With escalation queue config
|
|
52
52
|
*
|
|
53
53
|
* Pass a {@link ConditionQueueConfig} as the second argument to surface the
|
|
54
|
-
* pause as a claimable row in `public.hmsh_escalations`. The INSERT
|
|
55
|
-
*
|
|
56
|
-
*
|
|
54
|
+
* pause as a claimable row in `public.hmsh_escalations`. The INSERT — every
|
|
55
|
+
* field of the config, including `metadata` facets — is committed atomically
|
|
56
|
+
* with the workflow checkpoint: one write, one commit, crash-safe. From the
|
|
57
|
+
* row's first visible moment it carries its complete routing context and
|
|
58
|
+
* metadata, so claim-by-metadata routing and version-pinned facets (e.g. a
|
|
59
|
+
* `schema_version` the resolver UI renders) can trust every row they read.
|
|
57
60
|
*
|
|
58
61
|
* ```typescript
|
|
59
62
|
* const decision = await Durable.workflow.condition<{ approved: boolean }>(
|
|
@@ -86,6 +89,14 @@ const didRun_1 = require("./didRun");
|
|
|
86
89
|
* SLA-gated human waits in the workflow body and let hook functions report
|
|
87
90
|
* back via `signal()`.
|
|
88
91
|
*
|
|
92
|
+
* ## Early signals are buffered
|
|
93
|
+
*
|
|
94
|
+
* A signal delivered before its `condition()` registers — a fast signaler,
|
|
95
|
+
* or a payload deposited before the workflow starts — is buffered as a
|
|
96
|
+
* pending signal and delivered when the wait registers. The buffer holds a
|
|
97
|
+
* signal for 10 minutes by default; pass `expire` to `signal()` (e.g.
|
|
98
|
+
* `'1h'`, `'30d'`) to hold it longer when signaling early on purpose.
|
|
99
|
+
*
|
|
89
100
|
* ## Fan-in: wait for multiple signals in parallel
|
|
90
101
|
*
|
|
91
102
|
* ```typescript
|
|
@@ -95,6 +106,12 @@ const didRun_1 = require("./didRun");
|
|
|
95
106
|
* ]);
|
|
96
107
|
* ```
|
|
97
108
|
*
|
|
109
|
+
* Harvest fan-out scales the same way: open N waits with `Promise.all` and
|
|
110
|
+
* signal all of them at once. Buffering covers every signal that races
|
|
111
|
+
* ahead of registration, so size fan-out by the pending-signal TTL — how
|
|
112
|
+
* long a racing signal may wait for its condition to register — with no
|
|
113
|
+
* separate bound on the number of concurrent waits.
|
|
114
|
+
*
|
|
98
115
|
* ## Paired with hook: spawn work, wait for its signal
|
|
99
116
|
*
|
|
100
117
|
* ```typescript
|
|
@@ -111,6 +111,20 @@ declare function wrapActivity<T>(activityName: string, options?: ActivityConfig)
|
|
|
111
111
|
* }
|
|
112
112
|
* ```
|
|
113
113
|
*
|
|
114
|
+
* ## Long-running activities execute exactly once
|
|
115
|
+
*
|
|
116
|
+
* `startToCloseTimeout` bounds an activity's run and is honored for long
|
|
117
|
+
* work (a 30–120s batch loop is a supported shape: poll → reconcile → act,
|
|
118
|
+
* one durable checkpoint per call). While the activity runs, the consumer
|
|
119
|
+
* heartbeats its stream reservation at half the base window, so the message
|
|
120
|
+
* stays leased for the full run — however long — and is redelivered to
|
|
121
|
+
* another worker only when the owning consumer crashes and stops
|
|
122
|
+
* heartbeating. The collation ledger then guarantees any redelivered
|
|
123
|
+
* message settles as a duplicate before re-executing the activity. With a
|
|
124
|
+
* `securedWorker` connection (SECURITY DEFINER stored-proc mode), lease
|
|
125
|
+
* extension is unavailable and an activity must finish within the adaptive
|
|
126
|
+
* reservation window instead.
|
|
127
|
+
*
|
|
114
128
|
* @template ACT - The activity type map (use `typeof activities` for inline registration).
|
|
115
129
|
* @param {ActivityConfig} [options] - Activity configuration including retry policy and routing.
|
|
116
130
|
* @returns {ProxyType<ACT>} A typed proxy object mapping activity names to their durable wrappers.
|
|
@@ -230,6 +230,20 @@ exports.wrapActivity = wrapActivity;
|
|
|
230
230
|
* }
|
|
231
231
|
* ```
|
|
232
232
|
*
|
|
233
|
+
* ## Long-running activities execute exactly once
|
|
234
|
+
*
|
|
235
|
+
* `startToCloseTimeout` bounds an activity's run and is honored for long
|
|
236
|
+
* work (a 30–120s batch loop is a supported shape: poll → reconcile → act,
|
|
237
|
+
* one durable checkpoint per call). While the activity runs, the consumer
|
|
238
|
+
* heartbeats its stream reservation at half the base window, so the message
|
|
239
|
+
* stays leased for the full run — however long — and is redelivered to
|
|
240
|
+
* another worker only when the owning consumer crashes and stops
|
|
241
|
+
* heartbeating. The collation ledger then guarantees any redelivered
|
|
242
|
+
* message settles as a duplicate before re-executing the activity. With a
|
|
243
|
+
* `securedWorker` connection (SECURITY DEFINER stored-proc mode), lease
|
|
244
|
+
* extension is unavailable and an activity must finish within the adaptive
|
|
245
|
+
* reservation window instead.
|
|
246
|
+
*
|
|
233
247
|
* @template ACT - The activity type map (use `typeof activities` for inline registration).
|
|
234
248
|
* @param {ActivityConfig} [options] - Activity configuration including retry policy and routing.
|
|
235
249
|
* @returns {ProxyType<ACT>} A typed proxy object mapping activity names to their durable wrappers.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { HotMesh } from '../hotmesh';
|
|
2
2
|
import { EventsConfig } from '../../types/system_events';
|
|
3
3
|
import { Connection } from '../../types/durable';
|
|
4
|
-
import { EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, StatsEscalationsParams, EscalationStats, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, ClaimManyParams, EscalateManyToRoleParams, UpdateManyPriorityParams, ResolveManyParams } from '../../types/hmsh_escalations';
|
|
4
|
+
import { EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, StatsEscalationsParams, EscalationStats, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, ClaimManyParams, EscalateManyToRoleParams, UpdateManyPriorityParams, ResolveManyParams, ResolveAllOrNoneParams, ResolveAllOrNoneResult, PruneEscalationsParams, PruneEscalationsResult } from '../../types/hmsh_escalations';
|
|
5
5
|
export type GetHotMeshFn = (topic: string | null, namespace?: string) => Promise<HotMesh>;
|
|
6
6
|
export interface EscalationClientConfig {
|
|
7
7
|
/** Postgres connection options — used when creating a standalone EscalationClient. */
|
|
@@ -93,6 +93,18 @@ export declare class EscalationClientService {
|
|
|
93
93
|
* Uses the same filter parameters as `list()`.
|
|
94
94
|
*/
|
|
95
95
|
count(params?: ListEscalationsParams): Promise<number>;
|
|
96
|
+
/**
|
|
97
|
+
* Retention: deletes terminal escalation rows (`resolved`/`cancelled`/`expired`)
|
|
98
|
+
* whose `updated_at` is older than the horizon (a Postgres interval string,
|
|
99
|
+
* e.g. `'90 days'`). Terminal rows are inert — every engine state transition
|
|
100
|
+
* guards on `status = 'pending'` — so pruning them is safe for live waiters,
|
|
101
|
+
* claims, and signal delivery; this call is the engine-blessed way to age
|
|
102
|
+
* out the audit backlog. Each call deletes at most `limit` rows (default
|
|
103
|
+
* 10,000) in one atomic statement; loop until `deleted` is 0 to drain a
|
|
104
|
+
* large backlog. `list()`/`get()`/`stats()` reads over windows older than
|
|
105
|
+
* the pruning horizon reflect only the rows retained.
|
|
106
|
+
*/
|
|
107
|
+
prune(params: PruneEscalationsParams): Promise<PruneEscalationsResult>;
|
|
96
108
|
/** Returns a single escalation row by UUID. Returns `null` if not found. */
|
|
97
109
|
get(id: string, namespace?: string): Promise<EscalationEntry | null>;
|
|
98
110
|
/** Looks up an escalation by `signal_key` — the value passed to `condition()`. */
|
|
@@ -214,6 +226,26 @@ export declare class EscalationClientService {
|
|
|
214
226
|
* bulk resolution carries no wake and would strand the workflow.
|
|
215
227
|
*/
|
|
216
228
|
resolveMany(params: ResolveManyParams): Promise<EscalationEntry[]>;
|
|
229
|
+
/**
|
|
230
|
+
* All-or-none bulk resolve with per-row payloads. Every item's escalation
|
|
231
|
+
* must be pending (and, when `assertAssignee` is set, currently assigned to
|
|
232
|
+
* that assignee) or NOTHING resolves — one atomic SQL statement locks the
|
|
233
|
+
* rows in deterministic order, applies each row's own `resolverPayload`,
|
|
234
|
+
* and commits each waiter's wake WITH its resolve (same durability contract
|
|
235
|
+
* as `resolve()`). Unlike `resolveMany()`, rows backing a live `condition()`
|
|
236
|
+
* waiter are first-class here: each receives its own payload as
|
|
237
|
+
* `condition()`'s return value.
|
|
238
|
+
*
|
|
239
|
+
* On `ok: false` no row was written and `failed` lists exactly the blocking
|
|
240
|
+
* rows with reasons (`not-found`, `already-resolved`, `already-cancelled`,
|
|
241
|
+
* `already-expired`, `assignee-mismatch`) — rows that were themselves
|
|
242
|
+
* resolvable stay pending and are not listed.
|
|
243
|
+
*
|
|
244
|
+
* Pass `params.metadata` to merge one shared outcome patch into every row's
|
|
245
|
+
* GIN-indexed `metadata` in the same statement. Ids must be unique; an empty
|
|
246
|
+
* `items` array returns `ok: true` with no entries.
|
|
247
|
+
*/
|
|
248
|
+
resolveAllOrNone(params: ResolveAllOrNoneParams, namespace?: string): Promise<ResolveAllOrNoneResult>;
|
|
217
249
|
/**
|
|
218
250
|
* Returns dashboard-ready escalation counts. `period` controls the window
|
|
219
251
|
* used for `created` and `resolved` counts (default `'24h'`). When `roles`
|
|
@@ -201,6 +201,21 @@ class EscalationClientService {
|
|
|
201
201
|
const hm = await this._engine(null, params?.namespace);
|
|
202
202
|
return hm.engine.store.countEscalations(params ?? {});
|
|
203
203
|
}
|
|
204
|
+
/**
|
|
205
|
+
* Retention: deletes terminal escalation rows (`resolved`/`cancelled`/`expired`)
|
|
206
|
+
* whose `updated_at` is older than the horizon (a Postgres interval string,
|
|
207
|
+
* e.g. `'90 days'`). Terminal rows are inert — every engine state transition
|
|
208
|
+
* guards on `status = 'pending'` — so pruning them is safe for live waiters,
|
|
209
|
+
* claims, and signal delivery; this call is the engine-blessed way to age
|
|
210
|
+
* out the audit backlog. Each call deletes at most `limit` rows (default
|
|
211
|
+
* 10,000) in one atomic statement; loop until `deleted` is 0 to drain a
|
|
212
|
+
* large backlog. `list()`/`get()`/`stats()` reads over windows older than
|
|
213
|
+
* the pruning horizon reflect only the rows retained.
|
|
214
|
+
*/
|
|
215
|
+
async prune(params) {
|
|
216
|
+
const hm = await this._engine(null, params.namespace);
|
|
217
|
+
return hm.engine.store.pruneEscalations(params);
|
|
218
|
+
}
|
|
204
219
|
/** Returns a single escalation row by UUID. Returns `null` if not found. */
|
|
205
220
|
async get(id, namespace) {
|
|
206
221
|
const hm = await this._engine(null, namespace);
|
|
@@ -461,6 +476,74 @@ class EscalationClientService {
|
|
|
461
476
|
this._emitMany('resolved', entries);
|
|
462
477
|
return entries;
|
|
463
478
|
}
|
|
479
|
+
/**
|
|
480
|
+
* All-or-none bulk resolve with per-row payloads. Every item's escalation
|
|
481
|
+
* must be pending (and, when `assertAssignee` is set, currently assigned to
|
|
482
|
+
* that assignee) or NOTHING resolves — one atomic SQL statement locks the
|
|
483
|
+
* rows in deterministic order, applies each row's own `resolverPayload`,
|
|
484
|
+
* and commits each waiter's wake WITH its resolve (same durability contract
|
|
485
|
+
* as `resolve()`). Unlike `resolveMany()`, rows backing a live `condition()`
|
|
486
|
+
* waiter are first-class here: each receives its own payload as
|
|
487
|
+
* `condition()`'s return value.
|
|
488
|
+
*
|
|
489
|
+
* On `ok: false` no row was written and `failed` lists exactly the blocking
|
|
490
|
+
* rows with reasons (`not-found`, `already-resolved`, `already-cancelled`,
|
|
491
|
+
* `already-expired`, `assignee-mismatch`) — rows that were themselves
|
|
492
|
+
* resolvable stay pending and are not listed.
|
|
493
|
+
*
|
|
494
|
+
* Pass `params.metadata` to merge one shared outcome patch into every row's
|
|
495
|
+
* GIN-indexed `metadata` in the same statement. Ids must be unique; an empty
|
|
496
|
+
* `items` array returns `ok: true` with no entries.
|
|
497
|
+
*/
|
|
498
|
+
async resolveAllOrNone(params, namespace) {
|
|
499
|
+
const ns = (params.namespace ?? namespace) ?? factory_1.APP_ID;
|
|
500
|
+
const items = params.items ?? [];
|
|
501
|
+
if (!items.length)
|
|
502
|
+
return { ok: true, entries: [] };
|
|
503
|
+
const ids = items.map((i) => i.id);
|
|
504
|
+
if (new Set(ids).size !== ids.length) {
|
|
505
|
+
throw new Error('resolveAllOrNone: duplicate escalation ids in items');
|
|
506
|
+
}
|
|
507
|
+
const hm = await this._engine(null, ns);
|
|
508
|
+
const store = hm.engine.store;
|
|
509
|
+
//pre-build each waiter's wake so it commits INSIDE the resolve
|
|
510
|
+
//statement; signal routing (signal_key, topic) is immutable after
|
|
511
|
+
//creation, so the preview rows are authoritative for wake targeting
|
|
512
|
+
const payloadById = new Map(items.map((i) => [i.id, i.resolverPayload ?? {}]));
|
|
513
|
+
const previews = await store.listEscalations({
|
|
514
|
+
ids,
|
|
515
|
+
namespace: params.namespace,
|
|
516
|
+
});
|
|
517
|
+
const wakeCommands = [];
|
|
518
|
+
for (const row of previews) {
|
|
519
|
+
if (!row.signal_key)
|
|
520
|
+
continue;
|
|
521
|
+
const cmd = await this._buildWakeCommand(ns, row.topic, row.signal_key, payloadById.get(row.id) ?? {});
|
|
522
|
+
if (cmd)
|
|
523
|
+
wakeCommands.push(cmd);
|
|
524
|
+
}
|
|
525
|
+
const dbResult = await store.resolveAllOrNoneEscalations({
|
|
526
|
+
items,
|
|
527
|
+
namespace: params.namespace,
|
|
528
|
+
metadata: params.metadata,
|
|
529
|
+
assertAssignee: params.assertAssignee,
|
|
530
|
+
}, wakeCommands);
|
|
531
|
+
if (!dbResult.ok)
|
|
532
|
+
return { ok: false, failed: dbResult.failed };
|
|
533
|
+
//rows whose wake was not part of the commit (no hook rule deployed, or
|
|
534
|
+
//signal routing enriched after the preview) — deliver post-commit
|
|
535
|
+
const enqueuedKeys = new Set(wakeCommands.map((w) => w.forSignalKey));
|
|
536
|
+
for (const entry of dbResult.entries) {
|
|
537
|
+
if (entry.signal_key && !enqueuedKeys.has(entry.signal_key)) {
|
|
538
|
+
await this._deliverEscalationSignal(ns, entry.topic, {
|
|
539
|
+
id: entry.signal_key,
|
|
540
|
+
data: payloadById.get(entry.id) ?? {},
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
this._emitMany('resolved', dbResult.entries);
|
|
545
|
+
return { ok: true, entries: dbResult.entries };
|
|
546
|
+
}
|
|
464
547
|
// ─── Aggregates ─────────────────────────────────────────────────────────────
|
|
465
548
|
/**
|
|
466
549
|
* Returns dashboard-ready escalation counts. `period` controls the window
|
|
@@ -328,8 +328,43 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
|
|
|
328
328
|
escalateManyEscalationsToRole(params: import('../../../../types/hmsh_escalations').EscalateManyToRoleParams): Promise<number>;
|
|
329
329
|
updateManyEscalationsPriority(params: import('../../../../types/hmsh_escalations').UpdateManyPriorityParams): Promise<number>;
|
|
330
330
|
resolveManyEscalations(params: import('../../../../types/hmsh_escalations').ResolveManyParams): Promise<import('../../../../types/hmsh_escalations').EscalationEntry[]>;
|
|
331
|
+
/**
|
|
332
|
+
* All-or-none bulk resolve with per-row payloads. One SQL statement:
|
|
333
|
+
* the FOR UPDATE lock (ordered by id — deterministic acquisition, no
|
|
334
|
+
* deadlock between overlapping batches), the count-gated UPDATE, and
|
|
335
|
+
* the per-row wake INSERTs are one atomic unit. When ANY row is
|
|
336
|
+
* missing, non-pending, or fails the assignee assertion, the gate
|
|
337
|
+
* count mismatches, the UPDATE matches zero rows, no wake is written,
|
|
338
|
+
* and the statement degrades to a pure read — the returned snapshot
|
|
339
|
+
* names each blocking row and why. Same durability contract as
|
|
340
|
+
* `resolveEscalation`: each wake commits WITH its row's resolve.
|
|
341
|
+
*/
|
|
342
|
+
resolveAllOrNoneEscalations(params: import('../../../../types/hmsh_escalations').ResolveAllOrNoneParams, wakeCommands?: import('../../../../types/hmsh_escalations').EscalationWakeCommand[]): Promise<{
|
|
343
|
+
ok: true;
|
|
344
|
+
entries: import('../../../../types/hmsh_escalations').EscalationEntry[];
|
|
345
|
+
wakeCount: number;
|
|
346
|
+
} | {
|
|
347
|
+
ok: false;
|
|
348
|
+
failed: Array<{
|
|
349
|
+
id: string;
|
|
350
|
+
reason: import('../../../../types/hmsh_escalations').ResolveAllOrNoneBlockReason;
|
|
351
|
+
}>;
|
|
352
|
+
}>;
|
|
331
353
|
escalationStats(params?: import('../../../../types/hmsh_escalations').StatsEscalationsParams): Promise<import('../../../../types/hmsh_escalations').EscalationStats>;
|
|
332
354
|
listDistinctEscalationTypes(namespace?: string): Promise<string[]>;
|
|
333
355
|
releaseExpiredEscalations(_namespace?: string): Promise<number>;
|
|
356
|
+
/**
|
|
357
|
+
* Deletes terminal escalation rows (`resolved`/`cancelled`/`expired`) whose
|
|
358
|
+
* `updated_at` is older than the given horizon. Terminal rows are inert —
|
|
359
|
+
* every state transition guards `status = 'pending'` — so pruning them is
|
|
360
|
+
* the engine-blessed retention path for the audit backlog.
|
|
361
|
+
*
|
|
362
|
+
* Single statement: the candidate SELECT (`FOR UPDATE SKIP LOCKED`) and the
|
|
363
|
+
* DELETE are one atomic unit, so concurrent pruners and readers never block
|
|
364
|
+
* each other and a row is either fully gone or fully present. `limit` bounds
|
|
365
|
+
* rows per call to keep vacuum pressure flat; callers loop until `deleted`
|
|
366
|
+
* returns 0.
|
|
367
|
+
*/
|
|
368
|
+
pruneEscalations(params: import('../../../../types/hmsh_escalations').PruneEscalationsParams): Promise<import('../../../../types/hmsh_escalations').PruneEscalationsResult>;
|
|
334
369
|
}
|
|
335
370
|
export { PostgresStoreService };
|
|
@@ -2222,6 +2222,117 @@ class PostgresStoreService extends __1.StoreService {
|
|
|
2222
2222
|
RETURNING *`, namespace ? [payloadJson, ids, metaJson, namespace] : [payloadJson, ids, metaJson]);
|
|
2223
2223
|
return result.rows;
|
|
2224
2224
|
}
|
|
2225
|
+
/**
|
|
2226
|
+
* All-or-none bulk resolve with per-row payloads. One SQL statement:
|
|
2227
|
+
* the FOR UPDATE lock (ordered by id — deterministic acquisition, no
|
|
2228
|
+
* deadlock between overlapping batches), the count-gated UPDATE, and
|
|
2229
|
+
* the per-row wake INSERTs are one atomic unit. When ANY row is
|
|
2230
|
+
* missing, non-pending, or fails the assignee assertion, the gate
|
|
2231
|
+
* count mismatches, the UPDATE matches zero rows, no wake is written,
|
|
2232
|
+
* and the statement degrades to a pure read — the returned snapshot
|
|
2233
|
+
* names each blocking row and why. Same durability contract as
|
|
2234
|
+
* `resolveEscalation`: each wake commits WITH its row's resolve.
|
|
2235
|
+
*/
|
|
2236
|
+
async resolveAllOrNoneEscalations(params, wakeCommands) {
|
|
2237
|
+
const { items, namespace, metadata, assertAssignee } = params;
|
|
2238
|
+
if (!items?.length)
|
|
2239
|
+
return { ok: true, entries: [], wakeCount: 0 };
|
|
2240
|
+
const ids = items.map((i) => i.id);
|
|
2241
|
+
if (new Set(ids).size !== ids.length) {
|
|
2242
|
+
throw new Error('resolveAllOrNoneEscalations: duplicate escalation ids in items');
|
|
2243
|
+
}
|
|
2244
|
+
const payloads = items.map((i) => i.resolverPayload ? JSON.stringify(i.resolverPayload) : null);
|
|
2245
|
+
const metaJson = metadata ? JSON.stringify(metadata) : null;
|
|
2246
|
+
const sqlParams = [ids, payloads, metaJson, assertAssignee ?? null];
|
|
2247
|
+
let nsClause = '';
|
|
2248
|
+
if (namespace) {
|
|
2249
|
+
sqlParams.push(namespace);
|
|
2250
|
+
nsClause = `AND e.namespace = $${sqlParams.length}`;
|
|
2251
|
+
}
|
|
2252
|
+
//per-row wakes: one engine_streams INSERT per resolved row whose
|
|
2253
|
+
//signal_key matches its pre-built command; zero rows when the gate fails
|
|
2254
|
+
let wakeCTE = '';
|
|
2255
|
+
let wakeCount = '0::int AS wake_count';
|
|
2256
|
+
if (wakeCommands?.length) {
|
|
2257
|
+
const schemaName = this.kvsql().safeName(this.appId);
|
|
2258
|
+
const base = sqlParams.length;
|
|
2259
|
+
sqlParams.push(this.appId, wakeCommands.map((w) => w.forSignalKey), wakeCommands.map((w) => w.message));
|
|
2260
|
+
wakeCTE = `,
|
|
2261
|
+
wake AS (
|
|
2262
|
+
INSERT INTO ${schemaName}.engine_streams (stream_name, message, priority)
|
|
2263
|
+
SELECT $${base + 1}, w.message, 5
|
|
2264
|
+
FROM unnest($${base + 2}::text[], $${base + 3}::text[]) AS w(signal_key, message)
|
|
2265
|
+
JOIN resolved r ON r.signal_key = w.signal_key
|
|
2266
|
+
RETURNING id
|
|
2267
|
+
)`;
|
|
2268
|
+
wakeCount = '(SELECT COUNT(*) FROM wake)::int AS wake_count';
|
|
2269
|
+
}
|
|
2270
|
+
const result = await this.pgClient.query(`
|
|
2271
|
+
WITH input AS MATERIALIZED (
|
|
2272
|
+
SELECT * FROM unnest($1::uuid[], $2::text[]) AS t(id, payload_text)
|
|
2273
|
+
),
|
|
2274
|
+
target AS MATERIALIZED (
|
|
2275
|
+
SELECT e.id, e.signal_key, e.topic, e.status, e.assigned_to
|
|
2276
|
+
FROM public.hmsh_escalations e
|
|
2277
|
+
WHERE e.id = ANY($1::uuid[]) ${nsClause}
|
|
2278
|
+
ORDER BY e.id
|
|
2279
|
+
FOR UPDATE
|
|
2280
|
+
),
|
|
2281
|
+
eligible AS (
|
|
2282
|
+
SELECT t.id FROM target t
|
|
2283
|
+
WHERE t.status = 'pending'
|
|
2284
|
+
AND ($4::text IS NULL OR t.assigned_to = $4::text)
|
|
2285
|
+
),
|
|
2286
|
+
resolved AS (
|
|
2287
|
+
UPDATE public.hmsh_escalations e
|
|
2288
|
+
SET status = 'resolved', resolved_at = NOW(),
|
|
2289
|
+
resolver_payload = i.payload_text::jsonb,
|
|
2290
|
+
metadata = CASE WHEN $3::jsonb IS NOT NULL
|
|
2291
|
+
THEN COALESCE(e.metadata, '{}'::jsonb) || $3::jsonb
|
|
2292
|
+
ELSE e.metadata END,
|
|
2293
|
+
updated_at = NOW()
|
|
2294
|
+
FROM input i
|
|
2295
|
+
WHERE e.id = i.id
|
|
2296
|
+
AND e.status = 'pending'
|
|
2297
|
+
AND (SELECT COUNT(*) FROM eligible) = (SELECT COUNT(*) FROM input)
|
|
2298
|
+
RETURNING e.*
|
|
2299
|
+
)${wakeCTE}
|
|
2300
|
+
SELECT
|
|
2301
|
+
i.id AS input_id,
|
|
2302
|
+
t.status AS prior_status,
|
|
2303
|
+
CASE
|
|
2304
|
+
WHEN r.id IS NOT NULL THEN 'resolved'
|
|
2305
|
+
WHEN t.id IS NULL THEN 'not-found'
|
|
2306
|
+
WHEN t.status = 'cancelled' THEN 'already-cancelled'
|
|
2307
|
+
WHEN t.status = 'expired' THEN 'already-expired'
|
|
2308
|
+
WHEN t.status <> 'pending' THEN 'already-resolved'
|
|
2309
|
+
WHEN $4::text IS NOT NULL AND t.assigned_to IS DISTINCT FROM $4::text THEN 'assignee-mismatch'
|
|
2310
|
+
ELSE 'blocked-by-batch'
|
|
2311
|
+
END AS outcome,
|
|
2312
|
+
row_to_json(r.*) AS entry_json,
|
|
2313
|
+
${wakeCount}
|
|
2314
|
+
FROM input i
|
|
2315
|
+
LEFT JOIN target t ON t.id = i.id
|
|
2316
|
+
LEFT JOIN resolved r ON r.id = i.id
|
|
2317
|
+
ORDER BY i.id
|
|
2318
|
+
`, sqlParams);
|
|
2319
|
+
const rows = result.rows;
|
|
2320
|
+
if (rows.length && rows.every((r) => r.outcome === 'resolved')) {
|
|
2321
|
+
return {
|
|
2322
|
+
ok: true,
|
|
2323
|
+
entries: rows.map((r) => r.entry_json),
|
|
2324
|
+
wakeCount: rows[0].wake_count,
|
|
2325
|
+
};
|
|
2326
|
+
}
|
|
2327
|
+
return {
|
|
2328
|
+
ok: false,
|
|
2329
|
+
//'blocked-by-batch' rows were themselves resolvable — only the true
|
|
2330
|
+
//blockers are reported; the caller re-gangs around exactly these
|
|
2331
|
+
failed: rows
|
|
2332
|
+
.filter((r) => r.outcome !== 'resolved' && r.outcome !== 'blocked-by-batch')
|
|
2333
|
+
.map((r) => ({ id: r.input_id, reason: r.outcome })),
|
|
2334
|
+
};
|
|
2335
|
+
}
|
|
2225
2336
|
async escalationStats(params = {}) {
|
|
2226
2337
|
const { namespace, roles, period = '24h' } = params;
|
|
2227
2338
|
if (roles !== undefined && roles.length === 0) {
|
|
@@ -2318,5 +2429,44 @@ class PostgresStoreService extends __1.StoreService {
|
|
|
2318
2429
|
// assigned_until, so no background sweep is needed to release expired claims.
|
|
2319
2430
|
return 0;
|
|
2320
2431
|
}
|
|
2432
|
+
/**
|
|
2433
|
+
* Deletes terminal escalation rows (`resolved`/`cancelled`/`expired`) whose
|
|
2434
|
+
* `updated_at` is older than the given horizon. Terminal rows are inert —
|
|
2435
|
+
* every state transition guards `status = 'pending'` — so pruning them is
|
|
2436
|
+
* the engine-blessed retention path for the audit backlog.
|
|
2437
|
+
*
|
|
2438
|
+
* Single statement: the candidate SELECT (`FOR UPDATE SKIP LOCKED`) and the
|
|
2439
|
+
* DELETE are one atomic unit, so concurrent pruners and readers never block
|
|
2440
|
+
* each other and a row is either fully gone or fully present. `limit` bounds
|
|
2441
|
+
* rows per call to keep vacuum pressure flat; callers loop until `deleted`
|
|
2442
|
+
* returns 0.
|
|
2443
|
+
*/
|
|
2444
|
+
async pruneEscalations(params) {
|
|
2445
|
+
const TERMINAL = ['resolved', 'cancelled', 'expired'];
|
|
2446
|
+
const statuses = params.statuses?.length
|
|
2447
|
+
? params.statuses.filter((s) => TERMINAL.includes(s))
|
|
2448
|
+
: [...TERMINAL];
|
|
2449
|
+
if (!statuses.length)
|
|
2450
|
+
return { deleted: 0 };
|
|
2451
|
+
const limit = Math.max(1, Math.min(params.limit ?? 10000, 100000));
|
|
2452
|
+
const values = [statuses, params.olderThan, limit];
|
|
2453
|
+
let nsClause = '';
|
|
2454
|
+
if (params.namespace) {
|
|
2455
|
+
values.push(params.namespace);
|
|
2456
|
+
nsClause = `AND namespace = $${values.length}`;
|
|
2457
|
+
}
|
|
2458
|
+
const result = await this.pgClient.query(`WITH doomed AS (
|
|
2459
|
+
SELECT ctid FROM public.hmsh_escalations
|
|
2460
|
+
WHERE status = ANY($1::text[])
|
|
2461
|
+
AND status IN ('resolved', 'cancelled', 'expired')
|
|
2462
|
+
AND updated_at < NOW() - $2::interval
|
|
2463
|
+
${nsClause}
|
|
2464
|
+
LIMIT $3
|
|
2465
|
+
FOR UPDATE SKIP LOCKED
|
|
2466
|
+
)
|
|
2467
|
+
DELETE FROM public.hmsh_escalations e
|
|
2468
|
+
USING doomed WHERE e.ctid = doomed.ctid`, values);
|
|
2469
|
+
return { deleted: result.rowCount ?? 0 };
|
|
2470
|
+
}
|
|
2321
2471
|
}
|
|
2322
2472
|
exports.PostgresStoreService = PostgresStoreService;
|
|
@@ -131,6 +131,29 @@ export type CancelEscalationResult = {
|
|
|
131
131
|
ok: false;
|
|
132
132
|
reason: 'not-found' | 'already-terminal';
|
|
133
133
|
};
|
|
134
|
+
/**
|
|
135
|
+
* Retention parameters for `pruneEscalations`. Prunes only terminal rows
|
|
136
|
+
* (`resolved`/`cancelled`/`expired`) — the statuses every engine state
|
|
137
|
+
* transition treats as final — older than the given horizon.
|
|
138
|
+
*/
|
|
139
|
+
export interface PruneEscalationsParams {
|
|
140
|
+
/**
|
|
141
|
+
* Age horizon as a Postgres interval string (e.g. `'90 days'`, `'12 hours'`).
|
|
142
|
+
* Rows qualify when `updated_at < NOW() - olderThan`.
|
|
143
|
+
*/
|
|
144
|
+
olderThan: string;
|
|
145
|
+
/** Terminal statuses to prune. Defaults to all three; non-terminal values are ignored. */
|
|
146
|
+
statuses?: Array<'resolved' | 'cancelled' | 'expired'>;
|
|
147
|
+
namespace?: string;
|
|
148
|
+
/**
|
|
149
|
+
* Max rows deleted per call (bounds lock time and vacuum pressure).
|
|
150
|
+
* Default 10,000, capped at 100,000. Loop until `deleted` is 0 to drain.
|
|
151
|
+
*/
|
|
152
|
+
limit?: number;
|
|
153
|
+
}
|
|
154
|
+
export interface PruneEscalationsResult {
|
|
155
|
+
deleted: number;
|
|
156
|
+
}
|
|
134
157
|
export interface ListEscalationsParams {
|
|
135
158
|
namespace?: string;
|
|
136
159
|
role?: string;
|
|
@@ -329,6 +352,47 @@ export interface ResolveManyParams {
|
|
|
329
352
|
*/
|
|
330
353
|
metadata?: Record<string, unknown>;
|
|
331
354
|
}
|
|
355
|
+
/** One member of a `resolveAllOrNone()` batch — its own `resolverPayload` is
|
|
356
|
+
* stored as that row's `resolver_payload` and delivered to that row's waiting
|
|
357
|
+
* workflow as `condition()`'s return value. */
|
|
358
|
+
export interface ResolveAllOrNoneItem {
|
|
359
|
+
id: string;
|
|
360
|
+
resolverPayload?: Record<string, unknown>;
|
|
361
|
+
}
|
|
362
|
+
export interface ResolveAllOrNoneParams {
|
|
363
|
+
/** The batch. Ids must be unique; each item carries its own payload. */
|
|
364
|
+
items: ResolveAllOrNoneItem[];
|
|
365
|
+
namespace?: string;
|
|
366
|
+
/**
|
|
367
|
+
* Shared outcome patch merged (not replaced) into EVERY row's GIN-indexed
|
|
368
|
+
* `metadata` in the single atomic statement. See {@link ResolveEscalationParams.metadata}.
|
|
369
|
+
*/
|
|
370
|
+
metadata?: Record<string, unknown>;
|
|
371
|
+
/**
|
|
372
|
+
* When provided, every row must currently be assigned to this assignee
|
|
373
|
+
* (`assigned_to` equality, asserted inside the same statement). Closes the
|
|
374
|
+
* claim-race window for claim-then-resolve flows: a row re-claimed by another
|
|
375
|
+
* principal between the caller's claim and this resolve blocks the batch.
|
|
376
|
+
*/
|
|
377
|
+
assertAssignee?: string;
|
|
378
|
+
}
|
|
379
|
+
/** Why a specific row blocked a `resolveAllOrNone()` batch. */
|
|
380
|
+
export type ResolveAllOrNoneBlockReason = 'not-found' | 'already-resolved' | 'already-cancelled' | 'already-expired' | 'assignee-mismatch';
|
|
381
|
+
/**
|
|
382
|
+
* Result of `resolveAllOrNone()`. On `ok: false` NOTHING was written; `failed`
|
|
383
|
+
* lists only the rows that blocked the batch (rows that were themselves
|
|
384
|
+
* resolvable are not listed — they remain pending, untouched).
|
|
385
|
+
*/
|
|
386
|
+
export type ResolveAllOrNoneResult = {
|
|
387
|
+
ok: true;
|
|
388
|
+
entries: EscalationEntry[];
|
|
389
|
+
} | {
|
|
390
|
+
ok: false;
|
|
391
|
+
failed: Array<{
|
|
392
|
+
id: string;
|
|
393
|
+
reason: ResolveAllOrNoneBlockReason;
|
|
394
|
+
}>;
|
|
395
|
+
};
|
|
332
396
|
/**
|
|
333
397
|
* Full-fidelity migration params. Extends `CreateEscalationParams` with:
|
|
334
398
|
* - `id` (required) — preserves the original UUID; no auto-generation
|
package/build/types/index.d.ts
CHANGED
|
@@ -25,5 +25,5 @@ export { ReclaimedMessageType, RetryPolicy, RouterConfig, StreamCode, StreamConf
|
|
|
25
25
|
export { context, Context, Counter, Meter, metrics, propagation, SpanContext, Span, SpanStatus, SpanStatusCode, SpanKind, trace, Tracer, ValueType, } from './telemetry';
|
|
26
26
|
export { WorkListTaskType } from './task';
|
|
27
27
|
export { TransitionMatch, TransitionRule, Transitions } from './transition';
|
|
28
|
-
export { ConditionQueueConfig, EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, EscalateToRoleParams, MigrateEscalationParams, } from './hmsh_escalations';
|
|
28
|
+
export { ConditionQueueConfig, EscalationEntry, ClaimEscalationResult, ClaimByMetadataResult, ReleaseEscalationResult, ResolveEscalationResult, CancelEscalationResult, ListEscalationsParams, CreateEscalationParams, UpdateEscalationParams, AppendMilestonesParams, ClaimEscalationParams, ClaimByMetadataParams, ReleaseEscalationParams, ResolveEscalationParams, ResolveByMetadataParams, ResolveAllOrNoneItem, ResolveAllOrNoneParams, ResolveAllOrNoneBlockReason, ResolveAllOrNoneResult, EscalateToRoleParams, MigrateEscalationParams, } from './hmsh_escalations';
|
|
29
29
|
export { EscalationVerb, EngineVerb, WorkerVerb, SystemEvent, EventsConfig } from './system_events';
|
|
@@ -143,8 +143,8 @@ export interface SystemEvent {
|
|
|
143
143
|
span_id?: string;
|
|
144
144
|
/**
|
|
145
145
|
* Full committed row for escalation events; lifecycle metadata for
|
|
146
|
-
* engine/worker events.
|
|
147
|
-
*
|
|
146
|
+
* engine/worker events. Each dependent cherry-picks fields for its
|
|
147
|
+
* own event shape.
|
|
148
148
|
*/
|
|
149
149
|
data: Record<string, unknown>;
|
|
150
150
|
}
|
package/package.json
CHANGED
package/ISSUE.md
DELETED
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
## EIGHTH ITEM — a claimed row's post-signal settle can fail, stranding pending state that contradicts the workflow (dashboard-visible)
|
|
2
|
-
|
|
3
|
-
Two order trails, same signature (hike-mono, 0.25.5/0.7.5):
|
|
4
|
-
- run A order `a0e68dba`: every station row resolved EXCEPT quality-control —
|
|
5
|
-
yet the QC stationWorker delivered (shipping minted 5s later, order
|
|
6
|
-
COMPLETED). The QC row sits `pending` + claimed forever.
|
|
7
|
-
- run B order `89942462`: identical, at post-print-qa (claimed 07:34:44,
|
|
8
|
-
gluing resolved 07:34:54, order COMPLETED 07:35:11; the qa row still
|
|
9
|
-
pending+claimed 20+ min later).
|
|
10
|
-
|
|
11
|
-
Trigger context (our side, fixed by us): two associates raced a shallow pond
|
|
12
|
-
and BOTH booked the same order in the same second (audit rows at 07:34:44) —
|
|
13
|
-
so a concurrent claim write was interleaved with the resolve.
|
|
14
|
-
|
|
15
|
-
The library-side gap: the resolve path delivered the SIGNAL (workflow
|
|
16
|
-
advanced, next station minted) but the row's own transition to `resolved`
|
|
17
|
-
was lost against the concurrent claim — no error surfaced anywhere. The
|
|
18
|
-
result is dashboard-visible corruption: a live, claimable, countdown-bearing
|
|
19
|
-
work item asserting QA is owed on a COMPLETED order (real associates would
|
|
20
|
-
work it). Whatever branch handled this (atomic signal_key resolve or the
|
|
21
|
-
legacy handle.signal → ltResolveEscalation), signal delivery and row settle
|
|
22
|
-
must be atomic — or the settle must retry until the row cannot lie.
|
|
23
|
-
|
|
24
|
-
Repro shape: claim a signal_key escalation from principal A, and within the
|
|
25
|
-
same second re-claim/book from principal B while principal C resolves by
|
|
26
|
-
metadata. Expect: signal delivered AND row resolved, always.
|
|
27
|
-
|
|
28
|
-
# RE-FILED — two reports lost when this file was replaced with your 0.25.6 reply (both found the same evening, before it)
|
|
29
|
-
|
|
30
|
-
## NINTH ITEM (re-filed seventh) — a timeout wake KILLS the collator of a collated wait set; parent hangs forever
|
|
31
|
-
|
|
32
|
-
`Promise.all(21 × condition(signalId, {timeout:'1320s'}))` in one parent
|
|
33
|
-
(hike-mono simulationDirector, 0.25.5). Signal-path delivery through the
|
|
34
|
-
collation worked (seat 18 resolved cleanly at 04:55:05, its collator_waiter
|
|
35
|
-
slot consumed). At 05:07:25 UTC the FIRST batch of SLA expirations landed
|
|
36
|
-
(vm0..vm4 rows flipped 'expired' in the same second) and the collator job
|
|
37
|
-
(`HfSVaQSgUaYu3XGsCCjyQWy$C`) went **status -1 in that same second**
|
|
38
|
-
(context={}, is_live=t). Every subsequent expiry wake then hit
|
|
39
|
-
`Inactive job …$C` at CollatorService.assertJobActive (collator/index.js:13)
|
|
40
|
-
via Hook.processTimeHookEvent (hook.js:650) and was dropped. End state,
|
|
41
|
-
stable 10+ min: 20 live collator_waiter registry rows, all rows 'expired',
|
|
42
|
-
zero pending engine_streams for the collator, parent wedged in Promise.all
|
|
43
|
-
forever with no surfaced error.
|
|
44
|
-
|
|
45
|
-
Repro: `await Promise.all([condition('k1',{timeout:'30s'}), condition('k2',{timeout:'30s'})])`,
|
|
46
|
-
no resolver — let both expire. Expect collator -1 on first expiry, parent wedged.
|
|
47
|
-
|
|
48
|
-
Our standing mitigation: each timeout-bearing wait moved into its own child
|
|
49
|
-
workflow (inline wait), parent collates ExecChild completions — works, but
|
|
50
|
-
collated waits + SLA should compose.
|
|
51
|
-
|
|
52
|
-
## TENTH ITEM (re-filed follow-up) — second of two near-simultaneous sibling wakes lost (no crash, no restart)
|
|
53
|
-
|
|
54
|
-
Clean floor, 10 orders: per order, TWO sibling printJobWorkers under one
|
|
55
|
-
dispatcher park signal_key waits; a machine resolves BOTH escalations
|
|
56
|
-
seconds apart. 3-for-3 wedged orders: LEFT worker woke and completed; RIGHT
|
|
57
|
-
worker (resolved moments later) never woke — job row untouched since
|
|
58
|
-
creation, 10+ min, stored-pending-signal path did not recover it. 14/14
|
|
59
|
-
wakes delivered for the 7 orders whose resolves were spread out. Process up
|
|
60
|
-
the whole time (distinct from the reclaim case). Plausibly fixed by commit
|
|
61
|
-
3's atomic wake INSERT ("no interleave window") — flagging so it gets a
|
|
62
|
-
sibling-burst regression test: resolve two waits of one parent within the
|
|
63
|
-
same tick, both must wake. Our mitigation: 0–4s jitter between sibling
|
|
64
|
-
resolves (also more realistic for print hardware).
|
package/RESPONSE-0.25.6.md
DELETED
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
# hotmesh 0.25.6 — fixes for the wake-loss and reservation reports (branch: fix/atomic-cancel-wake, three commits)
|
|
2
|
-
|
|
3
|
-
Everything below is implemented and fail-first tested (each fix was watched failing
|
|
4
|
-
before its change); the branch is in final full-suite verification before merge.
|
|
5
|
-
|
|
6
|
-
## Commit 1 — cancellation wakes are durable
|
|
7
|
-
|
|
8
|
-
**Your second addendum (3 of 6 pills slept after the 400ms cancel wave).** Your
|
|
9
|
-
watchers settle waits via `cancel()`, and cancel's wake was a post-commit,
|
|
10
|
-
best-effort publish whose failure was silently discarded. Now the cancellation
|
|
11
|
-
wake commits inside the cancel transaction: a cancel that is durable always
|
|
12
|
-
resumes its `condition()` with `null`. A native six-pill burst test pins your
|
|
13
|
-
wave shape.
|
|
14
|
-
|
|
15
|
-
Also adds a permanent regression test for your converge topology
|
|
16
|
-
(`Promise.all([executeChild x4, sleep branch])`): the sleep's timehook mints and
|
|
17
|
-
the converge settles.
|
|
18
|
-
|
|
19
|
-
## Commit 2 — dead reservations reclaim; resolved waits disarm their timers
|
|
20
|
-
|
|
21
|
-
**Your fifth data point (reservations never reclaimed after process death) — the
|
|
22
|
-
highest-leverage fix.** The fallback poller's discovery query filtered
|
|
23
|
-
`reserved_at IS NULL`, so a dead holder's reservation was invisible forever.
|
|
24
|
-
It now surfaces stale reservations (threshold tracks the configured window;
|
|
25
|
-
heartbeating work never matches), through dedicated index-backed branches, and
|
|
26
|
-
the fixed function re-deploys to existing databases on boot. A crash or deploy
|
|
27
|
-
now costs ≤ ~90 seconds of redelivery latency instead of a permanent wedge —
|
|
28
|
-
and this most plausibly explains your third data point too (wake messages
|
|
29
|
-
claimed by churning per-principal clients that died were never reclaimed).
|
|
30
|
-
|
|
31
|
-
**Your sixth item (armed-timeout drizzle).** When the signal wins an SLA-gated
|
|
32
|
-
wait, the armed timeout is now disarmed with the settled wait — root and cycled
|
|
33
|
-
dimensions both covered. New timers are jid-stamped so job-scoped purges
|
|
34
|
-
(terminate) remove them as well. Timers armed before the upgrade drain as the
|
|
35
|
-
one-shot benign events you observed.
|
|
36
|
-
|
|
37
|
-
Also hardens the pending-signal handoff and cancel wake into single atomic
|
|
38
|
-
statements (safe on pooled connections, no interleave window; an in-window
|
|
39
|
-
wake refreshed by a newer signal delivers the newest payload).
|
|
40
|
-
|
|
41
|
-
## Commit 3 — resolve paths made single-statement; bulk resolve protects waiters
|
|
42
|
-
|
|
43
|
-
- `resolve()` and `resolveByMetadata()` now use the same single-statement
|
|
44
|
-
atomic idiom as cancel: row lock, status guard, and the wake INSERT commit
|
|
45
|
-
as one unit on any connection type.
|
|
46
|
-
- `resolveMany()` now excludes rows backing a live `condition()` waiter — bulk
|
|
47
|
-
resolution carries no wake, so those rows stay pending for a targeted
|
|
48
|
-
`resolve()`/`cancel()` instead of silently stranding the workflow.
|
|
49
|
-
- Secured-mode worker responses stamp `jid`, so job-scoped purges address them.
|
|
50
|
-
|
|
51
|
-
## Status of the remaining items
|
|
52
|
-
|
|
53
|
-
- **Lost `sleep()` timer (your timer forensics):** still not reproduced — your
|
|
54
|
-
exact topology mints and converges cleanly under test, and every registration
|
|
55
|
-
path checks out atomic. The reclaim fix (commit 2) may retire this too if the
|
|
56
|
-
collator's driving message was claimed by a dying client. The decisive
|
|
57
|
-
forensic if it recurs: the wedged director's collator job
|
|
58
|
-
(`SELECT * FROM durable.jobs WHERE key LIKE '%$C'`) plus that collator jid's
|
|
59
|
-
`engine_streams` rows, no filters — it records exactly which item-cycle
|
|
60
|
-
stopped.
|
|
61
|
-
- **`createClient` construction churn (your fourth data point):** your
|
|
62
|
-
per-principal memoization is the right mitigation; an upstream answer
|
|
63
|
-
(internal per-auth memoization or an explicit `client.close()`) is under
|
|
64
|
-
design as its own change.
|
|
65
|
-
- **Crash windows on interrupt's parent notification** (found by our internal
|
|
66
|
-
audit, same family as your wake fixes): being engineered in a dedicated
|
|
67
|
-
follow-up PR — the straightforward implementation measurably slowed interrupt
|
|
68
|
-
dispatch, and interrupts race against live completions by design, so it is
|
|
69
|
-
getting the unhurried version.
|
|
70
|
-
|
|
71
|
-
## After upgrading
|
|
72
|
-
|
|
73
|
-
- Restarting workers drains any orphaned-reservation backlog automatically.
|
|
74
|
-
- The settlement guard can be retired again — cancel and resolve wakes are both
|
|
75
|
-
transactional now, and dead reservations self-heal.
|
|
76
|
-
- Watch for `hook-signal-pending-redelivered` (in-window wake handled) and the
|
|
77
|
-
reclaim redeliveries after any crash: both are the system working as intended.
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
jid,reserved_at,retry_attempt,created_at,message
|
|
2
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm3,2026-07-06 20:19:07.953447+00,0,2026-07-06 20:18:07.945876+00,"{""metadata"":{""guid"":""HxfFFVmLVwtk37194ysP1qQ"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm3"",""gid"":""HAXgFtL8ZicUsOywHCKUauE"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm3"",""operatorId"":""0dfa0000-0000-4000-8000-000000000004"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":6,""plan"":[],""advertId"":""0a4af7d1-2dc4-4e06-9d43-791acf3c49e5""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm3-$printerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm3-$printerBatch$1-5"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
3
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm4,2026-07-06 20:19:10.889217+00,0,2026-07-06 20:18:10.880925+00,"{""metadata"":{""guid"":""HXZnTUGkTix4uF7q14PuP0f"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm4"",""gid"":""HO9SAZEoZGiycaugLuwuUXv"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm4"",""operatorId"":""0dfa0000-0000-4000-8000-000000000005"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":6,""plan"":[],""advertId"":""a1d6d90a-a2cf-4621-a74f-e0ed1013cbc6""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm4-$printerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm4-$printerBatch$1-5"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
4
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm5,2026-07-06 20:19:16.082786+00,0,2026-07-06 20:18:16.072927+00,"{""metadata"":{""guid"":""HE7nFSVKoh2oFe6-lYtCJWV"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm5"",""gid"":""H697gK8YvdS-_waP4NLiL-L"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm5"",""operatorId"":""0dfa0000-0000-4000-8000-000000000006"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":4,""plan"":[],""advertId"":""df441cd6-1811-41f0-b646-3c794ebf71d4""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm5-$printerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm5-$printerBatch$1-5"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
5
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm1,2026-07-06 20:19:05.227375+00,0,2026-07-06 20:18:20.216145+00,"{""metadata"":{""guid"":""HTFIOwV5q0HYi8z04GwX9qj"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm1"",""gid"":""HSR3wD1ryGQ6f5vzzGcO8eV"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm1"",""operatorId"":""0dfa0000-0000-4000-8000-000000000002"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":13,""plan"":[],""advertId"":""e8ee662b-6de6-4a57-b8d6-ee9ed76e1f99""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm1-$printerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm1-$printerBatch$1-5"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
6
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm0,2026-07-06 20:19:09.068044+00,0,2026-07-06 20:18:39.059327+00,"{""metadata"":{""guid"":""HcFhO_I7Fqn_5ftBd4uUTuZ"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm0"",""gid"":""HyVUxkj0OkvEO2r0sdgtTan"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm0"",""operatorId"":""0dfa0000-0000-4000-8000-000000000001"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":12,""plan"":[],""advertId"":""53bd1b3a-8ec3-4c35-a839-da93d9e23bc0""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm0-$printerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm0-$printerBatch$1-5"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
7
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm2,2026-07-06 20:19:18.195319+00,0,2026-07-06 20:19:18.191286+00,"{""metadata"":{""guid"":""HWpXdbtZ-Bx5iqINxvAa6wf"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm2"",""gid"":""HWzNcP3ToTg27Qc5u5MljTT"",""dad"":"",0,16,0,0"",""aid"":""proxyer"",""topic"":""virtual-farm-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""printerBatch"",""arguments"":[{""machineId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm2"",""operatorId"":""0dfa0000-0000-4000-8000-000000000003"",""printSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""runsSoFar"":8,""plan"":[],""advertId"":""a0a84553-5031-456d-ae02-5d9746b606fc""}],""workflowDimension"":""$1"",""index"":6,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm2-$printerBatch$1-6"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-farm-vm2-$printerBatch$1-6"",""workflowTopic"":""virtual-farm-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
8
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-manager,2026-07-06 20:19:14.76557+00,0,2026-07-06 20:18:29.759824+00,"{""metadata"":{""guid"":""HzkL-DZTtXyIyDV8oZNZJxI"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-manager"",""gid"":""HCGavoqirvD8FwdwQOWgcov"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""managerBatch"",""arguments"":[{""laneId"":""9a93b820-dde3-425b-a320-260b91c08ec2"",""reviewSeconds"":3,""moveSeconds"":3,""idleTickSeconds"":3,""maxIterations"":25,""actorEmail"":""sim.manager.marco@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-manager-$managerBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-manager-$managerBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
9
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-0,2026-07-06 20:19:12.094467+00,0,2026-07-06 20:18:42.089308+00,"{""metadata"":{""guid"":""H2MwGXqBeP8-8nPX-T6ndnU"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-0"",""gid"":""H7nf5o8VvELz6NHWGWGNsXe"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_QUALITY_CONTROL"",""role"":""quality-inspector"",""toStatus"":""NEEDS_SHIPPING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":7,""unparkedRejects"":[],""troubledOrders"":[""931a1e99-2db6-4280-b034-57e3c42a9aba""],""actorEmail"":""sim.inspector.ines@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-0-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-0-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
10
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-1,2026-07-06 20:19:12.096894+00,0,2026-07-06 20:18:42.093836+00,"{""metadata"":{""guid"":""H4pQ4lScxaW8wUEiP8BQh8E"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-1"",""gid"":""HQUqSMYqzsvODyjMvRNPtvl"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_QUALITY_CONTROL"",""role"":""quality-inspector"",""toStatus"":""NEEDS_SHIPPING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":7,""unparkedRejects"":[],""troubledOrders"":[""931a1e99-2db6-4280-b034-57e3c42a9aba""],""actorEmail"":""sim.inspector.tomas@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-1-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_quality_control-1-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
11
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-draft,2026-07-06 20:19:16.150935+00,0,2026-07-06 20:18:46.146153+00,"{""metadata"":{""guid"":""HgtJMxOb1oYv6jJb2uErF0_"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-draft"",""gid"":""HN6b23imRu-wgDHA-DpxzLV"",""dad"":"",0,14,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""DRAFT"",""role"":""insole-designer"",""toStatus"":""NEEDS_MANUFACTURING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":24,""unparkedRejects"":[],""troubledOrders"":[""2b294913-f8ca-4639-9928-3abd53ea9616""],""actorEmail"":""sim.designer.dana@sim.hike.local"",""worklistStation"":""design-approval""}],""workflowDimension"":""$1"",""index"":4,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-draft-$associateBatch$1-4"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-draft-$associateBatch$1-4"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
12
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-1,2026-07-06 20:19:04.938838+00,0,2026-07-06 20:18:49.932504+00,"{""metadata"":{""guid"":""HQD-nuRyxBmH431uJ8noMBS"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-1"",""gid"":""HWM3gVxjxz9_K-EnQpCeFjY"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_FINISHING"",""role"":""finisher"",""toStatus"":""NEEDS_QUALITY_CONTROL"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":9,""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.finisher.kofi@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-1-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-1-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
13
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-0,2026-07-06 20:19:14.016307+00,0,2026-07-06 20:18:59.008759+00,"{""metadata"":{""guid"":""HHBHdHpX2Wear_cVJim70Ps"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-0"",""gid"":""HkZyt5-JhLoCNX26Q4cyqbq"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_FINISHING"",""role"":""finisher"",""toStatus"":""NEEDS_QUALITY_CONTROL"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":11,""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.finisher.ada@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-0-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_finishing-0-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
14
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_shipping,2026-07-06 20:19:15.654293+00,0,2026-07-06 20:19:00.646061+00,"{""metadata"":{""guid"":""HffnP49JTwmHAwk3BNRn22l"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_shipping"",""gid"":""HuK-iSfDdYXXBJWo2WwhCDF"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_SHIPPING"",""role"":""shipping-associate"",""toStatus"":""COMPLETED"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":12,""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.shipper.june@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_shipping-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_shipping-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
15
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-1,2026-07-06 20:19:05.381804+00,0,2026-07-06 20:19:05.3792+00,"{""metadata"":{""guid"":""Hx82JQ7g0Y4XT7nqPFK4km9"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-1"",""gid"":""HoT3fZZg04_z7BKimsTs-Ei"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_GLUING"",""role"":""gluer"",""toStatus"":""NEEDS_FINISHING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":13,""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.gluer.dev@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-1-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-1-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
16
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-0,2026-07-06 20:19:05.386589+00,0,2026-07-06 20:19:05.384446+00,"{""metadata"":{""guid"":""HvNFncRHNbqR_OSOOiQ0M12"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-0"",""gid"":""Hk7Wy8HO98wh45Clgp5-sUo"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""NEEDS_GLUING"",""role"":""gluer"",""toStatus"":""NEEDS_FINISHING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":13,""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.gluer.rosa@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-0-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-needs_gluing-0-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
17
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-1,2026-07-06 20:19:06.832852+00,0,2026-07-06 20:19:06.829652+00,"{""metadata"":{""guid"":""Hg1xKYket-ULnS-HdHaNuiD"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-1"",""gid"":""HLjxCrhPCUyqBaWbwSE0pxp"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""POST_PRINT_QA"",""role"":""post-print-qa"",""toStatus"":""NEEDS_GLUING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":14,""rejectRate"":{""pct"":15,""targetStation"":""PRINTING"",""reasons"":[""Layer separation on the arch — reprint"",""Surface finish below spec"",""Edge delamination at the heel cup"",""Warping on the medial side""],""leftQuantity"":1,""rightQuantity"":0},""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.qa.elena@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-1-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-1-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
18
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-0,2026-07-06 20:19:12.746444+00,0,2026-07-06 20:19:12.744474+00,"{""metadata"":{""guid"":""HTl4tkkOMhgso2QMlQlKYvn"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-0"",""gid"":""HY9mpKnlDbC3a1o3B4R-53h"",""dad"":"",0,15,0,0"",""aid"":""proxyer"",""topic"":""actor-crew-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""associateBatch"",""arguments"":[{""station"":""POST_PRINT_QA"",""role"":""post-print-qa"",""toStatus"":""NEEDS_GLUING"",""dwellSeconds"":4,""betweenSeconds"":2,""idleTickSeconds"":3,""maxIterations"":25,""touchesSoFar"":16,""rejectRate"":{""pct"":15,""targetStation"":""PRINTING"",""reasons"":[""Layer separation on the arch — reprint"",""Surface finish below spec"",""Edge delamination at the heel cup"",""Warping on the medial side""],""leftQuantity"":1,""rightQuantity"":0},""unparkedRejects"":[],""troubledOrders"":[],""actorEmail"":""sim.qa.marcus@sim.hike.local""}],""workflowDimension"":""$1"",""index"":5,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-0-$associateBatch$1-5"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-crew-post_print_qa-0-$associateBatch$1-5"",""workflowTopic"":""actor-crew-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|
|
19
|
-
simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-design,2026-07-06 20:19:10.594054+00,0,2026-07-06 20:18:55.585455+00,"{""metadata"":{""guid"":""HeSdgdXIms_1Lhh2nNgc4EP"",""jid"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-design"",""gid"":""HU01R82t5rnplNLKWryLJPz"",""dad"":"",0,40,0,0"",""aid"":""proxyer"",""topic"":""simulation-activity"",""wfn"":""durable.execute""},""data"":{""activityName"":""designTwinBatch"",""arguments"":[{""user"":""sim.taika.iris@sim.hike.local"",""designSeconds"":2,""idleTickSeconds"":3,""maxIterations"":10,""concurrency"":4,""designedOrders"":[""0f596ab9-f449-485d-b06d-e74864750ae7"",""a5cedbc2-f726-4411-9ac3-10fd113532d2"",""c1a09bd7-bd4b-4eb1-bd3e-91d6b45b479f"",""94d74e0e-e4ce-4db7-91f3-02c1c081c2ba"",""0ed99be7-83b6-4a91-9aa5-560d5c110c6b"",""33b987bb-9253-483f-baa4-37d4ffd0903c"",""2c4bc240-1335-423d-96af-e5855b6ccc0f"",""0e7f24bb-3fba-40e5-a5a8-666f5c91fd49"",""ad0c05b6-4528-4f2a-9569-d06c76589b54"",""de2a02e2-33ca-4c0c-843c-5ad422c7c517"",""c008abd0-8b3b-44f7-9f1c-7d314f72f587"",""04a28a34-20f3-4d10-a636-3cb8f504f56e"",""52295fa8-452b-48ca-862b-64cf3ff8cae6"",""cb1dd956-639d-4ff8-9f56-c12f9c5278e8"",""fca4a3f0-fc49-47cd-8285-bf7e11c60dbe"",""bcd42ec0-f047-4acc-8e70-33575ae7535a"",""3b73fb95-2366-4498-ac9a-3f2d9eb4d1c0"",""3c9d174f-7928-42e7-9edb-2f3b38ba0ddc"",""1e61d445-77ba-4013-af89-1a6de07ff407"",""0b066160-06ab-4681-af20-8c36866c9987"",""fa9f996c-2327-4f30-8fdc-e63239483f23"",""ff31ec18-c7e3-4907-85f0-5c4eda42841f"",""f17a825c-8905-4166-b687-54015ddeae6c"",""931a1e99-2db6-4280-b034-57e3c42a9aba"",""2b294913-f8ca-4639-9928-3abd53ea9616"",""04caa323-662a-4c28-8de0-3655d435f206"",""b2eac688-d074-45d9-8a60-e87719b18c81"",""a11092fe-31e5-4cfe-815c-230a09f2165e"",""176e3d03-0813-4816-83fd-a785798a83bb"",""e1d887bd-1ad4-4436-b728-d75e7df2c39a"",""c8a0ffdc-91b0-443a-a8f9-c01c2e3935c0"",""83cbe6eb-ea9b-492e-8657-4f138e3df05e"",""e8e56685-82a3-4f88-a804-b6a8c21f0a4c"",""89d231bf-600b-40ed-8161-e58dbcc7157c"",""6904b329-7a63-465f-aa6a-82a1095638fb"",""9ca2abe9-ae5f-459a-a003-9ea4ad5cafc0"",""cae10b57-439a-4e98-b0d0-1bb172a9d79f"",""f90af7cb-401b-4e68-9f22-a45d34581d64"",""86a1d1b2-7287-48ab-94b3-544a743fda41"",""76793c77-67a0-4148-b83b-fa50f900111c""]}],""workflowDimension"":""$3"",""index"":8,""originJobId"":""simulationDirector-H5zQndzkcfIGHhUM-QXFzcG"",""parentWorkflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-design-$designTwinBatch$3-8"",""workflowId"":""-simulationDirector-H5zQndzkcfIGHhUM-QXFzcG-design-$designTwinBatch$3-8"",""workflowTopic"":""simulation-activity"",""expire"":2592000,""backoffCoefficient"":5,""initialInterval"":1,""maximumAttempts"":3,""maximumInterval"":120}}"
|