@parall/agent-core 1.44.0 → 1.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channel-capability.d.ts +2 -0
- package/dist/channel-capability.d.ts.map +1 -1
- package/dist/channel-capability.js +15 -0
- package/dist/dispatch-adapter.d.ts +9 -0
- package/dist/dispatch-adapter.d.ts.map +1 -1
- package/dist/event-format.d.ts.map +1 -1
- package/dist/event-format.js +27 -7
- package/dist/fork-session-finalizer.d.ts +65 -0
- package/dist/fork-session-finalizer.d.ts.map +1 -0
- package/dist/fork-session-finalizer.js +70 -0
- package/dist/gateway-base.d.ts +55 -0
- package/dist/gateway-base.d.ts.map +1 -1
- package/dist/gateway-base.js +640 -263
- package/dist/gateway-lane-flow.d.ts +75 -5
- package/dist/gateway-lane-flow.d.ts.map +1 -1
- package/dist/gateway-lane-flow.js +240 -18
- package/dist/http-keepalive.d.ts +4 -0
- package/dist/http-keepalive.d.ts.map +1 -0
- package/dist/http-keepalive.js +33 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/lane-ledger.d.ts +8 -0
- package/dist/lane-ledger.d.ts.map +1 -1
- package/dist/lane-ledger.js +14 -0
- package/dist/session-lifecycle.d.ts +198 -0
- package/dist/session-lifecycle.d.ts.map +1 -0
- package/dist/session-lifecycle.js +446 -0
- package/dist/skills/parall-clips.d.ts +1 -1
- package/dist/skills/parall-clips.d.ts.map +1 -1
- package/dist/skills/parall-clips.js +3 -0
- package/dist/skills/parall-schedules.d.ts +1 -1
- package/dist/skills/parall-schedules.d.ts.map +1 -1
- package/dist/skills/parall-schedules.js +1 -1
- package/dist/skills/parall-tasks.d.ts +1 -1
- package/dist/skills/parall-tasks.d.ts.map +1 -1
- package/dist/skills/parall-tasks.js +21 -5
- package/dist/step-persister.d.ts +66 -0
- package/dist/step-persister.d.ts.map +1 -0
- package/dist/step-persister.js +116 -0
- package/dist/step-retry-queue.d.ts +91 -0
- package/dist/step-retry-queue.d.ts.map +1 -0
- package/dist/step-retry-queue.js +259 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -2
- package/src/channel-capability.ts +16 -0
- package/src/dispatch-adapter.ts +10 -0
- package/src/event-format.ts +27 -7
- package/src/fork-session-finalizer.ts +122 -0
- package/src/gateway-base.ts +747 -331
- package/src/gateway-lane-flow.ts +275 -18
- package/src/http-keepalive.ts +36 -0
- package/src/index.ts +7 -1
- package/src/lane-ledger.ts +14 -0
- package/src/session-lifecycle.ts +552 -0
- package/src/skills/parall-clips.ts +3 -0
- package/src/skills/parall-schedules.ts +1 -1
- package/src/skills/parall-tasks.ts +21 -5
- package/src/step-persister.ts +161 -0
- package/src/step-retry-queue.ts +296 -0
- package/src/types.ts +2 -1
package/src/gateway-base.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as os from 'node:os';
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
+
import { randomUUID } from 'node:crypto';
|
|
4
5
|
import { ApiError, MENTION_ALL_USER_ID, ParallClient, ParallWs } from '@parall/sdk';
|
|
5
6
|
import type {
|
|
6
7
|
AgentConfigUpdateData,
|
|
@@ -28,6 +29,7 @@ import {
|
|
|
28
29
|
buildForkResultPrefix,
|
|
29
30
|
buildForkScopePrefix,
|
|
30
31
|
} from './event-format.js';
|
|
32
|
+
import { CAPABILITY_SLACK_SEND, channelCapabilityKeyFor } from './channel-capability.js';
|
|
31
33
|
import {
|
|
32
34
|
buildErrorStepContent,
|
|
33
35
|
type CleanupForkOpts,
|
|
@@ -38,13 +40,20 @@ import {
|
|
|
38
40
|
type RuntimeEvent,
|
|
39
41
|
} from './dispatch-adapter.js';
|
|
40
42
|
import {
|
|
43
|
+
clearTypedDedupeForEvent,
|
|
41
44
|
consumeMessageWorkItem,
|
|
42
45
|
consumeTypedDispatch,
|
|
43
46
|
dispatchLaneGroup,
|
|
47
|
+
resolveDispatchByID,
|
|
48
|
+
settleDrainedTypedGroup,
|
|
49
|
+
typedLedgerEventIds,
|
|
44
50
|
} from './gateway-lane-flow.js';
|
|
45
|
-
import type { LaneFlowHost } from './gateway-lane-flow.js';
|
|
51
|
+
import type { LaneFlowHost, TypedConsumeHooks } from './gateway-lane-flow.js';
|
|
46
52
|
import { LaneLedger } from './lane-ledger.js';
|
|
47
53
|
import { routeTrigger } from './routing.js';
|
|
54
|
+
import { StepPersister, isRetryableStepError } from './step-persister.js';
|
|
55
|
+
import { SessionLifecycleCoordinator, type TurnHandle } from './session-lifecycle.js';
|
|
56
|
+
import { ForkSessionFinalizer } from './fork-session-finalizer.js';
|
|
48
57
|
import {
|
|
49
58
|
clearDispatchMessageId,
|
|
50
59
|
clearDispatchMetrics,
|
|
@@ -138,6 +147,10 @@ export type ParallGatewayOptions = {
|
|
|
138
147
|
shutdownDeadlineMs?: number;
|
|
139
148
|
forkDeadlineMs?: number;
|
|
140
149
|
dispatchDeadlineMs?: number;
|
|
150
|
+
/** Test port: step retry-queue backoff override (StepRetryQueue schedule). */
|
|
151
|
+
stepRetryDelaysMs?: number[];
|
|
152
|
+
/** Test port: session lifecycle reconcile backoff override. */
|
|
153
|
+
lifecycleRetryDelaysMs?: number[];
|
|
141
154
|
contextFilePathForSession?: (sessionKey: string) => string | undefined;
|
|
142
155
|
/** @deprecated Use contextFilePathForSession. Kept for runtimes that haven't migrated. */
|
|
143
156
|
stepIdFilePathForSession?: (sessionKey: string) => string | undefined;
|
|
@@ -210,6 +223,36 @@ export function parseDispatchDeadlineMs(raw: string | undefined): number | undef
|
|
|
210
223
|
return Math.floor(n);
|
|
211
224
|
}
|
|
212
225
|
|
|
226
|
+
/**
|
|
227
|
+
* Stable input-step idempotency key: a re-delivered event (catch-up replay,
|
|
228
|
+
* lane re-drive) replays to the same row instead of duplicating it.
|
|
229
|
+
*
|
|
230
|
+
* Message-shaped events key on the message id — it is the logical identity
|
|
231
|
+
* and stays stable across delivery paths (a live WS delivery carries no
|
|
232
|
+
* dispatchEventId; its catch-up replay does — keying on the WorkItem there
|
|
233
|
+
* would split the two into different keys and duplicate the input step).
|
|
234
|
+
*
|
|
235
|
+
* Typed events key on the WorkItem id instead: their messageId can be
|
|
236
|
+
* REUSED across distinct work items — task events carry the task id for
|
|
237
|
+
* both task_assign and a later task_update, so keying on messageId would
|
|
238
|
+
* silently dedupe the second, legitimate, input step away. For the same
|
|
239
|
+
* reason a typed event WITHOUT a WorkItem id (legacy server) must NOT fall
|
|
240
|
+
* back to messageId — it gets a random UUID per logical step instead. The
|
|
241
|
+
* key rides the CreateAgentStepRequest object, so the same UUID is reused
|
|
242
|
+
* across that step's HTTP retries and queue redrives (replay-safe), while
|
|
243
|
+
* a redelivered legacy event writes a fresh row (rare duplicate beats
|
|
244
|
+
* silently losing a legitimate step). See protocol-vectors/agent-steps.json.
|
|
245
|
+
*/
|
|
246
|
+
export function inputStepIdempotencyKey(
|
|
247
|
+
event: Pick<ParallEvent, 'type' | 'dispatchEventId' | 'messageId'>,
|
|
248
|
+
): string {
|
|
249
|
+
if (event.type === 'message' || event.type === 'channel_message') {
|
|
250
|
+
return event.messageId ? `input:${event.messageId}` : randomUUID();
|
|
251
|
+
}
|
|
252
|
+
if (event.dispatchEventId) return `input:${event.dispatchEventId}`;
|
|
253
|
+
return randomUUID();
|
|
254
|
+
}
|
|
255
|
+
|
|
213
256
|
function resolveStepTarget(event: ParallEvent): { target_type: string; target_id?: string } {
|
|
214
257
|
if (event.type === 'task' || event.targetId.startsWith('tsk_')) {
|
|
215
258
|
return { target_type: 'task', target_id: event.targetId };
|
|
@@ -319,6 +362,13 @@ export class ParallAgentGateway {
|
|
|
319
362
|
|
|
320
363
|
private lastHeartbeatAt = Date.now();
|
|
321
364
|
private draining = false;
|
|
365
|
+
/**
|
|
366
|
+
* Typed WorkItem ids whose drain group left the buffer but has not settled
|
|
367
|
+
* yet. isBufferedTypedWorkItem treats them as still buffered — a re-drive
|
|
368
|
+
* claim taken inside this window would turn the drain's fence-less close
|
|
369
|
+
* stale (its dedupe cleared, its release re-driving handled work) (#1149).
|
|
370
|
+
*/
|
|
371
|
+
private drainingTypedIds = new Set<string>();
|
|
322
372
|
|
|
323
373
|
// Graceful shutdown state. When SIGTERM / abort fires, `shuttingDown` flips
|
|
324
374
|
// to true so no new dispatches start, and `inFlightDispatches` counts runs
|
|
@@ -330,6 +380,16 @@ export class ParallAgentGateway {
|
|
|
330
380
|
private pendingRestartNotification: string | null = null;
|
|
331
381
|
|
|
332
382
|
private readonly laneLedger?: LaneLedger;
|
|
383
|
+
private readonly stepPersister: StepPersister;
|
|
384
|
+
// Single entry point for session active/idle writes — serialized
|
|
385
|
+
// desired-state reconciler (see session-lifecycle.ts). The gateway only
|
|
386
|
+
// declares turn boundaries; ordering, retries and stale-finish rejection
|
|
387
|
+
// live in the coordinator.
|
|
388
|
+
private readonly sessionLifecycle: SessionLifecycleCoordinator;
|
|
389
|
+
// Normal fork teardown use case: seal → drain → close → release. The
|
|
390
|
+
// gateway only triggers it; ordering and ownership live in the finalizer
|
|
391
|
+
// (see fork-session-finalizer.ts).
|
|
392
|
+
private readonly forkFinalizer: ForkSessionFinalizer;
|
|
333
393
|
// Sticky fallback: flipped when the server predates the ledger (claim
|
|
334
394
|
// endpoint 404) so every subsequent dispatch uses the legacy flow.
|
|
335
395
|
private ledgerDisabled = false;
|
|
@@ -358,6 +418,35 @@ export class ParallAgentGateway {
|
|
|
358
418
|
this.SHUTDOWN_DEADLINE_MS = opts.shutdownDeadlineMs ?? 60_000;
|
|
359
419
|
this.FORK_DEADLINE_MS = opts.forkDeadlineMs ?? 2 * 60 * 60_000;
|
|
360
420
|
this.DISPATCH_DEADLINE_MS = opts.dispatchDeadlineMs ?? 20 * 60_000;
|
|
421
|
+
this.stepPersister = new StepPersister({
|
|
422
|
+
client: opts.client,
|
|
423
|
+
orgId: opts.config.org_id,
|
|
424
|
+
agentUserId: opts.agentUserId,
|
|
425
|
+
log: { warn: (msg) => this.opts.log?.warn(msg) },
|
|
426
|
+
isSessionStale: (err) => this.isSessionNotLiveError(err),
|
|
427
|
+
retryDelaysMs: opts.stepRetryDelaysMs,
|
|
428
|
+
});
|
|
429
|
+
this.sessionLifecycle = new SessionLifecycleCoordinator({
|
|
430
|
+
write: (sessionId, payload) =>
|
|
431
|
+
this.opts.client
|
|
432
|
+
.updateAgentSession(this.opts.config.org_id, this.opts.agentUserId, sessionId, payload)
|
|
433
|
+
.then(() => undefined),
|
|
434
|
+
// Same classification as step writes: transient (timeout/network/5xx)
|
|
435
|
+
// retries with backoff; a 4xx is permanent (never loop against it);
|
|
436
|
+
// a terminal-session 409 drops the session for good.
|
|
437
|
+
isRetryable: isRetryableStepError,
|
|
438
|
+
isSessionStale: (err) => this.isSessionNotLiveError(err),
|
|
439
|
+
log: { warn: (msg) => this.opts.log?.warn(msg) },
|
|
440
|
+
retryDelaysMs: opts.lifecycleRetryDelaysMs,
|
|
441
|
+
});
|
|
442
|
+
this.forkFinalizer = new ForkSessionFinalizer({
|
|
443
|
+
steps: this.stepPersister,
|
|
444
|
+
lifecycle: this.sessionLifecycle,
|
|
445
|
+
log: {
|
|
446
|
+
warn: (msg) => this.opts.log?.warn(msg),
|
|
447
|
+
error: (msg) => this.opts.log?.error(msg),
|
|
448
|
+
},
|
|
449
|
+
});
|
|
361
450
|
if (opts.coldStartWindowMs != null) {
|
|
362
451
|
opts.log?.warn?.(
|
|
363
452
|
'coldStartWindowMs is deprecated and ignored — cold-start time filter has been removed',
|
|
@@ -417,6 +506,14 @@ export class ParallAgentGateway {
|
|
|
417
506
|
const prevId = data.previous_session_id ?? '';
|
|
418
507
|
this.opts.log?.info(`new session signal received (previous=${prevId})`);
|
|
419
508
|
this.sessionBindings.clear();
|
|
509
|
+
// The server closed EVERY open session — not just `previous_session_id`
|
|
510
|
+
// (active forks have their own ase_ ids, and sessionBindings was just
|
|
511
|
+
// cleared, so they would otherwise keep retrying against closed rows
|
|
512
|
+
// until a 409 or the age budget). Any parked lifecycle write or step
|
|
513
|
+
// write would now 409 against a terminal row: drop them all rather than
|
|
514
|
+
// burn retry budget and log noise discovering it.
|
|
515
|
+
this.sessionLifecycle.dropAllSessions();
|
|
516
|
+
this.stepPersister.dropAllSessions();
|
|
420
517
|
if (prevId) {
|
|
421
518
|
this.pendingRestartNotification = `[Harness Notification] This is a fresh session. Your previous session (${prevId}) was ended by the user and you have been restarted.`;
|
|
422
519
|
}
|
|
@@ -447,30 +544,33 @@ export class ParallAgentGateway {
|
|
|
447
544
|
? { dispatchEventId: data.dispatch_event_id }
|
|
448
545
|
: { sourceType: 'task_activity', sourceId: data.id },
|
|
449
546
|
(dispatchEventId) => this.handleTaskAssignment(data, data.id, dispatchEventId),
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
this.
|
|
454
|
-
});
|
|
455
|
-
}
|
|
456
|
-
return this.opts.client
|
|
457
|
-
.ackDispatch(this.opts.config.org_id, {
|
|
458
|
-
source_type: 'task_activity',
|
|
459
|
-
source_id: data.id,
|
|
460
|
-
})
|
|
461
|
-
.then(
|
|
462
|
-
() => true,
|
|
463
|
-
(err) => {
|
|
464
|
-
// Same contract as ackDispatchEvent: a failed ack is a
|
|
465
|
-
// failed consume (arms backoff) and must free the hot-path
|
|
466
|
-
// dedupe so the re-drive isn't rejected by this pod forever.
|
|
547
|
+
{
|
|
548
|
+
legacyAck: (dispatchEventId) => {
|
|
549
|
+
if (dispatchEventId) {
|
|
550
|
+
return this.ackDispatchEvent(dispatchEventId, () => {
|
|
467
551
|
this.dispatchedTasks.delete(`${data.id}:${data.updated_at}`);
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
return this.opts.client
|
|
555
|
+
.ackDispatch(this.opts.config.org_id, {
|
|
556
|
+
source_type: 'task_activity',
|
|
557
|
+
source_id: data.id,
|
|
558
|
+
})
|
|
559
|
+
.then(
|
|
560
|
+
() => true,
|
|
561
|
+
(err) => {
|
|
562
|
+
// Same contract as ackDispatchEvent: a failed ack is a
|
|
563
|
+
// failed consume (arms backoff) and must free the hot-path
|
|
564
|
+
// dedupe so the re-drive isn't rejected by this pod forever.
|
|
565
|
+
this.dispatchedTasks.delete(`${data.id}:${data.updated_at}`);
|
|
566
|
+
this.opts.log?.warn(
|
|
567
|
+
`dispatch ack failed for task ${data.id}, releasing for re-drive: ${String(err)}`,
|
|
568
|
+
);
|
|
569
|
+
return false;
|
|
570
|
+
},
|
|
571
|
+
);
|
|
572
|
+
},
|
|
573
|
+
clearDedupe: () => this.dispatchedTasks.delete(`${data.id}:${data.updated_at}`),
|
|
474
574
|
},
|
|
475
575
|
);
|
|
476
576
|
} catch (err) {
|
|
@@ -479,6 +579,15 @@ export class ParallAgentGateway {
|
|
|
479
579
|
});
|
|
480
580
|
|
|
481
581
|
ws.on('dispatch.new', async (data: DispatchNewData) => {
|
|
582
|
+
// Re-drive of a WorkItem whose event copy is already buffered: skip
|
|
583
|
+
// BEFORE the typed claim (see isBufferedTypedWorkItem). The renotify
|
|
584
|
+
// pacing re-checks after the drain settles.
|
|
585
|
+
if (data.event_type !== 'message' && this.isBufferedTypedWorkItem(data.id)) {
|
|
586
|
+
this.opts.log?.info(
|
|
587
|
+
`typed dispatch ${data.id} already buffered for the drain — skipping re-claim`,
|
|
588
|
+
);
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
482
591
|
if (data.event_type === 'task_assign') {
|
|
483
592
|
if (!data.task_id) return;
|
|
484
593
|
try {
|
|
@@ -493,14 +602,19 @@ export class ParallAgentGateway {
|
|
|
493
602
|
try {
|
|
494
603
|
await this.consumeTypedDispatch(
|
|
495
604
|
{ dispatchEventId: data.id },
|
|
496
|
-
() =>
|
|
605
|
+
(dispatchEventId) =>
|
|
497
606
|
this.handleTaskComment(
|
|
498
607
|
data.source_id,
|
|
499
608
|
data.task_id ?? '',
|
|
500
609
|
data.actor_id,
|
|
501
610
|
data.delivery_reason,
|
|
611
|
+
dispatchEventId,
|
|
502
612
|
),
|
|
503
|
-
|
|
613
|
+
{
|
|
614
|
+
legacyAck: () =>
|
|
615
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
616
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
617
|
+
},
|
|
504
618
|
);
|
|
505
619
|
} catch (err) {
|
|
506
620
|
this.opts.log?.error(
|
|
@@ -512,8 +626,18 @@ export class ParallAgentGateway {
|
|
|
512
626
|
try {
|
|
513
627
|
await this.consumeTypedDispatch(
|
|
514
628
|
{ dispatchEventId: data.id },
|
|
515
|
-
() =>
|
|
516
|
-
|
|
629
|
+
(dispatchEventId) =>
|
|
630
|
+
this.handleWikiComment(
|
|
631
|
+
data.source_id,
|
|
632
|
+
data.actor_id,
|
|
633
|
+
data.delivery_reason,
|
|
634
|
+
dispatchEventId,
|
|
635
|
+
),
|
|
636
|
+
{
|
|
637
|
+
legacyAck: () =>
|
|
638
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
639
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
640
|
+
},
|
|
517
641
|
);
|
|
518
642
|
} catch (err) {
|
|
519
643
|
this.opts.log?.error(
|
|
@@ -530,7 +654,11 @@ export class ParallAgentGateway {
|
|
|
530
654
|
allowCreator: true,
|
|
531
655
|
dispatchEventId,
|
|
532
656
|
}),
|
|
533
|
-
|
|
657
|
+
{
|
|
658
|
+
legacyAck: () =>
|
|
659
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
660
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
661
|
+
},
|
|
534
662
|
);
|
|
535
663
|
} catch (err) {
|
|
536
664
|
this.opts.log?.error(`task update dispatch failed for ${data.task_id}: ${String(err)}`);
|
|
@@ -540,8 +668,13 @@ export class ParallAgentGateway {
|
|
|
540
668
|
try {
|
|
541
669
|
await this.consumeTypedDispatch(
|
|
542
670
|
{ dispatchEventId: data.id },
|
|
543
|
-
() =>
|
|
544
|
-
|
|
671
|
+
(dispatchEventId) =>
|
|
672
|
+
this.fetchAndHandleScheduleFire(data.source_id, data.actor_id, dispatchEventId),
|
|
673
|
+
{
|
|
674
|
+
legacyAck: () =>
|
|
675
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
676
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
677
|
+
},
|
|
545
678
|
);
|
|
546
679
|
} catch (err) {
|
|
547
680
|
this.opts.log?.error(
|
|
@@ -553,8 +686,13 @@ export class ParallAgentGateway {
|
|
|
553
686
|
try {
|
|
554
687
|
await this.consumeTypedDispatch(
|
|
555
688
|
{ dispatchEventId: data.id },
|
|
556
|
-
() =>
|
|
557
|
-
|
|
689
|
+
(dispatchEventId) =>
|
|
690
|
+
this.fetchAndHandleExternalTriggerRun(data.source_id, dispatchEventId),
|
|
691
|
+
{
|
|
692
|
+
legacyAck: () =>
|
|
693
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
694
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
695
|
+
},
|
|
558
696
|
);
|
|
559
697
|
} catch (err) {
|
|
560
698
|
this.opts.log?.error(
|
|
@@ -566,8 +704,12 @@ export class ParallAgentGateway {
|
|
|
566
704
|
try {
|
|
567
705
|
await this.consumeTypedDispatch(
|
|
568
706
|
{ dispatchEventId: data.id },
|
|
569
|
-
() => this.fetchAndHandleChannelMessage(data.source_id),
|
|
570
|
-
|
|
707
|
+
(dispatchEventId) => this.fetchAndHandleChannelMessage(data.source_id, dispatchEventId),
|
|
708
|
+
{
|
|
709
|
+
legacyAck: () =>
|
|
710
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
711
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
712
|
+
},
|
|
571
713
|
);
|
|
572
714
|
} catch (err) {
|
|
573
715
|
this.opts.log?.error(
|
|
@@ -579,13 +721,18 @@ export class ParallAgentGateway {
|
|
|
579
721
|
try {
|
|
580
722
|
await this.consumeTypedDispatch(
|
|
581
723
|
{ dispatchEventId: data.id },
|
|
582
|
-
() =>
|
|
724
|
+
(dispatchEventId) =>
|
|
583
725
|
this.fetchAndHandleApprovalDecided(
|
|
584
726
|
data.source_id,
|
|
585
727
|
data.actor_id,
|
|
586
728
|
data.chat_id ?? null,
|
|
729
|
+
dispatchEventId,
|
|
587
730
|
),
|
|
588
|
-
|
|
731
|
+
{
|
|
732
|
+
legacyAck: () =>
|
|
733
|
+
this.ackDispatchEvent(data.id, () => this.clearTypedDispatchDedupe(data)),
|
|
734
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(data),
|
|
735
|
+
},
|
|
589
736
|
);
|
|
590
737
|
} catch (err) {
|
|
591
738
|
this.opts.log?.error(
|
|
@@ -689,6 +836,26 @@ export class ParallAgentGateway {
|
|
|
689
836
|
});
|
|
690
837
|
}
|
|
691
838
|
|
|
839
|
+
/**
|
|
840
|
+
* Sticky: the server answered a by-id complete with 400 (predates the
|
|
841
|
+
* form). Typed resolution falls back to the legacy ack for the rest of
|
|
842
|
+
* the process lifetime.
|
|
843
|
+
*/
|
|
844
|
+
typedByIdCompleteUnsupported = false;
|
|
845
|
+
|
|
846
|
+
// Typed-face ledger helpers live in gateway-lane-flow.ts; thin delegates
|
|
847
|
+
// keep the site code and tests on the class surface.
|
|
848
|
+
private typedLedgerEventIds(events: ParallEvent[]): string[] | null {
|
|
849
|
+
return typedLedgerEventIds(this.laneFlowHost(), events);
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
private resolveDispatchByID(
|
|
853
|
+
dispatchEventId: string,
|
|
854
|
+
lane?: string,
|
|
855
|
+
): ReturnType<typeof resolveDispatchByID> {
|
|
856
|
+
return resolveDispatchByID(this.laneFlowHost(), dispatchEventId, lane);
|
|
857
|
+
}
|
|
858
|
+
|
|
692
859
|
/** True when this event's lifecycle is owned by the dispatch lane ledger. */
|
|
693
860
|
private usesLaneLedger(event: ParallEvent): boolean {
|
|
694
861
|
return this.laneLedger != null && !this.ledgerDisabled && this.laneLedger.handles(event);
|
|
@@ -716,7 +883,14 @@ export class ParallAgentGateway {
|
|
|
716
883
|
// into one turn.
|
|
717
884
|
return this.laneLedger!.laneKeyFor(event);
|
|
718
885
|
}
|
|
719
|
-
|
|
886
|
+
// Typed events never share a drain group with messages. On the
|
|
887
|
+
// ledger-disabled fallback both group by target id, and a typed event
|
|
888
|
+
// whose target is a chat (approval) could batch behind that chat's
|
|
889
|
+
// messages — the drain body carries only the trailing message while the
|
|
890
|
+
// legacy ack sweeps the whole group, silently dropping the typed body
|
|
891
|
+
// (typed events are never steer-injected, so the drain body is their
|
|
892
|
+
// only route to the model). Split the groups instead (#1149).
|
|
893
|
+
return event.type === 'message' ? event.targetId : `typed:${event.targetId}`;
|
|
720
894
|
}
|
|
721
895
|
|
|
722
896
|
// Lane-flow protocols live in gateway-lane-flow.ts; these thin delegates
|
|
@@ -739,17 +913,18 @@ export class ParallAgentGateway {
|
|
|
739
913
|
private consumeTypedDispatch(
|
|
740
914
|
ref: { dispatchEventId?: string; sourceType?: string; sourceId?: string },
|
|
741
915
|
run: (dispatchEventId?: string) => Promise<boolean>,
|
|
742
|
-
|
|
916
|
+
hooks: TypedConsumeHooks,
|
|
743
917
|
): Promise<void> {
|
|
744
|
-
return consumeTypedDispatch(this.laneFlowHost(), ref, run,
|
|
918
|
+
return consumeTypedDispatch(this.laneFlowHost(), ref, run, hooks);
|
|
745
919
|
}
|
|
746
920
|
|
|
747
|
-
//
|
|
748
|
-
//
|
|
749
|
-
//
|
|
750
|
-
//
|
|
751
|
-
//
|
|
752
|
-
//
|
|
921
|
+
// Legacy administrative ack (ledger-disabled fallback only). Typed
|
|
922
|
+
// completion must wait until the ack has either committed or failed.
|
|
923
|
+
// Errors stay best-effort: a failed ack leaves the row received, so
|
|
924
|
+
// Complete releases and re-drives it safely. The boolean outcome feeds the
|
|
925
|
+
// typed-consume backoff — an ack that failed must count as a failed
|
|
926
|
+
// consume, or an ack outage would clear the backoff entry and let the
|
|
927
|
+
// release re-drive spin at wire speed.
|
|
753
928
|
private ackDispatchEvent(dispatchEventId: string, onFailure?: () => void): Promise<boolean> {
|
|
754
929
|
return this.opts.client.ackDispatchByID(this.opts.config.org_id, dispatchEventId).then(
|
|
755
930
|
() => true,
|
|
@@ -766,6 +941,31 @@ export class ParallAgentGateway {
|
|
|
766
941
|
);
|
|
767
942
|
}
|
|
768
943
|
|
|
944
|
+
/**
|
|
945
|
+
* True while a typed WorkItem's event copy sits in the main buffer waiting
|
|
946
|
+
* for the drain. A buffered copy keeps its hot-path dedupe claim: the drain
|
|
947
|
+
* owns its settlement, and the released row's re-drive must short-circuit
|
|
948
|
+
* BEFORE claiming (a duplicate claim opens a lane that races the drain's
|
|
949
|
+
* fence-less by-id close — 409 STALE, and the claimant's release would
|
|
950
|
+
* re-drive already-handled work) (#1149). The claim's lifetime equals the
|
|
951
|
+
* buffer stay: settlement clears it on every drain outcome — success,
|
|
952
|
+
* failure, or a thrown turn.
|
|
953
|
+
*/
|
|
954
|
+
private isBufferedTypedWorkItem(dispatchEventId: string | undefined | null): boolean {
|
|
955
|
+
if (!dispatchEventId) return false;
|
|
956
|
+
return (
|
|
957
|
+
this.drainingTypedIds.has(dispatchEventId) ||
|
|
958
|
+
this.dispatchState.mainBuffer.some((e) => e.dispatchEventId === dispatchEventId)
|
|
959
|
+
);
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
private isTypedEventBuffered(event: ParallEvent): boolean {
|
|
963
|
+
return this.isBufferedTypedWorkItem(event.dispatchEventId);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// PARITY: this switch and gateway-lane-flow's clearTypedDedupeForEvent must
|
|
967
|
+
// handle the same typed source families — extend BOTH when adding a typed
|
|
968
|
+
// event type (same dedupe entries, keyed from different event shapes).
|
|
769
969
|
private clearTypedDispatchDedupe(item: DispatchNewData): void {
|
|
770
970
|
switch (item.event_type) {
|
|
771
971
|
case 'task_assign':
|
|
@@ -828,69 +1028,60 @@ export class ParallAgentGateway {
|
|
|
828
1028
|
|
|
829
1029
|
private async createInputStep(sessionId: string, event: ParallEvent) {
|
|
830
1030
|
const target = resolveStepTarget(event);
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
event.type === '
|
|
843
|
-
? '
|
|
844
|
-
: event.type === '
|
|
845
|
-
? '
|
|
846
|
-
: event.type === '
|
|
847
|
-
? '
|
|
848
|
-
: event.type === '
|
|
849
|
-
? '
|
|
850
|
-
: event.type === '
|
|
851
|
-
? '
|
|
852
|
-
:
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
event.type === '
|
|
859
|
-
? {
|
|
860
|
-
: event.type === '
|
|
861
|
-
? {
|
|
862
|
-
: event.type === '
|
|
863
|
-
? {
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
summary: event.body.substring(0, 200),
|
|
886
|
-
...(event.sentAt ? { sent_at: event.sentAt } : {}),
|
|
887
|
-
},
|
|
888
|
-
},
|
|
889
|
-
);
|
|
890
|
-
} catch (err) {
|
|
891
|
-
if (this.isSessionNotLiveError(err)) throw err;
|
|
892
|
-
this.opts.log?.warn(`failed to create input step: ${String(err)}`);
|
|
893
|
-
}
|
|
1031
|
+
await this.stepPersister.persist(sessionId, 'input', {
|
|
1032
|
+
step_type: 'input',
|
|
1033
|
+
target_type: target.target_type,
|
|
1034
|
+
target_id: target.target_id,
|
|
1035
|
+
idempotency_key: inputStepIdempotencyKey(event),
|
|
1036
|
+
content: {
|
|
1037
|
+
trigger_type:
|
|
1038
|
+
event.type === 'task'
|
|
1039
|
+
? 'task_assign'
|
|
1040
|
+
: event.type === 'task_comment'
|
|
1041
|
+
? 'task_comment'
|
|
1042
|
+
: event.type === 'wiki_comment'
|
|
1043
|
+
? 'wiki_comment'
|
|
1044
|
+
: event.type === 'schedule'
|
|
1045
|
+
? 'schedule_fire'
|
|
1046
|
+
: event.type === 'external_trigger'
|
|
1047
|
+
? 'external_trigger'
|
|
1048
|
+
: event.type === 'channel_message'
|
|
1049
|
+
? 'channel_message'
|
|
1050
|
+
: event.type === 'approval'
|
|
1051
|
+
? 'approval_decided'
|
|
1052
|
+
: 'mention',
|
|
1053
|
+
trigger_ref:
|
|
1054
|
+
event.type === 'task'
|
|
1055
|
+
? { task_id: event.targetId }
|
|
1056
|
+
: event.type === 'task_comment'
|
|
1057
|
+
? { comment_id: event.messageId, task_id: event.targetId }
|
|
1058
|
+
: event.type === 'wiki_comment'
|
|
1059
|
+
? { comment_id: event.messageId, target_uri: event.replyTargetUri }
|
|
1060
|
+
: event.type === 'schedule'
|
|
1061
|
+
? { schedule_id: event.targetId, run_id: event.messageId }
|
|
1062
|
+
: event.type === 'external_trigger'
|
|
1063
|
+
? {
|
|
1064
|
+
trigger_id: event.targetId,
|
|
1065
|
+
run_id: event.messageId,
|
|
1066
|
+
connection_id: event.externalConnectionId,
|
|
1067
|
+
ingress_event_id: event.externalIngressEventId,
|
|
1068
|
+
}
|
|
1069
|
+
: event.type === 'channel_message'
|
|
1070
|
+
? {
|
|
1071
|
+
conversation_id: event.targetId,
|
|
1072
|
+
channel_message_id: event.messageId,
|
|
1073
|
+
provider: event.channelProvider,
|
|
1074
|
+
external_conversation_id: event.channelExternalConversationId,
|
|
1075
|
+
}
|
|
1076
|
+
: event.type === 'approval'
|
|
1077
|
+
? { approval_id: event.messageId }
|
|
1078
|
+
: { message_id: event.messageId },
|
|
1079
|
+
sender_id: event.senderId,
|
|
1080
|
+
sender_name: event.senderName,
|
|
1081
|
+
summary: event.body.substring(0, 200),
|
|
1082
|
+
...(event.sentAt ? { sent_at: event.sentAt } : {}),
|
|
1083
|
+
},
|
|
1084
|
+
});
|
|
894
1085
|
}
|
|
895
1086
|
|
|
896
1087
|
private async createRuntimeStep(
|
|
@@ -902,62 +1093,57 @@ export class ParallAgentGateway {
|
|
|
902
1093
|
laneContextFilePath?: string,
|
|
903
1094
|
) {
|
|
904
1095
|
const target = resolveStepTarget(event);
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
content: { text: runtimeEvent.text },
|
|
917
|
-
group_key: runtimeEvent.groupKey,
|
|
918
|
-
},
|
|
919
|
-
);
|
|
920
|
-
break;
|
|
1096
|
+
switch (runtimeEvent.type) {
|
|
1097
|
+
case 'thinking':
|
|
1098
|
+
await this.stepPersister.persist(sessionId, 'thinking', {
|
|
1099
|
+
step_type: 'thinking',
|
|
1100
|
+
target_type: target.target_type,
|
|
1101
|
+
target_id: target.target_id,
|
|
1102
|
+
idempotency_key: randomUUID(),
|
|
1103
|
+
content: { text: runtimeEvent.text },
|
|
1104
|
+
group_key: runtimeEvent.groupKey,
|
|
1105
|
+
});
|
|
1106
|
+
break;
|
|
921
1107
|
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
group_key: runtimeEvent.groupKey,
|
|
937
|
-
},
|
|
938
|
-
);
|
|
939
|
-
break;
|
|
1108
|
+
case 'text':
|
|
1109
|
+
await this.stepPersister.persist(sessionId, 'text', {
|
|
1110
|
+
step_type: 'text',
|
|
1111
|
+
target_type: target.target_type,
|
|
1112
|
+
target_id: target.target_id,
|
|
1113
|
+
idempotency_key: randomUUID(),
|
|
1114
|
+
content: {
|
|
1115
|
+
text: runtimeEvent.text,
|
|
1116
|
+
suppressed: runtimeEvent.project !== true,
|
|
1117
|
+
},
|
|
1118
|
+
projection: runtimeEvent.project === true,
|
|
1119
|
+
group_key: runtimeEvent.groupKey,
|
|
1120
|
+
});
|
|
1121
|
+
break;
|
|
940
1122
|
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
1123
|
+
case 'tool_call': {
|
|
1124
|
+
const step = await this.stepPersister.persist(sessionId, 'tool_call', {
|
|
1125
|
+
step_type: 'tool_call',
|
|
1126
|
+
target_type: target.target_type,
|
|
1127
|
+
target_id: target.target_id,
|
|
1128
|
+
// call_id is session-unique for bridge runtimes (server-enforced),
|
|
1129
|
+
// so the bare form anchors the tool step pair across retries —
|
|
1130
|
+
// unlike parel's turn-scoped `tc:{turnId}:{callId}` (see
|
|
1131
|
+
// protocol-vectors/agent-steps.json).
|
|
1132
|
+
idempotency_key: `tc:${runtimeEvent.callId}`,
|
|
1133
|
+
content: {
|
|
1134
|
+
call_id: runtimeEvent.callId,
|
|
1135
|
+
tool_name: runtimeEvent.toolName,
|
|
1136
|
+
tool_input: runtimeEvent.input,
|
|
1137
|
+
status: 'running',
|
|
1138
|
+
started_at: runtimeEvent.startedAt ?? new Date().toISOString(),
|
|
1139
|
+
},
|
|
1140
|
+
group_key: runtimeEvent.groupKey,
|
|
1141
|
+
runtime_key: runtimeEvent.callId,
|
|
1142
|
+
});
|
|
1143
|
+
// step is null when the write was queued for background retry — the
|
|
1144
|
+
// CLI step-id linkage window has then passed, same as a failed write
|
|
1145
|
+
// before the queue existed.
|
|
1146
|
+
if (step) {
|
|
961
1147
|
if (contextFilePath) {
|
|
962
1148
|
this.updateContextFileStepId(contextFilePath, step.id);
|
|
963
1149
|
} else if (stepIdFilePath) {
|
|
@@ -966,57 +1152,48 @@ export class ParallAgentGateway {
|
|
|
966
1152
|
if (laneContextFilePath) {
|
|
967
1153
|
this.updateContextFileStepId(laneContextFilePath, step.id);
|
|
968
1154
|
}
|
|
969
|
-
break;
|
|
970
1155
|
}
|
|
1156
|
+
break;
|
|
1157
|
+
}
|
|
971
1158
|
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
);
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
}
|
|
1000
|
-
break;
|
|
1159
|
+
case 'tool_result':
|
|
1160
|
+
await this.stepPersister.persist(sessionId, 'tool_result', {
|
|
1161
|
+
step_type: 'tool_result',
|
|
1162
|
+
target_type: target.target_type,
|
|
1163
|
+
target_id: target.target_id,
|
|
1164
|
+
idempotency_key: `tr:${runtimeEvent.callId}`,
|
|
1165
|
+
content: {
|
|
1166
|
+
call_id: runtimeEvent.callId,
|
|
1167
|
+
tool_name: runtimeEvent.toolName,
|
|
1168
|
+
status: runtimeEvent.error ? 'error' : 'success',
|
|
1169
|
+
output: runtimeEvent.output,
|
|
1170
|
+
duration_ms: runtimeEvent.durationMs ?? 0,
|
|
1171
|
+
collapsible: true,
|
|
1172
|
+
},
|
|
1173
|
+
group_key: runtimeEvent.groupKey,
|
|
1174
|
+
});
|
|
1175
|
+
// The tool has finished regardless of whether the step write landed
|
|
1176
|
+
// inline or was queued — always clear the step-id linkage.
|
|
1177
|
+
if (contextFilePath) {
|
|
1178
|
+
this.updateContextFileStepId(contextFilePath, null);
|
|
1179
|
+
} else if (stepIdFilePath) {
|
|
1180
|
+
this.clearStepIdFile(stepIdFilePath);
|
|
1181
|
+
}
|
|
1182
|
+
if (laneContextFilePath) {
|
|
1183
|
+
this.updateContextFileStepId(laneContextFilePath, null);
|
|
1184
|
+
}
|
|
1185
|
+
break;
|
|
1001
1186
|
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
projection: false,
|
|
1013
|
-
},
|
|
1014
|
-
);
|
|
1015
|
-
break;
|
|
1016
|
-
}
|
|
1017
|
-
} catch (err) {
|
|
1018
|
-
if (this.isSessionNotLiveError(err)) throw err;
|
|
1019
|
-
this.opts.log?.warn(`failed to create ${runtimeEvent.type} step: ${String(err)}`);
|
|
1187
|
+
case 'error':
|
|
1188
|
+
await this.stepPersister.persist(sessionId, 'error', {
|
|
1189
|
+
step_type: 'text',
|
|
1190
|
+
target_type: target.target_type,
|
|
1191
|
+
target_id: target.target_id,
|
|
1192
|
+
idempotency_key: randomUUID(),
|
|
1193
|
+
content: buildErrorStepContent(runtimeEvent.message),
|
|
1194
|
+
projection: false,
|
|
1195
|
+
});
|
|
1196
|
+
break;
|
|
1020
1197
|
}
|
|
1021
1198
|
}
|
|
1022
1199
|
|
|
@@ -1239,9 +1416,19 @@ export class ParallAgentGateway {
|
|
|
1239
1416
|
|
|
1240
1417
|
let binding = this.sessionBindings.get(sessionKey);
|
|
1241
1418
|
let inputStepsCreated = false;
|
|
1242
|
-
let
|
|
1419
|
+
let turnHandle: TurnHandle | undefined;
|
|
1243
1420
|
let dispatchError: unknown;
|
|
1244
1421
|
const pendingSendCallIds = new Set<string>();
|
|
1422
|
+
// Turn boundary: must complete one bounded active reconciliation
|
|
1423
|
+
// BEFORE this turn's first AgentStep persists — a reused idle session
|
|
1424
|
+
// otherwise races the status write and the server presence guard
|
|
1425
|
+
// swallows the turn's first activity. Resolves even when the write
|
|
1426
|
+
// fails (warned; the coordinator keeps reconciling in the background)
|
|
1427
|
+
// so a degraded link never blocks the step flow indefinitely.
|
|
1428
|
+
const ensureTurnBegun = async (): Promise<void> => {
|
|
1429
|
+
if (turnHandle || !binding) return;
|
|
1430
|
+
turnHandle = await this.sessionLifecycle.beginTurn(binding.agentSessionId, event.messageId);
|
|
1431
|
+
};
|
|
1245
1432
|
try {
|
|
1246
1433
|
dispatchSpan = startDispatchSpan(event, this.opts.runtimeType, sessionKey);
|
|
1247
1434
|
for await (const runtimeEvent of this.opts.dispatchAdapter.dispatch({
|
|
@@ -1283,6 +1470,7 @@ export class ParallAgentGateway {
|
|
|
1283
1470
|
// while a dispatch was in flight) inside the in-flight window so a
|
|
1284
1471
|
// shutdown short-circuit BEFORE this point cannot leave orphan input
|
|
1285
1472
|
// steps that the replacement pod would duplicate on replay.
|
|
1473
|
+
await ensureTurnBegun();
|
|
1286
1474
|
if (earlierEvents.length > 0) {
|
|
1287
1475
|
await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
|
|
1288
1476
|
}
|
|
@@ -1296,17 +1484,7 @@ export class ParallAgentGateway {
|
|
|
1296
1484
|
const detail = runtimeEvent.type === 'error' ? `: ${runtimeEvent.message}` : '';
|
|
1297
1485
|
throw new Error(`runtime emitted ${runtimeEvent.type} before runtime_session${detail}`);
|
|
1298
1486
|
}
|
|
1299
|
-
|
|
1300
|
-
triggerMessageSet = true;
|
|
1301
|
-
this.opts.client
|
|
1302
|
-
.updateAgentSession(
|
|
1303
|
-
this.opts.config.org_id,
|
|
1304
|
-
this.opts.agentUserId,
|
|
1305
|
-
binding.agentSessionId,
|
|
1306
|
-
{ status: 'active', trigger_message_id: event.messageId },
|
|
1307
|
-
)
|
|
1308
|
-
.catch((err) => this.opts.log?.warn?.(`failed to set session active: ${err}`));
|
|
1309
|
-
}
|
|
1487
|
+
await ensureTurnBegun();
|
|
1310
1488
|
if (!inputStepsCreated) {
|
|
1311
1489
|
if (earlierEvents.length > 0) {
|
|
1312
1490
|
await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
|
|
@@ -1358,6 +1536,7 @@ export class ParallAgentGateway {
|
|
|
1358
1536
|
throw new Error('runtime completed without runtime_session');
|
|
1359
1537
|
}
|
|
1360
1538
|
if (!inputStepsCreated) {
|
|
1539
|
+
await ensureTurnBegun();
|
|
1361
1540
|
if (earlierEvents.length > 0) {
|
|
1362
1541
|
await this.createInputStepsForEarlierEvents(binding.agentSessionId, earlierEvents);
|
|
1363
1542
|
}
|
|
@@ -1368,6 +1547,7 @@ export class ParallAgentGateway {
|
|
|
1368
1547
|
let staleDetected = this.isSessionNotLiveError(err);
|
|
1369
1548
|
if (!staleDetected && binding) {
|
|
1370
1549
|
try {
|
|
1550
|
+
await ensureTurnBegun();
|
|
1371
1551
|
await this.createRuntimeStep(
|
|
1372
1552
|
binding.agentSessionId,
|
|
1373
1553
|
event,
|
|
@@ -1388,6 +1568,8 @@ export class ParallAgentGateway {
|
|
|
1388
1568
|
`session ${binding.agentSessionId} is stale (mid-dispatch), triggering recovery for ${sessionKey}`,
|
|
1389
1569
|
);
|
|
1390
1570
|
this.sessionBindings.delete(sessionKey);
|
|
1571
|
+
this.stepPersister.dropSession(binding.agentSessionId);
|
|
1572
|
+
this.sessionLifecycle.dropSession(binding.agentSessionId);
|
|
1391
1573
|
if (sessionKey === this.opts.runtimeKey) {
|
|
1392
1574
|
this.activeSessionId = undefined;
|
|
1393
1575
|
}
|
|
@@ -1424,15 +1606,10 @@ export class ParallAgentGateway {
|
|
|
1424
1606
|
}
|
|
1425
1607
|
|
|
1426
1608
|
clearDispatchMetrics(sessionKey);
|
|
1427
|
-
if (
|
|
1428
|
-
this
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
this.opts.agentUserId,
|
|
1432
|
-
binding.agentSessionId,
|
|
1433
|
-
{ status: 'idle' },
|
|
1434
|
-
)
|
|
1435
|
-
.catch((err) => this.opts.log?.warn?.(`failed to set session idle: ${err}`));
|
|
1609
|
+
if (turnHandle) {
|
|
1610
|
+
// Stale-handle safe: if a newer turn already began on this
|
|
1611
|
+
// session, the coordinator ignores this finish outright.
|
|
1612
|
+
this.sessionLifecycle.finishTurn(turnHandle);
|
|
1436
1613
|
}
|
|
1437
1614
|
clearSessionMessageId(sessionKey);
|
|
1438
1615
|
clearDispatchMessageId(sessionKey);
|
|
@@ -1570,6 +1747,22 @@ export class ParallAgentGateway {
|
|
|
1570
1747
|
earlier,
|
|
1571
1748
|
batchText,
|
|
1572
1749
|
);
|
|
1750
|
+
if (
|
|
1751
|
+
dispatched &&
|
|
1752
|
+
this.typedLedgerEventIds(events) &&
|
|
1753
|
+
this.consumeTurnError(fork.fork.sessionKey)
|
|
1754
|
+
) {
|
|
1755
|
+
// Typed error turn in a fork: report failure to the awaiting
|
|
1756
|
+
// typed consumes so the members release for a budgeted retry,
|
|
1757
|
+
// and do NOT record the events as handled — the redrive must
|
|
1758
|
+
// not arrive wearing an "already handled" prefix. Keep draining
|
|
1759
|
+
// (an error turn is not a shutdown).
|
|
1760
|
+
this.opts.log?.info(
|
|
1761
|
+
`typed fork turn for ${last.messageId} surfaced a runtime error — releasing for retry`,
|
|
1762
|
+
);
|
|
1763
|
+
for (const item of items) item.resolve(false);
|
|
1764
|
+
continue;
|
|
1765
|
+
}
|
|
1573
1766
|
}
|
|
1574
1767
|
if (!dispatched) {
|
|
1575
1768
|
// Shutdown short-circuit — resolve un-acked so the server requeues
|
|
@@ -1665,15 +1858,18 @@ export class ParallAgentGateway {
|
|
|
1665
1858
|
}
|
|
1666
1859
|
}
|
|
1667
1860
|
if (forkBinding) {
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1861
|
+
// Normal fork teardown is the ForkSessionFinalizer use case:
|
|
1862
|
+
// seal → drain parked/in-flight step writes → close (serialized
|
|
1863
|
+
// behind the turn's idle, ownership held through close retries) →
|
|
1864
|
+
// release the binding. The gateway only triggers it. Notably this
|
|
1865
|
+
// must NOT drop the session's step queue — parked steps drain to
|
|
1866
|
+
// the server before the close is issued (the identity check on the
|
|
1867
|
+
// release keeps a replacement fork's binding intact).
|
|
1868
|
+
await this.forkFinalizer.finalize(forkBinding.agentSessionId, () => {
|
|
1869
|
+
if (this.sessionBindings.get(fork.fork.sessionKey) === forkBinding) {
|
|
1870
|
+
this.sessionBindings.delete(fork.fork.sessionKey);
|
|
1871
|
+
}
|
|
1872
|
+
});
|
|
1677
1873
|
}
|
|
1678
1874
|
}
|
|
1679
1875
|
}
|
|
@@ -1755,41 +1951,115 @@ export class ParallAgentGateway {
|
|
|
1755
1951
|
// no_action sweep); no legacy acks.
|
|
1756
1952
|
continue;
|
|
1757
1953
|
}
|
|
1954
|
+
// Buffered typed events are the one wrapper-less dispatch path: their
|
|
1955
|
+
// consumeTypedDispatch guard returned long ago (buffer-main resolves
|
|
1956
|
+
// false) and released the claims, so THIS site owns their resolution.
|
|
1957
|
+
// Message groups here are the ledger-disabled legacy flow.
|
|
1958
|
+
const typedRefs = this.typedLedgerEventIds(events);
|
|
1959
|
+
// Guard the settlement window: from the moment the group leaves the
|
|
1960
|
+
// buffer until settlement finishes, a re-drive must still be
|
|
1961
|
+
// absorbed by isBufferedTypedWorkItem — see drainingTypedIds. Keyed
|
|
1962
|
+
// on the events' own WorkItem ids, NOT typedRefs: the legacy
|
|
1963
|
+
// (ledger-disabled) fallback settles through per-event acks but its
|
|
1964
|
+
// re-drives converge through the same buffered-WorkItem check.
|
|
1965
|
+
const drainingIds = events
|
|
1966
|
+
.map((ev) => ev.dispatchEventId)
|
|
1967
|
+
.filter((id): id is string => !!id);
|
|
1968
|
+
for (const id of drainingIds) this.drainingTypedIds.add(id);
|
|
1758
1969
|
try {
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
//
|
|
1776
|
-
//
|
|
1777
|
-
//
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
const
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1970
|
+
if (!typedRefs) {
|
|
1971
|
+
try {
|
|
1972
|
+
await this.emitDispatchReceived(event);
|
|
1973
|
+
} catch (err) {
|
|
1974
|
+
this.opts.log?.warn?.(
|
|
1975
|
+
`mark-received failed for buffered dispatch, leaving unacked for retry: ${String(err)}`,
|
|
1976
|
+
);
|
|
1977
|
+
this.dispatchState.mainBuffer.unshift(...events);
|
|
1978
|
+
this.dispatchState.pendingForkResults.unshift(...pendingFork);
|
|
1979
|
+
break;
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
// A typed group dispatches ALL buffered bodies, not just the
|
|
1983
|
+
// newest: typed events are never steer-injected (unlike messages,
|
|
1984
|
+
// which the model already saw mid-turn), so a comment burst folded
|
|
1985
|
+
// into one drain turn would otherwise surface only its last member
|
|
1986
|
+
// to the LLM — earlier ones exist solely as input steps the model
|
|
1987
|
+
// never reads. Keyed on the event kind, NOT on typedRefs: the
|
|
1988
|
+
// ledger-disabled fallback buffers the same bursts and owes the
|
|
1989
|
+
// model the same visibility (groups are homogeneous — see
|
|
1990
|
+
// dispatchGroupKey). Adapters that present earlierEvents natively
|
|
1991
|
+
// (OpenClaw InboundHistory) are exempt — concatenating would show
|
|
1992
|
+
// every earlier member twice.
|
|
1993
|
+
const isTypedGroup = events.every((ev) => ev.type !== 'message');
|
|
1994
|
+
const body =
|
|
1995
|
+
isTypedGroup &&
|
|
1996
|
+
events.length > 1 &&
|
|
1997
|
+
this.opts.dispatchAdapter.earlierEventsInPrompt !== true
|
|
1998
|
+
? events.map((ev) => buildEventBody(ev)).join('\n\n')
|
|
1999
|
+
: buildEventBody(event);
|
|
2000
|
+
let dispatched: boolean;
|
|
2001
|
+
try {
|
|
2002
|
+
dispatched = await this.runDispatch(
|
|
2003
|
+
event,
|
|
2004
|
+
this.opts.runtimeKey,
|
|
2005
|
+
forkPrefix + body,
|
|
2006
|
+
earlier,
|
|
2007
|
+
);
|
|
2008
|
+
} catch (err) {
|
|
2009
|
+
if (!isTypedGroup) throw err;
|
|
2010
|
+
// The retained dedupe claims live exactly as long as the buffer
|
|
2011
|
+
// stay — a thrown turn dropped these events without settlement,
|
|
2012
|
+
// so free the claims here or every re-drive is rejected at the
|
|
2013
|
+
// dedupe gate until restart (#1149).
|
|
2014
|
+
this.opts.log?.error(
|
|
2015
|
+
`typed drain dispatch failed for ${event.messageId} (group of ${events.length}, claims freed for re-drive): ${String(err)}`,
|
|
2016
|
+
);
|
|
2017
|
+
for (const ev of events) clearTypedDedupeForEvent(this.laneFlowHost(), ev);
|
|
2018
|
+
this.dispatchState.pendingForkResults.unshift(...pendingFork);
|
|
2019
|
+
continue;
|
|
2020
|
+
}
|
|
2021
|
+
if (!dispatched) {
|
|
2022
|
+
// Shutdown: skip the ack so the server redelivers these buffered
|
|
2023
|
+
// events to the replacement pod via dispatch catch-up. Put both
|
|
2024
|
+
// the buffered events and the fork results back so nothing is
|
|
2025
|
+
// lost.
|
|
2026
|
+
this.dispatchState.mainBuffer.unshift(...events);
|
|
2027
|
+
this.dispatchState.pendingForkResults.unshift(...pendingFork);
|
|
2028
|
+
break;
|
|
2029
|
+
}
|
|
2030
|
+
if (typedRefs) {
|
|
2031
|
+
// Wrapper-less resolution of the buffered typed group — protocol
|
|
2032
|
+
// lives in gateway-lane-flow.ts. The turn-error marker is
|
|
2033
|
+
// consumed HERE, before this loop can start another turn on the
|
|
2034
|
+
// session.
|
|
2035
|
+
await settleDrainedTypedGroup(
|
|
2036
|
+
this.laneFlowHost(),
|
|
2037
|
+
events,
|
|
2038
|
+
typedRefs,
|
|
2039
|
+
this.consumeTurnError(this.opts.runtimeKey),
|
|
2040
|
+
);
|
|
2041
|
+
continue;
|
|
2042
|
+
}
|
|
2043
|
+
for (const bufferedEvent of events) {
|
|
2044
|
+
const sourceType =
|
|
2045
|
+
bufferedEvent.ackSourceType ??
|
|
2046
|
+
(bufferedEvent.type === 'task' ? 'task_activity' : 'message');
|
|
2047
|
+
const sourceId = bufferedEvent.ackSourceId ?? bufferedEvent.messageId;
|
|
2048
|
+
this.opts.client
|
|
2049
|
+
.ackDispatch(this.opts.config.org_id, {
|
|
2050
|
+
source_type: sourceType,
|
|
2051
|
+
source_id: sourceId,
|
|
2052
|
+
})
|
|
2053
|
+
.catch(() => {});
|
|
2054
|
+
// Retained-claim lifetime is the buffer stay on the legacy face
|
|
2055
|
+
// too: the fire-and-forget ack may fail (the row re-drives and
|
|
2056
|
+
// must not be self-rejected), and a shared-key task sibling must
|
|
2057
|
+
// not be blocked by this drained copy's claim. No-op for message
|
|
2058
|
+
// events (no typed dedupe entry).
|
|
2059
|
+
clearTypedDedupeForEvent(this.laneFlowHost(), bufferedEvent);
|
|
2060
|
+
}
|
|
2061
|
+
} finally {
|
|
2062
|
+
for (const id of drainingIds) this.drainingTypedIds.delete(id);
|
|
1793
2063
|
}
|
|
1794
2064
|
}
|
|
1795
2065
|
} finally {
|
|
@@ -1866,16 +2136,24 @@ export class ParallAgentGateway {
|
|
|
1866
2136
|
}
|
|
1867
2137
|
return outcome === 'dispatched';
|
|
1868
2138
|
}
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
2139
|
+
// Ledger-claimed typed events skip the legacy received write — the
|
|
2140
|
+
// claim already marked the row received, and a second mark here would
|
|
2141
|
+
// wipe its lease owner (the 0713 black-hole shape on the typed face).
|
|
2142
|
+
const typedRefs = this.typedLedgerEventIds([event]);
|
|
2143
|
+
if (!typedRefs) {
|
|
2144
|
+
try {
|
|
2145
|
+
await this.emitDispatchReceived(event);
|
|
2146
|
+
} catch (err) {
|
|
2147
|
+
this.opts.log?.warn?.(
|
|
2148
|
+
`mark-received failed, leaving unacked for retry: ${String(err)}`,
|
|
2149
|
+
);
|
|
2150
|
+
this.dispatchState.mainDispatching = false;
|
|
2151
|
+
this.dispatchState.mainCurrentTargetId = undefined;
|
|
2152
|
+
this.mainCurrentGroupKey = undefined;
|
|
2153
|
+
this.dispatchState.mainPreDispatchBranchPoint = undefined;
|
|
2154
|
+
this.dispatchState.pendingForkResults.unshift(...pendingFork);
|
|
2155
|
+
return false;
|
|
2156
|
+
}
|
|
1879
2157
|
}
|
|
1880
2158
|
let dispatched = false;
|
|
1881
2159
|
try {
|
|
@@ -1884,6 +2162,17 @@ export class ParallAgentGateway {
|
|
|
1884
2162
|
this.opts.runtimeKey,
|
|
1885
2163
|
forkPrefix + buildEventBody(event),
|
|
1886
2164
|
);
|
|
2165
|
+
if (dispatched && typedRefs && this.consumeTurnError(this.opts.runtimeKey)) {
|
|
2166
|
+
// Typed error turn: report failure to the enclosing typed consume
|
|
2167
|
+
// so the member releases for a budgeted retry instead of being
|
|
2168
|
+
// terminally resolved (tech-debt: typed-dispatch-error-outcome).
|
|
2169
|
+
// Consumed HERE, before the finally's drain can start another
|
|
2170
|
+
// turn on this session and clear the marker.
|
|
2171
|
+
this.opts.log?.info(
|
|
2172
|
+
`typed dispatch turn for ${event.messageId} surfaced a runtime error — releasing for retry`,
|
|
2173
|
+
);
|
|
2174
|
+
dispatched = false;
|
|
2175
|
+
}
|
|
1887
2176
|
if (!dispatched) {
|
|
1888
2177
|
// Shutdown short-circuit — restore the fork results so a future
|
|
1889
2178
|
// pod can replay them, and return false so handleMessage skips ack.
|
|
@@ -1899,10 +2188,27 @@ export class ParallAgentGateway {
|
|
|
1899
2188
|
if (this.shuttingDown) {
|
|
1900
2189
|
return false;
|
|
1901
2190
|
}
|
|
2191
|
+
// A re-driven typed WorkItem may already be buffered from a prior
|
|
2192
|
+
// consume attempt (claim → busy main → buffer → release → server
|
|
2193
|
+
// re-drive): the buffered copy is the one the drain settles, so a
|
|
2194
|
+
// second copy would double the drain group's input steps and prompt
|
|
2195
|
+
// content. Drop the duplicate; the caller releases the row again and
|
|
2196
|
+
// the re-drive keeps converging on the buffered copy (#1149).
|
|
2197
|
+
if (this.isBufferedTypedWorkItem(event.dispatchEventId)) {
|
|
2198
|
+
return false;
|
|
2199
|
+
}
|
|
1902
2200
|
// Push synchronously BEFORE the (possibly async) steer attempt so
|
|
1903
2201
|
// arrival order is preserved and the event cannot be orphaned in a
|
|
1904
2202
|
// gap between the steer await and the push.
|
|
1905
2203
|
this.dispatchState.mainBuffer.push(event);
|
|
2204
|
+
// FIFO fence for BOTH injection branches below: a buffered typed
|
|
2205
|
+
// event is never injected, but the adapters track pending injections
|
|
2206
|
+
// as a COUNT, not by identity — if a message injected behind a
|
|
2207
|
+
// buffered typed event, the drain (typed group first, FIFO) would
|
|
2208
|
+
// consume the message's steer output as the typed group's turn: the
|
|
2209
|
+
// typed body never reaches the model yet resolves, and the message
|
|
2210
|
+
// replays. When anything un-injected sits ahead, buffer only.
|
|
2211
|
+
const typedAheadInBuffer = this.dispatchState.mainBuffer.some((e) => e.type !== 'message');
|
|
1906
2212
|
if (this.usesLaneLedger(event)) {
|
|
1907
2213
|
// Ledger flow: fold into the live lane server-side FIRST, then
|
|
1908
2214
|
// inject. An un-folded injection is forbidden (the pending WorkItem
|
|
@@ -1917,6 +2223,7 @@ export class ParallAgentGateway {
|
|
|
1917
2223
|
// the event un-folded keeps it buffered; the drain claims it as
|
|
1918
2224
|
// its own turn and folds it there.
|
|
1919
2225
|
if (
|
|
2226
|
+
!typedAheadInBuffer &&
|
|
1920
2227
|
this.mainCurrentGroupKey === this.dispatchGroupKey(event) &&
|
|
1921
2228
|
this.opts.dispatchAdapter.enqueueDuringDispatch != null &&
|
|
1922
2229
|
(await this.laneLedger?.steerLive(event)) &&
|
|
@@ -1930,6 +2237,15 @@ export class ParallAgentGateway {
|
|
|
1930
2237
|
);
|
|
1931
2238
|
}
|
|
1932
2239
|
} else if (
|
|
2240
|
+
// Message events only. A typed event (task_comment/schedule/…)
|
|
2241
|
+
// rides the typed-consume contract — buffer-main resolves false and
|
|
2242
|
+
// the claim releases for re-drive — so an injection here is exactly
|
|
2243
|
+
// the forbidden un-folded injection: the LLM sees the content while
|
|
2244
|
+
// the WorkItem stays live, and every re-drive injects it AGAIN (the
|
|
2245
|
+
// 7/16 watcher duplicate-delivery loop, #1149). Typed events stay
|
|
2246
|
+
// buffered; the drain claims them as their own turn.
|
|
2247
|
+
event.type === 'message' &&
|
|
2248
|
+
!typedAheadInBuffer &&
|
|
1933
2249
|
this.dispatchState.mainCurrentTargetId === event.targetId &&
|
|
1934
2250
|
(await this.opts.dispatchAdapter.enqueueDuringDispatch?.(
|
|
1935
2251
|
this.opts.runtimeKey,
|
|
@@ -1973,8 +2289,9 @@ export class ParallAgentGateway {
|
|
|
1973
2289
|
// — the fork's lane claim marks them received. A mark-received here
|
|
1974
2290
|
// outruns the claim, strands the row ownerless, and the chat goes
|
|
1975
2291
|
// silent (the 0713 black hole; dispatch-convergence-design.md §6).
|
|
1976
|
-
//
|
|
1977
|
-
|
|
2292
|
+
// Same rule for ledger-claimed typed events: their dsp-lane claim
|
|
2293
|
+
// owns received-ness, and a re-mark would wipe the lease owner.
|
|
2294
|
+
if (!this.usesLaneLedger(event) && !this.typedLedgerEventIds([event])) {
|
|
1978
2295
|
try {
|
|
1979
2296
|
await this.emitDispatchReceived(event);
|
|
1980
2297
|
} catch (err) {
|
|
@@ -2185,10 +2502,13 @@ export class ParallAgentGateway {
|
|
|
2185
2502
|
this.handleTaskDispatch(item.task_id ?? '', item.source_id ?? item.task_id ?? '', {
|
|
2186
2503
|
dispatchEventId,
|
|
2187
2504
|
}),
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
this.
|
|
2191
|
-
|
|
2505
|
+
{
|
|
2506
|
+
legacyAck: (dispatchEventId) =>
|
|
2507
|
+
this.ackDispatchEvent(dispatchEventId ?? item.id, () =>
|
|
2508
|
+
this.clearTypedDispatchDedupe(item),
|
|
2509
|
+
),
|
|
2510
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(item),
|
|
2511
|
+
},
|
|
2192
2512
|
);
|
|
2193
2513
|
}
|
|
2194
2514
|
|
|
@@ -2236,7 +2556,7 @@ export class ParallAgentGateway {
|
|
|
2236
2556
|
};
|
|
2237
2557
|
|
|
2238
2558
|
const dispatched = await this.handleInboundEvent(event);
|
|
2239
|
-
if (!dispatched) {
|
|
2559
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2240
2560
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2241
2561
|
}
|
|
2242
2562
|
return dispatched;
|
|
@@ -2271,6 +2591,7 @@ export class ParallAgentGateway {
|
|
|
2271
2591
|
taskId: string,
|
|
2272
2592
|
actorId: string | null,
|
|
2273
2593
|
deliveryReason?: DispatchDeliveryReason | null,
|
|
2594
|
+
dispatchEventId?: string,
|
|
2274
2595
|
): Promise<boolean> {
|
|
2275
2596
|
if (this.shuttingDown) return false; // drain window — let server requeue via catch-up
|
|
2276
2597
|
const dedupeKey = `comment:${commentId}`;
|
|
@@ -2335,6 +2656,7 @@ export class ParallAgentGateway {
|
|
|
2335
2656
|
deliveryReason: deliveryReason ?? undefined,
|
|
2336
2657
|
ackSourceType: 'comment',
|
|
2337
2658
|
ackSourceId: commentId,
|
|
2659
|
+
dispatchEventId,
|
|
2338
2660
|
};
|
|
2339
2661
|
|
|
2340
2662
|
let dispatched: boolean;
|
|
@@ -2345,7 +2667,7 @@ export class ParallAgentGateway {
|
|
|
2345
2667
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2346
2668
|
throw err;
|
|
2347
2669
|
}
|
|
2348
|
-
if (!dispatched) {
|
|
2670
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2349
2671
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2350
2672
|
}
|
|
2351
2673
|
return dispatched;
|
|
@@ -2355,6 +2677,7 @@ export class ParallAgentGateway {
|
|
|
2355
2677
|
commentId: string,
|
|
2356
2678
|
actorId: string | null,
|
|
2357
2679
|
deliveryReason?: DispatchDeliveryReason | null,
|
|
2680
|
+
dispatchEventId?: string,
|
|
2358
2681
|
): Promise<boolean> {
|
|
2359
2682
|
if (this.shuttingDown) return false; // drain window — let server requeue via catch-up
|
|
2360
2683
|
// Shares the comment dedupe namespace with handleTaskComment; comment IDs
|
|
@@ -2410,6 +2733,7 @@ export class ParallAgentGateway {
|
|
|
2410
2733
|
replyTargetUri: comment.target_uri,
|
|
2411
2734
|
ackSourceType: 'comment',
|
|
2412
2735
|
ackSourceId: commentId,
|
|
2736
|
+
dispatchEventId,
|
|
2413
2737
|
};
|
|
2414
2738
|
|
|
2415
2739
|
let dispatched: boolean;
|
|
@@ -2420,7 +2744,7 @@ export class ParallAgentGateway {
|
|
|
2420
2744
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2421
2745
|
throw err;
|
|
2422
2746
|
}
|
|
2423
|
-
if (!dispatched) {
|
|
2747
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2424
2748
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2425
2749
|
}
|
|
2426
2750
|
return dispatched;
|
|
@@ -2437,6 +2761,7 @@ export class ParallAgentGateway {
|
|
|
2437
2761
|
private async fetchAndHandleScheduleFire(
|
|
2438
2762
|
runId: string,
|
|
2439
2763
|
actorId: string | null,
|
|
2764
|
+
dispatchEventId?: string,
|
|
2440
2765
|
): Promise<boolean> {
|
|
2441
2766
|
let run: ScheduleRun | null = null;
|
|
2442
2767
|
try {
|
|
@@ -2458,10 +2783,14 @@ export class ParallAgentGateway {
|
|
|
2458
2783
|
return false;
|
|
2459
2784
|
}
|
|
2460
2785
|
if (!run) return true;
|
|
2461
|
-
return this.handleScheduleFire(run, actorId);
|
|
2786
|
+
return this.handleScheduleFire(run, actorId, dispatchEventId);
|
|
2462
2787
|
}
|
|
2463
2788
|
|
|
2464
|
-
private async handleScheduleFire(
|
|
2789
|
+
private async handleScheduleFire(
|
|
2790
|
+
run: ScheduleRun,
|
|
2791
|
+
actorId: string | null,
|
|
2792
|
+
dispatchEventId?: string,
|
|
2793
|
+
): Promise<boolean> {
|
|
2465
2794
|
if (this.shuttingDown) return false; // drain window — let server requeue via catch-up
|
|
2466
2795
|
const dedupeKey = `schedule_run:${run.id}`;
|
|
2467
2796
|
if (this.dispatchedTasks.has(dedupeKey)) return false;
|
|
@@ -2484,6 +2813,7 @@ export class ParallAgentGateway {
|
|
|
2484
2813
|
attachedUri: run.fired_attached_uri ?? undefined,
|
|
2485
2814
|
ackSourceType: 'schedule_run',
|
|
2486
2815
|
ackSourceId: run.id,
|
|
2816
|
+
dispatchEventId,
|
|
2487
2817
|
};
|
|
2488
2818
|
|
|
2489
2819
|
let dispatched: boolean;
|
|
@@ -2493,13 +2823,16 @@ export class ParallAgentGateway {
|
|
|
2493
2823
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2494
2824
|
throw err;
|
|
2495
2825
|
}
|
|
2496
|
-
if (!dispatched) {
|
|
2826
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2497
2827
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2498
2828
|
}
|
|
2499
2829
|
return dispatched;
|
|
2500
2830
|
}
|
|
2501
2831
|
|
|
2502
|
-
private async fetchAndHandleExternalTriggerRun(
|
|
2832
|
+
private async fetchAndHandleExternalTriggerRun(
|
|
2833
|
+
runId: string,
|
|
2834
|
+
dispatchEventId?: string,
|
|
2835
|
+
): Promise<boolean> {
|
|
2503
2836
|
let run: ExternalTriggerRun | null = null;
|
|
2504
2837
|
try {
|
|
2505
2838
|
run = await this.opts.client.getExternalTriggerRun(this.opts.config.org_id, runId);
|
|
@@ -2517,7 +2850,7 @@ export class ParallAgentGateway {
|
|
|
2517
2850
|
return false;
|
|
2518
2851
|
}
|
|
2519
2852
|
if (!run) return true;
|
|
2520
|
-
return this.handleExternalTriggerRun(run);
|
|
2853
|
+
return this.handleExternalTriggerRun(run, dispatchEventId);
|
|
2521
2854
|
}
|
|
2522
2855
|
|
|
2523
2856
|
// fetchAndHandleChannelMessage resolves a channel_message dispatch to its
|
|
@@ -2525,7 +2858,10 @@ export class ParallAgentGateway {
|
|
|
2525
2858
|
// pipeline. targetId = the ChannelConversation id, so per-conversation
|
|
2526
2859
|
// multi-turn continuity rides the same per-target session mechanics as
|
|
2527
2860
|
// chats. Design: docs/engineering-design/external-im-channel-design.md.
|
|
2528
|
-
private async fetchAndHandleChannelMessage(
|
|
2861
|
+
private async fetchAndHandleChannelMessage(
|
|
2862
|
+
messageId: string,
|
|
2863
|
+
dispatchEventId?: string,
|
|
2864
|
+
): Promise<boolean> {
|
|
2529
2865
|
if (this.shuttingDown) return false;
|
|
2530
2866
|
// Capped dedupe (the chat-message path, not the unbounded task set): a busy
|
|
2531
2867
|
// external IM conversation would otherwise retain one key per message ever
|
|
@@ -2579,18 +2915,20 @@ export class ParallAgentGateway {
|
|
|
2579
2915
|
}
|
|
2580
2916
|
}
|
|
2581
2917
|
|
|
2582
|
-
// The reply hint routes on the live capability grant
|
|
2583
|
-
//
|
|
2584
|
-
//
|
|
2585
|
-
//
|
|
2586
|
-
//
|
|
2587
|
-
//
|
|
2588
|
-
//
|
|
2589
|
-
//
|
|
2918
|
+
// The reply hint routes on the live capability grant, keyed PER
|
|
2919
|
+
// PROVIDER: feishu's affordance is the vendor CLI on PATH (`feishu-cli`
|
|
2920
|
+
// → lark-cli, tier A) and slack's is the platform verb (`slack-send` →
|
|
2921
|
+
// `parall slack send`, tier B — there is no `slack-cli`). Absent →
|
|
2922
|
+
// outbound is disabled for this org (flag/connection off) and the hint
|
|
2923
|
+
// must say so instead of pointing at a retired clip. The provider label
|
|
2924
|
+
// lookup above is best-effort/cosmetic — when it fails, ANY granted
|
|
2925
|
+
// channel capability keeps the hint on the capability path: a transient
|
|
2926
|
+
// metadata miss must not flip an actively granted agent's hint to
|
|
2927
|
+
// "outbound disabled" and strand a valid external message.
|
|
2590
2928
|
const keys = this.opts.getCapabilityKeys?.() ?? [];
|
|
2591
2929
|
const cliCapable = provider
|
|
2592
|
-
? keys.includes(
|
|
2593
|
-
: keys.some((k) => k.endsWith('-cli'));
|
|
2930
|
+
? keys.includes(channelCapabilityKeyFor(provider))
|
|
2931
|
+
: keys.some((k) => k.endsWith('-cli') || k === CAPABILITY_SLACK_SEND);
|
|
2594
2932
|
|
|
2595
2933
|
const event: ParallEvent = {
|
|
2596
2934
|
type: 'channel_message',
|
|
@@ -2609,6 +2947,7 @@ export class ParallAgentGateway {
|
|
|
2609
2947
|
channelCliCapable: cliCapable,
|
|
2610
2948
|
ackSourceType: 'channel_message',
|
|
2611
2949
|
ackSourceId: msg.id,
|
|
2950
|
+
dispatchEventId,
|
|
2612
2951
|
};
|
|
2613
2952
|
|
|
2614
2953
|
// Release the claim if the event isn't actually dispatched (or throws) so
|
|
@@ -2620,13 +2959,16 @@ export class ParallAgentGateway {
|
|
|
2620
2959
|
this.dispatchedMessages.delete(claimKey);
|
|
2621
2960
|
throw err;
|
|
2622
2961
|
}
|
|
2623
|
-
if (!dispatched) {
|
|
2962
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2624
2963
|
this.dispatchedMessages.delete(claimKey);
|
|
2625
2964
|
}
|
|
2626
2965
|
return dispatched;
|
|
2627
2966
|
}
|
|
2628
2967
|
|
|
2629
|
-
private async handleExternalTriggerRun(
|
|
2968
|
+
private async handleExternalTriggerRun(
|
|
2969
|
+
run: ExternalTriggerRun,
|
|
2970
|
+
dispatchEventId?: string,
|
|
2971
|
+
): Promise<boolean> {
|
|
2630
2972
|
if (this.shuttingDown) return false;
|
|
2631
2973
|
const dedupeKey = `external_trigger_run:${run.id}`;
|
|
2632
2974
|
if (this.dispatchedTasks.has(dedupeKey)) return false;
|
|
@@ -2654,6 +2996,7 @@ export class ParallAgentGateway {
|
|
|
2654
2996
|
attachedUri,
|
|
2655
2997
|
ackSourceType: 'external_trigger_run',
|
|
2656
2998
|
ackSourceId: run.id,
|
|
2999
|
+
dispatchEventId,
|
|
2657
3000
|
};
|
|
2658
3001
|
|
|
2659
3002
|
let dispatched: boolean;
|
|
@@ -2663,7 +3006,7 @@ export class ParallAgentGateway {
|
|
|
2663
3006
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2664
3007
|
throw err;
|
|
2665
3008
|
}
|
|
2666
|
-
if (!dispatched) {
|
|
3009
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2667
3010
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2668
3011
|
}
|
|
2669
3012
|
return dispatched;
|
|
@@ -2673,6 +3016,7 @@ export class ParallAgentGateway {
|
|
|
2673
3016
|
approvalId: string,
|
|
2674
3017
|
actorId: string | null,
|
|
2675
3018
|
chatId: string | null,
|
|
3019
|
+
dispatchEventId?: string,
|
|
2676
3020
|
): Promise<boolean> {
|
|
2677
3021
|
let approval: Approval | null = null;
|
|
2678
3022
|
try {
|
|
@@ -2709,6 +3053,11 @@ export class ParallAgentGateway {
|
|
|
2709
3053
|
senderName: 'approver',
|
|
2710
3054
|
messageId: approval.id,
|
|
2711
3055
|
body,
|
|
3056
|
+
// The WorkItem's real source pair. Without it the legacy fallback
|
|
3057
|
+
// guessed ('message', approval_id) — a pair no row matches.
|
|
3058
|
+
ackSourceType: 'approval',
|
|
3059
|
+
ackSourceId: approval.id,
|
|
3060
|
+
dispatchEventId,
|
|
2712
3061
|
};
|
|
2713
3062
|
|
|
2714
3063
|
let dispatched: boolean;
|
|
@@ -2718,7 +3067,7 @@ export class ParallAgentGateway {
|
|
|
2718
3067
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2719
3068
|
throw err;
|
|
2720
3069
|
}
|
|
2721
|
-
if (!dispatched) {
|
|
3070
|
+
if (!dispatched && !this.isTypedEventBuffered(event)) {
|
|
2722
3071
|
this.dispatchedTasks.delete(dedupeKey);
|
|
2723
3072
|
}
|
|
2724
3073
|
return dispatched;
|
|
@@ -2767,7 +3116,24 @@ export class ParallAgentGateway {
|
|
|
2767
3116
|
|
|
2768
3117
|
if (overflowMode && processed >= CATCHUP_MAX) {
|
|
2769
3118
|
try {
|
|
2770
|
-
|
|
3119
|
+
// Administrative drop of the overflow backlog (summarized to the
|
|
3120
|
+
// agent instead of dispatched). Ledger path: by-id complete —
|
|
3121
|
+
// closes the pending row and refuses (stale) one a live lane
|
|
3122
|
+
// owns; the legacy ack stays as the ledger-disabled / pre-by-id
|
|
3123
|
+
// fallback.
|
|
3124
|
+
if (this.laneLedger && !this.ledgerDisabled && !this.typedByIdCompleteUnsupported) {
|
|
3125
|
+
const outcome = await this.resolveDispatchByID(item.id);
|
|
3126
|
+
if (outcome === 'unsupported') {
|
|
3127
|
+
this.typedByIdCompleteUnsupported = true;
|
|
3128
|
+
await this.opts.client.ackDispatchByID(this.opts.config.org_id, item.id);
|
|
3129
|
+
} else if (outcome === 'failed') {
|
|
3130
|
+
throw new Error('by-id complete failed');
|
|
3131
|
+
}
|
|
3132
|
+
// 'stale' counts as skipped: a live lane owns the row and its
|
|
3133
|
+
// turn will resolve it — nothing left for catch-up to do.
|
|
3134
|
+
} else {
|
|
3135
|
+
await this.opts.client.ackDispatchByID(this.opts.config.org_id, item.id);
|
|
3136
|
+
}
|
|
2771
3137
|
skipped++;
|
|
2772
3138
|
const key = item.event_type;
|
|
2773
3139
|
skippedByType.set(key, (skippedByType.get(key) ?? 0) + 1);
|
|
@@ -2777,10 +3143,23 @@ export class ParallAgentGateway {
|
|
|
2777
3143
|
continue;
|
|
2778
3144
|
}
|
|
2779
3145
|
|
|
3146
|
+
// Same pre-claim guard as the dispatch.new handler: a WorkItem whose
|
|
3147
|
+
// event copy is already buffered belongs to the drain — claiming it
|
|
3148
|
+
// here would race the drain's fence-less settlement.
|
|
3149
|
+
if (item.event_type !== 'message' && this.isBufferedTypedWorkItem(item.id)) {
|
|
3150
|
+
this.opts.log?.info(
|
|
3151
|
+
`typed dispatch ${item.id} already buffered for the drain — skipping catch-up claim`,
|
|
3152
|
+
);
|
|
3153
|
+
continue;
|
|
3154
|
+
}
|
|
3155
|
+
|
|
2780
3156
|
processed++;
|
|
2781
3157
|
try {
|
|
2782
|
-
const
|
|
2783
|
-
|
|
3158
|
+
const typedHooks: TypedConsumeHooks = {
|
|
3159
|
+
legacyAck: () =>
|
|
3160
|
+
this.ackDispatchEvent(item.id, () => this.clearTypedDispatchDedupe(item)),
|
|
3161
|
+
clearDedupe: () => this.clearTypedDispatchDedupe(item),
|
|
3162
|
+
};
|
|
2784
3163
|
if (item.event_type === 'task_assign' && item.task_id) {
|
|
2785
3164
|
try {
|
|
2786
3165
|
await this.consumeTypedDispatch(
|
|
@@ -2793,7 +3172,7 @@ export class ParallAgentGateway {
|
|
|
2793
3172
|
dispatchEventId,
|
|
2794
3173
|
},
|
|
2795
3174
|
),
|
|
2796
|
-
|
|
3175
|
+
typedHooks,
|
|
2797
3176
|
);
|
|
2798
3177
|
} catch (err: unknown) {
|
|
2799
3178
|
this.opts.log?.warn(
|
|
@@ -2814,7 +3193,7 @@ export class ParallAgentGateway {
|
|
|
2814
3193
|
dispatchEventId,
|
|
2815
3194
|
},
|
|
2816
3195
|
),
|
|
2817
|
-
|
|
3196
|
+
typedHooks,
|
|
2818
3197
|
);
|
|
2819
3198
|
} catch (err: unknown) {
|
|
2820
3199
|
this.opts.log?.warn(
|
|
@@ -2825,49 +3204,60 @@ export class ParallAgentGateway {
|
|
|
2825
3204
|
} else if (item.event_type === 'task_comment' && item.source_id && item.task_id) {
|
|
2826
3205
|
await this.consumeTypedDispatch(
|
|
2827
3206
|
{ dispatchEventId: item.id },
|
|
2828
|
-
() =>
|
|
3207
|
+
(dispatchEventId) =>
|
|
2829
3208
|
this.handleTaskComment(
|
|
2830
3209
|
item.source_id,
|
|
2831
3210
|
item.task_id ?? '',
|
|
2832
3211
|
item.actor_id,
|
|
2833
3212
|
item.delivery_reason,
|
|
3213
|
+
dispatchEventId,
|
|
2834
3214
|
),
|
|
2835
|
-
|
|
3215
|
+
typedHooks,
|
|
2836
3216
|
);
|
|
2837
3217
|
} else if (item.event_type === 'wiki_comment' && item.source_id) {
|
|
2838
3218
|
await this.consumeTypedDispatch(
|
|
2839
3219
|
{ dispatchEventId: item.id },
|
|
2840
|
-
() =>
|
|
2841
|
-
|
|
3220
|
+
(dispatchEventId) =>
|
|
3221
|
+
this.handleWikiComment(
|
|
3222
|
+
item.source_id,
|
|
3223
|
+
item.actor_id,
|
|
3224
|
+
item.delivery_reason,
|
|
3225
|
+
dispatchEventId,
|
|
3226
|
+
),
|
|
3227
|
+
typedHooks,
|
|
2842
3228
|
);
|
|
2843
3229
|
} else if (item.event_type === 'schedule.fire' && item.source_id) {
|
|
2844
3230
|
await this.consumeTypedDispatch(
|
|
2845
3231
|
{ dispatchEventId: item.id },
|
|
2846
|
-
() =>
|
|
2847
|
-
|
|
3232
|
+
(dispatchEventId) =>
|
|
3233
|
+
this.fetchAndHandleScheduleFire(item.source_id, item.actor_id, dispatchEventId),
|
|
3234
|
+
typedHooks,
|
|
2848
3235
|
);
|
|
2849
3236
|
} else if (item.event_type === 'external_trigger' && item.source_id) {
|
|
2850
3237
|
await this.consumeTypedDispatch(
|
|
2851
3238
|
{ dispatchEventId: item.id },
|
|
2852
|
-
() =>
|
|
2853
|
-
|
|
3239
|
+
(dispatchEventId) =>
|
|
3240
|
+
this.fetchAndHandleExternalTriggerRun(item.source_id, dispatchEventId),
|
|
3241
|
+
typedHooks,
|
|
2854
3242
|
);
|
|
2855
3243
|
} else if (item.event_type === 'channel_message' && item.source_id) {
|
|
2856
3244
|
await this.consumeTypedDispatch(
|
|
2857
3245
|
{ dispatchEventId: item.id },
|
|
2858
|
-
() =>
|
|
2859
|
-
|
|
3246
|
+
(dispatchEventId) =>
|
|
3247
|
+
this.fetchAndHandleChannelMessage(item.source_id, dispatchEventId),
|
|
3248
|
+
typedHooks,
|
|
2860
3249
|
);
|
|
2861
3250
|
} else if (item.event_type === 'approval_decided' && item.source_id) {
|
|
2862
3251
|
await this.consumeTypedDispatch(
|
|
2863
3252
|
{ dispatchEventId: item.id },
|
|
2864
|
-
() =>
|
|
3253
|
+
(dispatchEventId) =>
|
|
2865
3254
|
this.fetchAndHandleApprovalDecided(
|
|
2866
3255
|
item.source_id,
|
|
2867
3256
|
item.actor_id,
|
|
2868
3257
|
item.chat_id ?? null,
|
|
3258
|
+
dispatchEventId,
|
|
2869
3259
|
),
|
|
2870
|
-
|
|
3260
|
+
typedHooks,
|
|
2871
3261
|
);
|
|
2872
3262
|
} else if (item.event_type === 'message' && item.source_id && item.chat_id) {
|
|
2873
3263
|
await this.consumeMessageWorkItem({
|
|
@@ -3070,6 +3460,32 @@ export class ParallAgentGateway {
|
|
|
3070
3460
|
|
|
3071
3461
|
await this.opts.onBeforeDisconnect?.();
|
|
3072
3462
|
|
|
3463
|
+
// Parked step writes are process-local and their WorkItems are already
|
|
3464
|
+
// resolved — restart catch-up will NOT re-drive them, so anything still
|
|
3465
|
+
// parked at exit is permanently lost. Spend a slice of the shutdown
|
|
3466
|
+
// budget on one flush pass first: the common shutdown (idle-stop,
|
|
3467
|
+
// deploy) happens on a healthy network where these writes just succeed.
|
|
3468
|
+
// The 10s cap is hard — a write still in flight at the deadline is
|
|
3469
|
+
// abandoned to the background (see StepRetryQueue.flush).
|
|
3470
|
+
if (this.stepPersister.pendingTotal() > 0) {
|
|
3471
|
+
const remaining = await this.stepPersister.flush(10_000);
|
|
3472
|
+
if (remaining > 0) {
|
|
3473
|
+
this.opts.log?.warn(
|
|
3474
|
+
`${remaining} parked step write(s) could not be flushed at shutdown; they are permanently lost`,
|
|
3475
|
+
);
|
|
3476
|
+
}
|
|
3477
|
+
}
|
|
3478
|
+
this.stepPersister.dispose();
|
|
3479
|
+
// Lifecycle last: idle writes land after the flushed steps, so the
|
|
3480
|
+
// server clears activity once and no flushed step can relight it.
|
|
3481
|
+
const lifecycleRemaining = await this.sessionLifecycle.flush(5_000);
|
|
3482
|
+
if (lifecycleRemaining > 0) {
|
|
3483
|
+
this.opts.log?.warn(
|
|
3484
|
+
`${lifecycleRemaining} session lifecycle write(s) unreconciled at shutdown`,
|
|
3485
|
+
);
|
|
3486
|
+
}
|
|
3487
|
+
this.sessionLifecycle.dispose();
|
|
3488
|
+
|
|
3073
3489
|
this.opts.ws.disconnect();
|
|
3074
3490
|
this.opts.log?.info(`disconnected`);
|
|
3075
3491
|
}
|