@dsh-enhanced/assistant-delivery 0.1.62 → 0.1.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -10
- package/bin/dsh-restart-supervisor.js +76 -0
- package/lib/agent-runtime.d.ts +21 -1
- package/lib/agent-runtime.d.ts.map +1 -1
- package/lib/agent-runtime.js +192 -23
- package/lib/agent-runtime.js.map +1 -1
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +10 -0
- package/lib/coordinator.js.map +1 -1
- package/lib/operator-snapshot.d.ts +1 -1
- package/lib/operator-snapshot.d.ts.map +1 -1
- package/lib/operator-snapshot.js +5 -5
- package/lib/operator-snapshot.js.map +1 -1
- package/lib/restart-native.d.ts +47 -0
- package/lib/restart-native.d.ts.map +1 -0
- package/lib/restart-native.js +170 -0
- package/lib/restart-native.js.map +1 -0
- package/lib/restart-runtime.d.ts +38 -0
- package/lib/restart-runtime.d.ts.map +1 -0
- package/lib/restart-runtime.js +335 -0
- package/lib/restart-runtime.js.map +1 -0
- package/lib/restart-store.d.ts +132 -0
- package/lib/restart-store.d.ts.map +1 -0
- package/lib/restart-store.js +757 -0
- package/lib/restart-store.js.map +1 -0
- package/lib/restart-supervisor.d.ts +129 -0
- package/lib/restart-supervisor.d.ts.map +1 -0
- package/lib/restart-supervisor.js +615 -0
- package/lib/restart-supervisor.js.map +1 -0
- package/lib/service.d.ts +45 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +488 -6
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +76 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +41 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +433 -85
- package/lib/store.js.map +1 -1
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +21 -2
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +33 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +11 -10
package/lib/service.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import { chmodSync, mkdirSync } from 'node:fs';
|
|
1
|
+
import { chmodSync, mkdirSync, readFileSync } from 'node:fs';
|
|
2
2
|
import { isAbsolute } from 'node:path';
|
|
3
3
|
import { createHash, randomUUID } from 'node:crypto';
|
|
4
4
|
import { Context, Service } from '@deepseek-ai/cordis';
|
|
5
5
|
import Schema from '@deepseek-ai/schemastery';
|
|
6
|
+
import { RestartStore } from './restart-store.js';
|
|
7
|
+
import { restartDigest } from './restart-runtime.js';
|
|
8
|
+
import { version } from './version.js';
|
|
6
9
|
import { acceptanceCanonicalJson, goalDefinitionSituation } from '@dsh-enhanced/task-acceptance-contract';
|
|
7
10
|
import { approvalReviewerOf, } from '@dsh-enhanced/assistant-policy';
|
|
8
11
|
// Cross-package wire literal, kept locally so this bundle can load without the
|
|
@@ -63,6 +66,8 @@ const goalOutcomeFeedbackFooter = [
|
|
|
63
66
|
'`helpful` 等只记录偏好,不会被当成目标成败。',
|
|
64
67
|
].join('\n');
|
|
65
68
|
const goalWakeCompletionNotice = '目标已通过独立验收并完成。';
|
|
69
|
+
const interruptedTurnNotice = '这次请求在执行过程中被中断,当前已停止等待;已有操作的结果尚未确认,因此没有自动重跑。你可以继续发送消息;涉及写入的任务需要先核对现有结果。';
|
|
70
|
+
const timedOutTurnNotice = '这次请求等待超时;已有操作的结果尚未确认,因此不会自动重跑。';
|
|
66
71
|
function isGoalOutcomeFeedbackAuthority(value) {
|
|
67
72
|
return typeof value === 'object' && value !== null
|
|
68
73
|
&& typeof value.issueOwnerGoalOutcomeFeedbackTarget === 'function'
|
|
@@ -268,8 +273,17 @@ function isOwnerGoalSnapshotLocator(value) {
|
|
|
268
273
|
const configSchema = Schema.object({
|
|
269
274
|
databasePath: Schema.string().required(),
|
|
270
275
|
spoolPath: Schema.string().required(),
|
|
276
|
+
restart: Schema.union([Schema.object({
|
|
277
|
+
databasePath: Schema.string().required(),
|
|
278
|
+
ownerRouteId: Schema.string().min(1).required(),
|
|
279
|
+
maxModelRequests: Schema.number().step(1).min(1).max(256).required(),
|
|
280
|
+
maxToolCalls: Schema.number().step(1).min(1).max(1024).required(),
|
|
281
|
+
taskTimeoutMs: Schema.number().step(1).min(60_000).max(3_600_000).required(),
|
|
282
|
+
allowedTools: Schema.array(Schema.string().min(1)).required(),
|
|
283
|
+
})]),
|
|
271
284
|
schedulerEnabled: Schema.boolean().default(true),
|
|
272
285
|
tickIntervalMs: Schema.number().step(1).min(100).default(1_000),
|
|
286
|
+
agentTaskTimeoutMs: Schema.number().step(1).min(1_000).max(86_400_000).default(1_800_000),
|
|
273
287
|
leaseMs: Schema.number().step(1).min(1_000).default(30_000),
|
|
274
288
|
maxAttempts: Schema.number().step(1).min(1).max(100).default(5),
|
|
275
289
|
maxConcurrency: Schema.number().step(1).min(1).max(100).default(4),
|
|
@@ -569,6 +583,13 @@ function exactToolApprovalCall(events, openTurn, callId, toolName) {
|
|
|
569
583
|
export class AssistantDeliveryService extends Service {
|
|
570
584
|
static Config = configSchema;
|
|
571
585
|
deliveryStore;
|
|
586
|
+
restartStore;
|
|
587
|
+
restartInvocationId = process.env.INVOCATION_ID ?? `standalone-${randomUUID()}`;
|
|
588
|
+
restartCodeDigest = '';
|
|
589
|
+
restartNoticeCursor;
|
|
590
|
+
restartFlight;
|
|
591
|
+
restartController = new AbortController();
|
|
592
|
+
restartSafeDefinitions = new Set();
|
|
572
593
|
context;
|
|
573
594
|
policy;
|
|
574
595
|
registry;
|
|
@@ -610,6 +631,8 @@ export class AssistantDeliveryService extends Service {
|
|
|
610
631
|
preferenceProjectionFlight;
|
|
611
632
|
inboundAdmissionRecoveryFlight;
|
|
612
633
|
inboundAdmissionRecoveryAfterSequence = 0;
|
|
634
|
+
interruptedInboxNoticeFlight;
|
|
635
|
+
interruptedInboxNoticeAfterSequence = 0;
|
|
613
636
|
inboundAdmissionRecoveryController = new AbortController();
|
|
614
637
|
workflowTraceRetryTimer;
|
|
615
638
|
preferenceProjectionRetryTimer;
|
|
@@ -666,6 +689,36 @@ export class AssistantDeliveryService extends Service {
|
|
|
666
689
|
};
|
|
667
690
|
this.ownerRouteGuard = Object.freeze(ownerRouteGuard);
|
|
668
691
|
this.deliveryStore = new DeliveryStore({ path: config.databasePath, maxTextBytes: config.maxTextBytes });
|
|
692
|
+
if (config.restart !== undefined && (!isAbsolute(config.restart.databasePath)
|
|
693
|
+
|| config.restart.databasePath === config.databasePath || !this.ownerRoutes.has(config.restart.ownerRouteId))) {
|
|
694
|
+
throw new Error('assistant-delivery: restart requires a separate absolute journal and an exact configured owner route');
|
|
695
|
+
}
|
|
696
|
+
try {
|
|
697
|
+
this.restartStore = config.restart === undefined ? undefined : new RestartStore({ path: config.restart.databasePath });
|
|
698
|
+
}
|
|
699
|
+
catch (error) {
|
|
700
|
+
this.restartStore = undefined;
|
|
701
|
+
ctx.logger.warn(`assistant-delivery: planned restart disabled; journal unavailable: ${String(error)}`);
|
|
702
|
+
}
|
|
703
|
+
if (this.restartStore !== undefined) {
|
|
704
|
+
const extension = import.meta.url.endsWith('.ts') ? 'ts' : 'js';
|
|
705
|
+
this.restartCodeDigest = restartDigest(['service', 'agent-runtime', 'restart-runtime', 'restart-native', 'restart-store', 'tools']
|
|
706
|
+
.map(name => createHash('sha256').update(readFileSync(new URL(`./${name}.${extension}`, import.meta.url))).digest('hex')));
|
|
707
|
+
}
|
|
708
|
+
if (this.restartStore !== undefined) {
|
|
709
|
+
const permitted = () => this.restartDispatchPermitted(ctx.get('agents')?.currentInitiator());
|
|
710
|
+
ctx.on('agent/pre-step', async (_payload, next) => permitted() ? await next() : { kind: 'reject' }, { prepend: true });
|
|
711
|
+
ctx.on('llm/stream', async function* (_options, next) {
|
|
712
|
+
if (!permitted())
|
|
713
|
+
throw new Error('assistant-delivery: Host is draining for planned maintenance');
|
|
714
|
+
yield* next();
|
|
715
|
+
}, { prepend: true });
|
|
716
|
+
ctx.on('tools/execute', async (execution, next) => {
|
|
717
|
+
if (!this.restartDispatchPermitted(execution.agent))
|
|
718
|
+
throw new Error('assistant-delivery: Host is draining for planned maintenance');
|
|
719
|
+
return await next();
|
|
720
|
+
}, { prepend: true });
|
|
721
|
+
}
|
|
669
722
|
ctx.on('agent/inbox/claimed', ({ agent, message, turn }) => {
|
|
670
723
|
this.capturePreferenceTurn(agent, message.source, turn);
|
|
671
724
|
});
|
|
@@ -686,6 +739,24 @@ export class AssistantDeliveryService extends Service {
|
|
|
686
739
|
return;
|
|
687
740
|
this.deliveryStore.recordReceipt(receipt);
|
|
688
741
|
},
|
|
742
|
+
progressJournal: {
|
|
743
|
+
begin: async (intent, fingerprint, sequence, showDetails) => this.progressAuthorized(intent)
|
|
744
|
+
? this.deliveryStore.beginProgress(intent, fingerprint, sequence, showDetails) : undefined,
|
|
745
|
+
get: async (intent, fingerprint) => this.progressAuthorized(intent)
|
|
746
|
+
? this.deliveryStore.getProgress(intent, fingerprint) : undefined,
|
|
747
|
+
created: async (intent, fingerprint, handle) => this.progressAuthorized(intent)
|
|
748
|
+
? this.deliveryStore.attachProgressHandle(intent, fingerprint, handle) : undefined,
|
|
749
|
+
advance: async (intent, fingerprint) => this.progressAuthorized(intent)
|
|
750
|
+
? this.deliveryStore.advanceProgress(intent, fingerprint) : undefined,
|
|
751
|
+
stageTerminal: async (intent, fingerprint) => this.progressAuthorized(intent)
|
|
752
|
+
? this.deliveryStore.stageProgressTerminal(intent, fingerprint) : undefined,
|
|
753
|
+
delivered: async (intent, fingerprint) => {
|
|
754
|
+
if (this.progressAuthorized(intent))
|
|
755
|
+
this.deliveryStore.confirmProgressTerminal(intent, fingerprint);
|
|
756
|
+
},
|
|
757
|
+
recover: async (channel, account, fingerprint, limit) => this.active
|
|
758
|
+
? this.deliveryStore.recoverProgress(channel, account, fingerprint, limit, (binding, inbox) => this.isInboundAuthorized(binding, inbox.envelope)) : [],
|
|
759
|
+
},
|
|
689
760
|
});
|
|
690
761
|
const imageMaterializer = new InboundImageMaterializer({
|
|
691
762
|
store: this.deliveryStore,
|
|
@@ -717,6 +788,13 @@ export class AssistantDeliveryService extends Service {
|
|
|
717
788
|
runtimeCtx.effect(() => () => { if (this.nativeWebRuntime === webRuntime)
|
|
718
789
|
this.nativeWebRuntime = undefined; });
|
|
719
790
|
const unregister = this.registerInboundRuntime(new DshDeliveryRuntime(runtimeCtx, policy, {
|
|
791
|
+
...(this.restartStore === undefined ? {} : { restart: { store: this.restartStore, config: config.restart,
|
|
792
|
+
invocationId: this.restartInvocationId,
|
|
793
|
+
authorize: (record) => this.restartAuthorized(record),
|
|
794
|
+
authorizeTask: (inboxId) => this.restartAuthorized({ inboxId }),
|
|
795
|
+
onContinuationAdmitted: (record, sessionId) => this.releaseRestartFenceOnAdmission(record, sessionId),
|
|
796
|
+
isRestartSafeTool: (agent, name) => this.isRestartSafeTool(agent, name),
|
|
797
|
+
} }),
|
|
720
798
|
sessionLeases,
|
|
721
799
|
sessionNamespace: this.deliveryStore.instanceId(),
|
|
722
800
|
workspace: config.defaultWorkspace, agentPreset: config.defaultAgentPreset, policyRef: config.policyRef,
|
|
@@ -724,13 +802,28 @@ export class AssistantDeliveryService extends Service {
|
|
|
724
802
|
readOwnerApprovalStatus: agent => this.readOwnerApprovalStatus(agent),
|
|
725
803
|
provider: config.agentProvider, model: config.agentModel, maxOutputTokens: config.agentMaxOutputTokens,
|
|
726
804
|
maxAutoContinuationTurns: config.agentMaxAutoContinuationTurns,
|
|
805
|
+
taskTimeoutMs: config.agentTaskTimeoutMs,
|
|
727
806
|
goalContinuationTimeoutMs: config.agentGoalContinuationTimeoutMs,
|
|
728
807
|
maxTextBytes: config.maxTextBytes,
|
|
729
808
|
prepareForegroundTaskAcceptance: (binding, envelope) => this.prepareForegroundTaskAcceptance(binding, envelope),
|
|
730
809
|
completeForegroundTaskAcceptance: (handle, input) => this.completeForegroundTaskAcceptance(handle, input),
|
|
731
810
|
recordForegroundTaskModelSelection: (handle, input) => this.deliveryStore.recordForegroundTaskModelSelection({ contractId: handle.contractId, ...input }),
|
|
732
811
|
beginForegroundTaskExecution: (binding, envelope) => this.beginForegroundTaskExecution(binding, envelope),
|
|
733
|
-
recordForegroundExecutionModelSelection: (inboxId, input) =>
|
|
812
|
+
recordForegroundExecutionModelSelection: (inboxId, input) => {
|
|
813
|
+
this.deliveryStore.recordForegroundExecutionModelSelection({ inboxId, ...input });
|
|
814
|
+
if (this.restartStore?.getTask(inboxId) !== undefined) {
|
|
815
|
+
if (this.restartStore.getRequestForTask(inboxId) === undefined) {
|
|
816
|
+
try {
|
|
817
|
+
this.restartStore.observeModel(inboxId, input);
|
|
818
|
+
}
|
|
819
|
+
catch (error) {
|
|
820
|
+
this.context.logger.warn(`assistant-delivery: restart model observation unavailable: ${String(error)}`);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
else
|
|
824
|
+
this.restartStore.captureModel(inboxId, input);
|
|
825
|
+
}
|
|
826
|
+
},
|
|
734
827
|
completeForegroundTaskExecution: (inboxId, input) => this.completeForegroundTaskExecution(inboxId, input),
|
|
735
828
|
modelPickerTtlMs: config.modelPickerTtlMs,
|
|
736
829
|
permissionPickerTtlMs: config.permissionPickerTtlMs,
|
|
@@ -783,7 +876,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
783
876
|
}
|
|
784
877
|
return approvalCtx.assistantPolicy.registerHumanApprovalAnswerer((request, next) => this.requestToolApproval(approvalCtx, request, next));
|
|
785
878
|
}, 'assistant-delivery.tool-approval-route'));
|
|
786
|
-
ctx.inject(['tools'], toolsCtx => registerDeliveryTools(toolsCtx, this));
|
|
879
|
+
ctx.inject(['tools', 'assistantPolicy'], toolsCtx => registerDeliveryTools(toolsCtx, this));
|
|
787
880
|
// rc.1 exposes questions as an Agent-scoped waterfall. Delivery is one
|
|
788
881
|
// answerer in that chain, so Web and other frontends may call `next()` and
|
|
789
882
|
// coexist without a competing service provider or a private ApiProxy seam.
|
|
@@ -834,6 +927,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
834
927
|
}
|
|
835
928
|
await this.stopInternal();
|
|
836
929
|
await this.registry.stop();
|
|
930
|
+
this.restartStore?.close();
|
|
837
931
|
this.deliveryStore.close();
|
|
838
932
|
}, 'assistant-delivery.runtime');
|
|
839
933
|
}
|
|
@@ -1579,6 +1673,28 @@ export class AssistantDeliveryService extends Service {
|
|
|
1579
1673
|
const dispatchedAt = Date.now();
|
|
1580
1674
|
this.deliveryStore.bindForegroundTaskExecution({ inboxId: inbox.id,
|
|
1581
1675
|
scope: { workspace: binding.workspace, preset: binding.agentPreset }, owner, binding, dispatchedAt });
|
|
1676
|
+
try {
|
|
1677
|
+
if (this.restartStore !== undefined) {
|
|
1678
|
+
const route = this.resolveOwnerRoute(this.config.restart.ownerRouteId);
|
|
1679
|
+
if (route.binding.id === binding.id)
|
|
1680
|
+
this.restartStore.beginTask({
|
|
1681
|
+
inboxId: inbox.id, eventId: inbox.eventId, owner,
|
|
1682
|
+
binding: { id: binding.id, version: binding.version, generation: binding.generation,
|
|
1683
|
+
sessionId: binding.sessionId, workspace: binding.workspace, preset: binding.agentPreset,
|
|
1684
|
+
conversation: binding.conversation, principal: binding.principal, policyRef: binding.policyRef,
|
|
1685
|
+
authorityHash: route.snapshot.authorityHash },
|
|
1686
|
+
sourceInvocationId: this.restartInvocationId,
|
|
1687
|
+
maxModelRequests: this.config.restart.maxModelRequests,
|
|
1688
|
+
maxToolCalls: this.config.restart.maxToolCalls,
|
|
1689
|
+
maxOutputTokens: this.config.agentMaxOutputTokens,
|
|
1690
|
+
deadline: dispatchedAt + this.config.restart.taskTimeoutMs,
|
|
1691
|
+
capabilityDigest: this.restartCapabilityDigest(),
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
catch (error) {
|
|
1696
|
+
this.context.logger.warn(`assistant-delivery: restart eligibility observation unavailable: ${String(error)}`);
|
|
1697
|
+
}
|
|
1582
1698
|
this.observeForegroundBegin(Object.freeze({ protocol: 'assistant-delivery/foreground-task/v1', inboxId: inbox.id,
|
|
1583
1699
|
sessionId: binding.sessionId, scope: Object.freeze({ workspace: binding.workspace, preset: binding.agentPreset }),
|
|
1584
1700
|
owner: Object.freeze({ principalRecordId: owner.principalRecordId, principalVersion: owner.principalVersion }),
|
|
@@ -2555,6 +2671,14 @@ export class AssistantDeliveryService extends Service {
|
|
|
2555
2671
|
void this.drainDeliveryPresentations();
|
|
2556
2672
|
return stored;
|
|
2557
2673
|
}
|
|
2674
|
+
progressAuthorized(intent) {
|
|
2675
|
+
if (!this.active)
|
|
2676
|
+
return false;
|
|
2677
|
+
const inbox = this.deliveryStore.getInboxByProviderEvent(intent.target.conversation.channel, intent.target.conversation.account, intent.eventId);
|
|
2678
|
+
const binding = this.deliveryStore.getBinding(intent.bindingId);
|
|
2679
|
+
return inbox !== undefined && binding !== undefined && inbox.bindingId === binding.id
|
|
2680
|
+
&& this.isInboundAuthorized(binding, inbox.envelope);
|
|
2681
|
+
}
|
|
2558
2682
|
isInboundAuthorized(binding, envelope) {
|
|
2559
2683
|
if (!this.active)
|
|
2560
2684
|
return false;
|
|
@@ -3900,6 +4024,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
3900
4024
|
beforeInboxId: record.id,
|
|
3901
4025
|
failureCode: 'new-session-before-dispatch',
|
|
3902
4026
|
});
|
|
4027
|
+
this.restartStore?.cancelForBinding(binding.id);
|
|
3903
4028
|
await this.runtime?.cancelActive?.(binding, 'new');
|
|
3904
4029
|
await this.inbound.cancelUndispatchedClaims(fence.claimedInboxIds, 'new');
|
|
3905
4030
|
const rotated = await this.rotateBinding(envelope, binding, record.id, recoverySignal);
|
|
@@ -3934,6 +4059,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
3934
4059
|
});
|
|
3935
4060
|
// Holding the transition while the live turn drains prevents a later
|
|
3936
4061
|
// message from overtaking the durable stop acknowledgement.
|
|
4062
|
+
this.restartStore?.cancelForBinding(binding.id);
|
|
3937
4063
|
await this.runtime?.cancelActive?.(binding, 'stop');
|
|
3938
4064
|
await this.inbound.cancelUndispatchedClaims(fence.claimedInboxIds, 'stop');
|
|
3939
4065
|
record = this.deliveryStore.queueInbox(record.id, binding.id);
|
|
@@ -3992,6 +4118,62 @@ export class AssistantDeliveryService extends Service {
|
|
|
3992
4118
|
...(input.metadata === undefined ? {} : { metadata: input.metadata }) });
|
|
3993
4119
|
}
|
|
3994
4120
|
#notificationGuard(record) {
|
|
4121
|
+
const key = record.intent.idempotencyKey;
|
|
4122
|
+
if (key.startsWith('inbound:') && (key.endsWith(':maintenance') || key.endsWith(':maintenance-failed'))) {
|
|
4123
|
+
try {
|
|
4124
|
+
const failed = key.endsWith(':maintenance-failed');
|
|
4125
|
+
const inboxId = key.slice('inbound:'.length, -(failed ? ':maintenance-failed'.length : ':maintenance'.length));
|
|
4126
|
+
const request = this.restartStore?.getRequestForTask(inboxId);
|
|
4127
|
+
const task = this.restartStore?.getTask(inboxId);
|
|
4128
|
+
const inbox = this.deliveryStore.getInbox(inboxId);
|
|
4129
|
+
const binding = this.deliveryStore.getBinding(record.intent.bindingId);
|
|
4130
|
+
if (request === undefined || task === undefined || inbox === undefined || binding === undefined
|
|
4131
|
+
|| request.status === 'cancelled' || (failed ? request.status !== 'failed' : !this.restartAuthorized(request))
|
|
4132
|
+
|| task.binding.id !== binding.id || task.binding.version !== binding.version || task.binding.generation !== binding.generation
|
|
4133
|
+
|| task.owner.principalRecordId !== this.ownerLineageForBinding(binding)?.principalRecordId
|
|
4134
|
+
|| task.owner.principalVersion !== this.ownerLineageForBinding(binding)?.principalVersion
|
|
4135
|
+
|| inbox.bindingId !== binding.id || inbox.eventId !== record.intent.replyToEventId
|
|
4136
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)
|
|
4137
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
4138
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
4139
|
+
throw new Error('maintenance notice is stale');
|
|
4140
|
+
const decision = this.policy.evaluate({ subject: { kind: 'agent', id: binding.agentPreset,
|
|
4141
|
+
workspace: binding.workspace, principal: externalPrincipalId(binding.principal) },
|
|
4142
|
+
action: 'reply', resource: { kind: 'message', id: binding.id }, context: { initiator: 'external' } });
|
|
4143
|
+
if (decision.effect !== 'allow')
|
|
4144
|
+
throw new Error('maintenance notice permission changed');
|
|
4145
|
+
}
|
|
4146
|
+
catch {
|
|
4147
|
+
return { outcome: 'not-sent', failureCode: 'maintenance-notice-stale', retryable: false };
|
|
4148
|
+
}
|
|
4149
|
+
return undefined;
|
|
4150
|
+
}
|
|
4151
|
+
if (key.startsWith('inbound:') && key.endsWith(':interrupted')) {
|
|
4152
|
+
try {
|
|
4153
|
+
const inboxId = key.slice('inbound:'.length, -':interrupted'.length);
|
|
4154
|
+
const inbox = this.deliveryStore.getInbox(inboxId);
|
|
4155
|
+
const binding = this.deliveryStore.getBinding(record.intent.bindingId);
|
|
4156
|
+
const expectedNotice = inbox?.failureCode === 'agent-task-timeout'
|
|
4157
|
+
? timedOutTurnNotice : interruptedTurnNotice;
|
|
4158
|
+
if (record.intent.text !== expectedNotice || inbox === undefined || binding === undefined
|
|
4159
|
+
|| !this.deliveryStore.canSendInterruptedInboxNotice(record)
|
|
4160
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)) {
|
|
4161
|
+
throw new Error('interrupted Inbox authority changed');
|
|
4162
|
+
}
|
|
4163
|
+
const reply = this.policy.evaluate({
|
|
4164
|
+
subject: { kind: 'agent', id: binding.agentPreset, workspace: binding.workspace,
|
|
4165
|
+
principal: externalPrincipalId(binding.principal) },
|
|
4166
|
+
action: 'reply', resource: { kind: 'message', id: binding.id },
|
|
4167
|
+
context: { initiator: 'external' },
|
|
4168
|
+
});
|
|
4169
|
+
if (reply.effect !== 'allow')
|
|
4170
|
+
throw new Error('interrupted Inbox reply authority changed');
|
|
4171
|
+
}
|
|
4172
|
+
catch {
|
|
4173
|
+
return { outcome: 'not-sent', failureCode: 'interrupted-inbox-authority-revoked', retryable: false };
|
|
4174
|
+
}
|
|
4175
|
+
return undefined;
|
|
4176
|
+
}
|
|
3995
4177
|
const metadata = record.intent.metadata;
|
|
3996
4178
|
const nativeKeys = Object.keys(metadata ?? {}).filter(key => key.startsWith('dsh.native-notice'));
|
|
3997
4179
|
if (nativeKeys.length > 0) {
|
|
@@ -4604,7 +4786,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
4604
4786
|
throw new AssistantDeliveryError('missing-binding', 'control command binding principal is no longer active');
|
|
4605
4787
|
}
|
|
4606
4788
|
const format = this.replyFormat(binding.conversation, input.format) ?? 'plain';
|
|
4607
|
-
|
|
4789
|
+
const intent = {
|
|
4608
4790
|
idempotencyKey,
|
|
4609
4791
|
bindingId: binding.id,
|
|
4610
4792
|
target: { conversation: binding.conversation, principal: binding.principal },
|
|
@@ -4615,7 +4797,11 @@ export class AssistantDeliveryService extends Service {
|
|
|
4615
4797
|
? {}
|
|
4616
4798
|
: { permissionPicker: input.permissionPicker }),
|
|
4617
4799
|
replyToEventId: input.replyToEventId,
|
|
4618
|
-
}
|
|
4800
|
+
};
|
|
4801
|
+
return input.interruptedInboxId === undefined
|
|
4802
|
+
? this.deliveryStore.enqueue(intent)
|
|
4803
|
+
: this.deliveryStore.enqueueInterruptedInboxNotice({ inboxId: input.interruptedInboxId,
|
|
4804
|
+
binding, intent });
|
|
4619
4805
|
}
|
|
4620
4806
|
/**
|
|
4621
4807
|
* Answers are authored as Markdown, but the coordinator drops an intent whose format the adapter
|
|
@@ -4783,10 +4969,249 @@ export class AssistantDeliveryService extends Service {
|
|
|
4783
4969
|
wiki: this.policy.listProposalStatesForRoute(wikiRoute, 5),
|
|
4784
4970
|
};
|
|
4785
4971
|
}
|
|
4972
|
+
restartCapabilityDigest() {
|
|
4973
|
+
return restartDigest({ protocol: 'assistant-delivery/restart/v1', version, code: this.restartCodeDigest,
|
|
4974
|
+
config: this.config.restart, maxOutputTokens: this.config.agentMaxOutputTokens });
|
|
4975
|
+
}
|
|
4976
|
+
restartAuthorized(record) {
|
|
4977
|
+
try {
|
|
4978
|
+
if (!this.active || this.restartStore === undefined || this.restartController.signal.aborted)
|
|
4979
|
+
return false;
|
|
4980
|
+
const current = record.id === undefined ? this.restartStore.getRequestForTask(record.inboxId) : this.restartStore.getRequest(record.id);
|
|
4981
|
+
const task = this.restartStore.getTask(record.inboxId);
|
|
4982
|
+
if ((record.id !== undefined && current === undefined) || task === undefined
|
|
4983
|
+
|| (current !== undefined && ['cancelled', 'failed', 'completed'].includes(current.status))
|
|
4984
|
+
|| Date.now() >= task.deadline || task.capabilityDigest !== this.restartCapabilityDigest())
|
|
4985
|
+
return false;
|
|
4986
|
+
const route = this.resolveOwnerRoute(this.config.restart.ownerRouteId);
|
|
4987
|
+
const binding = route.binding;
|
|
4988
|
+
const owner = this.ownerLineageForBinding(binding);
|
|
4989
|
+
const inbox = this.deliveryStore.getInbox(record.inboxId);
|
|
4990
|
+
if (owner?.principalRecordId !== task.owner.principalRecordId || owner.principalVersion !== task.owner.principalVersion
|
|
4991
|
+
|| binding.id !== task.binding.id || binding.version !== task.binding.version || binding.generation !== task.binding.generation
|
|
4992
|
+
|| binding.sessionId !== task.binding.sessionId || binding.workspace !== task.binding.workspace
|
|
4993
|
+
|| binding.agentPreset !== task.binding.preset || binding.policyRef !== task.binding.policyRef
|
|
4994
|
+
|| route.snapshot.authorityHash !== task.binding.authorityHash
|
|
4995
|
+
|| inbox?.bindingId !== binding.id || inbox.eventId !== task.eventId
|
|
4996
|
+
|| !['claimed', 'processed'].includes(inbox.status) || !this.isInboundAuthorized(binding, inbox.envelope)
|
|
4997
|
+
|| this.deliveryStore.hasRestartCancellationAfter(inbox.id, binding)
|
|
4998
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
4999
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
5000
|
+
return false;
|
|
5001
|
+
const decision = this.policy.evaluate({ subject: { kind: 'agent', id: binding.agentPreset,
|
|
5002
|
+
workspace: binding.workspace, principal: externalPrincipalId(binding.principal) },
|
|
5003
|
+
action: 'maintenance.restart', resource: { kind: 'message', id: `maintenance:${this.config.restart.ownerRouteId}` }, context: { initiator: 'external' } });
|
|
5004
|
+
return decision.effect === 'allow';
|
|
5005
|
+
}
|
|
5006
|
+
catch {
|
|
5007
|
+
return false;
|
|
5008
|
+
}
|
|
5009
|
+
}
|
|
5010
|
+
/** Host-only maintenance gate shared by the native background scheduler. */
|
|
5011
|
+
isMaintenanceDraining() { return this.active && this.deliveryStore.getRestartFence() !== undefined; }
|
|
5012
|
+
restartDispatchPermitted(agent) {
|
|
5013
|
+
if (!this.active)
|
|
5014
|
+
return false;
|
|
5015
|
+
const fence = this.deliveryStore.getRestartFence();
|
|
5016
|
+
if (fence === undefined)
|
|
5017
|
+
return true;
|
|
5018
|
+
const request = this.restartStore?.getRequest(fence.requestId);
|
|
5019
|
+
const task = request === undefined ? undefined : this.restartStore?.getTask(request.inboxId);
|
|
5020
|
+
return request?.status === 'resuming' && task !== undefined && agent?.session.id === task.binding.sessionId
|
|
5021
|
+
&& this.restartAuthorized(request);
|
|
5022
|
+
}
|
|
5023
|
+
/** Run only from native maintenance after the continuation notice is durably flushed. */
|
|
5024
|
+
releaseRestartFenceOnAdmission(record, sessionId) {
|
|
5025
|
+
const current = this.restartStore?.getRequest(record.id);
|
|
5026
|
+
const task = this.restartStore?.getTask(record.inboxId);
|
|
5027
|
+
const fence = this.deliveryStore.getRestartFence();
|
|
5028
|
+
if (current?.status !== 'resuming' || current.continuationHolderId !== this.ownerId
|
|
5029
|
+
|| current.restartInvocationId !== this.restartInvocationId
|
|
5030
|
+
|| current.sourceInvocationId === this.restartInvocationId
|
|
5031
|
+
|| current.inboxId !== record.inboxId || task?.binding.sessionId !== sessionId
|
|
5032
|
+
|| fence?.requestId !== record.id || fence.sourceInvocationId !== record.sourceInvocationId
|
|
5033
|
+
|| !this.restartAuthorized(current)) {
|
|
5034
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart admission no longer owns the maintenance fence');
|
|
5035
|
+
}
|
|
5036
|
+
this.deliveryStore.clearRestartFence(record.id);
|
|
5037
|
+
if (this.deliveryStore.getRestartFence()?.requestId === record.id) {
|
|
5038
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart maintenance fence could not be released');
|
|
5039
|
+
}
|
|
5040
|
+
}
|
|
5041
|
+
otherRestartWorkQuiescent(agent) {
|
|
5042
|
+
if (this.context.get('agents')?.list().some(other => other !== agent))
|
|
5043
|
+
return false;
|
|
5044
|
+
const automations = this.context.get('assistantAutomations');
|
|
5045
|
+
return automations === undefined || automations.isRestartQuiescent?.() === true;
|
|
5046
|
+
}
|
|
5047
|
+
canRequestRestart(agent) {
|
|
5048
|
+
return this.active && agent !== undefined && this.otherRestartWorkQuiescent(agent)
|
|
5049
|
+
&& this.runtime?.canRequestRestart?.(agent) === true;
|
|
5050
|
+
}
|
|
5051
|
+
requestRestart(agent, reason) {
|
|
5052
|
+
this.assertActive();
|
|
5053
|
+
if (agent === undefined || this.restartStore === undefined || this.runtime?.requestRestart === undefined) {
|
|
5054
|
+
throw new AssistantDeliveryError('runtime-unavailable', 'planned restart is not configured for this owner');
|
|
5055
|
+
}
|
|
5056
|
+
if (!this.otherRestartWorkQuiescent(agent))
|
|
5057
|
+
throw new AssistantDeliveryError('runtime-conflict', 'other Host work has not drained');
|
|
5058
|
+
const result = this.runtime.requestRestart(agent, reason);
|
|
5059
|
+
try {
|
|
5060
|
+
this.deliveryStore.setRestartFence(result.id, this.restartInvocationId);
|
|
5061
|
+
}
|
|
5062
|
+
catch (error) {
|
|
5063
|
+
this.restartStore.fail(result.id, 'restart-drain-fence-unavailable');
|
|
5064
|
+
throw error;
|
|
5065
|
+
}
|
|
5066
|
+
return result;
|
|
5067
|
+
}
|
|
5068
|
+
get supportsPlannedRestart() { return this.restartStore !== undefined; }
|
|
5069
|
+
/** Host-side registration of an exact, currently mounted read-only definition. */
|
|
5070
|
+
registerRestartSafeTool(caller, definition) {
|
|
5071
|
+
if (this.restartStore === undefined)
|
|
5072
|
+
return () => { };
|
|
5073
|
+
const trusted = definition.name === 'delivery_status'
|
|
5074
|
+
? caller.fiber.name === 'AssistantDeliveryService'
|
|
5075
|
+
&& caller.fiber.parent.fiber === this.context.fiber
|
|
5076
|
+
: definition.name === 'wiki_read' || definition.name === 'wiki_search'
|
|
5077
|
+
? caller.fiber.name === 'PersonalWikiService' || caller.fiber.name === 'dsh-enhanced-personal-wiki'
|
|
5078
|
+
: definition.name === 'memory_search'
|
|
5079
|
+
&& (caller.fiber.name === 'PersonalMemoryService' || caller.fiber.name === 'dsh-enhanced-personal-memory');
|
|
5080
|
+
// The installed Cordis fork declares FiberState as an ambient const enum;
|
|
5081
|
+
// its ACTIVE and LOADING numeric values are 2 and 1, respectively.
|
|
5082
|
+
if (!trusted || caller.fiber.uid === null || (caller.fiber.state !== 1 && caller.fiber.state !== 2)) {
|
|
5083
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart-safe tool registration requires a trusted live provider');
|
|
5084
|
+
}
|
|
5085
|
+
const entry = { caller, definition };
|
|
5086
|
+
this.restartSafeDefinitions.add(entry);
|
|
5087
|
+
const remove = () => { this.restartSafeDefinitions.delete(entry); };
|
|
5088
|
+
try {
|
|
5089
|
+
caller.effect(() => remove, 'assistant-delivery.restart-safe-tool');
|
|
5090
|
+
}
|
|
5091
|
+
catch (error) {
|
|
5092
|
+
remove();
|
|
5093
|
+
throw error;
|
|
5094
|
+
}
|
|
5095
|
+
return remove;
|
|
5096
|
+
}
|
|
5097
|
+
isRestartSafeTool(agent, name) {
|
|
5098
|
+
if (!this.active || this.restartStore === undefined)
|
|
5099
|
+
return false;
|
|
5100
|
+
let current;
|
|
5101
|
+
try {
|
|
5102
|
+
current = agent.ctx.tools.get(name, agent);
|
|
5103
|
+
}
|
|
5104
|
+
catch {
|
|
5105
|
+
return false;
|
|
5106
|
+
}
|
|
5107
|
+
if (current === undefined)
|
|
5108
|
+
return false;
|
|
5109
|
+
for (const entry of this.restartSafeDefinitions) {
|
|
5110
|
+
if (entry.definition === current && entry.definition.name === name
|
|
5111
|
+
&& entry.caller.fiber.uid !== null && entry.caller.fiber.state === 2)
|
|
5112
|
+
return true;
|
|
5113
|
+
}
|
|
5114
|
+
return false;
|
|
5115
|
+
}
|
|
5116
|
+
/** Startup/tick reconciliation uses the existing native runtime, never requeues the old Inbox. */
|
|
5117
|
+
drainRestartContinuations() {
|
|
5118
|
+
if (this.restartFlight !== undefined || this.restartStore === undefined || this.runtime?.resumeRestart === undefined)
|
|
5119
|
+
return;
|
|
5120
|
+
const store = this.restartStore;
|
|
5121
|
+
const pending = store.listRequests({ statuses: ['requested', 'checkpoint', 'ready', 'restarting', 'restarted', 'resuming'], limit: 100 });
|
|
5122
|
+
for (const record of pending) {
|
|
5123
|
+
if (Date.now() >= (store.getTask(record.inboxId)?.deadline ?? 0))
|
|
5124
|
+
store.fail(record.id, 'restart-deadline-expired');
|
|
5125
|
+
else if (['requested', 'checkpoint', 'ready'].includes(record.status) && Date.now() >= record.createdAt + 120_000) {
|
|
5126
|
+
store.fail(record.id, 'restart-handoff-timeout');
|
|
5127
|
+
}
|
|
5128
|
+
else if (record.sourceInvocationId !== this.restartInvocationId
|
|
5129
|
+
&& ['requested', 'checkpoint'].includes(record.status))
|
|
5130
|
+
store.fail(record.id, 'restart-checkpoint-not-quiescent');
|
|
5131
|
+
else if (record.status === 'resuming' && record.continuationHolderId !== this.ownerId) {
|
|
5132
|
+
// A later restart must not blindly execute a possibly dispatched continuation.
|
|
5133
|
+
store.fail(record.id, 'restart-continuation-outcome-unknown');
|
|
5134
|
+
}
|
|
5135
|
+
}
|
|
5136
|
+
const candidate = pending.find(record => record.status === 'restarted' && store.getRequest(record.id)?.status === 'restarted'
|
|
5137
|
+
&& record.restartInvocationId === this.restartInvocationId && record.sourceInvocationId !== this.restartInvocationId);
|
|
5138
|
+
if (candidate === undefined)
|
|
5139
|
+
return;
|
|
5140
|
+
if (!this.restartAuthorized(candidate)) {
|
|
5141
|
+
store.fail(candidate.id, 'restart-authority-changed');
|
|
5142
|
+
return;
|
|
5143
|
+
}
|
|
5144
|
+
const task = store.getTask(candidate.inboxId);
|
|
5145
|
+
const binding = this.deliveryStore.getBinding(task.binding.id);
|
|
5146
|
+
const inbox = this.deliveryStore.getInbox(candidate.inboxId);
|
|
5147
|
+
const claimed = store.claimContinuation(candidate.id, { holderId: this.ownerId, invocationId: this.restartInvocationId });
|
|
5148
|
+
const runtime = this.runtime;
|
|
5149
|
+
const flight = (async () => {
|
|
5150
|
+
try {
|
|
5151
|
+
await runtime.resumeRestart(claimed, binding, inbox.envelope, this.restartController.signal);
|
|
5152
|
+
}
|
|
5153
|
+
catch (error) {
|
|
5154
|
+
if (this.active) {
|
|
5155
|
+
const current = store.getRequest(claimed.id);
|
|
5156
|
+
if (current?.status === 'resuming')
|
|
5157
|
+
store.fail(claimed.id, 'restart-continuation-failed');
|
|
5158
|
+
this.context.logger.warn(`assistant-delivery: restart continuation failed: ${String(error)}`);
|
|
5159
|
+
}
|
|
5160
|
+
}
|
|
5161
|
+
})().finally(() => { if (this.restartFlight === flight)
|
|
5162
|
+
this.restartFlight = undefined; });
|
|
5163
|
+
this.restartFlight = flight;
|
|
5164
|
+
}
|
|
5165
|
+
drainRestartFailureNotices() {
|
|
5166
|
+
if (this.restartStore === undefined)
|
|
5167
|
+
return;
|
|
5168
|
+
let records = this.restartStore.listRequests({ statuses: ['failed'], limit: 100,
|
|
5169
|
+
...(this.restartNoticeCursor === undefined ? {} : { after: this.restartNoticeCursor }) });
|
|
5170
|
+
if (records.length === 0 && this.restartNoticeCursor !== undefined) {
|
|
5171
|
+
this.restartNoticeCursor = undefined;
|
|
5172
|
+
records = this.restartStore.listRequests({ statuses: ['failed'], limit: 100 });
|
|
5173
|
+
}
|
|
5174
|
+
for (const record of records) {
|
|
5175
|
+
this.restartNoticeCursor = { createdAt: record.createdAt, id: record.id };
|
|
5176
|
+
const task = this.restartStore.getTask(record.inboxId);
|
|
5177
|
+
const inbox = this.deliveryStore.getInbox(record.inboxId);
|
|
5178
|
+
const binding = task === undefined ? undefined : this.deliveryStore.getBinding(task.binding.id);
|
|
5179
|
+
const key = `inbound:${record.inboxId}:maintenance-failed`;
|
|
5180
|
+
if (task === undefined || inbox === undefined || binding?.status !== 'active'
|
|
5181
|
+
|| binding.version !== task.binding.version || binding.generation !== task.binding.generation
|
|
5182
|
+
|| this.deliveryStore.getActiveBinding(binding.conversation)?.id !== binding.id
|
|
5183
|
+
|| this.ownerLineageForBinding(binding)?.principalRecordId !== task.owner.principalRecordId
|
|
5184
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)
|
|
5185
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(key) !== undefined
|
|
5186
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
5187
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
5188
|
+
continue;
|
|
5189
|
+
try {
|
|
5190
|
+
this.replyCommand(binding, { idempotencyKey: key, replyToEventId: inbox.eventId,
|
|
5191
|
+
text: '维护恢复未能安全完成,当前任务已停止自动继续,服务仍可接收新请求。已保存原会话;可能已经执行的操作不会自动重做。' });
|
|
5192
|
+
}
|
|
5193
|
+
catch (error) {
|
|
5194
|
+
if (this.active)
|
|
5195
|
+
this.context.logger.warn(`assistant-delivery: restart failure notice: ${String(error)}`);
|
|
5196
|
+
}
|
|
5197
|
+
}
|
|
5198
|
+
}
|
|
4786
5199
|
async tick() {
|
|
4787
5200
|
this.assertActive();
|
|
4788
5201
|
await this.drainInboundAdmissions();
|
|
4789
|
-
|
|
5202
|
+
this.drainRestartContinuations();
|
|
5203
|
+
const restartFence = this.deliveryStore.getRestartFence();
|
|
5204
|
+
if (restartFence !== undefined) {
|
|
5205
|
+
const record = this.restartStore?.getRequest(restartFence.requestId);
|
|
5206
|
+
if (record === undefined || ['failed', 'cancelled', 'completed'].includes(record.status)) {
|
|
5207
|
+
this.deliveryStore.clearRestartFence(restartFence.requestId);
|
|
5208
|
+
}
|
|
5209
|
+
}
|
|
5210
|
+
const maintenance = this.restartStore?.listRequests({ statuses: ['requested', 'checkpoint', 'ready', 'restarting', 'restarted'], limit: 1 });
|
|
5211
|
+
if (maintenance === undefined || maintenance.length === 0)
|
|
5212
|
+
await this.inbound.tick();
|
|
5213
|
+
this.drainRestartFailureNotices();
|
|
5214
|
+
await this.drainInterruptedInboxNotices();
|
|
4790
5215
|
await this.drainModelSelectionSettlements();
|
|
4791
5216
|
this.drainApprovalDispatches();
|
|
4792
5217
|
await this.outbound.tick();
|
|
@@ -4806,6 +5231,8 @@ export class AssistantDeliveryService extends Service {
|
|
|
4806
5231
|
this.naturalObjectiveFlight,
|
|
4807
5232
|
this.workflowTraceFlight,
|
|
4808
5233
|
this.inboundAdmissionRecoveryFlight,
|
|
5234
|
+
this.interruptedInboxNoticeFlight,
|
|
5235
|
+
this.restartFlight,
|
|
4809
5236
|
]);
|
|
4810
5237
|
}
|
|
4811
5238
|
start() {
|
|
@@ -4855,6 +5282,59 @@ export class AssistantDeliveryService extends Service {
|
|
|
4855
5282
|
this.inboundAdmissionRecoveryFlight = flight;
|
|
4856
5283
|
return flight;
|
|
4857
5284
|
}
|
|
5285
|
+
drainInterruptedInboxNotices() {
|
|
5286
|
+
if (this.interruptedInboxNoticeFlight !== undefined)
|
|
5287
|
+
return this.interruptedInboxNoticeFlight;
|
|
5288
|
+
const flight = this.runInterruptedInboxNotices()
|
|
5289
|
+
.finally(() => { if (this.interruptedInboxNoticeFlight === flight)
|
|
5290
|
+
this.interruptedInboxNoticeFlight = undefined; });
|
|
5291
|
+
this.interruptedInboxNoticeFlight = flight;
|
|
5292
|
+
return flight;
|
|
5293
|
+
}
|
|
5294
|
+
async runInterruptedInboxNotices() {
|
|
5295
|
+
let pending = this.deliveryStore.listInterruptedInboxNotices({
|
|
5296
|
+
afterSequence: this.interruptedInboxNoticeAfterSequence, limit: 100,
|
|
5297
|
+
});
|
|
5298
|
+
if (pending.length === 0 && this.interruptedInboxNoticeAfterSequence !== 0) {
|
|
5299
|
+
this.interruptedInboxNoticeAfterSequence = 0;
|
|
5300
|
+
pending = this.deliveryStore.listInterruptedInboxNotices({ limit: 100 });
|
|
5301
|
+
}
|
|
5302
|
+
for (const inbox of pending) {
|
|
5303
|
+
if (!this.active)
|
|
5304
|
+
return;
|
|
5305
|
+
this.interruptedInboxNoticeAfterSequence = inbox.admissionCursor.sequence;
|
|
5306
|
+
try {
|
|
5307
|
+
const current = this.deliveryStore.getInbox(inbox.id);
|
|
5308
|
+
if (current?.status !== 'dead_letter'
|
|
5309
|
+
|| (current.failureCode !== 'dispatch-ambiguous' && current.failureCode !== 'agent-task-timeout')
|
|
5310
|
+
|| current.bindingId === undefined
|
|
5311
|
+
|| this.deliveryStore.getDeadLetterResolution({ kind: 'inbox', id: current.id,
|
|
5312
|
+
attemptCount: current.attemptCount }) !== undefined
|
|
5313
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.id}:reply`) !== undefined
|
|
5314
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.eventId}:reply`) !== undefined
|
|
5315
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.id}:interrupted`) !== undefined)
|
|
5316
|
+
continue;
|
|
5317
|
+
const binding = this.deliveryStore.getBinding(current.bindingId);
|
|
5318
|
+
if (binding?.status !== 'active'
|
|
5319
|
+
|| this.deliveryStore.getActiveBinding(current.envelope.conversation)?.id !== binding.id
|
|
5320
|
+
|| this.deliveryStore.getPrincipal(binding.principal)?.role !== 'owner'
|
|
5321
|
+
|| !this.isInboundAuthorized(binding, current.envelope)
|
|
5322
|
+
|| this.registry.get(binding.conversation.channel, binding.conversation.account) === undefined)
|
|
5323
|
+
continue;
|
|
5324
|
+
this.replyCommand(binding, { idempotencyKey: `inbound:${current.id}:interrupted`,
|
|
5325
|
+
interruptedInboxId: current.id, replyToEventId: current.eventId,
|
|
5326
|
+
text: current.failureCode === 'agent-task-timeout' ? timedOutTurnNotice : interruptedTurnNotice });
|
|
5327
|
+
}
|
|
5328
|
+
catch (error) {
|
|
5329
|
+
if (!this.active)
|
|
5330
|
+
return;
|
|
5331
|
+
if ((error instanceof AssistantDeliveryError && (error.code === 'policy-denied' || error.code === 'missing-binding'))
|
|
5332
|
+
|| (error instanceof DeliveryStoreError && error.code === 'version-conflict'))
|
|
5333
|
+
continue;
|
|
5334
|
+
this.context.logger.warn(`assistant-delivery: interrupted Inbox notice ${inbox.id}: ${String(error)}`);
|
|
5335
|
+
}
|
|
5336
|
+
}
|
|
5337
|
+
}
|
|
4858
5338
|
async runInboundAdmissionRecovery() {
|
|
4859
5339
|
if (this.runtime === undefined)
|
|
4860
5340
|
return;
|
|
@@ -5040,6 +5520,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
5040
5520
|
this.preferenceProjectionRetryTimer = undefined;
|
|
5041
5521
|
this.cancelModelSelectionWaiters();
|
|
5042
5522
|
this.inboundAdmissionRecoveryController.abort(new Error('assistant-delivery is stopping'));
|
|
5523
|
+
this.restartController.abort(new Error('assistant-delivery is stopping'));
|
|
5043
5524
|
const admissionRecovery = this.inboundAdmissionRecoveryFlight === undefined
|
|
5044
5525
|
? undefined
|
|
5045
5526
|
: Promise.race([
|
|
@@ -5058,6 +5539,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
5058
5539
|
this.preferenceProjectionFlight,
|
|
5059
5540
|
this.naturalObjectiveFlight,
|
|
5060
5541
|
this.workflowTraceFlight,
|
|
5542
|
+
this.restartFlight,
|
|
5061
5543
|
]);
|
|
5062
5544
|
}
|
|
5063
5545
|
drainPreferenceProjections() {
|