@dsh-enhanced/assistant-delivery 0.1.62 → 0.1.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -10
- package/bin/dsh-restart-supervisor.js +76 -0
- package/lib/agent-runtime.d.ts +19 -1
- package/lib/agent-runtime.d.ts.map +1 -1
- package/lib/agent-runtime.js +122 -15
- package/lib/agent-runtime.js.map +1 -1
- package/lib/operator-snapshot.d.ts +1 -1
- package/lib/operator-snapshot.d.ts.map +1 -1
- package/lib/operator-snapshot.js +4 -4
- package/lib/operator-snapshot.js.map +1 -1
- package/lib/restart-native.d.ts +47 -0
- package/lib/restart-native.d.ts.map +1 -0
- package/lib/restart-native.js +170 -0
- package/lib/restart-native.js.map +1 -0
- package/lib/restart-runtime.d.ts +38 -0
- package/lib/restart-runtime.d.ts.map +1 -0
- package/lib/restart-runtime.js +335 -0
- package/lib/restart-runtime.js.map +1 -0
- package/lib/restart-store.d.ts +132 -0
- package/lib/restart-store.d.ts.map +1 -0
- package/lib/restart-store.js +757 -0
- package/lib/restart-store.js.map +1 -0
- package/lib/restart-supervisor.d.ts +129 -0
- package/lib/restart-supervisor.d.ts.map +1 -0
- package/lib/restart-supervisor.js +615 -0
- package/lib/restart-supervisor.js.map +1 -0
- package/lib/service.d.ts +42 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +455 -6
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +44 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +23 -0
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +222 -85
- package/lib/store.js.map +1 -1
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +21 -2
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +2 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +11 -10
package/lib/service.js
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
import { chmodSync, mkdirSync } from 'node:fs';
|
|
1
|
+
import { chmodSync, mkdirSync, readFileSync } from 'node:fs';
|
|
2
2
|
import { isAbsolute } from 'node:path';
|
|
3
3
|
import { createHash, randomUUID } from 'node:crypto';
|
|
4
4
|
import { Context, Service } from '@deepseek-ai/cordis';
|
|
5
5
|
import Schema from '@deepseek-ai/schemastery';
|
|
6
|
+
import { RestartStore } from './restart-store.js';
|
|
7
|
+
import { restartDigest } from './restart-runtime.js';
|
|
8
|
+
import { version } from './version.js';
|
|
6
9
|
import { acceptanceCanonicalJson, goalDefinitionSituation } from '@dsh-enhanced/task-acceptance-contract';
|
|
7
10
|
import { approvalReviewerOf, } from '@dsh-enhanced/assistant-policy';
|
|
8
11
|
// Cross-package wire literal, kept locally so this bundle can load without the
|
|
@@ -63,6 +66,7 @@ const goalOutcomeFeedbackFooter = [
|
|
|
63
66
|
'`helpful` 等只记录偏好,不会被当成目标成败。',
|
|
64
67
|
].join('\n');
|
|
65
68
|
const goalWakeCompletionNotice = '目标已通过独立验收并完成。';
|
|
69
|
+
const interruptedTurnNotice = '这次请求在执行过程中被中断,当前已停止等待;已有操作的结果尚未确认,因此没有自动重跑。你可以继续发送消息;涉及写入的任务需要先核对现有结果。';
|
|
66
70
|
function isGoalOutcomeFeedbackAuthority(value) {
|
|
67
71
|
return typeof value === 'object' && value !== null
|
|
68
72
|
&& typeof value.issueOwnerGoalOutcomeFeedbackTarget === 'function'
|
|
@@ -268,6 +272,14 @@ function isOwnerGoalSnapshotLocator(value) {
|
|
|
268
272
|
const configSchema = Schema.object({
|
|
269
273
|
databasePath: Schema.string().required(),
|
|
270
274
|
spoolPath: Schema.string().required(),
|
|
275
|
+
restart: Schema.union([Schema.object({
|
|
276
|
+
databasePath: Schema.string().required(),
|
|
277
|
+
ownerRouteId: Schema.string().min(1).required(),
|
|
278
|
+
maxModelRequests: Schema.number().step(1).min(1).max(256).required(),
|
|
279
|
+
maxToolCalls: Schema.number().step(1).min(1).max(1024).required(),
|
|
280
|
+
taskTimeoutMs: Schema.number().step(1).min(60_000).max(3_600_000).required(),
|
|
281
|
+
allowedTools: Schema.array(Schema.string().min(1)).required(),
|
|
282
|
+
})]),
|
|
271
283
|
schedulerEnabled: Schema.boolean().default(true),
|
|
272
284
|
tickIntervalMs: Schema.number().step(1).min(100).default(1_000),
|
|
273
285
|
leaseMs: Schema.number().step(1).min(1_000).default(30_000),
|
|
@@ -569,6 +581,13 @@ function exactToolApprovalCall(events, openTurn, callId, toolName) {
|
|
|
569
581
|
export class AssistantDeliveryService extends Service {
|
|
570
582
|
static Config = configSchema;
|
|
571
583
|
deliveryStore;
|
|
584
|
+
restartStore;
|
|
585
|
+
restartInvocationId = process.env.INVOCATION_ID ?? `standalone-${randomUUID()}`;
|
|
586
|
+
restartCodeDigest = '';
|
|
587
|
+
restartNoticeCursor;
|
|
588
|
+
restartFlight;
|
|
589
|
+
restartController = new AbortController();
|
|
590
|
+
restartSafeDefinitions = new Set();
|
|
572
591
|
context;
|
|
573
592
|
policy;
|
|
574
593
|
registry;
|
|
@@ -610,6 +629,8 @@ export class AssistantDeliveryService extends Service {
|
|
|
610
629
|
preferenceProjectionFlight;
|
|
611
630
|
inboundAdmissionRecoveryFlight;
|
|
612
631
|
inboundAdmissionRecoveryAfterSequence = 0;
|
|
632
|
+
interruptedInboxNoticeFlight;
|
|
633
|
+
interruptedInboxNoticeAfterSequence = 0;
|
|
613
634
|
inboundAdmissionRecoveryController = new AbortController();
|
|
614
635
|
workflowTraceRetryTimer;
|
|
615
636
|
preferenceProjectionRetryTimer;
|
|
@@ -666,6 +687,36 @@ export class AssistantDeliveryService extends Service {
|
|
|
666
687
|
};
|
|
667
688
|
this.ownerRouteGuard = Object.freeze(ownerRouteGuard);
|
|
668
689
|
this.deliveryStore = new DeliveryStore({ path: config.databasePath, maxTextBytes: config.maxTextBytes });
|
|
690
|
+
if (config.restart !== undefined && (!isAbsolute(config.restart.databasePath)
|
|
691
|
+
|| config.restart.databasePath === config.databasePath || !this.ownerRoutes.has(config.restart.ownerRouteId))) {
|
|
692
|
+
throw new Error('assistant-delivery: restart requires a separate absolute journal and an exact configured owner route');
|
|
693
|
+
}
|
|
694
|
+
try {
|
|
695
|
+
this.restartStore = config.restart === undefined ? undefined : new RestartStore({ path: config.restart.databasePath });
|
|
696
|
+
}
|
|
697
|
+
catch (error) {
|
|
698
|
+
this.restartStore = undefined;
|
|
699
|
+
ctx.logger.warn(`assistant-delivery: planned restart disabled; journal unavailable: ${String(error)}`);
|
|
700
|
+
}
|
|
701
|
+
if (this.restartStore !== undefined) {
|
|
702
|
+
const extension = import.meta.url.endsWith('.ts') ? 'ts' : 'js';
|
|
703
|
+
this.restartCodeDigest = restartDigest(['service', 'agent-runtime', 'restart-runtime', 'restart-native', 'restart-store', 'tools']
|
|
704
|
+
.map(name => createHash('sha256').update(readFileSync(new URL(`./${name}.${extension}`, import.meta.url))).digest('hex')));
|
|
705
|
+
}
|
|
706
|
+
if (this.restartStore !== undefined) {
|
|
707
|
+
const permitted = () => this.restartDispatchPermitted(ctx.get('agents')?.currentInitiator());
|
|
708
|
+
ctx.on('agent/pre-step', async (_payload, next) => permitted() ? await next() : { kind: 'reject' }, { prepend: true });
|
|
709
|
+
ctx.on('llm/stream', async function* (_options, next) {
|
|
710
|
+
if (!permitted())
|
|
711
|
+
throw new Error('assistant-delivery: Host is draining for planned maintenance');
|
|
712
|
+
yield* next();
|
|
713
|
+
}, { prepend: true });
|
|
714
|
+
ctx.on('tools/execute', async (execution, next) => {
|
|
715
|
+
if (!this.restartDispatchPermitted(execution.agent))
|
|
716
|
+
throw new Error('assistant-delivery: Host is draining for planned maintenance');
|
|
717
|
+
return await next();
|
|
718
|
+
}, { prepend: true });
|
|
719
|
+
}
|
|
669
720
|
ctx.on('agent/inbox/claimed', ({ agent, message, turn }) => {
|
|
670
721
|
this.capturePreferenceTurn(agent, message.source, turn);
|
|
671
722
|
});
|
|
@@ -717,6 +768,13 @@ export class AssistantDeliveryService extends Service {
|
|
|
717
768
|
runtimeCtx.effect(() => () => { if (this.nativeWebRuntime === webRuntime)
|
|
718
769
|
this.nativeWebRuntime = undefined; });
|
|
719
770
|
const unregister = this.registerInboundRuntime(new DshDeliveryRuntime(runtimeCtx, policy, {
|
|
771
|
+
...(this.restartStore === undefined ? {} : { restart: { store: this.restartStore, config: config.restart,
|
|
772
|
+
invocationId: this.restartInvocationId,
|
|
773
|
+
authorize: (record) => this.restartAuthorized(record),
|
|
774
|
+
authorizeTask: (inboxId) => this.restartAuthorized({ inboxId }),
|
|
775
|
+
onContinuationAdmitted: (record, sessionId) => this.releaseRestartFenceOnAdmission(record, sessionId),
|
|
776
|
+
isRestartSafeTool: (agent, name) => this.isRestartSafeTool(agent, name),
|
|
777
|
+
} }),
|
|
720
778
|
sessionLeases,
|
|
721
779
|
sessionNamespace: this.deliveryStore.instanceId(),
|
|
722
780
|
workspace: config.defaultWorkspace, agentPreset: config.defaultAgentPreset, policyRef: config.policyRef,
|
|
@@ -730,7 +788,21 @@ export class AssistantDeliveryService extends Service {
|
|
|
730
788
|
completeForegroundTaskAcceptance: (handle, input) => this.completeForegroundTaskAcceptance(handle, input),
|
|
731
789
|
recordForegroundTaskModelSelection: (handle, input) => this.deliveryStore.recordForegroundTaskModelSelection({ contractId: handle.contractId, ...input }),
|
|
732
790
|
beginForegroundTaskExecution: (binding, envelope) => this.beginForegroundTaskExecution(binding, envelope),
|
|
733
|
-
recordForegroundExecutionModelSelection: (inboxId, input) =>
|
|
791
|
+
recordForegroundExecutionModelSelection: (inboxId, input) => {
|
|
792
|
+
this.deliveryStore.recordForegroundExecutionModelSelection({ inboxId, ...input });
|
|
793
|
+
if (this.restartStore?.getTask(inboxId) !== undefined) {
|
|
794
|
+
if (this.restartStore.getRequestForTask(inboxId) === undefined) {
|
|
795
|
+
try {
|
|
796
|
+
this.restartStore.observeModel(inboxId, input);
|
|
797
|
+
}
|
|
798
|
+
catch (error) {
|
|
799
|
+
this.context.logger.warn(`assistant-delivery: restart model observation unavailable: ${String(error)}`);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
else
|
|
803
|
+
this.restartStore.captureModel(inboxId, input);
|
|
804
|
+
}
|
|
805
|
+
},
|
|
734
806
|
completeForegroundTaskExecution: (inboxId, input) => this.completeForegroundTaskExecution(inboxId, input),
|
|
735
807
|
modelPickerTtlMs: config.modelPickerTtlMs,
|
|
736
808
|
permissionPickerTtlMs: config.permissionPickerTtlMs,
|
|
@@ -783,7 +855,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
783
855
|
}
|
|
784
856
|
return approvalCtx.assistantPolicy.registerHumanApprovalAnswerer((request, next) => this.requestToolApproval(approvalCtx, request, next));
|
|
785
857
|
}, 'assistant-delivery.tool-approval-route'));
|
|
786
|
-
ctx.inject(['tools'], toolsCtx => registerDeliveryTools(toolsCtx, this));
|
|
858
|
+
ctx.inject(['tools', 'assistantPolicy'], toolsCtx => registerDeliveryTools(toolsCtx, this));
|
|
787
859
|
// rc.1 exposes questions as an Agent-scoped waterfall. Delivery is one
|
|
788
860
|
// answerer in that chain, so Web and other frontends may call `next()` and
|
|
789
861
|
// coexist without a competing service provider or a private ApiProxy seam.
|
|
@@ -834,6 +906,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
834
906
|
}
|
|
835
907
|
await this.stopInternal();
|
|
836
908
|
await this.registry.stop();
|
|
909
|
+
this.restartStore?.close();
|
|
837
910
|
this.deliveryStore.close();
|
|
838
911
|
}, 'assistant-delivery.runtime');
|
|
839
912
|
}
|
|
@@ -1579,6 +1652,28 @@ export class AssistantDeliveryService extends Service {
|
|
|
1579
1652
|
const dispatchedAt = Date.now();
|
|
1580
1653
|
this.deliveryStore.bindForegroundTaskExecution({ inboxId: inbox.id,
|
|
1581
1654
|
scope: { workspace: binding.workspace, preset: binding.agentPreset }, owner, binding, dispatchedAt });
|
|
1655
|
+
try {
|
|
1656
|
+
if (this.restartStore !== undefined) {
|
|
1657
|
+
const route = this.resolveOwnerRoute(this.config.restart.ownerRouteId);
|
|
1658
|
+
if (route.binding.id === binding.id)
|
|
1659
|
+
this.restartStore.beginTask({
|
|
1660
|
+
inboxId: inbox.id, eventId: inbox.eventId, owner,
|
|
1661
|
+
binding: { id: binding.id, version: binding.version, generation: binding.generation,
|
|
1662
|
+
sessionId: binding.sessionId, workspace: binding.workspace, preset: binding.agentPreset,
|
|
1663
|
+
conversation: binding.conversation, principal: binding.principal, policyRef: binding.policyRef,
|
|
1664
|
+
authorityHash: route.snapshot.authorityHash },
|
|
1665
|
+
sourceInvocationId: this.restartInvocationId,
|
|
1666
|
+
maxModelRequests: this.config.restart.maxModelRequests,
|
|
1667
|
+
maxToolCalls: this.config.restart.maxToolCalls,
|
|
1668
|
+
maxOutputTokens: this.config.agentMaxOutputTokens,
|
|
1669
|
+
deadline: dispatchedAt + this.config.restart.taskTimeoutMs,
|
|
1670
|
+
capabilityDigest: this.restartCapabilityDigest(),
|
|
1671
|
+
});
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
catch (error) {
|
|
1675
|
+
this.context.logger.warn(`assistant-delivery: restart eligibility observation unavailable: ${String(error)}`);
|
|
1676
|
+
}
|
|
1582
1677
|
this.observeForegroundBegin(Object.freeze({ protocol: 'assistant-delivery/foreground-task/v1', inboxId: inbox.id,
|
|
1583
1678
|
sessionId: binding.sessionId, scope: Object.freeze({ workspace: binding.workspace, preset: binding.agentPreset }),
|
|
1584
1679
|
owner: Object.freeze({ principalRecordId: owner.principalRecordId, principalVersion: owner.principalVersion }),
|
|
@@ -3900,6 +3995,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
3900
3995
|
beforeInboxId: record.id,
|
|
3901
3996
|
failureCode: 'new-session-before-dispatch',
|
|
3902
3997
|
});
|
|
3998
|
+
this.restartStore?.cancelForBinding(binding.id);
|
|
3903
3999
|
await this.runtime?.cancelActive?.(binding, 'new');
|
|
3904
4000
|
await this.inbound.cancelUndispatchedClaims(fence.claimedInboxIds, 'new');
|
|
3905
4001
|
const rotated = await this.rotateBinding(envelope, binding, record.id, recoverySignal);
|
|
@@ -3934,6 +4030,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
3934
4030
|
});
|
|
3935
4031
|
// Holding the transition while the live turn drains prevents a later
|
|
3936
4032
|
// message from overtaking the durable stop acknowledgement.
|
|
4033
|
+
this.restartStore?.cancelForBinding(binding.id);
|
|
3937
4034
|
await this.runtime?.cancelActive?.(binding, 'stop');
|
|
3938
4035
|
await this.inbound.cancelUndispatchedClaims(fence.claimedInboxIds, 'stop');
|
|
3939
4036
|
record = this.deliveryStore.queueInbox(record.id, binding.id);
|
|
@@ -3992,6 +4089,60 @@ export class AssistantDeliveryService extends Service {
|
|
|
3992
4089
|
...(input.metadata === undefined ? {} : { metadata: input.metadata }) });
|
|
3993
4090
|
}
|
|
3994
4091
|
#notificationGuard(record) {
|
|
4092
|
+
const key = record.intent.idempotencyKey;
|
|
4093
|
+
if (key.startsWith('inbound:') && (key.endsWith(':maintenance') || key.endsWith(':maintenance-failed'))) {
|
|
4094
|
+
try {
|
|
4095
|
+
const failed = key.endsWith(':maintenance-failed');
|
|
4096
|
+
const inboxId = key.slice('inbound:'.length, -(failed ? ':maintenance-failed'.length : ':maintenance'.length));
|
|
4097
|
+
const request = this.restartStore?.getRequestForTask(inboxId);
|
|
4098
|
+
const task = this.restartStore?.getTask(inboxId);
|
|
4099
|
+
const inbox = this.deliveryStore.getInbox(inboxId);
|
|
4100
|
+
const binding = this.deliveryStore.getBinding(record.intent.bindingId);
|
|
4101
|
+
if (request === undefined || task === undefined || inbox === undefined || binding === undefined
|
|
4102
|
+
|| request.status === 'cancelled' || (failed ? request.status !== 'failed' : !this.restartAuthorized(request))
|
|
4103
|
+
|| task.binding.id !== binding.id || task.binding.version !== binding.version || task.binding.generation !== binding.generation
|
|
4104
|
+
|| task.owner.principalRecordId !== this.ownerLineageForBinding(binding)?.principalRecordId
|
|
4105
|
+
|| task.owner.principalVersion !== this.ownerLineageForBinding(binding)?.principalVersion
|
|
4106
|
+
|| inbox.bindingId !== binding.id || inbox.eventId !== record.intent.replyToEventId
|
|
4107
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)
|
|
4108
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
4109
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
4110
|
+
throw new Error('maintenance notice is stale');
|
|
4111
|
+
const decision = this.policy.evaluate({ subject: { kind: 'agent', id: binding.agentPreset,
|
|
4112
|
+
workspace: binding.workspace, principal: externalPrincipalId(binding.principal) },
|
|
4113
|
+
action: 'reply', resource: { kind: 'message', id: binding.id }, context: { initiator: 'external' } });
|
|
4114
|
+
if (decision.effect !== 'allow')
|
|
4115
|
+
throw new Error('maintenance notice permission changed');
|
|
4116
|
+
}
|
|
4117
|
+
catch {
|
|
4118
|
+
return { outcome: 'not-sent', failureCode: 'maintenance-notice-stale', retryable: false };
|
|
4119
|
+
}
|
|
4120
|
+
return undefined;
|
|
4121
|
+
}
|
|
4122
|
+
if (key.startsWith('inbound:') && key.endsWith(':interrupted')) {
|
|
4123
|
+
try {
|
|
4124
|
+
const inboxId = key.slice('inbound:'.length, -':interrupted'.length);
|
|
4125
|
+
const inbox = this.deliveryStore.getInbox(inboxId);
|
|
4126
|
+
const binding = this.deliveryStore.getBinding(record.intent.bindingId);
|
|
4127
|
+
if (record.intent.text !== interruptedTurnNotice || inbox === undefined || binding === undefined
|
|
4128
|
+
|| !this.deliveryStore.canSendInterruptedInboxNotice(record)
|
|
4129
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)) {
|
|
4130
|
+
throw new Error('interrupted Inbox authority changed');
|
|
4131
|
+
}
|
|
4132
|
+
const reply = this.policy.evaluate({
|
|
4133
|
+
subject: { kind: 'agent', id: binding.agentPreset, workspace: binding.workspace,
|
|
4134
|
+
principal: externalPrincipalId(binding.principal) },
|
|
4135
|
+
action: 'reply', resource: { kind: 'message', id: binding.id },
|
|
4136
|
+
context: { initiator: 'external' },
|
|
4137
|
+
});
|
|
4138
|
+
if (reply.effect !== 'allow')
|
|
4139
|
+
throw new Error('interrupted Inbox reply authority changed');
|
|
4140
|
+
}
|
|
4141
|
+
catch {
|
|
4142
|
+
return { outcome: 'not-sent', failureCode: 'interrupted-inbox-authority-revoked', retryable: false };
|
|
4143
|
+
}
|
|
4144
|
+
return undefined;
|
|
4145
|
+
}
|
|
3995
4146
|
const metadata = record.intent.metadata;
|
|
3996
4147
|
const nativeKeys = Object.keys(metadata ?? {}).filter(key => key.startsWith('dsh.native-notice'));
|
|
3997
4148
|
if (nativeKeys.length > 0) {
|
|
@@ -4604,7 +4755,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
4604
4755
|
throw new AssistantDeliveryError('missing-binding', 'control command binding principal is no longer active');
|
|
4605
4756
|
}
|
|
4606
4757
|
const format = this.replyFormat(binding.conversation, input.format) ?? 'plain';
|
|
4607
|
-
|
|
4758
|
+
const intent = {
|
|
4608
4759
|
idempotencyKey,
|
|
4609
4760
|
bindingId: binding.id,
|
|
4610
4761
|
target: { conversation: binding.conversation, principal: binding.principal },
|
|
@@ -4615,7 +4766,11 @@ export class AssistantDeliveryService extends Service {
|
|
|
4615
4766
|
? {}
|
|
4616
4767
|
: { permissionPicker: input.permissionPicker }),
|
|
4617
4768
|
replyToEventId: input.replyToEventId,
|
|
4618
|
-
}
|
|
4769
|
+
};
|
|
4770
|
+
return input.interruptedInboxId === undefined
|
|
4771
|
+
? this.deliveryStore.enqueue(intent)
|
|
4772
|
+
: this.deliveryStore.enqueueInterruptedInboxNotice({ inboxId: input.interruptedInboxId,
|
|
4773
|
+
binding, intent });
|
|
4619
4774
|
}
|
|
4620
4775
|
/**
|
|
4621
4776
|
* Answers are authored as Markdown, but the coordinator drops an intent whose format the adapter
|
|
@@ -4783,10 +4938,249 @@ export class AssistantDeliveryService extends Service {
|
|
|
4783
4938
|
wiki: this.policy.listProposalStatesForRoute(wikiRoute, 5),
|
|
4784
4939
|
};
|
|
4785
4940
|
}
|
|
4941
|
+
restartCapabilityDigest() {
|
|
4942
|
+
return restartDigest({ protocol: 'assistant-delivery/restart/v1', version, code: this.restartCodeDigest,
|
|
4943
|
+
config: this.config.restart, maxOutputTokens: this.config.agentMaxOutputTokens });
|
|
4944
|
+
}
|
|
4945
|
+
restartAuthorized(record) {
|
|
4946
|
+
try {
|
|
4947
|
+
if (!this.active || this.restartStore === undefined || this.restartController.signal.aborted)
|
|
4948
|
+
return false;
|
|
4949
|
+
const current = record.id === undefined ? this.restartStore.getRequestForTask(record.inboxId) : this.restartStore.getRequest(record.id);
|
|
4950
|
+
const task = this.restartStore.getTask(record.inboxId);
|
|
4951
|
+
if ((record.id !== undefined && current === undefined) || task === undefined
|
|
4952
|
+
|| (current !== undefined && ['cancelled', 'failed', 'completed'].includes(current.status))
|
|
4953
|
+
|| Date.now() >= task.deadline || task.capabilityDigest !== this.restartCapabilityDigest())
|
|
4954
|
+
return false;
|
|
4955
|
+
const route = this.resolveOwnerRoute(this.config.restart.ownerRouteId);
|
|
4956
|
+
const binding = route.binding;
|
|
4957
|
+
const owner = this.ownerLineageForBinding(binding);
|
|
4958
|
+
const inbox = this.deliveryStore.getInbox(record.inboxId);
|
|
4959
|
+
if (owner?.principalRecordId !== task.owner.principalRecordId || owner.principalVersion !== task.owner.principalVersion
|
|
4960
|
+
|| binding.id !== task.binding.id || binding.version !== task.binding.version || binding.generation !== task.binding.generation
|
|
4961
|
+
|| binding.sessionId !== task.binding.sessionId || binding.workspace !== task.binding.workspace
|
|
4962
|
+
|| binding.agentPreset !== task.binding.preset || binding.policyRef !== task.binding.policyRef
|
|
4963
|
+
|| route.snapshot.authorityHash !== task.binding.authorityHash
|
|
4964
|
+
|| inbox?.bindingId !== binding.id || inbox.eventId !== task.eventId
|
|
4965
|
+
|| !['claimed', 'processed'].includes(inbox.status) || !this.isInboundAuthorized(binding, inbox.envelope)
|
|
4966
|
+
|| this.deliveryStore.hasRestartCancellationAfter(inbox.id, binding)
|
|
4967
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
4968
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
4969
|
+
return false;
|
|
4970
|
+
const decision = this.policy.evaluate({ subject: { kind: 'agent', id: binding.agentPreset,
|
|
4971
|
+
workspace: binding.workspace, principal: externalPrincipalId(binding.principal) },
|
|
4972
|
+
action: 'maintenance.restart', resource: { kind: 'message', id: `maintenance:${this.config.restart.ownerRouteId}` }, context: { initiator: 'external' } });
|
|
4973
|
+
return decision.effect === 'allow';
|
|
4974
|
+
}
|
|
4975
|
+
catch {
|
|
4976
|
+
return false;
|
|
4977
|
+
}
|
|
4978
|
+
}
|
|
4979
|
+
/** Host-only maintenance gate shared by the native background scheduler. */
|
|
4980
|
+
isMaintenanceDraining() { return this.active && this.deliveryStore.getRestartFence() !== undefined; }
|
|
4981
|
+
restartDispatchPermitted(agent) {
|
|
4982
|
+
if (!this.active)
|
|
4983
|
+
return false;
|
|
4984
|
+
const fence = this.deliveryStore.getRestartFence();
|
|
4985
|
+
if (fence === undefined)
|
|
4986
|
+
return true;
|
|
4987
|
+
const request = this.restartStore?.getRequest(fence.requestId);
|
|
4988
|
+
const task = request === undefined ? undefined : this.restartStore?.getTask(request.inboxId);
|
|
4989
|
+
return request?.status === 'resuming' && task !== undefined && agent?.session.id === task.binding.sessionId
|
|
4990
|
+
&& this.restartAuthorized(request);
|
|
4991
|
+
}
|
|
4992
|
+
/** Run only from native maintenance after the continuation notice is durably flushed. */
|
|
4993
|
+
releaseRestartFenceOnAdmission(record, sessionId) {
|
|
4994
|
+
const current = this.restartStore?.getRequest(record.id);
|
|
4995
|
+
const task = this.restartStore?.getTask(record.inboxId);
|
|
4996
|
+
const fence = this.deliveryStore.getRestartFence();
|
|
4997
|
+
if (current?.status !== 'resuming' || current.continuationHolderId !== this.ownerId
|
|
4998
|
+
|| current.restartInvocationId !== this.restartInvocationId
|
|
4999
|
+
|| current.sourceInvocationId === this.restartInvocationId
|
|
5000
|
+
|| current.inboxId !== record.inboxId || task?.binding.sessionId !== sessionId
|
|
5001
|
+
|| fence?.requestId !== record.id || fence.sourceInvocationId !== record.sourceInvocationId
|
|
5002
|
+
|| !this.restartAuthorized(current)) {
|
|
5003
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart admission no longer owns the maintenance fence');
|
|
5004
|
+
}
|
|
5005
|
+
this.deliveryStore.clearRestartFence(record.id);
|
|
5006
|
+
if (this.deliveryStore.getRestartFence()?.requestId === record.id) {
|
|
5007
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart maintenance fence could not be released');
|
|
5008
|
+
}
|
|
5009
|
+
}
|
|
5010
|
+
otherRestartWorkQuiescent(agent) {
|
|
5011
|
+
if (this.context.get('agents')?.list().some(other => other !== agent))
|
|
5012
|
+
return false;
|
|
5013
|
+
const automations = this.context.get('assistantAutomations');
|
|
5014
|
+
return automations === undefined || automations.isRestartQuiescent?.() === true;
|
|
5015
|
+
}
|
|
5016
|
+
canRequestRestart(agent) {
|
|
5017
|
+
return this.active && agent !== undefined && this.otherRestartWorkQuiescent(agent)
|
|
5018
|
+
&& this.runtime?.canRequestRestart?.(agent) === true;
|
|
5019
|
+
}
|
|
5020
|
+
requestRestart(agent, reason) {
|
|
5021
|
+
this.assertActive();
|
|
5022
|
+
if (agent === undefined || this.restartStore === undefined || this.runtime?.requestRestart === undefined) {
|
|
5023
|
+
throw new AssistantDeliveryError('runtime-unavailable', 'planned restart is not configured for this owner');
|
|
5024
|
+
}
|
|
5025
|
+
if (!this.otherRestartWorkQuiescent(agent))
|
|
5026
|
+
throw new AssistantDeliveryError('runtime-conflict', 'other Host work has not drained');
|
|
5027
|
+
const result = this.runtime.requestRestart(agent, reason);
|
|
5028
|
+
try {
|
|
5029
|
+
this.deliveryStore.setRestartFence(result.id, this.restartInvocationId);
|
|
5030
|
+
}
|
|
5031
|
+
catch (error) {
|
|
5032
|
+
this.restartStore.fail(result.id, 'restart-drain-fence-unavailable');
|
|
5033
|
+
throw error;
|
|
5034
|
+
}
|
|
5035
|
+
return result;
|
|
5036
|
+
}
|
|
5037
|
+
get supportsPlannedRestart() { return this.restartStore !== undefined; }
|
|
5038
|
+
/** Host-side registration of an exact, currently mounted read-only definition. */
|
|
5039
|
+
registerRestartSafeTool(caller, definition) {
|
|
5040
|
+
if (this.restartStore === undefined)
|
|
5041
|
+
return () => { };
|
|
5042
|
+
const trusted = definition.name === 'delivery_status'
|
|
5043
|
+
? caller.fiber.name === 'AssistantDeliveryService'
|
|
5044
|
+
&& caller.fiber.parent.fiber === this.context.fiber
|
|
5045
|
+
: definition.name === 'wiki_read' || definition.name === 'wiki_search'
|
|
5046
|
+
? caller.fiber.name === 'PersonalWikiService' || caller.fiber.name === 'dsh-enhanced-personal-wiki'
|
|
5047
|
+
: definition.name === 'memory_search'
|
|
5048
|
+
&& (caller.fiber.name === 'PersonalMemoryService' || caller.fiber.name === 'dsh-enhanced-personal-memory');
|
|
5049
|
+
// The installed Cordis fork declares FiberState as an ambient const enum;
|
|
5050
|
+
// its ACTIVE and LOADING numeric values are 2 and 1, respectively.
|
|
5051
|
+
if (!trusted || caller.fiber.uid === null || (caller.fiber.state !== 1 && caller.fiber.state !== 2)) {
|
|
5052
|
+
throw new AssistantDeliveryError('runtime-conflict', 'restart-safe tool registration requires a trusted live provider');
|
|
5053
|
+
}
|
|
5054
|
+
const entry = { caller, definition };
|
|
5055
|
+
this.restartSafeDefinitions.add(entry);
|
|
5056
|
+
const remove = () => { this.restartSafeDefinitions.delete(entry); };
|
|
5057
|
+
try {
|
|
5058
|
+
caller.effect(() => remove, 'assistant-delivery.restart-safe-tool');
|
|
5059
|
+
}
|
|
5060
|
+
catch (error) {
|
|
5061
|
+
remove();
|
|
5062
|
+
throw error;
|
|
5063
|
+
}
|
|
5064
|
+
return remove;
|
|
5065
|
+
}
|
|
5066
|
+
isRestartSafeTool(agent, name) {
|
|
5067
|
+
if (!this.active || this.restartStore === undefined)
|
|
5068
|
+
return false;
|
|
5069
|
+
let current;
|
|
5070
|
+
try {
|
|
5071
|
+
current = agent.ctx.tools.get(name, agent);
|
|
5072
|
+
}
|
|
5073
|
+
catch {
|
|
5074
|
+
return false;
|
|
5075
|
+
}
|
|
5076
|
+
if (current === undefined)
|
|
5077
|
+
return false;
|
|
5078
|
+
for (const entry of this.restartSafeDefinitions) {
|
|
5079
|
+
if (entry.definition === current && entry.definition.name === name
|
|
5080
|
+
&& entry.caller.fiber.uid !== null && entry.caller.fiber.state === 2)
|
|
5081
|
+
return true;
|
|
5082
|
+
}
|
|
5083
|
+
return false;
|
|
5084
|
+
}
|
|
5085
|
+
/** Startup/tick reconciliation uses the existing native runtime, never requeues the old Inbox. */
|
|
5086
|
+
drainRestartContinuations() {
|
|
5087
|
+
if (this.restartFlight !== undefined || this.restartStore === undefined || this.runtime?.resumeRestart === undefined)
|
|
5088
|
+
return;
|
|
5089
|
+
const store = this.restartStore;
|
|
5090
|
+
const pending = store.listRequests({ statuses: ['requested', 'checkpoint', 'ready', 'restarting', 'restarted', 'resuming'], limit: 100 });
|
|
5091
|
+
for (const record of pending) {
|
|
5092
|
+
if (Date.now() >= (store.getTask(record.inboxId)?.deadline ?? 0))
|
|
5093
|
+
store.fail(record.id, 'restart-deadline-expired');
|
|
5094
|
+
else if (['requested', 'checkpoint', 'ready'].includes(record.status) && Date.now() >= record.createdAt + 120_000) {
|
|
5095
|
+
store.fail(record.id, 'restart-handoff-timeout');
|
|
5096
|
+
}
|
|
5097
|
+
else if (record.sourceInvocationId !== this.restartInvocationId
|
|
5098
|
+
&& ['requested', 'checkpoint'].includes(record.status))
|
|
5099
|
+
store.fail(record.id, 'restart-checkpoint-not-quiescent');
|
|
5100
|
+
else if (record.status === 'resuming' && record.continuationHolderId !== this.ownerId) {
|
|
5101
|
+
// A later restart must not blindly execute a possibly dispatched continuation.
|
|
5102
|
+
store.fail(record.id, 'restart-continuation-outcome-unknown');
|
|
5103
|
+
}
|
|
5104
|
+
}
|
|
5105
|
+
const candidate = pending.find(record => record.status === 'restarted' && store.getRequest(record.id)?.status === 'restarted'
|
|
5106
|
+
&& record.restartInvocationId === this.restartInvocationId && record.sourceInvocationId !== this.restartInvocationId);
|
|
5107
|
+
if (candidate === undefined)
|
|
5108
|
+
return;
|
|
5109
|
+
if (!this.restartAuthorized(candidate)) {
|
|
5110
|
+
store.fail(candidate.id, 'restart-authority-changed');
|
|
5111
|
+
return;
|
|
5112
|
+
}
|
|
5113
|
+
const task = store.getTask(candidate.inboxId);
|
|
5114
|
+
const binding = this.deliveryStore.getBinding(task.binding.id);
|
|
5115
|
+
const inbox = this.deliveryStore.getInbox(candidate.inboxId);
|
|
5116
|
+
const claimed = store.claimContinuation(candidate.id, { holderId: this.ownerId, invocationId: this.restartInvocationId });
|
|
5117
|
+
const runtime = this.runtime;
|
|
5118
|
+
const flight = (async () => {
|
|
5119
|
+
try {
|
|
5120
|
+
await runtime.resumeRestart(claimed, binding, inbox.envelope, this.restartController.signal);
|
|
5121
|
+
}
|
|
5122
|
+
catch (error) {
|
|
5123
|
+
if (this.active) {
|
|
5124
|
+
const current = store.getRequest(claimed.id);
|
|
5125
|
+
if (current?.status === 'resuming')
|
|
5126
|
+
store.fail(claimed.id, 'restart-continuation-failed');
|
|
5127
|
+
this.context.logger.warn(`assistant-delivery: restart continuation failed: ${String(error)}`);
|
|
5128
|
+
}
|
|
5129
|
+
}
|
|
5130
|
+
})().finally(() => { if (this.restartFlight === flight)
|
|
5131
|
+
this.restartFlight = undefined; });
|
|
5132
|
+
this.restartFlight = flight;
|
|
5133
|
+
}
|
|
5134
|
+
drainRestartFailureNotices() {
|
|
5135
|
+
if (this.restartStore === undefined)
|
|
5136
|
+
return;
|
|
5137
|
+
let records = this.restartStore.listRequests({ statuses: ['failed'], limit: 100,
|
|
5138
|
+
...(this.restartNoticeCursor === undefined ? {} : { after: this.restartNoticeCursor }) });
|
|
5139
|
+
if (records.length === 0 && this.restartNoticeCursor !== undefined) {
|
|
5140
|
+
this.restartNoticeCursor = undefined;
|
|
5141
|
+
records = this.restartStore.listRequests({ statuses: ['failed'], limit: 100 });
|
|
5142
|
+
}
|
|
5143
|
+
for (const record of records) {
|
|
5144
|
+
this.restartNoticeCursor = { createdAt: record.createdAt, id: record.id };
|
|
5145
|
+
const task = this.restartStore.getTask(record.inboxId);
|
|
5146
|
+
const inbox = this.deliveryStore.getInbox(record.inboxId);
|
|
5147
|
+
const binding = task === undefined ? undefined : this.deliveryStore.getBinding(task.binding.id);
|
|
5148
|
+
const key = `inbound:${record.inboxId}:maintenance-failed`;
|
|
5149
|
+
if (task === undefined || inbox === undefined || binding?.status !== 'active'
|
|
5150
|
+
|| binding.version !== task.binding.version || binding.generation !== task.binding.generation
|
|
5151
|
+
|| this.deliveryStore.getActiveBinding(binding.conversation)?.id !== binding.id
|
|
5152
|
+
|| this.ownerLineageForBinding(binding)?.principalRecordId !== task.owner.principalRecordId
|
|
5153
|
+
|| !this.isInboundAuthorized(binding, inbox.envelope)
|
|
5154
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(key) !== undefined
|
|
5155
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.id}:reply`) !== undefined
|
|
5156
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${inbox.eventId}:reply`) !== undefined)
|
|
5157
|
+
continue;
|
|
5158
|
+
try {
|
|
5159
|
+
this.replyCommand(binding, { idempotencyKey: key, replyToEventId: inbox.eventId,
|
|
5160
|
+
text: '维护恢复未能安全完成,当前任务已停止自动继续,服务仍可接收新请求。已保存原会话;可能已经执行的操作不会自动重做。' });
|
|
5161
|
+
}
|
|
5162
|
+
catch (error) {
|
|
5163
|
+
if (this.active)
|
|
5164
|
+
this.context.logger.warn(`assistant-delivery: restart failure notice: ${String(error)}`);
|
|
5165
|
+
}
|
|
5166
|
+
}
|
|
5167
|
+
}
|
|
4786
5168
|
async tick() {
|
|
4787
5169
|
this.assertActive();
|
|
4788
5170
|
await this.drainInboundAdmissions();
|
|
4789
|
-
|
|
5171
|
+
this.drainRestartContinuations();
|
|
5172
|
+
const restartFence = this.deliveryStore.getRestartFence();
|
|
5173
|
+
if (restartFence !== undefined) {
|
|
5174
|
+
const record = this.restartStore?.getRequest(restartFence.requestId);
|
|
5175
|
+
if (record === undefined || ['failed', 'cancelled', 'completed'].includes(record.status)) {
|
|
5176
|
+
this.deliveryStore.clearRestartFence(restartFence.requestId);
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
const maintenance = this.restartStore?.listRequests({ statuses: ['requested', 'checkpoint', 'ready', 'restarting', 'restarted'], limit: 1 });
|
|
5180
|
+
if (maintenance === undefined || maintenance.length === 0)
|
|
5181
|
+
await this.inbound.tick();
|
|
5182
|
+
this.drainRestartFailureNotices();
|
|
5183
|
+
await this.drainInterruptedInboxNotices();
|
|
4790
5184
|
await this.drainModelSelectionSettlements();
|
|
4791
5185
|
this.drainApprovalDispatches();
|
|
4792
5186
|
await this.outbound.tick();
|
|
@@ -4806,6 +5200,8 @@ export class AssistantDeliveryService extends Service {
|
|
|
4806
5200
|
this.naturalObjectiveFlight,
|
|
4807
5201
|
this.workflowTraceFlight,
|
|
4808
5202
|
this.inboundAdmissionRecoveryFlight,
|
|
5203
|
+
this.interruptedInboxNoticeFlight,
|
|
5204
|
+
this.restartFlight,
|
|
4809
5205
|
]);
|
|
4810
5206
|
}
|
|
4811
5207
|
start() {
|
|
@@ -4855,6 +5251,57 @@ export class AssistantDeliveryService extends Service {
|
|
|
4855
5251
|
this.inboundAdmissionRecoveryFlight = flight;
|
|
4856
5252
|
return flight;
|
|
4857
5253
|
}
|
|
5254
|
+
drainInterruptedInboxNotices() {
|
|
5255
|
+
if (this.interruptedInboxNoticeFlight !== undefined)
|
|
5256
|
+
return this.interruptedInboxNoticeFlight;
|
|
5257
|
+
const flight = this.runInterruptedInboxNotices()
|
|
5258
|
+
.finally(() => { if (this.interruptedInboxNoticeFlight === flight)
|
|
5259
|
+
this.interruptedInboxNoticeFlight = undefined; });
|
|
5260
|
+
this.interruptedInboxNoticeFlight = flight;
|
|
5261
|
+
return flight;
|
|
5262
|
+
}
|
|
5263
|
+
async runInterruptedInboxNotices() {
|
|
5264
|
+
let pending = this.deliveryStore.listInterruptedInboxNotices({
|
|
5265
|
+
afterSequence: this.interruptedInboxNoticeAfterSequence, limit: 100,
|
|
5266
|
+
});
|
|
5267
|
+
if (pending.length === 0 && this.interruptedInboxNoticeAfterSequence !== 0) {
|
|
5268
|
+
this.interruptedInboxNoticeAfterSequence = 0;
|
|
5269
|
+
pending = this.deliveryStore.listInterruptedInboxNotices({ limit: 100 });
|
|
5270
|
+
}
|
|
5271
|
+
for (const inbox of pending) {
|
|
5272
|
+
if (!this.active)
|
|
5273
|
+
return;
|
|
5274
|
+
this.interruptedInboxNoticeAfterSequence = inbox.admissionCursor.sequence;
|
|
5275
|
+
try {
|
|
5276
|
+
const current = this.deliveryStore.getInbox(inbox.id);
|
|
5277
|
+
if (current?.status !== 'dead_letter' || current.failureCode !== 'dispatch-ambiguous'
|
|
5278
|
+
|| current.bindingId === undefined
|
|
5279
|
+
|| this.deliveryStore.getDeadLetterResolution({ kind: 'inbox', id: current.id,
|
|
5280
|
+
attemptCount: current.attemptCount }) !== undefined
|
|
5281
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.id}:reply`) !== undefined
|
|
5282
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.eventId}:reply`) !== undefined
|
|
5283
|
+
|| this.deliveryStore.getOutboxByIdempotencyKey(`inbound:${current.id}:interrupted`) !== undefined)
|
|
5284
|
+
continue;
|
|
5285
|
+
const binding = this.deliveryStore.getBinding(current.bindingId);
|
|
5286
|
+
if (binding?.status !== 'active'
|
|
5287
|
+
|| this.deliveryStore.getActiveBinding(current.envelope.conversation)?.id !== binding.id
|
|
5288
|
+
|| this.deliveryStore.getPrincipal(binding.principal)?.role !== 'owner'
|
|
5289
|
+
|| !this.isInboundAuthorized(binding, current.envelope)
|
|
5290
|
+
|| this.registry.get(binding.conversation.channel, binding.conversation.account) === undefined)
|
|
5291
|
+
continue;
|
|
5292
|
+
this.replyCommand(binding, { idempotencyKey: `inbound:${current.id}:interrupted`,
|
|
5293
|
+
interruptedInboxId: current.id, replyToEventId: current.eventId, text: interruptedTurnNotice });
|
|
5294
|
+
}
|
|
5295
|
+
catch (error) {
|
|
5296
|
+
if (!this.active)
|
|
5297
|
+
return;
|
|
5298
|
+
if ((error instanceof AssistantDeliveryError && (error.code === 'policy-denied' || error.code === 'missing-binding'))
|
|
5299
|
+
|| (error instanceof DeliveryStoreError && error.code === 'version-conflict'))
|
|
5300
|
+
continue;
|
|
5301
|
+
this.context.logger.warn(`assistant-delivery: interrupted Inbox notice ${inbox.id}: ${String(error)}`);
|
|
5302
|
+
}
|
|
5303
|
+
}
|
|
5304
|
+
}
|
|
4858
5305
|
async runInboundAdmissionRecovery() {
|
|
4859
5306
|
if (this.runtime === undefined)
|
|
4860
5307
|
return;
|
|
@@ -5040,6 +5487,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
5040
5487
|
this.preferenceProjectionRetryTimer = undefined;
|
|
5041
5488
|
this.cancelModelSelectionWaiters();
|
|
5042
5489
|
this.inboundAdmissionRecoveryController.abort(new Error('assistant-delivery is stopping'));
|
|
5490
|
+
this.restartController.abort(new Error('assistant-delivery is stopping'));
|
|
5043
5491
|
const admissionRecovery = this.inboundAdmissionRecoveryFlight === undefined
|
|
5044
5492
|
? undefined
|
|
5045
5493
|
: Promise.race([
|
|
@@ -5058,6 +5506,7 @@ export class AssistantDeliveryService extends Service {
|
|
|
5058
5506
|
this.preferenceProjectionFlight,
|
|
5059
5507
|
this.naturalObjectiveFlight,
|
|
5060
5508
|
this.workflowTraceFlight,
|
|
5509
|
+
this.restartFlight,
|
|
5061
5510
|
]);
|
|
5062
5511
|
}
|
|
5063
5512
|
drainPreferenceProjections() {
|