@selesai/code 0.13.38 → 0.13.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/core/intercom-rpc.d.ts +135 -0
- package/dist/core/intercom-rpc.js +64 -0
- package/dist/core/release-rpc.d.ts +137 -0
- package/dist/core/release-rpc.js +77 -0
- package/dist/core/resource-loader.d.ts +3 -0
- package/dist/core/resource-loader.js +3 -0
- package/dist/core/subagent-rpc.d.ts +110 -0
- package/dist/core/subagent-rpc.js +60 -0
- package/dist/extensions/agent-browser.ts +9 -0
- package/dist/extensions/pi-graft/cli.test.ts +5 -5
- package/dist/extensions/pi-graft/cli.ts +10 -5
- package/dist/extensions/pi-graft/commands.test.ts +5 -5
- package/dist/extensions/pi-graft/graft.integration.test.ts +1 -1
- package/dist/extensions/pi-graft/lifecycle.test.ts +26 -26
- package/dist/extensions/pi-graft/prompt.test.ts +1 -1
- package/dist/extensions/pi-graft/state.test.ts +9 -9
- package/dist/extensions/pi-graft/test-support.ts +1 -1
- package/dist/extensions/pi-graft/tools.test.ts +1 -1
- package/dist/extensions/pi-intercom/README.md +15 -0
- package/dist/extensions/pi-intercom/broker/broker.ts +277 -13
- package/dist/extensions/pi-intercom/broker/client.ts +6 -2
- package/dist/extensions/pi-intercom/broker/mailbox-persistence.test.ts +523 -0
- package/dist/extensions/pi-intercom/broker/paths.test.ts +4 -4
- package/dist/extensions/pi-intercom/broker/paths.ts +4 -2
- package/dist/extensions/pi-intercom/broker/spawn.test.ts +1 -3
- package/dist/extensions/pi-intercom/extension-api.ts +10 -0
- package/dist/extensions/pi-intercom/index.ts +188 -8
- package/dist/extensions/pi-intercom/intercom.integration.test.ts +280 -0
- package/dist/extensions/pi-intercom/package.json +1 -1
- package/dist/extensions/pi-intercom/release-readiness.test.ts +116 -0
- package/dist/extensions/pi-intercom/release-readiness.ts +93 -0
- package/dist/extensions/pi-subagents/docs/extension-api.md +8 -0
- package/dist/extensions/pi-subagents/docs/observability.md +8 -0
- package/dist/extensions/pi-subagents/src/extension/index.ts +51 -1
- package/dist/extensions/pi-subagents/src/extension/observability.ts +164 -0
- package/dist/extensions/pi-subagents/src/extension/release-readiness.ts +367 -0
- package/dist/extensions/pi-subagents/src/extension/rpc.ts +8 -1
- package/dist/extensions/pi-subagents/src/integrations/pi-web-session-liveness.ts +7 -1
- package/dist/extensions/pi-subagents/src/missions/workflow-state.ts +1 -44
- package/dist/extensions/pi-subagents/src/runs/background/async-job-tracker.ts +6 -0
- package/dist/extensions/pi-subagents/src/runs/background/notify.ts +13 -3
- package/dist/extensions/pi-subagents/src/runs/background/result-delivery-ownership.ts +269 -7
- package/dist/extensions/pi-subagents/src/runs/background/result-watcher.ts +11 -10
- package/dist/extensions/pi-subagents/src/runs/background/retained-nested-route-tracker.ts +15 -3
- package/dist/extensions/pi-subagents/src/runs/background/scheduled-runs.ts +64 -0
- package/dist/extensions/pi-subagents/src/runs/background/subagent-runner.ts +69 -6
- package/dist/extensions/pi-subagents/src/runs/shared/subagent-control.ts +14 -1
- package/dist/extensions/pi-subagents/src/shared/owner-record.ts +107 -0
- package/dist/extensions/pi-subagents/src/shared/process-identity.ts +87 -0
- package/dist/extensions/pi-subagents/src/shared/types.ts +9 -0
- package/dist/extensions/pi-subagents/test/integration/async-attention-clearing.test.ts +191 -0
- package/dist/extensions/pi-subagents/test/integration/result-watcher.test.ts +63 -0
- package/dist/extensions/pi-subagents/test/integration/stop-all-background.test.ts +142 -0
- package/dist/extensions/pi-subagents/test/support/fake-child-session.ts +8 -0
- package/dist/extensions/pi-subagents/test/unit/observability.test.ts +198 -0
- package/dist/extensions/pi-subagents/test/unit/process-identity.test.ts +75 -0
- package/dist/extensions/pi-subagents/test/unit/release-readiness.test.ts +715 -0
- package/dist/extensions/pi-subagents/test/unit/result-takeover.test.ts +332 -0
- package/dist/extensions/tokenin-onboarding.ts +142 -3
- package/dist/index.d.ts +6 -1
- package/dist/index.js +7 -0
- package/dist/main.d.ts +6 -0
- package/dist/main.js +1 -1
- package/dist/modes/index.d.ts +1 -1
- package/dist/modes/rpc/intercom-bridge.d.ts +14 -0
- package/dist/modes/rpc/intercom-bridge.js +75 -0
- package/dist/modes/rpc/intercom-bridge.test.d.ts +1 -0
- package/dist/modes/rpc/intercom-bridge.test.js +65 -0
- package/dist/modes/rpc/release-bridge.d.ts +35 -0
- package/dist/modes/rpc/release-bridge.js +159 -0
- package/dist/modes/rpc/release-bridge.test.d.ts +1 -0
- package/dist/modes/rpc/release-bridge.test.js +347 -0
- package/dist/modes/rpc/rpc-client.d.ts +34 -10
- package/dist/modes/rpc/rpc-client.js +45 -2
- package/dist/modes/rpc/rpc-intercom-transport.test.d.ts +1 -0
- package/dist/modes/rpc/rpc-intercom-transport.test.js +85 -0
- package/dist/modes/rpc/rpc-mode.d.ts +1 -13
- package/dist/modes/rpc/rpc-mode.js +239 -8
- package/dist/modes/rpc/rpc-release-transport.test.d.ts +1 -0
- package/dist/modes/rpc/rpc-release-transport.test.js +329 -0
- package/dist/modes/rpc/rpc-subagent-transport.test.d.ts +1 -0
- package/dist/modes/rpc/rpc-subagent-transport.test.js +60 -0
- package/dist/modes/rpc/rpc-types.d.ts +12 -2
- package/dist/modes/rpc/rpc-types.js +0 -6
- package/dist/modes/rpc/subagent-bridge.d.ts +23 -0
- package/dist/modes/rpc/subagent-bridge.js +93 -0
- package/dist/modes/rpc/test-fixtures/intercom-host.d.ts +1 -0
- package/dist/modes/rpc/test-fixtures/intercom-host.js +13 -0
- package/docs/rpc-subagent.schema.json +491 -0
- package/docs/rpc.md +210 -1
- package/package.json +2 -1
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
import * as fs from "node:fs";
|
|
2
|
+
import * as path from "node:path";
|
|
3
|
+
import type { ReleaseBlocker, ReleaseReadinessRequest, StopAllBackgroundItem, StopAllBackgroundRequest } from "@selesai/code";
|
|
4
|
+
import { hasLiveSubagentWork } from "../integrations/pi-web-session-liveness.ts";
|
|
5
|
+
import { reconcileAsyncRun } from "../runs/background/stale-run-reconciler.ts";
|
|
6
|
+
import { resultCandidateFilesForSession, resultPayloadPathForSessionRun } from "../runs/background/result-files.ts";
|
|
7
|
+
import { hasLiveNestedDescendants } from "../runs/shared/nested-events.ts";
|
|
8
|
+
import { DIRS, type AsyncJobState, type NestedRunSummary, type SubagentState } from "../shared/types.ts";
|
|
9
|
+
|
|
10
|
+
interface Events { on(name: string, handler: (payload: unknown) => void): (() => void) | void }
|
|
11
|
+
/**
|
|
12
|
+
* Mirrors `RELEASE_READINESS_EVENT` / `answerReleaseReadiness` from the host package. Kept local (types only are imported) so this
|
|
13
|
+
* module stays loadable under the test shim, like the other event-name constants in this extension; a host-side test pins them together.
|
|
14
|
+
*/
|
|
15
|
+
export const RELEASE_READINESS_EVENT = "release:readiness:v1";
|
|
16
|
+
function answerReleaseReadiness(raw: unknown, name: string, boundSessionId: string | null | undefined, compute: () => ReleaseBlocker[]): void {
|
|
17
|
+
const request = raw as Partial<ReleaseReadinessRequest> | null | undefined;
|
|
18
|
+
if (!request || request.version !== 1 || typeof request.contribute !== "function" || !boundSessionId || request.sessionId !== boundSessionId) return;
|
|
19
|
+
let blockers: ReleaseBlocker[];
|
|
20
|
+
try { blockers = compute(); }
|
|
21
|
+
catch (error) { blockers = [{ source: name, kind: "contributor_error", detail: (error instanceof Error ? error.message : String(error)).slice(0, 512) || "contributor failed" }]; }
|
|
22
|
+
request.contribute(name, blockers);
|
|
23
|
+
}
|
|
24
|
+
const TERMINAL = new Set(["complete", "failed", "partial", "paused", "stopped", "rejected"]);
|
|
25
|
+
const MAX_RESULT_BLOCKERS = 32;
|
|
26
|
+
const message = (error: unknown) => error instanceof Error ? error.message : String(error);
|
|
27
|
+
const stamp = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : undefined;
|
|
28
|
+
|
|
29
|
+
export interface SubagentReleaseDeps {
|
|
30
|
+
state: SubagentState;
|
|
31
|
+
/** `completionNotifier.hasPendingDelivery()`: completions queued for the parent but not yet delivered. */
|
|
32
|
+
hasPendingDelivery(): boolean;
|
|
33
|
+
/** Same predicate the result watcher uses to decide whether delivery still matters (`hasResultDeliveryDemand`). */
|
|
34
|
+
hasDeliveryDemand?(): boolean;
|
|
35
|
+
/** Run ids with a persisted result this process still owes the parent. */
|
|
36
|
+
listUndeliveredResults(): string[];
|
|
37
|
+
resultsDir?: string;
|
|
38
|
+
/** Injectable for tests; defaults to the real dead-runner repair. */
|
|
39
|
+
reconcile?: (asyncDir: string, options: Parameters<typeof reconcileAsyncRun>[1]) => ReturnType<typeof reconcileAsyncRun>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface SchedulerReleaseDeps {
|
|
43
|
+
armedSchedules(): Array<{ id: string; nextRunAt?: number; sessionOnly?: boolean }>;
|
|
44
|
+
observedCompletionRunIds(): Iterable<string>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Persisted results for the given sessions that this process still owes the parent. Results owned by another
|
|
49
|
+
* live (or unprovable) process are skipped; a result whose previous owner is provably dead is reported because this
|
|
50
|
+
* process will take it over and deliver it (`canTakeOver` is read-only: nothing is claimed here). Unreadable results
|
|
51
|
+
* fail closed.
|
|
52
|
+
*/
|
|
53
|
+
export function findUndeliveredResults(options: {
|
|
54
|
+
resultsDir: string;
|
|
55
|
+
sessionIds: Array<string | null | undefined>;
|
|
56
|
+
owns(sessionId: string, completionOwnerId: unknown): boolean;
|
|
57
|
+
canTakeOver?(sessionId: string, completionOwnerId: unknown, runKey: string): boolean;
|
|
58
|
+
}): string[] {
|
|
59
|
+
const runIds = new Set<string>();
|
|
60
|
+
for (const sessionId of options.sessionIds) {
|
|
61
|
+
if (!sessionId) continue;
|
|
62
|
+
for (const file of resultCandidateFilesForSession(options.resultsDir, sessionId)) {
|
|
63
|
+
const runId = file.replace(/\.json$/i, "");
|
|
64
|
+
let data: Record<string, unknown> | undefined;
|
|
65
|
+
try {
|
|
66
|
+
const payload = resultPayloadPathForSessionRun(options.resultsDir, sessionId, runId);
|
|
67
|
+
if (payload) data = JSON.parse(fs.readFileSync(payload, "utf-8")) as Record<string, unknown>;
|
|
68
|
+
} catch { runIds.add(runId); continue; }
|
|
69
|
+
if (!data || typeof data !== "object") continue;
|
|
70
|
+
if (typeof data.notificationDeliveredAt === "number") continue;
|
|
71
|
+
if (typeof data.sessionId === "string" && !options.owns(data.sessionId, data.completionOwnerId)
|
|
72
|
+
&& !options.canTakeOver?.(data.sessionId, data.completionOwnerId, runId)) continue;
|
|
73
|
+
runIds.add(runId);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
return [...runIds];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Blockers for background subagent work. Reconciles dead runners first (decision D5: a dead runner is not a
|
|
81
|
+
* blocker; its repaired failure result is an `undelivered_result` until delivered). Fail closed everywhere.
|
|
82
|
+
*/
|
|
83
|
+
export function subagentReleaseBlockers(deps: SubagentReleaseDeps): ReleaseBlocker[] {
|
|
84
|
+
const { state } = deps;
|
|
85
|
+
const blockers: ReleaseBlocker[] = [];
|
|
86
|
+
const add = (kind: string, detail: string, id?: string, since?: number) => {
|
|
87
|
+
const at = stamp(since);
|
|
88
|
+
blockers.push({ source: "subagents", kind, detail: detail.slice(0, 512), ...(id ? { id: id.slice(0, 256) } : {}), ...(at === undefined ? {} : { since: at }) });
|
|
89
|
+
};
|
|
90
|
+
const guard = (label: string, run: () => void) => {
|
|
91
|
+
try { run(); } catch (error) { add("contributor_error", `${label}: ${message(error)}`); }
|
|
92
|
+
};
|
|
93
|
+
const reconcile = deps.reconcile ?? reconcileAsyncRun;
|
|
94
|
+
const resultsDir = deps.resultsDir ?? DIRS.results;
|
|
95
|
+
const live = new Map<string, AsyncJobState>();
|
|
96
|
+
const blockedRunIds = new Set<string>();
|
|
97
|
+
const reconciledTerminal = new Set<string>();
|
|
98
|
+
for (const job of state.asyncJobs.values()) {
|
|
99
|
+
if (job.status !== "queued" && job.status !== "running") continue;
|
|
100
|
+
let settled = false;
|
|
101
|
+
try {
|
|
102
|
+
const reconciled = reconcile(job.asyncDir, {
|
|
103
|
+
resultsDir,
|
|
104
|
+
startedRun: {
|
|
105
|
+
runId: job.asyncId, pid: job.pid, sessionId: job.sessionId, completionOwnerId: job.completionOwnerId, mode: job.mode,
|
|
106
|
+
agents: job.agents, chainStepCount: job.chainStepCount, parallelGroups: job.parallelGroups, startedAt: job.startedAt, sessionFile: job.sessionFile,
|
|
107
|
+
},
|
|
108
|
+
});
|
|
109
|
+
if (reconciled.status && TERMINAL.has(reconciled.status.state)) {
|
|
110
|
+
settled = true;
|
|
111
|
+
reconciledTerminal.add(job.asyncId);
|
|
112
|
+
add("undelivered_result", reconciled.repaired
|
|
113
|
+
? `Async run ${job.asyncId} ended without a live runner (${reconciled.status.state}); its result has not been delivered`
|
|
114
|
+
: `Async run ${job.asyncId} finished (${reconciled.status.state}); the parent session has not observed it yet`, job.asyncId, reconciled.status.endedAt ?? job.updatedAt);
|
|
115
|
+
blockedRunIds.add(job.asyncId);
|
|
116
|
+
}
|
|
117
|
+
} catch (error) {
|
|
118
|
+
add("contributor_error", `Failed to reconcile async run ${job.asyncId}: ${message(error)}`, job.asyncId);
|
|
119
|
+
}
|
|
120
|
+
if (!settled) live.set(job.asyncId, job);
|
|
121
|
+
}
|
|
122
|
+
for (const job of live.values()) {
|
|
123
|
+
add("async_run", `Async ${job.mode ?? "run"} ${job.status}${job.agents?.length ? `: ${job.agents.join(", ")}` : ""}`, job.asyncId, job.startedAt);
|
|
124
|
+
blockedRunIds.add(job.asyncId);
|
|
125
|
+
}
|
|
126
|
+
guard("nested descendants", () => {
|
|
127
|
+
for (const job of state.asyncJobs.values()) {
|
|
128
|
+
if (live.has(job.asyncId) || !hasLiveNestedDescendants(job.nestedChildren)) continue;
|
|
129
|
+
add("async_run", `Async run ${job.asyncId} has live nested descendants`, job.asyncId, job.startedAt);
|
|
130
|
+
blockedRunIds.add(job.asyncId);
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
for (const control of state.foregroundControls.values()) {
|
|
134
|
+
add("foreground_run", `Foreground ${control.mode} run in progress${control.currentAgent ? `: ${control.currentAgent}` : ""}`, control.runId, control.startedAt);
|
|
135
|
+
}
|
|
136
|
+
for (const rootRunId of state.retainedForegroundNestedRoutes?.keys() ?? []) {
|
|
137
|
+
add("async_run", `Retained nested route for foreground run ${rootRunId} may still have live descendants`, rootRunId);
|
|
138
|
+
}
|
|
139
|
+
guard("completion notifier", () => {
|
|
140
|
+
if (deps.hasPendingDelivery()) add("undelivered_result", "Completion notifications are queued for delivery to the parent session");
|
|
141
|
+
});
|
|
142
|
+
guard("undelivered results", () => {
|
|
143
|
+
const runIds = deps.listUndeliveredResults().filter((runId) => !blockedRunIds.has(runId));
|
|
144
|
+
for (const runId of runIds.slice(0, MAX_RESULT_BLOCKERS)) add("undelivered_result", `Result for run ${runId} has not been delivered to the parent session`, runId);
|
|
145
|
+
if (runIds.length > MAX_RESULT_BLOCKERS) add("undelivered_result", `${runIds.length - MAX_RESULT_BLOCKERS} further undelivered results`);
|
|
146
|
+
});
|
|
147
|
+
// Catch-all so the total is never under-reported against the liveness predicate used by pi-web
|
|
148
|
+
// (`hasLiveSubagentWork(state) || hasPendingDelivery()`), evaluated after reconciliation.
|
|
149
|
+
guard("liveness", () => {
|
|
150
|
+
const afterReconcile = { asyncJobs: new Map([...state.asyncJobs].filter(([id]) => !reconciledTerminal.has(id))), foregroundControls: state.foregroundControls, retainedForegroundNestedRoutes: state.retainedForegroundNestedRoutes };
|
|
151
|
+
const unattributed = hasLiveSubagentWork(afterReconcile) && !blockers.some((b) => b.kind === "async_run" || b.kind === "foreground_run");
|
|
152
|
+
if (unattributed) add("async_run", "Live subagent work reported by the liveness predicate could not be attributed to a specific run");
|
|
153
|
+
});
|
|
154
|
+
guard("delivery demand", () => {
|
|
155
|
+
if (!blockers.length && deps.hasDeliveryDemand?.()) add("undelivered_result", "Result delivery is still pending for this session");
|
|
156
|
+
});
|
|
157
|
+
return blockers;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Armed SESSION-ONLY schedules (decisions D7 + D11: project schedules persist on disk and do not pin a process) and
|
|
162
|
+
* scheduled runs still awaiting completion (live work, whatever the schedule kind). An armed timer whose record could not
|
|
163
|
+
* be read (`sessionOnly` undefined) fails closed and blocks.
|
|
164
|
+
*/
|
|
165
|
+
export function schedulerReleaseBlockers(deps: SchedulerReleaseDeps): ReleaseBlocker[] {
|
|
166
|
+
const blockers: ReleaseBlocker[] = [];
|
|
167
|
+
for (const schedule of deps.armedSchedules()) {
|
|
168
|
+
if (schedule.sessionOnly === false) continue;
|
|
169
|
+
const next = stamp(schedule.nextRunAt);
|
|
170
|
+
blockers.push({
|
|
171
|
+
source: "scheduler", kind: "armed_schedule", id: schedule.id.slice(0, 256),
|
|
172
|
+
detail: `Armed ${schedule.sessionOnly === true ? "session-only" : "unreadable"} schedule${next === undefined ? "" : `, next run ${new Date(next).toISOString()}`}`,
|
|
173
|
+
...(next === undefined ? {} : { since: next }),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
for (const runId of deps.observedCompletionRunIds()) {
|
|
177
|
+
blockers.push({ source: "scheduler", kind: "scheduled_run_pending", id: runId.slice(0, 256), detail: `Scheduled run ${runId} is awaiting completion` });
|
|
178
|
+
}
|
|
179
|
+
return blockers;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Mirrors `STOP_ALL_BACKGROUND_EVENT` / `answerStopAllBackground` from the host package (same reason as the readiness mirror above). */
|
|
183
|
+
export const STOP_ALL_BACKGROUND_EVENT = "release:stop-all:v1";
|
|
184
|
+
function answerStopAllBackground(
|
|
185
|
+
raw: unknown, name: string, boundSessionId: string | null | undefined,
|
|
186
|
+
compute: (request: { deadline: number }) => StopAllBackgroundItem[] | PromiseLike<StopAllBackgroundItem[]>,
|
|
187
|
+
): void {
|
|
188
|
+
const request = raw as Partial<StopAllBackgroundRequest> | null | undefined;
|
|
189
|
+
if (!request || request.version !== 1 || typeof request.contribute !== "function" || !boundSessionId || request.sessionId !== boundSessionId) return;
|
|
190
|
+
const deadline = typeof request.deadline === "number" && Number.isFinite(request.deadline) ? request.deadline : Date.now() + 5_000;
|
|
191
|
+
let items: StopAllBackgroundItem[] | PromiseLike<StopAllBackgroundItem[]>;
|
|
192
|
+
try { items = compute({ deadline }); }
|
|
193
|
+
catch (error) {
|
|
194
|
+
const detail = message(error).slice(0, 400) || "contributor failed";
|
|
195
|
+
items = [{ source: name, kind: "contributor", detail: `Stop failed: ${detail}`, ok: false, error: detail.slice(0, 256) }];
|
|
196
|
+
}
|
|
197
|
+
request.contribute(name, items);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
export interface SubagentStopDeps {
|
|
201
|
+
state: SubagentState;
|
|
202
|
+
/** Run ids a scheduled run is still awaiting (`ScheduledRunManager.observedCompletionRunIds`): the `scheduled_run_pending` readiness blockers. */
|
|
203
|
+
observedCompletionRunIds(): Iterable<string>;
|
|
204
|
+
/**
|
|
205
|
+
* Stops ONE async run through the machinery behind the subagents RPC `stop` method (`stopAsyncRun` in rpc.ts): checks that the
|
|
206
|
+
* run belongs to the active session and drops a stop request into the run's control inbox, which the detached runner consumes by
|
|
207
|
+
* itself (nothing here depends on this process staying alive). Throws when the request is refused.
|
|
208
|
+
*/
|
|
209
|
+
stopRun(target: { dir: string }): unknown;
|
|
210
|
+
asyncDirRoot?: string;
|
|
211
|
+
resultsDir?: string;
|
|
212
|
+
reconcile?: SubagentReleaseDeps["reconcile"];
|
|
213
|
+
now?(): number;
|
|
214
|
+
sleep?(ms: number): Promise<void>;
|
|
215
|
+
pollIntervalMs?: number;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export interface SchedulerStopDeps {
|
|
219
|
+
/** `ScheduledRunManager.disarmSessionOnlySchedules`: pauses (persisted) and un-arms this session's session-only schedules; project schedules untouched. */
|
|
220
|
+
disarmSessionSchedules(): Array<{ id: string; ok: boolean; detail: string; error?: string }>;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const liveNested = (children: NestedRunSummary[] | undefined, found: Array<{ id: string; asyncDir: string }> = []): Array<{ id: string; asyncDir: string }> => {
|
|
224
|
+
for (const child of children ?? []) {
|
|
225
|
+
if (!TERMINAL.has(child.state) && child.asyncDir) found.push({ id: child.id, asyncDir: child.asyncDir });
|
|
226
|
+
liveNested(child.children, found);
|
|
227
|
+
liveNested(child.steps?.flatMap((step) => step.children ?? []), found);
|
|
228
|
+
}
|
|
229
|
+
return found;
|
|
230
|
+
};
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Kill switch for background subagent work of this session. Stops exactly what the readiness contributor reports as
|
|
234
|
+
* `async_run` / `scheduled_run_pending`: every live async run of the session (including detached runners), runs a scheduled
|
|
235
|
+
* run is awaiting, and live nested descendants; then waits (bounded by `request.deadline`) until each run's status is terminal.
|
|
236
|
+
* Foreground runs belong to the parent turn and are killed by the core abort (their tool signal); they are only awaited.
|
|
237
|
+
* The synchronous prefix (enumeration + delivering every stop request) runs before the first `await`, so the requests are on disk
|
|
238
|
+
* even if the process exits right after. Per-run failures are `ok:false` items and never throw. Runs that already ended
|
|
239
|
+
* (including a dead runner repaired by reconciliation) are not reported: there is nothing left to stop, which also makes the call idempotent.
|
|
240
|
+
*/
|
|
241
|
+
export async function stopSubagentWork(deps: SubagentStopDeps, request: { deadline: number }): Promise<StopAllBackgroundItem[]> {
|
|
242
|
+
const { state } = deps;
|
|
243
|
+
const items: StopAllBackgroundItem[] = [];
|
|
244
|
+
const add = (kind: string, detail: string, ok: boolean, id?: string, error?: string) => {
|
|
245
|
+
items.push({ source: "subagents", kind, detail: detail.slice(0, 512), ok, ...(id ? { id: id.slice(0, 256) } : {}), ...(error ? { error: error.slice(0, 256) } : {}) });
|
|
246
|
+
};
|
|
247
|
+
const now = deps.now ?? Date.now;
|
|
248
|
+
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
|
|
249
|
+
const reconcile = deps.reconcile ?? reconcileAsyncRun;
|
|
250
|
+
const resultsDir = deps.resultsDir ?? DIRS.results;
|
|
251
|
+
const asyncRoot = deps.asyncDirRoot ?? DIRS.async;
|
|
252
|
+
|
|
253
|
+
/** "live" and "unknown" both get a stop attempt (fail closed); only a provably finished run is skipped. */
|
|
254
|
+
const check = (asyncDir: string, job?: AsyncJobState): { settled: boolean; state?: string } => {
|
|
255
|
+
try {
|
|
256
|
+
const reconciled = reconcile(asyncDir, job ? { resultsDir, startedRun: {
|
|
257
|
+
runId: job.asyncId, pid: job.pid, sessionId: job.sessionId, completionOwnerId: job.completionOwnerId, mode: job.mode,
|
|
258
|
+
agents: job.agents, chainStepCount: job.chainStepCount, parallelGroups: job.parallelGroups, startedAt: job.startedAt, sessionFile: job.sessionFile,
|
|
259
|
+
} } : { resultsDir });
|
|
260
|
+
const status = reconciled.status;
|
|
261
|
+
if (!status) return { settled: job === undefined };
|
|
262
|
+
return { settled: TERMINAL.has(status.state), state: status.state };
|
|
263
|
+
} catch { return { settled: false }; }
|
|
264
|
+
};
|
|
265
|
+
|
|
266
|
+
const candidates = new Map<string, { asyncDir: string; label: string; kind: string }>();
|
|
267
|
+
for (const job of state.asyncJobs.values()) {
|
|
268
|
+
if (job.status !== "queued" && job.status !== "running") continue;
|
|
269
|
+
if (job.sessionId && state.currentSessionId && job.sessionId !== state.currentSessionId) {
|
|
270
|
+
add("async_run", `Async run ${job.asyncId} belongs to another session; left untouched`, false, job.asyncId, "foreign_session");
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
candidates.set(job.asyncId, { asyncDir: job.asyncDir, kind: "async_run", label: `Async ${job.mode ?? "run"}${job.agents?.length ? ` (${job.agents.join(", ")})` : ""} ${job.asyncId}` });
|
|
274
|
+
}
|
|
275
|
+
for (const runId of deps.observedCompletionRunIds()) {
|
|
276
|
+
if (candidates.has(runId)) continue;
|
|
277
|
+
candidates.set(runId, { asyncDir: path.join(asyncRoot, runId), kind: "scheduled_run", label: `Scheduled run ${runId}` });
|
|
278
|
+
}
|
|
279
|
+
const nested = [
|
|
280
|
+
...[...state.asyncJobs.values()].flatMap((job) => liveNested(job.nestedChildren)),
|
|
281
|
+
...[...(state.retainedForegroundNestedChildren?.values() ?? [])].flatMap((retained) => liveNested(retained.children)),
|
|
282
|
+
];
|
|
283
|
+
for (const child of nested) {
|
|
284
|
+
if (!candidates.has(child.id)) candidates.set(child.id, { asyncDir: child.asyncDir, kind: "nested_run", label: `Nested run ${child.id}` });
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const waiting: Array<{ id: string; asyncDir: string; label: string; kind: string }> = [];
|
|
288
|
+
for (const [id, candidate] of candidates) {
|
|
289
|
+
const job = state.asyncJobs.get(id);
|
|
290
|
+
if (check(candidate.asyncDir, job).settled) continue;
|
|
291
|
+
try {
|
|
292
|
+
deps.stopRun({ dir: candidate.asyncDir });
|
|
293
|
+
waiting.push({ id, ...candidate });
|
|
294
|
+
} catch (error) {
|
|
295
|
+
add(candidate.kind, `Failed to stop ${candidate.label}: ${message(error)}`, false, id, message(error));
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
const foreground = [...state.foregroundControls.values()].map((control) => ({ runId: control.runId, label: `Foreground ${control.mode} run${control.currentAgent ? ` (${control.currentAgent})` : ""} ${control.runId}` }));
|
|
299
|
+
|
|
300
|
+
const poll = deps.pollIntervalMs ?? 50;
|
|
301
|
+
let pendingForeground = foreground;
|
|
302
|
+
let pendingRuns = waiting;
|
|
303
|
+
while ((pendingRuns.length > 0 || pendingForeground.length > 0) && now() < request.deadline) {
|
|
304
|
+
await sleep(poll);
|
|
305
|
+
pendingRuns = pendingRuns.filter((run) => {
|
|
306
|
+
const result = check(run.asyncDir);
|
|
307
|
+
if (!result.settled) return true;
|
|
308
|
+
add(run.kind, result.state === "stopped" || result.state === undefined ? `Stopped ${run.label}` : `${run.label} ended (${result.state}) while stopping`, true, run.id);
|
|
309
|
+
return false;
|
|
310
|
+
});
|
|
311
|
+
pendingForeground = pendingForeground.filter((run) => {
|
|
312
|
+
if (state.foregroundControls.has(run.runId)) return true;
|
|
313
|
+
add("foreground_run", `${run.label} ended with the aborted turn`, true, run.runId);
|
|
314
|
+
return false;
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
for (const run of pendingRuns) add(run.kind, `Stop request delivered for ${run.label}, but the runner has not acknowledged it yet`, false, run.id, "timeout");
|
|
318
|
+
for (const run of pendingForeground) add("foreground_run", `${run.label} is still active after the turn abort`, false, run.runId, "timeout");
|
|
319
|
+
return items;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Disarms this session's session-only schedules (see `ScheduledRunManager.disarmSessionOnlySchedules`). */
|
|
323
|
+
export function stopSchedules(deps: SchedulerStopDeps): StopAllBackgroundItem[] {
|
|
324
|
+
return deps.disarmSessionSchedules().map((entry) => ({
|
|
325
|
+
source: "scheduler", kind: "schedule", id: entry.id.slice(0, 256), detail: entry.detail.slice(0, 512), ok: entry.ok,
|
|
326
|
+
...(entry.error ? { error: entry.error.slice(0, 256) } : {}),
|
|
327
|
+
}));
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* Registers the `subagents` and `scheduler` release-readiness contributors and, when `stopAll` is given, their kill-switch
|
|
332
|
+
* handlers on the same bus. All answer only for the session this extension hosts (`supervisorOwnerSessionId`, not
|
|
333
|
+
* `currentSessionId`); other sessions' requests are ignored.
|
|
334
|
+
*/
|
|
335
|
+
export function registerReleaseReadiness(options: {
|
|
336
|
+
events: Events;
|
|
337
|
+
getHostSessionId(): string | null;
|
|
338
|
+
subagents: SubagentReleaseDeps;
|
|
339
|
+
scheduler: SchedulerReleaseDeps;
|
|
340
|
+
stopAll?: { subagents: SubagentStopDeps; scheduler: SchedulerStopDeps };
|
|
341
|
+
}): { dispose(): void } {
|
|
342
|
+
let disposed = false;
|
|
343
|
+
const unsubscribes = [
|
|
344
|
+
options.events.on(RELEASE_READINESS_EVENT, (raw) => {
|
|
345
|
+
if (!disposed) answerReleaseReadiness(raw, "subagents", options.getHostSessionId(), () => subagentReleaseBlockers(options.subagents));
|
|
346
|
+
}),
|
|
347
|
+
options.events.on(RELEASE_READINESS_EVENT, (raw) => {
|
|
348
|
+
if (!disposed) answerReleaseReadiness(raw, "scheduler", options.getHostSessionId(), () => schedulerReleaseBlockers(options.scheduler));
|
|
349
|
+
}),
|
|
350
|
+
];
|
|
351
|
+
const stopAll = options.stopAll;
|
|
352
|
+
if (stopAll) {
|
|
353
|
+
// Schedules first: their timers are cleared synchronously, before the run stops below can complete a scheduled run and re-arm anything.
|
|
354
|
+
unsubscribes.push(
|
|
355
|
+
options.events.on(STOP_ALL_BACKGROUND_EVENT, (raw) => {
|
|
356
|
+
if (!disposed) answerStopAllBackground(raw, "scheduler", options.getHostSessionId(), () => stopSchedules(stopAll.scheduler));
|
|
357
|
+
}),
|
|
358
|
+
options.events.on(STOP_ALL_BACKGROUND_EVENT, (raw) => {
|
|
359
|
+
if (!disposed) answerStopAllBackground(raw, "subagents", options.getHostSessionId(), (request) => stopSubagentWork(stopAll.subagents, request));
|
|
360
|
+
}),
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
return { dispose() {
|
|
364
|
+
disposed = true;
|
|
365
|
+
for (const unsubscribe of unsubscribes) if (typeof unsubscribe === "function") unsubscribe();
|
|
366
|
+
} };
|
|
367
|
+
}
|
|
@@ -175,7 +175,7 @@ interface FleetCandidate {
|
|
|
175
175
|
|
|
176
176
|
type StatusRpcParams = Pick<SubagentParamsLike, "id" | "runId" | "dir" | "index" | "view" | "lines">;
|
|
177
177
|
|
|
178
|
-
function buildFleetStatus(
|
|
178
|
+
export function buildFleetStatus(
|
|
179
179
|
state: SubagentState | undefined,
|
|
180
180
|
keyState: FleetKeyState,
|
|
181
181
|
sessionId: string | null | undefined,
|
|
@@ -822,6 +822,8 @@ function errorReply(raw: unknown, error: unknown): SubagentRpcReplyEnvelope {
|
|
|
822
822
|
export function registerSubagentRpcBridge(options: RegisterSubagentRpcBridgeOptions): {
|
|
823
823
|
emitReady: (ctx?: ExtensionContext | null) => void;
|
|
824
824
|
dispose: () => void;
|
|
825
|
+
/** In-process entry to the same stop path as the RPC `stop` method (used by the release kill switch). Throws when refused. */
|
|
826
|
+
stopRun: (params: { runId?: string; dir?: string; childId?: string }) => ReturnType<typeof stopAsyncRun>;
|
|
825
827
|
} {
|
|
826
828
|
const fleetKeys: FleetKeyState = { sessionId: null, next: 0, keys: new Map() };
|
|
827
829
|
const unsubscribe = options.events.on(SUBAGENT_RPC_REQUEST_EVENT, async (raw) => {
|
|
@@ -849,5 +851,10 @@ export function registerSubagentRpcBridge(options: RegisterSubagentRpcBridgeOpti
|
|
|
849
851
|
dispose: () => {
|
|
850
852
|
if (typeof unsubscribe === "function") unsubscribe();
|
|
851
853
|
},
|
|
854
|
+
stopRun: (params) => {
|
|
855
|
+
const ctx = options.getContext();
|
|
856
|
+
if (!ctx) throw new SubagentRpcError("no_active_session", "No active extension context for subagent RPC.");
|
|
857
|
+
return stopAsyncRun(params, options, ctx);
|
|
858
|
+
},
|
|
852
859
|
};
|
|
853
860
|
}
|
|
@@ -34,11 +34,17 @@ function resolveRegistry(): PiWebSessionLivenessRegistry | null {
|
|
|
34
34
|
return registry as PiWebSessionLivenessRegistry;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
export function retainLiveForegroundNestedRoute(state: Pick<SubagentState, "retainedForegroundNestedRoutes"
|
|
37
|
+
export function retainLiveForegroundNestedRoute(state: Pick<SubagentState, "retainedForegroundNestedRoutes" | "retainedForegroundNestedChildren"> & Partial<Pick<SubagentState, "currentSessionId" | "foregroundControls">>, route: NestedRouteInfo): boolean {
|
|
38
38
|
const nested = projectNestedEvents(route);
|
|
39
39
|
if (!hasLiveNestedDescendants(nested.children)) return false;
|
|
40
40
|
state.retainedForegroundNestedRoutes ??= new Map();
|
|
41
41
|
state.retainedForegroundNestedRoutes.set(route.rootRunId, route);
|
|
42
|
+
if (state.currentSessionId) {
|
|
43
|
+
state.retainedForegroundNestedChildren ??= new Map();
|
|
44
|
+
const owner = state.foregroundControls?.get(route.rootRunId);
|
|
45
|
+
const workflowId = owner?.parentWorkflowRunId ?? (owner?.mode === "workflow" ? owner.runId : undefined);
|
|
46
|
+
state.retainedForegroundNestedChildren.set(route.rootRunId, { sessionId: state.currentSessionId, ...(workflowId ? { workflowId } : {}), children: nested.children });
|
|
47
|
+
}
|
|
42
48
|
return true;
|
|
43
49
|
}
|
|
44
50
|
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { execFileSync } from "node:child_process";
|
|
2
1
|
import { randomUUID } from "node:crypto";
|
|
3
2
|
import * as fs from "node:fs";
|
|
4
3
|
import * as path from "node:path";
|
|
5
4
|
import { writePrivateAtomicJson } from "../shared/atomic-json.ts";
|
|
6
5
|
import { DEFAULT_FILE_SYSTEM_RETRY_DELAYS_MS, isRetryableFileSystemError, waitForFileSystemRetry } from "../shared/file-system-retry.ts";
|
|
6
|
+
import { processStartKey } from "../shared/process-identity.ts";
|
|
7
7
|
import { assertWorkflowJsonValue } from "../workflows/scripted-workflow.ts";
|
|
8
8
|
import type { MissionStoreLocation } from "./types.ts";
|
|
9
9
|
import { validateMissionId } from "./store.ts";
|
|
@@ -44,49 +44,6 @@ interface StateLockOwner {
|
|
|
44
44
|
processKey?: string;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
function linuxProcessStartKey(pid: number): string | undefined {
|
|
48
|
-
try {
|
|
49
|
-
const raw = fs.readFileSync(`/proc/${pid}/stat`, "utf-8");
|
|
50
|
-
const tail = raw.slice(raw.lastIndexOf(")") + 2).trim().split(/\s+/);
|
|
51
|
-
return tail[19] ? `linux:${tail[19]}` : undefined;
|
|
52
|
-
} catch {
|
|
53
|
-
return undefined;
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
function psProcessStartKey(pid: number): string | undefined {
|
|
58
|
-
try {
|
|
59
|
-
const raw = execFileSync("ps", ["-p", String(pid), "-o", "lstart="], { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], timeout: 1000 }).trim();
|
|
60
|
-
return raw ? `ps:${raw}` : undefined;
|
|
61
|
-
} catch {
|
|
62
|
-
return undefined;
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
function windowsProcessStartKey(pid: number): string | undefined {
|
|
67
|
-
try {
|
|
68
|
-
const raw = execFileSync("powershell.exe", ["-NoProfile", "-Command", `(Get-CimInstance Win32_Process -Filter \"ProcessId=${pid}\").CreationDate`], { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"], timeout: 1000, windowsHide: true }).trim();
|
|
69
|
-
return raw ? `win:${raw}` : undefined;
|
|
70
|
-
} catch {
|
|
71
|
-
return undefined;
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
let currentProcessKey: string | undefined | null = null;
|
|
76
|
-
|
|
77
|
-
function processStartKey(pid: number): string | undefined {
|
|
78
|
-
// Foreign PIDs can be reused while this process lives, so resolve them afresh.
|
|
79
|
-
if (pid !== process.pid) return computeProcessStartKey(pid);
|
|
80
|
-
if (currentProcessKey === null) currentProcessKey = computeProcessStartKey(pid);
|
|
81
|
-
return currentProcessKey;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
function computeProcessStartKey(pid: number): string | undefined {
|
|
85
|
-
if (process.platform === "linux") return linuxProcessStartKey(pid) ?? psProcessStartKey(pid);
|
|
86
|
-
if (process.platform === "win32") return windowsProcessStartKey(pid);
|
|
87
|
-
return undefined;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
47
|
type StateLockOptions = Required<MissionWorkflowStateOptions>;
|
|
91
48
|
|
|
92
49
|
function readStateLockOwner(lockPath: string): StateLockOwner | undefined {
|
|
@@ -162,6 +162,9 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
|
|
|
162
162
|
asyncDir: run.asyncDir,
|
|
163
163
|
toolCallId: run.toolCallId,
|
|
164
164
|
status: run.state,
|
|
165
|
+
endedAt: run.endedAt,
|
|
166
|
+
processTerminal: run.processTerminal,
|
|
167
|
+
error: run.error,
|
|
165
168
|
sessionId: run.sessionId,
|
|
166
169
|
activityState: run.activityState,
|
|
167
170
|
lastActivityAt: run.lastActivityAt,
|
|
@@ -445,6 +448,9 @@ export function createAsyncJobTracker(pi: Pick<ExtensionAPI, "events">, state: S
|
|
|
445
448
|
if (status) {
|
|
446
449
|
const previousStatus = job.status;
|
|
447
450
|
job.status = status.state;
|
|
451
|
+
job.endedAt = status.endedAt;
|
|
452
|
+
job.error = status.error;
|
|
453
|
+
job.processTerminal = readProcessTerminal(job.asyncDir, { runId: job.asyncId, runnerProcessInstanceId: status.processTerminal?.runnerProcessInstanceId }) ?? status.processTerminal;
|
|
448
454
|
if (!terminalStatus(job.status)) terminalPublications.delete(job.asyncId);
|
|
449
455
|
if (job.status === "running") runningJobIds.add(job.asyncId);
|
|
450
456
|
else runningJobIds.delete(job.asyncId);
|
|
@@ -425,12 +425,21 @@ function traceNotification(reason: NotificationReason, result?: TraceIdentity):
|
|
|
425
425
|
}
|
|
426
426
|
}
|
|
427
427
|
|
|
428
|
+
function completionRunKey(result: CompletionNotification): string | undefined {
|
|
429
|
+
for (const candidate of [result.runId, result.id]) {
|
|
430
|
+
if (typeof candidate === "string" && candidate) return candidate;
|
|
431
|
+
}
|
|
432
|
+
return undefined;
|
|
433
|
+
}
|
|
434
|
+
|
|
428
435
|
interface PendingCompletion {
|
|
429
436
|
trace?: TraceIdentity;
|
|
430
437
|
key: string;
|
|
431
438
|
details: SubagentNotifyDetails;
|
|
432
439
|
sessionId: string;
|
|
433
440
|
completionOwnerId: unknown;
|
|
441
|
+
/** Run id used to take over a result left by a provably dead previous owner. */
|
|
442
|
+
runKey?: string;
|
|
434
443
|
triggerTurn: boolean;
|
|
435
444
|
resolve(accepted: boolean): void;
|
|
436
445
|
}
|
|
@@ -595,7 +604,7 @@ export default function registerSubagentNotify(
|
|
|
595
604
|
const batchers = new Map<string, CompletionBatcher<PendingCompletion>>();
|
|
596
605
|
let disposed = false;
|
|
597
606
|
const ownsResult = options.ownership?.owns
|
|
598
|
-
?? ((sessionId: string, completionOwnerId: unknown) => sessionId === state.currentSessionId
|
|
607
|
+
?? ((sessionId: string, completionOwnerId: unknown, _runKey?: string) => sessionId === state.currentSessionId
|
|
599
608
|
&& typeof completionOwnerId === "string"
|
|
600
609
|
&& completionOwnerId === state.completionOwnerId);
|
|
601
610
|
|
|
@@ -613,7 +622,7 @@ export default function registerSubagentNotify(
|
|
|
613
622
|
for (const item of items) {
|
|
614
623
|
const owned = item.details.source === "foreground"
|
|
615
624
|
? item.sessionId === state.currentSessionId
|
|
616
|
-
: ownsResult(item.sessionId, item.completionOwnerId);
|
|
625
|
+
: ownsResult(item.sessionId, item.completionOwnerId, item.runKey);
|
|
617
626
|
if (!owned) traceNotification(item.details.source === "foreground" ? "emit_foreground_session_mismatch" : "emit_not_owned", item.trace);
|
|
618
627
|
(owned ? accepted : rejected).push(item);
|
|
619
628
|
}
|
|
@@ -646,7 +655,7 @@ export default function registerSubagentNotify(
|
|
|
646
655
|
traceNotification("foreground_session_mismatch", result);
|
|
647
656
|
return Promise.resolve(false);
|
|
648
657
|
}
|
|
649
|
-
} else if (!ownsResult(result.sessionId, result.completionOwnerId)) {
|
|
658
|
+
} else if (!ownsResult(result.sessionId, result.completionOwnerId, completionRunKey(result))) {
|
|
650
659
|
traceNotification("not_owned", result);
|
|
651
660
|
return Promise.resolve(false);
|
|
652
661
|
}
|
|
@@ -675,6 +684,7 @@ export default function registerSubagentNotify(
|
|
|
675
684
|
details,
|
|
676
685
|
sessionId: result.sessionId,
|
|
677
686
|
completionOwnerId: result.completionOwnerId,
|
|
687
|
+
runKey: completionRunKey(result),
|
|
678
688
|
triggerTurn: result.triggerTurn !== false,
|
|
679
689
|
resolve,
|
|
680
690
|
};
|