@danceiny/gotry 0.2.0-rc.31 → 0.2.0-rc.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/bin/gotry-bootstrap.js +63 -53
- package/bin/gotry-hbcli-release.d.ts +3 -0
- package/bin/gotry-hbcli-release.js +4 -0
- package/client/client.js +66 -21
- package/dist/capabilities/doctor.js +7 -8
- package/dist/capabilities/effect.js +45 -12
- package/dist/capabilities/flyai-config.js +12 -10
- package/dist/capabilities/flyai.js +87 -56
- package/dist/capabilities/hbcli-config.js +156 -0
- package/dist/capabilities/hbcli.js +29 -2
- package/dist/capabilities/resilience.js +6 -6
- package/dist/capabilities/session/adapters/ctrip-flight.js +42 -7
- package/dist/capabilities/session/adapters/ctrip-hotel.js +52 -3
- package/dist/capabilities/session/bridge-ledger.js +168 -67
- package/dist/capabilities/session/ctrip-city.js +186 -0
- package/dist/capabilities/session/extension-bridge.js +148 -47
- package/dist/capabilities/session/gui-flight.js +949 -0
- package/dist/capabilities/session/gui-hotel.js +470 -0
- package/dist/capabilities/session-search.js +102 -14
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +129 -2
- package/dist/scripts/booking-copilot-dsh-readiness-proof.js +53 -5
- package/dist/scripts/booking-copilot-gateway-chat-adapter-tests.js +233 -0
- package/dist/scripts/booking-copilot-planner-live-probe.js +1 -0
- package/dist/scripts/booking-copilot-startup-proof-tests.js +1 -1
- package/dist/scripts/bootstrap-tests.js +23 -6
- package/dist/scripts/bridge-ledger-tests.js +20 -18
- package/dist/scripts/ctrip-city-tests.js +80 -0
- package/dist/scripts/effect-tests.js +602 -504
- package/dist/scripts/extension-tests.js +14 -8
- package/dist/scripts/fixtures/tenant-runaway-worker-fixture.mjs +73 -0
- package/dist/scripts/flight-malformed-tests.js +1 -1
- package/dist/scripts/flyai-setup-tests.js +19 -4
- package/dist/scripts/flyai-tests.js +768 -653
- package/dist/scripts/gotry-web-api-tests.js +5 -0
- package/dist/scripts/gotry-web-client-tests.js +20 -1
- package/dist/scripts/gui-flight-live.js +27 -0
- package/dist/scripts/hbcli-tests.js +1 -1
- package/dist/scripts/hbcli-upgrade-tests.js +107 -0
- package/dist/scripts/hbcli-web-setup-tests.js +114 -0
- package/dist/scripts/hotel-date-gate-tests.js +1 -1
- package/dist/scripts/hotel-result-tests.js +169 -0
- package/dist/scripts/issue-271-liveness-tests.js +1 -1
- package/dist/scripts/mutation-check.js +153 -0
- package/dist/scripts/nightly-evidence.js +4 -111
- package/dist/scripts/session-tests.js +211 -6
- package/dist/scripts/tenant-bridge-org-tests.js +1264 -0
- package/dist/scripts/tenant-capacity-probe.js +263 -0
- package/dist/scripts/tenant-compute-isolation-tests.js +707 -0
- package/dist/scripts/tenant-egress-credentials-tests.js +822 -0
- package/dist/scripts/tenant-executor-binding-tests.js +203 -0
- package/dist/scripts/tenant-prompt-partition-tests.js +265 -0
- package/dist/scripts/tenant-seam-adversarial-tests.js +957 -0
- package/dist/scripts/tenant-seam-quota-tests.js +383 -0
- package/dist/scripts/tenant-seam-test-kit.js +292 -0
- package/dist/scripts/tenant-state-copies-tests.js +607 -0
- package/dist/scripts/tenant-state-drill-tests.js +298 -0
- package/dist/scripts/tenant-state-drill-worker.js +164 -0
- package/dist/scripts/tenant-state-durable-conformance.js +101 -0
- package/dist/scripts/tenant-state-handoff-tests.js +854 -0
- package/dist/scripts/tenant-state-lease-conformance.js +142 -0
- package/dist/scripts/tenant-state-lease-worker.js +28 -0
- package/dist/scripts/tenant-state-router-tests.js +464 -0
- package/dist/scripts/tenant-state-test-kit.js +92 -0
- package/dist/scripts/tenant-state-tests.js +564 -0
- package/dist/src/backend/modules/booking-executor.js +61 -6
- package/dist/src/backend/modules/session-search.js +136 -39
- package/dist/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/dist/src/booking-surface/dsh-planner.js +150 -19
- package/dist/src/booking-surface/index.js +1 -0
- package/dist/src/booking-surface/managed-dsh-run-port.js +121 -7
- package/dist/src/booking-surface/server.js +193 -80
- package/dist/src/booking-surface/startup.js +153 -0
- package/dist/src/booking-surface/tenancy.js +188 -0
- package/dist/src/gotry-web-api.js +64 -2
- package/dist/src/index.js +70 -18
- package/dist/src/llm-price-table.js +148 -0
- package/dist/src/tenant-seam/index.js +8 -0
- package/dist/src/tenant-seam/llm-broker.js +508 -0
- package/dist/src/tenant-seam/process-governor.js +281 -0
- package/dist/src/tenant-seam/quotas.js +360 -0
- package/dist/src/tenant-seam/seam.js +92 -0
- package/dist/src/tenant-state/durable-store.js +158 -0
- package/dist/src/tenant-state/epoch-copies.js +422 -0
- package/dist/src/tenant-state/errors.js +108 -0
- package/dist/src/tenant-state/ids.js +101 -0
- package/dist/src/tenant-state/index.js +11 -0
- package/dist/src/tenant-state/lease-store.js +203 -0
- package/dist/src/tenant-state/tenant-lease.js +150 -0
- package/dist/src/tenant-state/tenant-router.js +650 -0
- package/extension/manifest.json +2 -2
- package/package.json +5 -1
- package/ts/capabilities/flyai-config.ts +12 -10
- package/ts/capabilities/flyai.ts +91 -50
- package/ts/capabilities/hbcli-config.ts +128 -0
- package/ts/capabilities/hbcli.ts +30 -3
- package/ts/capabilities/session/adapters/ctrip-flight.ts +43 -7
- package/ts/capabilities/session/adapters/ctrip-hotel.ts +51 -4
- package/ts/capabilities/session/bridge-ledger.ts +192 -78
- package/ts/capabilities/session/ctrip-city.ts +158 -0
- package/ts/capabilities/session/extension-bridge.ts +185 -67
- package/ts/capabilities/session/gui-flight.ts +1022 -0
- package/ts/capabilities/session/gui-hotel.ts +505 -0
- package/ts/capabilities/session-search.ts +130 -18
- package/ts/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/ts/src/booking-surface/dsh-planner.ts +169 -19
- package/ts/src/booking-surface/index.ts +1 -0
- package/ts/src/booking-surface/managed-dsh-run-port.ts +150 -7
- package/ts/src/booking-surface/server.ts +215 -84
- package/ts/src/booking-surface/startup.ts +155 -0
- package/ts/src/booking-surface/tenancy.ts +242 -0
- package/ts/src/gotry-web-api.ts +51 -2
- package/ts/src/index.ts +67 -19
- package/ts/src/llm-price-table.ts +180 -0
- package/ts/src/tenant-seam/index.ts +9 -0
- package/ts/src/tenant-seam/llm-broker.ts +520 -0
- package/ts/src/tenant-seam/process-governor.ts +343 -0
- package/ts/src/tenant-seam/quotas.ts +433 -0
- package/ts/src/tenant-seam/seam.ts +142 -0
- package/ts/src/tenant-state/durable-store.ts +171 -0
- package/ts/src/tenant-state/epoch-copies.ts +491 -0
- package/ts/src/tenant-state/errors.ts +128 -0
- package/ts/src/tenant-state/ids.ts +108 -0
- package/ts/src/tenant-state/index.ts +12 -0
- package/ts/src/tenant-state/lease-store.ts +266 -0
- package/ts/src/tenant-state/tenant-lease.ts +175 -0
- package/ts/src/tenant-state/tenant-router.ts +743 -0
|
@@ -14,11 +14,12 @@ import { join } from 'node:path';
|
|
|
14
14
|
import { fileURLToPath } from 'node:url';
|
|
15
15
|
import { BOOKING_READ_ACTION_KINDS } from './contracts.js';
|
|
16
16
|
import { buildTimeAnchor } from '../time-anchor.js';
|
|
17
|
+
import { assertPromptCachePartition } from '../tenant-state/ids.js';
|
|
17
18
|
export { formatUtcOffsetLabel } from '../time-anchor.js';
|
|
18
19
|
import { actionsForEmbeddedCapability, } from './profile.js';
|
|
19
20
|
import { bookingDigest } from './runtime.js';
|
|
20
21
|
import { validateBookingReadAction, validateBookingIntentProjection, bookingSurfaceSchema, } from './validation.js';
|
|
21
|
-
import { createManagedDshRunPort } from './managed-dsh-run-port.js';
|
|
22
|
+
import { assertManagedDshLimitsEnforceable, createManagedDshRunPort, ManagedDshResourceLimitError } from './managed-dsh-run-port.js';
|
|
22
23
|
import { BOOKING_INTENT_SCHEMA_VERSION, BOOKING_INTENT_TARGET_ACTION, } from './booking-intent.js';
|
|
23
24
|
export const DSH_EMBEDDED_BOOKING_TOOL_NAMES = [
|
|
24
25
|
'booking_search_hotels',
|
|
@@ -101,6 +102,11 @@ export function buildDshPlannerEnvironment(source = process.env) {
|
|
|
101
102
|
target.DEEPSEEK_MODEL = model;
|
|
102
103
|
if (maxTokens)
|
|
103
104
|
target.DEEPSEEK_MAX_TOKENS = maxTokens;
|
|
105
|
+
// Wire-protocol selection rides the same namespace: 'gateway-chat' routes the
|
|
106
|
+
// child at the chat-completions adapter plugin (dsh-llm-gateway-chat.js)
|
|
107
|
+
// instead of the SDK's bundled Messages adapter.
|
|
108
|
+
if (source.DEEPSEEK_PROVIDER)
|
|
109
|
+
target.DEEPSEEK_PROVIDER = source.DEEPSEEK_PROVIDER;
|
|
104
110
|
return target;
|
|
105
111
|
}
|
|
106
112
|
/** Typed against contracts so the persona example can never drift from the wire shape. */
|
|
@@ -119,8 +125,8 @@ function quoteYaml(value) {
|
|
|
119
125
|
return `'${value.replaceAll("'", "''")}'`;
|
|
120
126
|
}
|
|
121
127
|
/** Runtime patch over dsh's sdk-minimal profile: one typed planner tool per action kind. */
|
|
122
|
-
export function buildDshEmbeddedBookingPatch(pluginPath) {
|
|
123
|
-
|
|
128
|
+
export function buildDshEmbeddedBookingPatch(pluginPath, gatewayLlmPluginPath = '') {
|
|
129
|
+
const patchBody = `# GoTry embedded Booking Copilot: typed planning only; no browser, shell, files, Book, or persisted chat.\n\
|
|
124
130
|
- id: deepseek-llm-api-extensions\n disabled: true\n\
|
|
125
131
|
- id: session-log-deepseek\n disabled: true\n\
|
|
126
132
|
- id: plugin-package-inventory-deepseek\n disabled: true\n\
|
|
@@ -179,19 +185,37 @@ export function buildDshEmbeddedBookingPatch(pluginPath) {
|
|
|
179
185
|
skills:\n\
|
|
180
186
|
enabled: false\n\
|
|
181
187
|
toolBash: false\n\
|
|
182
|
-
toolJobs: false\n
|
|
183
|
-
|
|
184
|
-
- id: gotry-embedded-booking\n\
|
|
185
|
-
|
|
188
|
+
toolJobs: false\n`;
|
|
189
|
+
const inserts = [
|
|
190
|
+
`- insert:\n - id: gotry-embedded-booking\n name: ${quoteYaml(pluginPath)}\n`,
|
|
191
|
+
...(gatewayLlmPluginPath
|
|
192
|
+
? [`- insert:\n - id: llm-gateway-chat\n name: ${quoteYaml(gatewayLlmPluginPath)}\n`]
|
|
193
|
+
: []),
|
|
194
|
+
];
|
|
195
|
+
return patchBody + inserts.join('');
|
|
186
196
|
}
|
|
187
197
|
// Exported for the proof tests: this is the one place where a missing
|
|
188
198
|
// DEEPSEEK_MODEL could otherwise be absorbed silently.
|
|
189
199
|
export function resolveRealRunPortConfig(options) {
|
|
190
200
|
const sourceEnv = options.env ?? process.env;
|
|
191
|
-
|
|
192
|
-
|
|
201
|
+
// The route settings (provider, model, token budget) are read from the namespace the credential selects. A deployment
|
|
202
|
+
// that keeps the key in the broker may have none in its environment; the DeepSeek namespace is then the route's, and
|
|
203
|
+
// the placeholder below only selects it: it is removed from the child's environment before anything uses it.
|
|
204
|
+
const routeSource = options.llmTickets && !sourceEnv.DEEPSEEK_API_KEY && !sourceEnv.LLM_API_KEY
|
|
205
|
+
? { ...sourceEnv, DEEPSEEK_API_KEY: 'namespace-selector-only' }
|
|
206
|
+
: sourceEnv;
|
|
207
|
+
const childEnv = buildDshPlannerEnvironment(routeSource);
|
|
208
|
+
if (options.llmTickets) {
|
|
209
|
+
// The credential reaches the child only as a ticket, issued per port. Whatever the environment held is not forwarded,
|
|
210
|
+
// so an ambient key cannot slip past the broker; the ticket's own values are set where the port is created.
|
|
211
|
+
delete childEnv.DEEPSEEK_API_KEY;
|
|
212
|
+
delete childEnv.DEEPSEEK_BASE_URL;
|
|
213
|
+
}
|
|
214
|
+
else if (!childEnv.DEEPSEEK_API_KEY)
|
|
193
215
|
throw new Error('booking_planner_model_key_required');
|
|
194
|
-
const provider = options.provider?.trim() || 'deepseek-official';
|
|
216
|
+
const provider = options.provider?.trim() || childEnv.DEEPSEEK_PROVIDER || 'deepseek-official';
|
|
217
|
+
if (options.llmTickets && provider !== 'gateway-chat')
|
|
218
|
+
throw new Error('booking_planner_llm_tickets_require_gateway_chat');
|
|
195
219
|
const configuredModel = options.model ?? childEnv.DEEPSEEK_MODEL;
|
|
196
220
|
if (provider !== 'deepseek-official' && !configuredModel) {
|
|
197
221
|
throw new Error('booking_planner_model_required_for_nondefault_provider');
|
|
@@ -212,9 +236,31 @@ export function resolveRealRunPortConfig(options) {
|
|
|
212
236
|
}
|
|
213
237
|
return { childEnv, provider, model, modelSource, reasoningEffort, maxTokens };
|
|
214
238
|
}
|
|
239
|
+
/**
|
|
240
|
+
* The names under which an ambient LLM credential or address could reach a child through the runtime's inherited
|
|
241
|
+
* environment. The runtime merges the explicit environment onto the parent's with names that look secret removed, so an
|
|
242
|
+
* address (`LLM_BASE_URL`) would otherwise pass; an explicit `undefined` removes an inherited entry.
|
|
243
|
+
*/
|
|
244
|
+
const AMBIENT_LLM_CREDENTIAL_NAMES = ['LLM_API_KEY', 'LLM_BASE_URL'];
|
|
245
|
+
/**
|
|
246
|
+
* The environment of a real planner process tree. With a ticket it carries the ticket in place of the key and base URL,
|
|
247
|
+
* and removes the ambient names that could carry the real ones; without one it is the resolved environment, unchanged.
|
|
248
|
+
*/
|
|
249
|
+
export function plannerChildEnvironment(resolved, ticket) {
|
|
250
|
+
if (!ticket)
|
|
251
|
+
return resolved;
|
|
252
|
+
const tombstones = {};
|
|
253
|
+
for (const name of AMBIENT_LLM_CREDENTIAL_NAMES)
|
|
254
|
+
tombstones[name] = undefined;
|
|
255
|
+
return { ...resolved, ...tombstones, DEEPSEEK_API_KEY: ticket.token, DEEPSEEK_BASE_URL: ticket.baseUrl };
|
|
256
|
+
}
|
|
215
257
|
/** Shared by the planner and real-subprocess proofs; owns its scratch directory. */
|
|
216
258
|
export async function createRealRunPort(options) {
|
|
217
|
-
const
|
|
259
|
+
const resolved = resolveRealRunPortConfig(options);
|
|
260
|
+
const { provider, model, modelSource, reasoningEffort, maxTokens } = resolved;
|
|
261
|
+
// Every real port, not only the first one of a process, starts a child only where the tree limits can be enforced.
|
|
262
|
+
if (options.resourceLimits)
|
|
263
|
+
await assertManagedDshLimitsEnforceable(options.resourceLimits);
|
|
218
264
|
// Name the resolved route on every real port. A deployment that forgot
|
|
219
265
|
// DEEPSEEK_MODEL used to be indistinguishable from one that set it, and the
|
|
220
266
|
// substituted model is exactly what a later diagnosis needs to see.
|
|
@@ -225,8 +271,13 @@ export async function createRealRunPort(options) {
|
|
|
225
271
|
const dshHome = join(scratch, 'home');
|
|
226
272
|
mkdirSync(dshHome, { recursive: true });
|
|
227
273
|
const pluginPath = options.pluginPath ?? fileURLToPath(new URL('./dsh-plugin.js', import.meta.url));
|
|
274
|
+
const gatewayLlmPluginPath = options.gatewayLlmPluginPath ?? fileURLToPath(new URL('./dsh-llm-gateway-chat.js', import.meta.url));
|
|
228
275
|
const patchPath = join(scratch, 'embedded-booking.cordis.yml');
|
|
229
|
-
writeFileSync(patchPath, buildDshEmbeddedBookingPatch(pluginPath), { encoding: 'utf8', mode: 0o600 });
|
|
276
|
+
writeFileSync(patchPath, buildDshEmbeddedBookingPatch(pluginPath, gatewayLlmPluginPath), { encoding: 'utf8', mode: 0o600 });
|
|
277
|
+
// One ticket per port, shared with its warmer, revoked when the port closes or fails to come up. It is issued only now,
|
|
278
|
+
// after the steps above that can fail (a full disk, no descriptors), so none of them can leave a live ticket behind.
|
|
279
|
+
const ticket = options.llmTickets?.issue();
|
|
280
|
+
const childEnv = plannerChildEnvironment(resolved.childEnv, ticket);
|
|
230
281
|
let managedPort;
|
|
231
282
|
try {
|
|
232
283
|
managedPort = createManagedDshRunPort({
|
|
@@ -255,10 +306,12 @@ export async function createRealRunPort(options) {
|
|
|
255
306
|
disposeEofGraceMs: 500,
|
|
256
307
|
disposeGraceMs: 500,
|
|
257
308
|
env: childEnv,
|
|
309
|
+
...(options.resourceLimits ? { limits: options.resourceLimits } : {}),
|
|
258
310
|
...(options.dshBin ? { dshBin: options.dshBin } : {}),
|
|
259
311
|
});
|
|
260
312
|
}
|
|
261
313
|
catch (error) {
|
|
314
|
+
ticket?.revoke();
|
|
262
315
|
rmSync(scratch, { recursive: true, force: true });
|
|
263
316
|
throw error;
|
|
264
317
|
}
|
|
@@ -298,10 +351,12 @@ export async function createRealRunPort(options) {
|
|
|
298
351
|
disposeEofGraceMs: 500,
|
|
299
352
|
disposeGraceMs: 500,
|
|
300
353
|
env: childEnv,
|
|
354
|
+
...(options.resourceLimits ? { limits: options.resourceLimits } : {}),
|
|
301
355
|
...(options.dshBin ? { dshBin: options.dshBin } : {}),
|
|
302
356
|
});
|
|
303
357
|
}
|
|
304
358
|
catch (error) {
|
|
359
|
+
ticket?.revoke();
|
|
305
360
|
try {
|
|
306
361
|
await managedPort.close();
|
|
307
362
|
}
|
|
@@ -337,6 +392,8 @@ export async function createRealRunPort(options) {
|
|
|
337
392
|
return closePromise;
|
|
338
393
|
closePromise = (async () => {
|
|
339
394
|
const results = await Promise.allSettled([managedPort.close(), closeWarmer()]);
|
|
395
|
+
// Whatever the cleanup did, the credential stops working with the port.
|
|
396
|
+
ticket?.revoke();
|
|
340
397
|
const failures = results.flatMap((result) => result.status === 'rejected' ? [result.reason] : []);
|
|
341
398
|
if (failures.length) {
|
|
342
399
|
throw new AggregateError(failures, 'booking_planner_run_port_cleanup_failed');
|
|
@@ -358,7 +415,7 @@ function plannerVisibleAllowedActions(task) {
|
|
|
358
415
|
? task.allowedActions.filter((kind) => kind === 'checkout.prepare')
|
|
359
416
|
: [...task.allowedActions];
|
|
360
417
|
}
|
|
361
|
-
function plannerPrompt(turn, task, clock) {
|
|
418
|
+
function plannerPrompt(turn, task, clock, cachePartition) {
|
|
362
419
|
const availability = task.availability;
|
|
363
420
|
const allowedActions = plannerVisibleAllowedActions(task);
|
|
364
421
|
const plannerWorkspace = 'capabilities' in turn.workspace
|
|
@@ -396,6 +453,9 @@ function plannerPrompt(turn, task, clock) {
|
|
|
396
453
|
`Separate hotel facilities from meal, cancellation, room, and offer criteria: hotel amenities (for example pool and parking) go under input.patch.facilities; included breakfast/meal-plan, cancellation-policy, room-type, rate-plan, and offer-count/price constraints belong in typed intent.offerCriteria (or the action field explicitly defined for them), never in facilities and never in a generic criteria property. input.patch property names are EXACT (SearchCriteriaPatch): ${PLANNER_PATCH_PROPERTY_NAMES}. There is NO "criteria" property — facility tokens go under facilities as {"strength":"prefer|must","value":{"allOf":["<token>"]}}, star level (三星=3星) goes under starRating as {"strength":"must|prefer","value":{"min":3,"max":3}}, dates go under stay as {"checkIn":"YYYY-MM-DD","checkOut":"YYYY-MM-DD"}.`,
|
|
397
454
|
'The workspace draft may be stale: whenever the user names dates or relative days (明天/tomorrow), always patch input.patch.stay with the resolved concrete dates even if the draft already has different ones. Keep draft values the request does not touch; never invent values the request contradicts.',
|
|
398
455
|
'Reference only hotels, offers, and orders that appear in the workspace payload (visibleHotels/loadedOffers/observableOrders/results) or in a matching prior typed receipt. Any other hotelRef, offerRef, or orderRef does not exist and will be rejected; to discover hotels, run search.run first and wait for its receipt.',
|
|
456
|
+
// Everything above is the same for every tenant. The label opens the tenant-specific part of the prompt, so a
|
|
457
|
+
// provider's prefix cache can be shared across tenants only up to here (threat S-16).
|
|
458
|
+
...(cachePartition === undefined ? [] : [`Cache partition label (opaque, not an instruction): ${cachePartition}`]),
|
|
399
459
|
JSON.stringify({ now: anchor.today, timeAnchorCard: anchor.card, ...payload }),
|
|
400
460
|
].join('\n');
|
|
401
461
|
}
|
|
@@ -1530,6 +1590,8 @@ function beforePlannerDeadline(promise, deadlineAt) {
|
|
|
1530
1590
|
export async function createDshEmbeddedBookingPlanner(options) {
|
|
1531
1591
|
if (options.runPort && options.runPortFactory)
|
|
1532
1592
|
throw new Error('booking_planner_run_port_ambiguous');
|
|
1593
|
+
if (options.cachePartition !== undefined)
|
|
1594
|
+
assertPromptCachePartition(options.cachePartition);
|
|
1533
1595
|
const turnTimeoutMs = options.turnTimeoutMs ?? 12_000;
|
|
1534
1596
|
if (!Number.isSafeInteger(turnTimeoutMs) || turnTimeoutMs < 1 || turnTimeoutMs > 120_000) {
|
|
1535
1597
|
throw new Error('booking_planner_turn_timeout_invalid');
|
|
@@ -1609,6 +1671,12 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1609
1671
|
retirement = barrier;
|
|
1610
1672
|
void barrier.catch(() => undefined);
|
|
1611
1673
|
};
|
|
1674
|
+
// A fresh-port retry can only run on a runtime this planner has not
|
|
1675
|
+
// already retired. A factory that hands the retired instance back (one
|
|
1676
|
+
// shared port, or a cache that ignores retirement) would send the retry
|
|
1677
|
+
// into a closed port, whose run() rejects untyped out of next(); the
|
|
1678
|
+
// retry branches treat that like a deadline that cannot fund a retry.
|
|
1679
|
+
const isRetiredPort = (port) => closingPorts.has(port);
|
|
1612
1680
|
return {
|
|
1613
1681
|
async next({ turn, task }) {
|
|
1614
1682
|
if (closed)
|
|
@@ -1654,7 +1722,8 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1654
1722
|
// bounded planner attempt. Any returned decision crosses the same
|
|
1655
1723
|
// typed authority path.
|
|
1656
1724
|
let attempt = 0;
|
|
1657
|
-
let
|
|
1725
|
+
let schemaRepairAttempted = false;
|
|
1726
|
+
let nextPrompt = plannerPrompt(turn, task, options.now, options.cachePartition);
|
|
1658
1727
|
// A run that goes silent must not be allowed to silently consume the
|
|
1659
1728
|
// whole turn deadline: the idle stall budget fires only when the run
|
|
1660
1729
|
// stops emitting notifications entirely, the child is retired, and
|
|
@@ -1673,10 +1742,30 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1673
1742
|
try {
|
|
1674
1743
|
const result = await beforePlannerDeadline(runWithStallBudget(runPort, nextPrompt, sessionId, softStallBudgetMs), deadlineAt);
|
|
1675
1744
|
modelStepCount += observedModelStepCount(result.events);
|
|
1676
|
-
|
|
1745
|
+
const schemaRejectionsThisRun = result.events
|
|
1677
1746
|
.map((event, index) => toolResultObservation(event, index))
|
|
1678
1747
|
.filter((observation) => observation?.kind === 'schema-rejection').length;
|
|
1679
|
-
|
|
1748
|
+
schemaRejectedCallCount += schemaRejectionsThisRun;
|
|
1749
|
+
try {
|
|
1750
|
+
decisions = scanDshToolDecisionEvents(result.events, task);
|
|
1751
|
+
}
|
|
1752
|
+
catch (scanError) {
|
|
1753
|
+
// hotel-be#33640: a provider can emit a tool call whose
|
|
1754
|
+
// arguments miss the declared schema and then stop. Treat one
|
|
1755
|
+
// schema-shaped rejection as a repairable provider mistake:
|
|
1756
|
+
// retry once with a stricter prompt on the same session
|
|
1757
|
+
// instead of surfacing PLANNER_PROVIDER_RESPONSE_INVALID to the
|
|
1758
|
+
// user. Safety errors and unproven rejections still fail closed.
|
|
1759
|
+
if (!schemaRepairAttempted && schemaRejectionsThisRun > 0 && attempt < 3 && isSchemaShapeError(scanError)) {
|
|
1760
|
+
schemaRepairAttempted = true;
|
|
1761
|
+
console.error('[booking-copilot] planner tool call failed schema validation; retrying with a repair prompt:', JSON.stringify({
|
|
1762
|
+
attempt, schemaRejectionsThisRun,
|
|
1763
|
+
}));
|
|
1764
|
+
nextPrompt = 'Your previous booking capability tool call was rejected because its arguments did not match the declared parameter schema. Emit exactly one corrected booking capability tool call that satisfies the schema; do not repeat the rejected arguments and do not add prose.';
|
|
1765
|
+
continue;
|
|
1766
|
+
}
|
|
1767
|
+
throw scanError;
|
|
1768
|
+
}
|
|
1680
1769
|
// A valid capability decision is already receipt-gated and is
|
|
1681
1770
|
// the authority for this interval. Some providers fail a later
|
|
1682
1771
|
// post-tool model step; that must not erase the accepted action.
|
|
@@ -1726,9 +1815,13 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1726
1815
|
attempt, softStallBudgetMs, remainingMs,
|
|
1727
1816
|
}));
|
|
1728
1817
|
retire(runPort);
|
|
1729
|
-
|
|
1818
|
+
const stallRetryPort = await beforePlannerDeadline(taskPort(), deadlineAt);
|
|
1730
1819
|
if (closed || sessionClosePromise)
|
|
1731
1820
|
throw new Error(closed ? 'planner_closed' : 'planner_session_closed');
|
|
1821
|
+
if (isRetiredPort(stallRetryPort)) {
|
|
1822
|
+
throw bootNeverCompleted(runPort) ? new PlannerBootTimeoutExceeded() : new PlannerTurnDeadlineExceeded();
|
|
1823
|
+
}
|
|
1824
|
+
runPort = stallRetryPort;
|
|
1732
1825
|
// The stalled run produced no decision; it must not consume a
|
|
1733
1826
|
// prose-nudge attempt.
|
|
1734
1827
|
attempt -= 1;
|
|
@@ -1736,8 +1829,30 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1736
1829
|
}
|
|
1737
1830
|
// The SDK reports an expired local handshake as its own typed
|
|
1738
1831
|
// failure; no provider request was ever made.
|
|
1739
|
-
if (error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT')
|
|
1740
|
-
|
|
1832
|
+
if (error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT') {
|
|
1833
|
+
// A cold OS page cache routinely pushes the first boot past
|
|
1834
|
+
// the local budget (observed 6.5–8s after idle on UAT,
|
|
1835
|
+
// Danceiny/gotry#660): the failed boot warms the very pages
|
|
1836
|
+
// the next port reads, so one fresh-port retry converges and
|
|
1837
|
+
// only a genuinely stuck runtime burns both. Same shape as
|
|
1838
|
+
// the soft-stall retry above; a boot that never reached a
|
|
1839
|
+
// provider request must not consume an attempt.
|
|
1840
|
+
const bootRemainingMs = deadlineAt - Date.now();
|
|
1841
|
+
if (bootRemainingMs < PLANNER_MIN_VIABLE_RUN_MS)
|
|
1842
|
+
throw new PlannerBootTimeoutExceeded();
|
|
1843
|
+
console.error('[booking-copilot] planner boot exceeded the local budget; retrying on a fresh run port:', JSON.stringify({
|
|
1844
|
+
attempt, bootRemainingMs,
|
|
1845
|
+
}));
|
|
1846
|
+
retire(runPort);
|
|
1847
|
+
const bootRetryPort = await beforePlannerDeadline(taskPort(), deadlineAt);
|
|
1848
|
+
if (closed || sessionClosePromise)
|
|
1849
|
+
throw new Error(closed ? 'planner_closed' : 'planner_session_closed');
|
|
1850
|
+
if (isRetiredPort(bootRetryPort))
|
|
1851
|
+
throw new PlannerBootTimeoutExceeded();
|
|
1852
|
+
runPort = bootRetryPort;
|
|
1853
|
+
attempt -= 1;
|
|
1854
|
+
continue;
|
|
1855
|
+
}
|
|
1741
1856
|
if (error instanceof PlannerTurnDeadlineExceeded) {
|
|
1742
1857
|
// The turn deadline can expire while our own runtime is still
|
|
1743
1858
|
// starting; that phase is ours, not the provider's.
|
|
@@ -1779,6 +1894,14 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1779
1894
|
return [{ kind: 'error', error: { code: 'PLANNER_ATTEMPT_BUDGET_EXHAUSTED', message: 'GoTry exhausted the planner attempt budget without a decision.', retryable: true } }];
|
|
1780
1895
|
}
|
|
1781
1896
|
catch (error) {
|
|
1897
|
+
// A port that fails its own initialize (createManagedDshRunPort with
|
|
1898
|
+
// initializeTimeoutMs) rejects with the SDK's raw typed message,
|
|
1899
|
+
// never reaching the run loop; classify it as the boot timeout it is
|
|
1900
|
+
// instead of surfacing an untyped failure.
|
|
1901
|
+
if (!(error instanceof PlannerBootTimeoutExceeded)
|
|
1902
|
+
&& error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT') {
|
|
1903
|
+
error = new PlannerBootTimeoutExceeded();
|
|
1904
|
+
}
|
|
1782
1905
|
if (error instanceof PlannerBootTimeoutExceeded) {
|
|
1783
1906
|
metricOutcome = 'boot_timeout';
|
|
1784
1907
|
logPlannerBootTimeout(runPort);
|
|
@@ -1790,6 +1913,14 @@ export async function createDshEmbeddedBookingPlanner(options) {
|
|
|
1790
1913
|
retire(runPort);
|
|
1791
1914
|
return [{ kind: 'error', error: { code: 'PLANNER_PROVIDER_TIMEOUT', message: providerFailureMessage('PLANNER_PROVIDER_TIMEOUT'), retryable: true } }];
|
|
1792
1915
|
}
|
|
1916
|
+
if (error instanceof ManagedDshResourceLimitError) {
|
|
1917
|
+
// The tree was killed for taking too much of the replica. Running the same turn again on a fresh port would
|
|
1918
|
+
// only let it take that much again, so this is terminal for the turn: not retried, not a provider fault.
|
|
1919
|
+
metricOutcome = 'resource_limit';
|
|
1920
|
+
console.error('[booking-copilot] planner child exceeded a resource limit:', JSON.stringify(error.breach));
|
|
1921
|
+
retire(runPort);
|
|
1922
|
+
return [{ kind: 'error', error: { code: 'PLANNER_FAILED', message: providerFailureMessage('PLANNER_FAILED'), retryable: false } }];
|
|
1923
|
+
}
|
|
1793
1924
|
retire(runPort);
|
|
1794
1925
|
throw error;
|
|
1795
1926
|
}
|
|
@@ -5,17 +5,49 @@ import { createRequire } from 'node:module';
|
|
|
5
5
|
import { Context } from '@deepseek-ai/cordis';
|
|
6
6
|
import LocalSubprocessRuntime from '@deepseek-ai/dsh-subprocess-local';
|
|
7
7
|
import { ManagedDshCleanupError, managedDshGroupIsQuiescent, readManagedDshWorkerStart, snapshotManagedDshGroup, } from './managed-dsh-cleanup-diagnostic.js';
|
|
8
|
+
import { sharedProcessTreeGovernor } from '../tenant-seam/process-governor.js';
|
|
9
|
+
/** The worker tree went over a resource limit and was killed. Closed shape: kind and numbers only, never worker output. */
|
|
10
|
+
export class ManagedDshResourceLimitError extends Error {
|
|
11
|
+
breach;
|
|
12
|
+
constructor(breach) {
|
|
13
|
+
super(breach.kind === 'blind' ? 'managed DSH worker tree could not be observed' : `managed DSH worker exceeded its ${breach.kind} limit`);
|
|
14
|
+
this.name = 'ManagedDshResourceLimitError';
|
|
15
|
+
this.breach = breach;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Proves that the sampler can see before a child is started: with a tree limit set, a process table that cannot be read
|
|
20
|
+
* would let the limit govern nothing, so it fails first and by name. A port with no tree limit has nothing to check.
|
|
21
|
+
*/
|
|
22
|
+
export async function assertManagedDshLimitsEnforceable(limits) {
|
|
23
|
+
if (limits.maxRssBytes === undefined && limits.maxCpuSeconds === undefined)
|
|
24
|
+
return;
|
|
25
|
+
await (limits.governor ?? sharedProcessTreeGovernor()).probe();
|
|
26
|
+
}
|
|
27
|
+
function assertPositiveLimit(name, value) {
|
|
28
|
+
if (value !== undefined && (!Number.isFinite(value) || value <= 0))
|
|
29
|
+
throw new RangeError(`managed DSH limit ${name} must be a positive number`);
|
|
30
|
+
}
|
|
31
|
+
function limitedEnvironment(env, heapMb) {
|
|
32
|
+
if (heapMb === undefined)
|
|
33
|
+
return env;
|
|
34
|
+
const existing = (env?.NODE_OPTIONS ?? '').split(/\s+/).filter((option) => option !== '' && !option.startsWith('--max-old-space-size='));
|
|
35
|
+
return { ...env, NODE_OPTIONS: [...existing, `--max-old-space-size=${Math.floor(heapMb)}`].join(' ') };
|
|
36
|
+
}
|
|
8
37
|
/** The only worker failure classifications allowed to cross the process boundary. */
|
|
9
38
|
const WORKER_FAILURE_CODES = new Set(['HARNESS_BOOT_TIMEOUT', 'HARNESS_START_FAILED', 'HARNESS_RUN_FAILED']);
|
|
10
39
|
const requireFromModule = createRequire(import.meta.url);
|
|
11
|
-
function workerLaunch(path) {
|
|
40
|
+
function workerLaunch(path, heapMb) {
|
|
41
|
+
// The V8 ceiling is given on the worker's own command line as well as in NODE_OPTIONS (which carries it to the
|
|
42
|
+
// processes below the worker), so the worker's ceiling does not depend on its environment surviving a scrub.
|
|
43
|
+
const heap = heapMb === undefined ? [] : [`--max-old-space-size=${Math.floor(heapMb)}`];
|
|
12
44
|
// The worker cwd is the isolated booking state root, so a bare `tsx/esm`
|
|
13
45
|
// loader would resolve from the wrong dependency tree and exit before the
|
|
14
46
|
// first request. Resolve the loader from this package once and pass its
|
|
15
47
|
// absolute installed path to Node.
|
|
16
48
|
return path.endsWith('.ts')
|
|
17
|
-
? [process.execPath, '--import', requireFromModule.resolve('tsx/esm'), path]
|
|
18
|
-
: [process.execPath, path];
|
|
49
|
+
? [process.execPath, ...heap, '--import', requireFromModule.resolve('tsx/esm'), path]
|
|
50
|
+
: [process.execPath, ...heap, path];
|
|
19
51
|
}
|
|
20
52
|
/** Run DeepSeekHarness in a child tree owned by dsh-subprocess-local. */
|
|
21
53
|
export class ManagedDshRunPort {
|
|
@@ -34,7 +66,13 @@ export class ManagedDshRunPort {
|
|
|
34
66
|
constructor(options = {}) {
|
|
35
67
|
const defaultWorker = join(dirname(fileURLToPath(import.meta.url)), 'dsh-worker.js');
|
|
36
68
|
const workerPath = options.workerPath ?? (existsSync(defaultWorker) ? defaultWorker : defaultWorker.replace(/\.js$/, '.ts'));
|
|
37
|
-
const { workerPath: _worker, graceMs = 500, cleanupRole, env, groupObserver, bootLog, ...harnessOptions } = options;
|
|
69
|
+
const { workerPath: _worker, graceMs = 500, cleanupRole, env, groupObserver, bootLog, limits, ...harnessOptions } = options;
|
|
70
|
+
assertPositiveLimit('heapMb', limits?.heapMb);
|
|
71
|
+
assertPositiveLimit('maxRssBytes', limits?.maxRssBytes);
|
|
72
|
+
assertPositiveLimit('maxCpuSeconds', limits?.maxCpuSeconds);
|
|
73
|
+
assertPositiveLimit('maxLineChars', limits?.maxLineChars);
|
|
74
|
+
assertPositiveLimit('attachDeadlineMs', limits?.attachDeadlineMs);
|
|
75
|
+
this.limits = limits;
|
|
38
76
|
this.cleanupRole = cleanupRole ?? 'unspecified';
|
|
39
77
|
this.graceMs = graceMs;
|
|
40
78
|
this.groupObserver = groupObserver ?? snapshotManagedDshGroup;
|
|
@@ -42,20 +80,30 @@ export class ManagedDshRunPort {
|
|
|
42
80
|
this.constructedAtMs = Date.now();
|
|
43
81
|
this.runtime = new LocalSubprocessRuntime(new Context());
|
|
44
82
|
this.handle = this.runtime.spawn({
|
|
45
|
-
argv: workerLaunch(workerPath),
|
|
83
|
+
argv: workerLaunch(workerPath, limits?.heapMb),
|
|
46
84
|
cwd: options.cwd ?? process.cwd(),
|
|
47
85
|
// Provider diagnostics may contain request fragments. Keep a bounded
|
|
48
86
|
// private tail for ownership without inheriting it into service logs.
|
|
49
87
|
stdio: { stdin: 'pipe', stdout: 'pipe', stderr: { maxBytes: 64 * 1024 } },
|
|
50
88
|
graceMs,
|
|
51
|
-
env,
|
|
89
|
+
env: limitedEnvironment(env, limits?.heapMb),
|
|
52
90
|
});
|
|
53
91
|
this.handle.stdout.setEncoding('utf8');
|
|
54
92
|
this.handle.stdout.on('data', (chunk) => this.consume(chunk));
|
|
93
|
+
if (limits && (limits.maxRssBytes !== undefined || limits.maxCpuSeconds !== undefined)) {
|
|
94
|
+
const deadlineMs = limits.attachDeadlineMs ?? 10_000;
|
|
95
|
+
this.attachTimer = setTimeout(() => {
|
|
96
|
+
this.emitBootLog(`RESOURCE_GOVERNOR_UNATTACHED ${JSON.stringify({ role: this.cleanupRole, afterMs: deadlineMs })}`);
|
|
97
|
+
}, deadlineMs);
|
|
98
|
+
this.attachTimer.unref();
|
|
99
|
+
}
|
|
55
100
|
this.handle.done.then((outcome) => {
|
|
101
|
+
this.stopWatching();
|
|
56
102
|
this.workerOutcome = { state: 'exited', exitCode: outcome.exitCode, signal: outcome.signal };
|
|
103
|
+
this.noteHeapExhaustion();
|
|
57
104
|
this.markTerminal('managed DSH worker exited');
|
|
58
105
|
}, () => {
|
|
106
|
+
this.stopWatching();
|
|
59
107
|
this.workerOutcome = { state: 'failed' };
|
|
60
108
|
this.markTerminal('managed DSH worker failed');
|
|
61
109
|
});
|
|
@@ -64,6 +112,9 @@ export class ManagedDshRunPort {
|
|
|
64
112
|
this.harnessOptions = harnessOptions;
|
|
65
113
|
this.cleanupDeadlineMs = Math.max(1_000, graceMs * 4);
|
|
66
114
|
}
|
|
115
|
+
limits;
|
|
116
|
+
attachTimer;
|
|
117
|
+
unwatchTree;
|
|
67
118
|
harnessOptions;
|
|
68
119
|
cleanupDeadlineMs;
|
|
69
120
|
groupObserver;
|
|
@@ -92,12 +143,73 @@ export class ManagedDshRunPort {
|
|
|
92
143
|
bootObservation() {
|
|
93
144
|
return this.booted;
|
|
94
145
|
}
|
|
146
|
+
/**
|
|
147
|
+
* A resource ceiling was crossed: say why before anything else can say something vaguer, fail what is waiting with
|
|
148
|
+
* that reason, and let the provider-owned range cleanup run. The governor (or the line cap) then kills the tree.
|
|
149
|
+
*/
|
|
150
|
+
resourceBreach(breach) {
|
|
151
|
+
this.terminalError ??= new ManagedDshResourceLimitError(breach);
|
|
152
|
+
this.failPending(this.terminalError);
|
|
153
|
+
this.stopWatching();
|
|
154
|
+
this.inputBuffer = '';
|
|
155
|
+
try {
|
|
156
|
+
this.handle.terminate();
|
|
157
|
+
}
|
|
158
|
+
catch { /* the governor's kill below is what must happen */ }
|
|
159
|
+
}
|
|
160
|
+
stopWatching() {
|
|
161
|
+
clearTimeout(this.attachTimer);
|
|
162
|
+
this.attachTimer = undefined;
|
|
163
|
+
const unwatch = this.unwatchTree;
|
|
164
|
+
this.unwatchTree = undefined;
|
|
165
|
+
unwatch?.();
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* A worker that died of its own V8 ceiling is a resource-limit death, not a mystery crash. Only the fixed V8 message is
|
|
169
|
+
* looked for in the bounded stderr tail; nothing from it is kept or reported.
|
|
170
|
+
*/
|
|
171
|
+
noteHeapExhaustion() {
|
|
172
|
+
const heapMb = this.limits?.heapMb;
|
|
173
|
+
if (heapMb === undefined || this.terminalError)
|
|
174
|
+
return;
|
|
175
|
+
let tail = '';
|
|
176
|
+
try {
|
|
177
|
+
tail = this.handle.collected.stderr?.readFrom(0).text ?? '';
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
if (!tail.includes('JavaScript heap out of memory'))
|
|
183
|
+
return;
|
|
184
|
+
const bytes = Math.floor(heapMb) * 1024 * 1024;
|
|
185
|
+
this.terminalError = new ManagedDshResourceLimitError({ kind: 'heap', limit: bytes, observed: bytes });
|
|
186
|
+
}
|
|
187
|
+
watchTree(workerPid) {
|
|
188
|
+
const limits = this.limits;
|
|
189
|
+
if (!limits || (limits.maxRssBytes === undefined && limits.maxCpuSeconds === undefined))
|
|
190
|
+
return;
|
|
191
|
+
clearTimeout(this.attachTimer);
|
|
192
|
+
this.attachTimer = undefined;
|
|
193
|
+
const governor = limits.governor ?? sharedProcessTreeGovernor();
|
|
194
|
+
// The worker is this process's own child (the lifecycle line was checked for that), so the governor also requires
|
|
195
|
+
// the parent to match: a pid the operating system reused for something else is never sampled or killed.
|
|
196
|
+
this.unwatchTree = governor.watch({ rootPid: workerPid, parentPid: process.pid }, { maxRssBytes: limits.maxRssBytes, maxCpuSeconds: limits.maxCpuSeconds }, (breach) => this.resourceBreach(breach));
|
|
197
|
+
}
|
|
95
198
|
consume(chunk) {
|
|
199
|
+
if (this.terminalError instanceof ManagedDshResourceLimitError)
|
|
200
|
+
return;
|
|
96
201
|
this.inputBuffer += chunk;
|
|
97
202
|
for (;;) {
|
|
98
203
|
const newline = this.inputBuffer.indexOf('\n');
|
|
99
|
-
if (newline < 0)
|
|
204
|
+
if (newline < 0) {
|
|
205
|
+
// The parent holds a line until it ends; a worker that never ends one would grow this process without bound.
|
|
206
|
+
const maxLineChars = this.limits?.maxLineChars;
|
|
207
|
+
if (maxLineChars !== undefined && this.inputBuffer.length > maxLineChars) {
|
|
208
|
+
this.resourceBreach({ kind: 'output', limit: maxLineChars, observed: this.inputBuffer.length });
|
|
209
|
+
// The worker stays up until the provider range cleanup lands; this port no longer reads from it.
|
|
210
|
+
}
|
|
100
211
|
return;
|
|
212
|
+
}
|
|
101
213
|
const line = this.inputBuffer.slice(0, newline);
|
|
102
214
|
this.inputBuffer = this.inputBuffer.slice(newline + 1);
|
|
103
215
|
try {
|
|
@@ -105,6 +217,7 @@ export class ManagedDshRunPort {
|
|
|
105
217
|
if (message.lifecycle === 'worker_started') {
|
|
106
218
|
if (this.workerPid === null && Number.isSafeInteger(message.workerPid) && message.workerPid > 0 && message.parentPid === process.pid) {
|
|
107
219
|
this.workerPid = message.workerPid;
|
|
220
|
+
this.watchTree(this.workerPid);
|
|
108
221
|
// One bounded read-only start-time anchor, captured while the worker
|
|
109
222
|
// is alive, so a later failure snapshot can disprove PID reuse.
|
|
110
223
|
void readManagedDshWorkerStart(this.workerPid).then((startedAt) => {
|
|
@@ -199,6 +312,7 @@ export class ManagedDshRunPort {
|
|
|
199
312
|
return this.closePromise;
|
|
200
313
|
this.closePromise = (async () => {
|
|
201
314
|
const started = performance.now();
|
|
315
|
+
this.stopWatching();
|
|
202
316
|
this.failPending(new Error('managed DSH run port closed'));
|
|
203
317
|
this.handle.terminate();
|
|
204
318
|
// Observe worker failure even if range observation rejects or times out
|