@danceiny/gotry 0.2.0-rc.31 → 0.2.0-rc.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/bin/gotry-bootstrap.js +63 -53
- package/bin/gotry-hbcli-release.d.ts +3 -0
- package/bin/gotry-hbcli-release.js +4 -0
- package/client/client.js +66 -21
- package/dist/capabilities/doctor.js +7 -8
- package/dist/capabilities/effect.js +45 -12
- package/dist/capabilities/flyai-config.js +12 -10
- package/dist/capabilities/flyai.js +87 -56
- package/dist/capabilities/hbcli-config.js +156 -0
- package/dist/capabilities/hbcli.js +29 -2
- package/dist/capabilities/resilience.js +6 -6
- package/dist/capabilities/session/adapters/ctrip-flight.js +42 -7
- package/dist/capabilities/session/adapters/ctrip-hotel.js +52 -3
- package/dist/capabilities/session/bridge-ledger.js +168 -67
- package/dist/capabilities/session/ctrip-city.js +186 -0
- package/dist/capabilities/session/extension-bridge.js +148 -47
- package/dist/capabilities/session/gui-flight.js +949 -0
- package/dist/capabilities/session/gui-hotel.js +470 -0
- package/dist/capabilities/session-search.js +102 -14
- package/dist/scripts/booking-copilot-dsh-planner-proof-tests.js +129 -2
- package/dist/scripts/booking-copilot-dsh-readiness-proof.js +53 -5
- package/dist/scripts/booking-copilot-gateway-chat-adapter-tests.js +233 -0
- package/dist/scripts/booking-copilot-planner-live-probe.js +1 -0
- package/dist/scripts/booking-copilot-startup-proof-tests.js +1 -1
- package/dist/scripts/bootstrap-tests.js +23 -6
- package/dist/scripts/bridge-ledger-tests.js +20 -18
- package/dist/scripts/ctrip-city-tests.js +80 -0
- package/dist/scripts/effect-tests.js +602 -504
- package/dist/scripts/extension-tests.js +14 -8
- package/dist/scripts/fixtures/tenant-runaway-worker-fixture.mjs +73 -0
- package/dist/scripts/flight-malformed-tests.js +1 -1
- package/dist/scripts/flyai-setup-tests.js +19 -4
- package/dist/scripts/flyai-tests.js +768 -653
- package/dist/scripts/gotry-web-api-tests.js +5 -0
- package/dist/scripts/gotry-web-client-tests.js +20 -1
- package/dist/scripts/gui-flight-live.js +27 -0
- package/dist/scripts/hbcli-tests.js +1 -1
- package/dist/scripts/hbcli-upgrade-tests.js +107 -0
- package/dist/scripts/hbcli-web-setup-tests.js +114 -0
- package/dist/scripts/hotel-date-gate-tests.js +1 -1
- package/dist/scripts/hotel-result-tests.js +169 -0
- package/dist/scripts/issue-271-liveness-tests.js +1 -1
- package/dist/scripts/mutation-check.js +153 -0
- package/dist/scripts/nightly-evidence.js +4 -111
- package/dist/scripts/session-tests.js +211 -6
- package/dist/scripts/tenant-bridge-org-tests.js +1264 -0
- package/dist/scripts/tenant-capacity-probe.js +263 -0
- package/dist/scripts/tenant-compute-isolation-tests.js +707 -0
- package/dist/scripts/tenant-egress-credentials-tests.js +822 -0
- package/dist/scripts/tenant-executor-binding-tests.js +203 -0
- package/dist/scripts/tenant-prompt-partition-tests.js +265 -0
- package/dist/scripts/tenant-seam-adversarial-tests.js +957 -0
- package/dist/scripts/tenant-seam-quota-tests.js +383 -0
- package/dist/scripts/tenant-seam-test-kit.js +292 -0
- package/dist/scripts/tenant-state-copies-tests.js +607 -0
- package/dist/scripts/tenant-state-drill-tests.js +298 -0
- package/dist/scripts/tenant-state-drill-worker.js +164 -0
- package/dist/scripts/tenant-state-durable-conformance.js +101 -0
- package/dist/scripts/tenant-state-handoff-tests.js +854 -0
- package/dist/scripts/tenant-state-lease-conformance.js +142 -0
- package/dist/scripts/tenant-state-lease-worker.js +28 -0
- package/dist/scripts/tenant-state-router-tests.js +464 -0
- package/dist/scripts/tenant-state-test-kit.js +92 -0
- package/dist/scripts/tenant-state-tests.js +564 -0
- package/dist/src/backend/modules/booking-executor.js +61 -6
- package/dist/src/backend/modules/session-search.js +136 -39
- package/dist/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/dist/src/booking-surface/dsh-planner.js +150 -19
- package/dist/src/booking-surface/index.js +1 -0
- package/dist/src/booking-surface/managed-dsh-run-port.js +121 -7
- package/dist/src/booking-surface/server.js +193 -80
- package/dist/src/booking-surface/startup.js +153 -0
- package/dist/src/booking-surface/tenancy.js +188 -0
- package/dist/src/gotry-web-api.js +64 -2
- package/dist/src/index.js +70 -18
- package/dist/src/llm-price-table.js +148 -0
- package/dist/src/tenant-seam/index.js +8 -0
- package/dist/src/tenant-seam/llm-broker.js +508 -0
- package/dist/src/tenant-seam/process-governor.js +281 -0
- package/dist/src/tenant-seam/quotas.js +360 -0
- package/dist/src/tenant-seam/seam.js +92 -0
- package/dist/src/tenant-state/durable-store.js +158 -0
- package/dist/src/tenant-state/epoch-copies.js +422 -0
- package/dist/src/tenant-state/errors.js +108 -0
- package/dist/src/tenant-state/ids.js +101 -0
- package/dist/src/tenant-state/index.js +11 -0
- package/dist/src/tenant-state/lease-store.js +203 -0
- package/dist/src/tenant-state/tenant-lease.js +150 -0
- package/dist/src/tenant-state/tenant-router.js +650 -0
- package/extension/manifest.json +2 -2
- package/package.json +5 -1
- package/ts/capabilities/flyai-config.ts +12 -10
- package/ts/capabilities/flyai.ts +91 -50
- package/ts/capabilities/hbcli-config.ts +128 -0
- package/ts/capabilities/hbcli.ts +30 -3
- package/ts/capabilities/session/adapters/ctrip-flight.ts +43 -7
- package/ts/capabilities/session/adapters/ctrip-hotel.ts +51 -4
- package/ts/capabilities/session/bridge-ledger.ts +192 -78
- package/ts/capabilities/session/ctrip-city.ts +158 -0
- package/ts/capabilities/session/extension-bridge.ts +185 -67
- package/ts/capabilities/session/gui-flight.ts +1022 -0
- package/ts/capabilities/session/gui-hotel.ts +505 -0
- package/ts/capabilities/session-search.ts +130 -18
- package/ts/src/booking-surface/dsh-llm-gateway-chat.js +434 -0
- package/ts/src/booking-surface/dsh-planner.ts +169 -19
- package/ts/src/booking-surface/index.ts +1 -0
- package/ts/src/booking-surface/managed-dsh-run-port.ts +150 -7
- package/ts/src/booking-surface/server.ts +215 -84
- package/ts/src/booking-surface/startup.ts +155 -0
- package/ts/src/booking-surface/tenancy.ts +242 -0
- package/ts/src/gotry-web-api.ts +51 -2
- package/ts/src/index.ts +67 -19
- package/ts/src/llm-price-table.ts +180 -0
- package/ts/src/tenant-seam/index.ts +9 -0
- package/ts/src/tenant-seam/llm-broker.ts +520 -0
- package/ts/src/tenant-seam/process-governor.ts +343 -0
- package/ts/src/tenant-seam/quotas.ts +433 -0
- package/ts/src/tenant-seam/seam.ts +142 -0
- package/ts/src/tenant-state/durable-store.ts +171 -0
- package/ts/src/tenant-state/epoch-copies.ts +491 -0
- package/ts/src/tenant-state/errors.ts +128 -0
- package/ts/src/tenant-state/ids.ts +108 -0
- package/ts/src/tenant-state/index.ts +12 -0
- package/ts/src/tenant-state/lease-store.ts +266 -0
- package/ts/src/tenant-state/tenant-lease.ts +175 -0
- package/ts/src/tenant-state/tenant-router.ts +743 -0
|
@@ -15,6 +15,8 @@ import { join } from 'node:path'
|
|
|
15
15
|
import { fileURLToPath } from 'node:url'
|
|
16
16
|
import { BOOKING_READ_ACTION_KINDS, type ActionReceipt, type BookingCopilotTurn, type BookingReadAction, type SearchCriteriaPatch } from './contracts.ts'
|
|
17
17
|
import { buildTimeAnchor } from '../time-anchor.ts'
|
|
18
|
+
import { assertPromptCachePartition } from '../tenant-state/ids.ts'
|
|
19
|
+
import type { LlmTicketIssuer } from '../tenant-seam/llm-broker.ts'
|
|
18
20
|
export { formatUtcOffsetLabel } from '../time-anchor.ts'
|
|
19
21
|
import {
|
|
20
22
|
actionsForEmbeddedCapability,
|
|
@@ -26,7 +28,7 @@ import {
|
|
|
26
28
|
validateBookingIntentProjection,
|
|
27
29
|
bookingSurfaceSchema,
|
|
28
30
|
} from './validation.ts'
|
|
29
|
-
import { createManagedDshRunPort } from './managed-dsh-run-port.ts'
|
|
31
|
+
import { assertManagedDshLimitsEnforceable, createManagedDshRunPort, ManagedDshResourceLimitError, type ManagedDshResourceLimits } from './managed-dsh-run-port.ts'
|
|
30
32
|
import {
|
|
31
33
|
BOOKING_INTENT_SCHEMA_VERSION,
|
|
32
34
|
BOOKING_INTENT_TARGET_ACTION,
|
|
@@ -103,7 +105,7 @@ export interface DshPlannerRunPort {
|
|
|
103
105
|
export type DshPlannerClock = Date | (() => Date)
|
|
104
106
|
|
|
105
107
|
export interface DshPlannerTurnMetric {
|
|
106
|
-
outcome: 'operation' | 'terminal' | 'provider_error' | 'timeout' | 'boot_timeout' | 'typed_decision_required' | 'failed'
|
|
108
|
+
outcome: 'operation' | 'terminal' | 'provider_error' | 'timeout' | 'boot_timeout' | 'typed_decision_required' | 'failed' | 'resource_limit'
|
|
107
109
|
decisionSource: 'runtime' | 'model'
|
|
108
110
|
elapsedMs: number
|
|
109
111
|
/** Calls across the Harness run/session seam; one run can contain repairs. */
|
|
@@ -138,6 +140,8 @@ export interface DshEmbeddedBookingPlannerOptions {
|
|
|
138
140
|
dshBin?: string
|
|
139
141
|
provider?: string
|
|
140
142
|
model?: string
|
|
143
|
+
/** Explicit path for the chat-completions gateway LLM plugin; defaults to the sibling dsh-llm-gateway-chat.js. */
|
|
144
|
+
gatewayLlmPluginPath?: string
|
|
141
145
|
reasoningEffort?: 'off' | 'low' | 'high' | 'max'
|
|
142
146
|
maxTokens?: number
|
|
143
147
|
/** Total wall-clock budget across all repair attempts for one planner turn. */
|
|
@@ -154,6 +158,25 @@ export interface DshEmbeddedBookingPlannerOptions {
|
|
|
154
158
|
onMetric?: (metric: DshPlannerTurnMetric) => void
|
|
155
159
|
env?: Record<string, string | undefined>
|
|
156
160
|
pluginPath?: string
|
|
161
|
+
/**
|
|
162
|
+
* Credential custody (AaaS, docs/rfc/aaas-rfc.md §4.8, threat S-18). When set, no Harness process ever holds the
|
|
163
|
+
* upstream key or base URL: each real port is given a short-lived loopback ticket from this issuer instead, revoked
|
|
164
|
+
* when the port closes, and any key in `env` is stripped from the child. Requires the chat-completions route
|
|
165
|
+
* (`DEEPSEEK_PROVIDER=gateway-chat`), the only wire dialect the broker carries. Absent: the child holds the key, as before.
|
|
166
|
+
*/
|
|
167
|
+
llmTickets?: LlmTicketIssuer
|
|
168
|
+
/**
|
|
169
|
+
* The tenant's prompt cache partition label (AaaS, docs/rfc/aaas-rfc.md §4.8, threat S-16). When set, every prompt
|
|
170
|
+
* carries it right after the shared instructions and ahead of all tenant-derived content, so a provider that caches
|
|
171
|
+
* by prefix cannot share a cached prefix between two tenants past that point. Absent (the shared form): no label,
|
|
172
|
+
* and the prompt is byte-identical to what it was.
|
|
173
|
+
*/
|
|
174
|
+
cachePartition?: string
|
|
175
|
+
/**
|
|
176
|
+
* Ceilings for each real Harness process tree (AaaS, docs/rfc/aaas-rfc.md §4.8). A tree over a ceiling is killed and
|
|
177
|
+
* the turn ends as a non-retryable `PLANNER_FAILED`; it is never retried on a fresh port. Absent: none, as before.
|
|
178
|
+
*/
|
|
179
|
+
resourceLimits?: ManagedDshResourceLimits
|
|
157
180
|
}
|
|
158
181
|
|
|
159
182
|
export interface DshEmbeddedBookingPlannerHandle {
|
|
@@ -208,6 +231,10 @@ export function buildDshPlannerEnvironment(
|
|
|
208
231
|
if (baseUrl) target.DEEPSEEK_BASE_URL = baseUrl
|
|
209
232
|
if (model) target.DEEPSEEK_MODEL = model
|
|
210
233
|
if (maxTokens) target.DEEPSEEK_MAX_TOKENS = maxTokens
|
|
234
|
+
// Wire-protocol selection rides the same namespace: 'gateway-chat' routes the
|
|
235
|
+
// child at the chat-completions adapter plugin (dsh-llm-gateway-chat.js)
|
|
236
|
+
// instead of the SDK's bundled Messages adapter.
|
|
237
|
+
if (source.DEEPSEEK_PROVIDER) target.DEEPSEEK_PROVIDER = source.DEEPSEEK_PROVIDER
|
|
211
238
|
return target
|
|
212
239
|
}
|
|
213
240
|
|
|
@@ -232,8 +259,8 @@ function quoteYaml(value: string): string {
|
|
|
232
259
|
}
|
|
233
260
|
|
|
234
261
|
/** Runtime patch over dsh's sdk-minimal profile: one typed planner tool per action kind. */
|
|
235
|
-
export function buildDshEmbeddedBookingPatch(pluginPath: string): string {
|
|
236
|
-
|
|
262
|
+
export function buildDshEmbeddedBookingPatch(pluginPath: string, gatewayLlmPluginPath = ''): string {
|
|
263
|
+
const patchBody = `# GoTry embedded Booking Copilot: typed planning only; no browser, shell, files, Book, or persisted chat.\n\
|
|
237
264
|
- id: deepseek-llm-api-extensions\n disabled: true\n\
|
|
238
265
|
- id: session-log-deepseek\n disabled: true\n\
|
|
239
266
|
- id: plugin-package-inventory-deepseek\n disabled: true\n\
|
|
@@ -292,10 +319,14 @@ export function buildDshEmbeddedBookingPatch(pluginPath: string): string {
|
|
|
292
319
|
skills:\n\
|
|
293
320
|
enabled: false\n\
|
|
294
321
|
toolBash: false\n\
|
|
295
|
-
toolJobs: false\n
|
|
296
|
-
|
|
297
|
-
- id: gotry-embedded-booking\n\
|
|
298
|
-
|
|
322
|
+
toolJobs: false\n`
|
|
323
|
+
const inserts = [
|
|
324
|
+
`- insert:\n - id: gotry-embedded-booking\n name: ${quoteYaml(pluginPath)}\n`,
|
|
325
|
+
...(gatewayLlmPluginPath
|
|
326
|
+
? [`- insert:\n - id: llm-gateway-chat\n name: ${quoteYaml(gatewayLlmPluginPath)}\n`]
|
|
327
|
+
: []),
|
|
328
|
+
]
|
|
329
|
+
return patchBody + inserts.join('')
|
|
299
330
|
}
|
|
300
331
|
|
|
301
332
|
interface ResolvedRealRunPortConfig {
|
|
@@ -312,9 +343,21 @@ interface ResolvedRealRunPortConfig {
|
|
|
312
343
|
// DEEPSEEK_MODEL could otherwise be absorbed silently.
|
|
313
344
|
export function resolveRealRunPortConfig(options: DshEmbeddedBookingPlannerOptions): ResolvedRealRunPortConfig {
|
|
314
345
|
const sourceEnv = options.env ?? process.env
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
346
|
+
// The route settings (provider, model, token budget) are read from the namespace the credential selects. A deployment
|
|
347
|
+
// that keeps the key in the broker may have none in its environment; the DeepSeek namespace is then the route's, and
|
|
348
|
+
// the placeholder below only selects it: it is removed from the child's environment before anything uses it.
|
|
349
|
+
const routeSource = options.llmTickets && !sourceEnv.DEEPSEEK_API_KEY && !sourceEnv.LLM_API_KEY
|
|
350
|
+
? { ...sourceEnv, DEEPSEEK_API_KEY: 'namespace-selector-only' }
|
|
351
|
+
: sourceEnv
|
|
352
|
+
const childEnv = buildDshPlannerEnvironment(routeSource)
|
|
353
|
+
if (options.llmTickets) {
|
|
354
|
+
// The credential reaches the child only as a ticket, issued per port. Whatever the environment held is not forwarded,
|
|
355
|
+
// so an ambient key cannot slip past the broker; the ticket's own values are set where the port is created.
|
|
356
|
+
delete childEnv.DEEPSEEK_API_KEY
|
|
357
|
+
delete childEnv.DEEPSEEK_BASE_URL
|
|
358
|
+
} else if (!childEnv.DEEPSEEK_API_KEY) throw new Error('booking_planner_model_key_required')
|
|
359
|
+
const provider = options.provider?.trim() || childEnv.DEEPSEEK_PROVIDER || 'deepseek-official'
|
|
360
|
+
if (options.llmTickets && provider !== 'gateway-chat') throw new Error('booking_planner_llm_tickets_require_gateway_chat')
|
|
318
361
|
const configuredModel = options.model ?? childEnv.DEEPSEEK_MODEL
|
|
319
362
|
if (provider !== 'deepseek-official' && !configuredModel) {
|
|
320
363
|
throw new Error('booking_planner_model_required_for_nondefault_provider')
|
|
@@ -336,9 +379,33 @@ export function resolveRealRunPortConfig(options: DshEmbeddedBookingPlannerOptio
|
|
|
336
379
|
return { childEnv, provider, model, modelSource, reasoningEffort, maxTokens }
|
|
337
380
|
}
|
|
338
381
|
|
|
382
|
+
/**
|
|
383
|
+
* The names under which an ambient LLM credential or address could reach a child through the runtime's inherited
|
|
384
|
+
* environment. The runtime merges the explicit environment onto the parent's with names that look secret removed, so an
|
|
385
|
+
* address (`LLM_BASE_URL`) would otherwise pass; an explicit `undefined` removes an inherited entry.
|
|
386
|
+
*/
|
|
387
|
+
const AMBIENT_LLM_CREDENTIAL_NAMES = ['LLM_API_KEY', 'LLM_BASE_URL'] as const
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* The environment of a real planner process tree. With a ticket it carries the ticket in place of the key and base URL,
|
|
391
|
+
* and removes the ambient names that could carry the real ones; without one it is the resolved environment, unchanged.
|
|
392
|
+
*/
|
|
393
|
+
export function plannerChildEnvironment(
|
|
394
|
+
resolved: Record<string, string>,
|
|
395
|
+
ticket?: { readonly token: string; readonly baseUrl: string },
|
|
396
|
+
): NodeJS.ProcessEnv {
|
|
397
|
+
if (!ticket) return resolved
|
|
398
|
+
const tombstones: NodeJS.ProcessEnv = {}
|
|
399
|
+
for (const name of AMBIENT_LLM_CREDENTIAL_NAMES) tombstones[name] = undefined
|
|
400
|
+
return { ...resolved, ...tombstones, DEEPSEEK_API_KEY: ticket.token, DEEPSEEK_BASE_URL: ticket.baseUrl }
|
|
401
|
+
}
|
|
402
|
+
|
|
339
403
|
/** Shared by the planner and real-subprocess proofs; owns its scratch directory. */
|
|
340
404
|
export async function createRealRunPort(options: DshEmbeddedBookingPlannerOptions): Promise<DshPlannerRunPort> {
|
|
341
|
-
const
|
|
405
|
+
const resolved = resolveRealRunPortConfig(options)
|
|
406
|
+
const { provider, model, modelSource, reasoningEffort, maxTokens } = resolved
|
|
407
|
+
// Every real port, not only the first one of a process, starts a child only where the tree limits can be enforced.
|
|
408
|
+
if (options.resourceLimits) await assertManagedDshLimitsEnforceable(options.resourceLimits)
|
|
342
409
|
// Name the resolved route on every real port. A deployment that forgot
|
|
343
410
|
// DEEPSEEK_MODEL used to be indistinguishable from one that set it, and the
|
|
344
411
|
// substituted model is exactly what a later diagnosis needs to see.
|
|
@@ -350,8 +417,14 @@ export async function createRealRunPort(options: DshEmbeddedBookingPlannerOption
|
|
|
350
417
|
const dshHome = join(scratch, 'home')
|
|
351
418
|
mkdirSync(dshHome, { recursive: true })
|
|
352
419
|
const pluginPath = options.pluginPath ?? fileURLToPath(new URL('./dsh-plugin.js', import.meta.url))
|
|
420
|
+
const gatewayLlmPluginPath = options.gatewayLlmPluginPath ?? fileURLToPath(new URL('./dsh-llm-gateway-chat.js', import.meta.url))
|
|
353
421
|
const patchPath = join(scratch, 'embedded-booking.cordis.yml')
|
|
354
|
-
writeFileSync(patchPath, buildDshEmbeddedBookingPatch(pluginPath), { encoding: 'utf8', mode: 0o600 })
|
|
422
|
+
writeFileSync(patchPath, buildDshEmbeddedBookingPatch(pluginPath, gatewayLlmPluginPath), { encoding: 'utf8', mode: 0o600 })
|
|
423
|
+
|
|
424
|
+
// One ticket per port, shared with its warmer, revoked when the port closes or fails to come up. It is issued only now,
|
|
425
|
+
// after the steps above that can fail (a full disk, no descriptors), so none of them can leave a live ticket behind.
|
|
426
|
+
const ticket = options.llmTickets?.issue()
|
|
427
|
+
const childEnv = plannerChildEnvironment(resolved.childEnv, ticket)
|
|
355
428
|
|
|
356
429
|
let managedPort: DshPlannerRunPort
|
|
357
430
|
try {
|
|
@@ -381,9 +454,11 @@ export async function createRealRunPort(options: DshEmbeddedBookingPlannerOption
|
|
|
381
454
|
disposeEofGraceMs: 500,
|
|
382
455
|
disposeGraceMs: 500,
|
|
383
456
|
env: childEnv,
|
|
457
|
+
...(options.resourceLimits ? { limits: options.resourceLimits } : {}),
|
|
384
458
|
...(options.dshBin ? { dshBin: options.dshBin } : {}),
|
|
385
459
|
})
|
|
386
460
|
} catch (error) {
|
|
461
|
+
ticket?.revoke()
|
|
387
462
|
rmSync(scratch, { recursive: true, force: true })
|
|
388
463
|
throw error
|
|
389
464
|
}
|
|
@@ -424,9 +499,11 @@ export async function createRealRunPort(options: DshEmbeddedBookingPlannerOption
|
|
|
424
499
|
disposeEofGraceMs: 500,
|
|
425
500
|
disposeGraceMs: 500,
|
|
426
501
|
env: childEnv,
|
|
502
|
+
...(options.resourceLimits ? { limits: options.resourceLimits } : {}),
|
|
427
503
|
...(options.dshBin ? { dshBin: options.dshBin } : {}),
|
|
428
504
|
})
|
|
429
505
|
} catch (error) {
|
|
506
|
+
ticket?.revoke()
|
|
430
507
|
try {
|
|
431
508
|
await managedPort.close()
|
|
432
509
|
} catch (cleanupError) {
|
|
@@ -459,6 +536,8 @@ export async function createRealRunPort(options: DshEmbeddedBookingPlannerOption
|
|
|
459
536
|
if (closePromise) return closePromise
|
|
460
537
|
closePromise = (async () => {
|
|
461
538
|
const results = await Promise.allSettled([managedPort.close(), closeWarmer()])
|
|
539
|
+
// Whatever the cleanup did, the credential stops working with the port.
|
|
540
|
+
ticket?.revoke()
|
|
462
541
|
const failures = results.flatMap((result) => result.status === 'rejected' ? [result.reason] : [])
|
|
463
542
|
if (failures.length) {
|
|
464
543
|
throw new AggregateError(failures, 'booking_planner_run_port_cleanup_failed')
|
|
@@ -483,7 +562,7 @@ function plannerVisibleAllowedActions(task: BookingCopilotTaskState): BookingRea
|
|
|
483
562
|
: [...task.allowedActions]
|
|
484
563
|
}
|
|
485
564
|
|
|
486
|
-
function plannerPrompt(turn: BookingCopilotTurn, task: BookingCopilotTaskState, clock?: DshPlannerClock): string {
|
|
565
|
+
function plannerPrompt(turn: BookingCopilotTurn, task: BookingCopilotTaskState, clock?: DshPlannerClock, cachePartition?: string): string {
|
|
487
566
|
const availability = task.availability
|
|
488
567
|
const allowedActions = plannerVisibleAllowedActions(task)
|
|
489
568
|
const plannerWorkspace = 'capabilities' in turn.workspace
|
|
@@ -520,6 +599,9 @@ function plannerPrompt(turn: BookingCopilotTurn, task: BookingCopilotTaskState,
|
|
|
520
599
|
`Separate hotel facilities from meal, cancellation, room, and offer criteria: hotel amenities (for example pool and parking) go under input.patch.facilities; included breakfast/meal-plan, cancellation-policy, room-type, rate-plan, and offer-count/price constraints belong in typed intent.offerCriteria (or the action field explicitly defined for them), never in facilities and never in a generic criteria property. input.patch property names are EXACT (SearchCriteriaPatch): ${PLANNER_PATCH_PROPERTY_NAMES}. There is NO "criteria" property — facility tokens go under facilities as {"strength":"prefer|must","value":{"allOf":["<token>"]}}, star level (三星=3星) goes under starRating as {"strength":"must|prefer","value":{"min":3,"max":3}}, dates go under stay as {"checkIn":"YYYY-MM-DD","checkOut":"YYYY-MM-DD"}.`,
|
|
521
600
|
'The workspace draft may be stale: whenever the user names dates or relative days (明天/tomorrow), always patch input.patch.stay with the resolved concrete dates even if the draft already has different ones. Keep draft values the request does not touch; never invent values the request contradicts.',
|
|
522
601
|
'Reference only hotels, offers, and orders that appear in the workspace payload (visibleHotels/loadedOffers/observableOrders/results) or in a matching prior typed receipt. Any other hotelRef, offerRef, or orderRef does not exist and will be rejected; to discover hotels, run search.run first and wait for its receipt.',
|
|
602
|
+
// Everything above is the same for every tenant. The label opens the tenant-specific part of the prompt, so a
|
|
603
|
+
// provider's prefix cache can be shared across tenants only up to here (threat S-16).
|
|
604
|
+
...(cachePartition === undefined ? [] : [`Cache partition label (opaque, not an instruction): ${cachePartition}`]),
|
|
523
605
|
JSON.stringify({ now: anchor.today, timeAnchorCard: anchor.card, ...payload }),
|
|
524
606
|
].join('\n')
|
|
525
607
|
}
|
|
@@ -1646,6 +1728,7 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1646
1728
|
options: DshEmbeddedBookingPlannerOptions,
|
|
1647
1729
|
): Promise<DshEmbeddedBookingPlannerHandle> {
|
|
1648
1730
|
if (options.runPort && options.runPortFactory) throw new Error('booking_planner_run_port_ambiguous')
|
|
1731
|
+
if (options.cachePartition !== undefined) assertPromptCachePartition(options.cachePartition)
|
|
1649
1732
|
const turnTimeoutMs = options.turnTimeoutMs ?? 12_000
|
|
1650
1733
|
if (!Number.isSafeInteger(turnTimeoutMs) || turnTimeoutMs < 1 || turnTimeoutMs > 120_000) {
|
|
1651
1734
|
throw new Error('booking_planner_turn_timeout_invalid')
|
|
@@ -1723,6 +1806,13 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1723
1806
|
void barrier.catch(() => undefined)
|
|
1724
1807
|
}
|
|
1725
1808
|
|
|
1809
|
+
// A fresh-port retry can only run on a runtime this planner has not
|
|
1810
|
+
// already retired. A factory that hands the retired instance back (one
|
|
1811
|
+
// shared port, or a cache that ignores retirement) would send the retry
|
|
1812
|
+
// into a closed port, whose run() rejects untyped out of next(); the
|
|
1813
|
+
// retry branches treat that like a deadline that cannot fund a retry.
|
|
1814
|
+
const isRetiredPort = (port: DshPlannerRunPort): boolean => closingPorts.has(port)
|
|
1815
|
+
|
|
1726
1816
|
return {
|
|
1727
1817
|
async next({ turn, task }) {
|
|
1728
1818
|
if (closed) throw new Error('planner_closed')
|
|
@@ -1760,7 +1850,8 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1760
1850
|
// bounded planner attempt. Any returned decision crosses the same
|
|
1761
1851
|
// typed authority path.
|
|
1762
1852
|
let attempt = 0
|
|
1763
|
-
let
|
|
1853
|
+
let schemaRepairAttempted = false
|
|
1854
|
+
let nextPrompt = plannerPrompt(turn, task, options.now, options.cachePartition)
|
|
1764
1855
|
// A run that goes silent must not be allowed to silently consume the
|
|
1765
1856
|
// whole turn deadline: the idle stall budget fires only when the run
|
|
1766
1857
|
// stops emitting notifications entirely, the child is retired, and
|
|
@@ -1782,10 +1873,29 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1782
1873
|
deadlineAt,
|
|
1783
1874
|
)
|
|
1784
1875
|
modelStepCount += observedModelStepCount(result.events)
|
|
1785
|
-
|
|
1876
|
+
const schemaRejectionsThisRun = result.events
|
|
1786
1877
|
.map((event, index) => toolResultObservation(event, index))
|
|
1787
1878
|
.filter((observation) => observation?.kind === 'schema-rejection').length
|
|
1788
|
-
|
|
1879
|
+
schemaRejectedCallCount += schemaRejectionsThisRun
|
|
1880
|
+
try {
|
|
1881
|
+
decisions = scanDshToolDecisionEvents(result.events, task)
|
|
1882
|
+
} catch (scanError) {
|
|
1883
|
+
// hotel-be#33640: a provider can emit a tool call whose
|
|
1884
|
+
// arguments miss the declared schema and then stop. Treat one
|
|
1885
|
+
// schema-shaped rejection as a repairable provider mistake:
|
|
1886
|
+
// retry once with a stricter prompt on the same session
|
|
1887
|
+
// instead of surfacing PLANNER_PROVIDER_RESPONSE_INVALID to the
|
|
1888
|
+
// user. Safety errors and unproven rejections still fail closed.
|
|
1889
|
+
if (!schemaRepairAttempted && schemaRejectionsThisRun > 0 && attempt < 3 && isSchemaShapeError(scanError)) {
|
|
1890
|
+
schemaRepairAttempted = true
|
|
1891
|
+
console.error('[booking-copilot] planner tool call failed schema validation; retrying with a repair prompt:', JSON.stringify({
|
|
1892
|
+
attempt, schemaRejectionsThisRun,
|
|
1893
|
+
}))
|
|
1894
|
+
nextPrompt = 'Your previous booking capability tool call was rejected because its arguments did not match the declared parameter schema. Emit exactly one corrected booking capability tool call that satisfies the schema; do not repeat the rejected arguments and do not add prose.'
|
|
1895
|
+
continue
|
|
1896
|
+
}
|
|
1897
|
+
throw scanError
|
|
1898
|
+
}
|
|
1789
1899
|
// A valid capability decision is already receipt-gated and is
|
|
1790
1900
|
// the authority for this interval. Some providers fail a later
|
|
1791
1901
|
// post-tool model step; that must not erase the accepted action.
|
|
@@ -1834,8 +1944,12 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1834
1944
|
attempt, softStallBudgetMs, remainingMs,
|
|
1835
1945
|
}))
|
|
1836
1946
|
retire(runPort)
|
|
1837
|
-
|
|
1947
|
+
const stallRetryPort = await beforePlannerDeadline(taskPort(), deadlineAt)
|
|
1838
1948
|
if (closed || sessionClosePromise) throw new Error(closed ? 'planner_closed' : 'planner_session_closed')
|
|
1949
|
+
if (isRetiredPort(stallRetryPort)) {
|
|
1950
|
+
throw bootNeverCompleted(runPort) ? new PlannerBootTimeoutExceeded() : new PlannerTurnDeadlineExceeded()
|
|
1951
|
+
}
|
|
1952
|
+
runPort = stallRetryPort
|
|
1839
1953
|
// The stalled run produced no decision; it must not consume a
|
|
1840
1954
|
// prose-nudge attempt.
|
|
1841
1955
|
attempt -= 1
|
|
@@ -1843,7 +1957,27 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1843
1957
|
}
|
|
1844
1958
|
// The SDK reports an expired local handshake as its own typed
|
|
1845
1959
|
// failure; no provider request was ever made.
|
|
1846
|
-
if (error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT')
|
|
1960
|
+
if (error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT') {
|
|
1961
|
+
// A cold OS page cache routinely pushes the first boot past
|
|
1962
|
+
// the local budget (observed 6.5–8s after idle on UAT,
|
|
1963
|
+
// Danceiny/gotry#660): the failed boot warms the very pages
|
|
1964
|
+
// the next port reads, so one fresh-port retry converges and
|
|
1965
|
+
// only a genuinely stuck runtime burns both. Same shape as
|
|
1966
|
+
// the soft-stall retry above; a boot that never reached a
|
|
1967
|
+
// provider request must not consume an attempt.
|
|
1968
|
+
const bootRemainingMs = deadlineAt - Date.now()
|
|
1969
|
+
if (bootRemainingMs < PLANNER_MIN_VIABLE_RUN_MS) throw new PlannerBootTimeoutExceeded()
|
|
1970
|
+
console.error('[booking-copilot] planner boot exceeded the local budget; retrying on a fresh run port:', JSON.stringify({
|
|
1971
|
+
attempt, bootRemainingMs,
|
|
1972
|
+
}))
|
|
1973
|
+
retire(runPort)
|
|
1974
|
+
const bootRetryPort = await beforePlannerDeadline(taskPort(), deadlineAt)
|
|
1975
|
+
if (closed || sessionClosePromise) throw new Error(closed ? 'planner_closed' : 'planner_session_closed')
|
|
1976
|
+
if (isRetiredPort(bootRetryPort)) throw new PlannerBootTimeoutExceeded()
|
|
1977
|
+
runPort = bootRetryPort
|
|
1978
|
+
attempt -= 1
|
|
1979
|
+
continue
|
|
1980
|
+
}
|
|
1847
1981
|
if (error instanceof PlannerTurnDeadlineExceeded) {
|
|
1848
1982
|
// The turn deadline can expire while our own runtime is still
|
|
1849
1983
|
// starting; that phase is ours, not the provider's.
|
|
@@ -1883,6 +2017,14 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1883
2017
|
retire(runPort)
|
|
1884
2018
|
return [{ kind: 'error', error: { code: 'PLANNER_ATTEMPT_BUDGET_EXHAUSTED', message: 'GoTry exhausted the planner attempt budget without a decision.', retryable: true } }]
|
|
1885
2019
|
} catch (error) {
|
|
2020
|
+
// A port that fails its own initialize (createManagedDshRunPort with
|
|
2021
|
+
// initializeTimeoutMs) rejects with the SDK's raw typed message,
|
|
2022
|
+
// never reaching the run loop; classify it as the boot timeout it is
|
|
2023
|
+
// instead of surfacing an untyped failure.
|
|
2024
|
+
if (!(error instanceof PlannerBootTimeoutExceeded)
|
|
2025
|
+
&& error instanceof Error && error.message === 'HARNESS_BOOT_TIMEOUT') {
|
|
2026
|
+
error = new PlannerBootTimeoutExceeded()
|
|
2027
|
+
}
|
|
1886
2028
|
if (error instanceof PlannerBootTimeoutExceeded) {
|
|
1887
2029
|
metricOutcome = 'boot_timeout'
|
|
1888
2030
|
logPlannerBootTimeout(runPort)
|
|
@@ -1894,6 +2036,14 @@ export async function createDshEmbeddedBookingPlanner(
|
|
|
1894
2036
|
retire(runPort)
|
|
1895
2037
|
return [{ kind: 'error', error: { code: 'PLANNER_PROVIDER_TIMEOUT', message: providerFailureMessage('PLANNER_PROVIDER_TIMEOUT'), retryable: true } }]
|
|
1896
2038
|
}
|
|
2039
|
+
if (error instanceof ManagedDshResourceLimitError) {
|
|
2040
|
+
// The tree was killed for taking too much of the replica. Running the same turn again on a fresh port would
|
|
2041
|
+
// only let it take that much again, so this is terminal for the turn: not retried, not a provider fault.
|
|
2042
|
+
metricOutcome = 'resource_limit'
|
|
2043
|
+
console.error('[booking-copilot] planner child exceeded a resource limit:', JSON.stringify(error.breach))
|
|
2044
|
+
retire(runPort)
|
|
2045
|
+
return [{ kind: 'error', error: { code: 'PLANNER_FAILED', message: providerFailureMessage('PLANNER_FAILED'), retryable: false } }]
|
|
2046
|
+
}
|
|
1897
2047
|
retire(runPort)
|
|
1898
2048
|
throw error
|
|
1899
2049
|
} finally {
|
|
@@ -14,6 +14,65 @@ import {
|
|
|
14
14
|
type ManagedDshProcessGroupSnapshot,
|
|
15
15
|
type ManagedDshWorkerOutcome,
|
|
16
16
|
} from './managed-dsh-cleanup-diagnostic.ts'
|
|
17
|
+
import { sharedProcessTreeGovernor, type ProcessTreeGovernor, type TreeBreach } from '../tenant-seam/process-governor.ts'
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* What one worker tree may take (AaaS, docs/rfc/aaas-rfc.md §4.8, threat S-17). Every field is opt-in: a port built
|
|
21
|
+
* without `limits` behaves exactly as before.
|
|
22
|
+
*/
|
|
23
|
+
export interface ManagedDshResourceLimits {
|
|
24
|
+
/** V8 old-space ceiling in MiB for the worker and every Node process below it (`NODE_OPTIONS`). An allocation spike dies in-process. */
|
|
25
|
+
heapMb?: number
|
|
26
|
+
/** Resident set of the whole worker tree in bytes (sampled; see `ProcessTreeGovernor`). Covers what the heap cap cannot see, such as buffers. */
|
|
27
|
+
maxRssBytes?: number
|
|
28
|
+
/** CPU seconds of the whole worker tree (sampled). */
|
|
29
|
+
maxCpuSeconds?: number
|
|
30
|
+
/** The longest line, in characters, the worker may write to the parent without a newline. The parent buffers a line until it ends. */
|
|
31
|
+
maxLineChars?: number
|
|
32
|
+
/**
|
|
33
|
+
* How long a worker may take to say who it is (its `worker_started` line) before the missing governor is reported. The
|
|
34
|
+
* sampler learns which process to watch from that line alone: without it the tree limits are silently off. Default 10 s.
|
|
35
|
+
*/
|
|
36
|
+
attachDeadlineMs?: number
|
|
37
|
+
/** Sampler shared by ports. Default: the process-wide governor. */
|
|
38
|
+
governor?: ProcessTreeGovernor
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* `memory` and `cpu` come from the tree sampler, `output` from the line cap, `heap` from the worker's own V8 ceiling
|
|
43
|
+
* (recognised from its stderr tail after it died, where only the ceiling is known and `observed` repeats it).
|
|
44
|
+
*/
|
|
45
|
+
export type ManagedDshResourceBreach = TreeBreach | { readonly kind: 'output' | 'heap'; readonly limit: number; readonly observed: number }
|
|
46
|
+
|
|
47
|
+
/** The worker tree went over a resource limit and was killed. Closed shape: kind and numbers only, never worker output. */
|
|
48
|
+
export class ManagedDshResourceLimitError extends Error {
|
|
49
|
+
readonly breach: ManagedDshResourceBreach
|
|
50
|
+
|
|
51
|
+
constructor(breach: ManagedDshResourceBreach) {
|
|
52
|
+
super(breach.kind === 'blind' ? 'managed DSH worker tree could not be observed' : `managed DSH worker exceeded its ${breach.kind} limit`)
|
|
53
|
+
this.name = 'ManagedDshResourceLimitError'
|
|
54
|
+
this.breach = breach
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Proves that the sampler can see before a child is started: with a tree limit set, a process table that cannot be read
|
|
60
|
+
* would let the limit govern nothing, so it fails first and by name. A port with no tree limit has nothing to check.
|
|
61
|
+
*/
|
|
62
|
+
export async function assertManagedDshLimitsEnforceable(limits: ManagedDshResourceLimits): Promise<void> {
|
|
63
|
+
if (limits.maxRssBytes === undefined && limits.maxCpuSeconds === undefined) return
|
|
64
|
+
await (limits.governor ?? sharedProcessTreeGovernor()).probe()
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function assertPositiveLimit(name: string, value: number | undefined): void {
|
|
68
|
+
if (value !== undefined && (!Number.isFinite(value) || value <= 0)) throw new RangeError(`managed DSH limit ${name} must be a positive number`)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function limitedEnvironment(env: NodeJS.ProcessEnv | undefined, heapMb: number | undefined): NodeJS.ProcessEnv | undefined {
|
|
72
|
+
if (heapMb === undefined) return env
|
|
73
|
+
const existing = (env?.NODE_OPTIONS ?? '').split(/\s+/).filter((option) => option !== '' && !option.startsWith('--max-old-space-size='))
|
|
74
|
+
return { ...env, NODE_OPTIONS: [...existing, `--max-old-space-size=${Math.floor(heapMb)}`].join(' ') }
|
|
75
|
+
}
|
|
17
76
|
|
|
18
77
|
export interface ManagedDshRunPortOptions {
|
|
19
78
|
cwd?: string
|
|
@@ -21,6 +80,8 @@ export interface ManagedDshRunPortOptions {
|
|
|
21
80
|
graceMs?: number
|
|
22
81
|
cleanupRole?: 'task' | 'warmer'
|
|
23
82
|
env?: NodeJS.ProcessEnv
|
|
83
|
+
/** Resource ceilings for the worker tree. Absent: none, as before. */
|
|
84
|
+
limits?: ManagedDshResourceLimits
|
|
24
85
|
/** Test seam overriding the failure-path group snapshot; production uses the safe ps snapshot. */
|
|
25
86
|
groupObserver?: typeof snapshotManagedDshGroup
|
|
26
87
|
/**
|
|
@@ -38,14 +99,17 @@ const WORKER_FAILURE_CODES = new Set(['HARNESS_BOOT_TIMEOUT', 'HARNESS_START_FAI
|
|
|
38
99
|
|
|
39
100
|
const requireFromModule = createRequire(import.meta.url)
|
|
40
101
|
|
|
41
|
-
function workerLaunch(path: string): readonly string[] {
|
|
102
|
+
function workerLaunch(path: string, heapMb?: number): readonly string[] {
|
|
103
|
+
// The V8 ceiling is given on the worker's own command line as well as in NODE_OPTIONS (which carries it to the
|
|
104
|
+
// processes below the worker), so the worker's ceiling does not depend on its environment surviving a scrub.
|
|
105
|
+
const heap = heapMb === undefined ? [] : [`--max-old-space-size=${Math.floor(heapMb)}`]
|
|
42
106
|
// The worker cwd is the isolated booking state root, so a bare `tsx/esm`
|
|
43
107
|
// loader would resolve from the wrong dependency tree and exit before the
|
|
44
108
|
// first request. Resolve the loader from this package once and pass its
|
|
45
109
|
// absolute installed path to Node.
|
|
46
110
|
return path.endsWith('.ts')
|
|
47
|
-
? [process.execPath, '--import', requireFromModule.resolve('tsx/esm'), path]
|
|
48
|
-
: [process.execPath, path]
|
|
111
|
+
? [process.execPath, ...heap, '--import', requireFromModule.resolve('tsx/esm'), path]
|
|
112
|
+
: [process.execPath, ...heap, path]
|
|
49
113
|
}
|
|
50
114
|
|
|
51
115
|
/** Run DeepSeekHarness in a child tree owned by dsh-subprocess-local. */
|
|
@@ -66,7 +130,13 @@ export class ManagedDshRunPort implements DshPlannerRunPort {
|
|
|
66
130
|
constructor(options: ManagedDshRunPortOptions = {}) {
|
|
67
131
|
const defaultWorker = join(dirname(fileURLToPath(import.meta.url)), 'dsh-worker.js')
|
|
68
132
|
const workerPath = options.workerPath ?? (existsSync(defaultWorker) ? defaultWorker : defaultWorker.replace(/\.js$/, '.ts'))
|
|
69
|
-
const { workerPath: _worker, graceMs = 500, cleanupRole, env, groupObserver, bootLog, ...harnessOptions } = options
|
|
133
|
+
const { workerPath: _worker, graceMs = 500, cleanupRole, env, groupObserver, bootLog, limits, ...harnessOptions } = options
|
|
134
|
+
assertPositiveLimit('heapMb', limits?.heapMb)
|
|
135
|
+
assertPositiveLimit('maxRssBytes', limits?.maxRssBytes)
|
|
136
|
+
assertPositiveLimit('maxCpuSeconds', limits?.maxCpuSeconds)
|
|
137
|
+
assertPositiveLimit('maxLineChars', limits?.maxLineChars)
|
|
138
|
+
assertPositiveLimit('attachDeadlineMs', limits?.attachDeadlineMs)
|
|
139
|
+
this.limits = limits
|
|
70
140
|
this.cleanupRole = cleanupRole ?? 'unspecified'
|
|
71
141
|
this.graceMs = graceMs
|
|
72
142
|
this.groupObserver = groupObserver ?? snapshotManagedDshGroup
|
|
@@ -74,22 +144,32 @@ export class ManagedDshRunPort implements DshPlannerRunPort {
|
|
|
74
144
|
this.constructedAtMs = Date.now()
|
|
75
145
|
this.runtime = new LocalSubprocessRuntime(new Context())
|
|
76
146
|
this.handle = this.runtime.spawn({
|
|
77
|
-
argv: workerLaunch(workerPath),
|
|
147
|
+
argv: workerLaunch(workerPath, limits?.heapMb),
|
|
78
148
|
cwd: options.cwd ?? process.cwd(),
|
|
79
149
|
// Provider diagnostics may contain request fragments. Keep a bounded
|
|
80
150
|
// private tail for ownership without inheriting it into service logs.
|
|
81
151
|
stdio: { stdin: 'pipe', stdout: 'pipe', stderr: { maxBytes: 64 * 1024 } },
|
|
82
152
|
graceMs,
|
|
83
|
-
env,
|
|
153
|
+
env: limitedEnvironment(env, limits?.heapMb),
|
|
84
154
|
})
|
|
85
155
|
this.handle.stdout!.setEncoding('utf8')
|
|
86
156
|
this.handle.stdout!.on('data', (chunk: string) => this.consume(chunk))
|
|
157
|
+
if (limits && (limits.maxRssBytes !== undefined || limits.maxCpuSeconds !== undefined)) {
|
|
158
|
+
const deadlineMs = limits.attachDeadlineMs ?? 10_000
|
|
159
|
+
this.attachTimer = setTimeout(() => {
|
|
160
|
+
this.emitBootLog(`RESOURCE_GOVERNOR_UNATTACHED ${JSON.stringify({ role: this.cleanupRole, afterMs: deadlineMs })}`)
|
|
161
|
+
}, deadlineMs)
|
|
162
|
+
this.attachTimer.unref()
|
|
163
|
+
}
|
|
87
164
|
this.handle.done.then(
|
|
88
165
|
(outcome) => {
|
|
166
|
+
this.stopWatching()
|
|
89
167
|
this.workerOutcome = { state: 'exited', exitCode: outcome.exitCode, signal: outcome.signal }
|
|
168
|
+
this.noteHeapExhaustion()
|
|
90
169
|
this.markTerminal('managed DSH worker exited')
|
|
91
170
|
},
|
|
92
171
|
() => {
|
|
172
|
+
this.stopWatching()
|
|
93
173
|
this.workerOutcome = { state: 'failed' }
|
|
94
174
|
this.markTerminal('managed DSH worker failed')
|
|
95
175
|
},
|
|
@@ -100,6 +180,9 @@ export class ManagedDshRunPort implements DshPlannerRunPort {
|
|
|
100
180
|
this.cleanupDeadlineMs = Math.max(1_000, graceMs * 4)
|
|
101
181
|
}
|
|
102
182
|
|
|
183
|
+
private readonly limits: ManagedDshResourceLimits | undefined
|
|
184
|
+
private attachTimer: NodeJS.Timeout | undefined
|
|
185
|
+
private unwatchTree: (() => void) | undefined
|
|
103
186
|
private readonly harnessOptions: Record<string, unknown>
|
|
104
187
|
private readonly cleanupDeadlineMs: number
|
|
105
188
|
private readonly groupObserver: typeof snapshotManagedDshGroup
|
|
@@ -130,17 +213,76 @@ export class ManagedDshRunPort implements DshPlannerRunPort {
|
|
|
130
213
|
return this.booted
|
|
131
214
|
}
|
|
132
215
|
|
|
216
|
+
/**
|
|
217
|
+
* A resource ceiling was crossed: say why before anything else can say something vaguer, fail what is waiting with
|
|
218
|
+
* that reason, and let the provider-owned range cleanup run. The governor (or the line cap) then kills the tree.
|
|
219
|
+
*/
|
|
220
|
+
private resourceBreach(breach: ManagedDshResourceBreach): void {
|
|
221
|
+
this.terminalError ??= new ManagedDshResourceLimitError(breach)
|
|
222
|
+
this.failPending(this.terminalError)
|
|
223
|
+
this.stopWatching()
|
|
224
|
+
this.inputBuffer = ''
|
|
225
|
+
try { this.handle.terminate() } catch { /* the governor's kill below is what must happen */ }
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
private stopWatching(): void {
|
|
229
|
+
clearTimeout(this.attachTimer)
|
|
230
|
+
this.attachTimer = undefined
|
|
231
|
+
const unwatch = this.unwatchTree
|
|
232
|
+
this.unwatchTree = undefined
|
|
233
|
+
unwatch?.()
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* A worker that died of its own V8 ceiling is a resource-limit death, not a mystery crash. Only the fixed V8 message is
|
|
238
|
+
* looked for in the bounded stderr tail; nothing from it is kept or reported.
|
|
239
|
+
*/
|
|
240
|
+
private noteHeapExhaustion(): void {
|
|
241
|
+
const heapMb = this.limits?.heapMb
|
|
242
|
+
if (heapMb === undefined || this.terminalError) return
|
|
243
|
+
let tail = ''
|
|
244
|
+
try { tail = this.handle.collected.stderr?.readFrom(0).text ?? '' } catch { return }
|
|
245
|
+
if (!tail.includes('JavaScript heap out of memory')) return
|
|
246
|
+
const bytes = Math.floor(heapMb) * 1024 * 1024
|
|
247
|
+
this.terminalError = new ManagedDshResourceLimitError({ kind: 'heap', limit: bytes, observed: bytes })
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
private watchTree(workerPid: number): void {
|
|
251
|
+
const limits = this.limits
|
|
252
|
+
if (!limits || (limits.maxRssBytes === undefined && limits.maxCpuSeconds === undefined)) return
|
|
253
|
+
clearTimeout(this.attachTimer)
|
|
254
|
+
this.attachTimer = undefined
|
|
255
|
+
const governor = limits.governor ?? sharedProcessTreeGovernor()
|
|
256
|
+
// The worker is this process's own child (the lifecycle line was checked for that), so the governor also requires
|
|
257
|
+
// the parent to match: a pid the operating system reused for something else is never sampled or killed.
|
|
258
|
+
this.unwatchTree = governor.watch(
|
|
259
|
+
{ rootPid: workerPid, parentPid: process.pid },
|
|
260
|
+
{ maxRssBytes: limits.maxRssBytes, maxCpuSeconds: limits.maxCpuSeconds },
|
|
261
|
+
(breach) => this.resourceBreach(breach),
|
|
262
|
+
)
|
|
263
|
+
}
|
|
264
|
+
|
|
133
265
|
private consume(chunk: string): void {
|
|
266
|
+
if (this.terminalError instanceof ManagedDshResourceLimitError) return
|
|
134
267
|
this.inputBuffer += chunk
|
|
135
268
|
for (;;) {
|
|
136
269
|
const newline = this.inputBuffer.indexOf('\n')
|
|
137
|
-
if (newline < 0)
|
|
270
|
+
if (newline < 0) {
|
|
271
|
+
// The parent holds a line until it ends; a worker that never ends one would grow this process without bound.
|
|
272
|
+
const maxLineChars = this.limits?.maxLineChars
|
|
273
|
+
if (maxLineChars !== undefined && this.inputBuffer.length > maxLineChars) {
|
|
274
|
+
this.resourceBreach({ kind: 'output', limit: maxLineChars, observed: this.inputBuffer.length })
|
|
275
|
+
// The worker stays up until the provider range cleanup lands; this port no longer reads from it.
|
|
276
|
+
}
|
|
277
|
+
return
|
|
278
|
+
}
|
|
138
279
|
const line = this.inputBuffer.slice(0, newline); this.inputBuffer = this.inputBuffer.slice(newline + 1)
|
|
139
280
|
try {
|
|
140
281
|
const message = JSON.parse(line) as { id: number; ok?: boolean; progress?: boolean; result?: DshPlannerRunResult; error?: string; lifecycle?: string; workerPid?: number; parentPid?: number; boot?: DshBootObservation }
|
|
141
282
|
if (message.lifecycle === 'worker_started') {
|
|
142
283
|
if (this.workerPid === null && Number.isSafeInteger(message.workerPid) && message.workerPid! > 0 && message.parentPid === process.pid) {
|
|
143
284
|
this.workerPid = message.workerPid!
|
|
285
|
+
this.watchTree(this.workerPid)
|
|
144
286
|
// One bounded read-only start-time anchor, captured while the worker
|
|
145
287
|
// is alive, so a later failure snapshot can disprove PID reuse.
|
|
146
288
|
void readManagedDshWorkerStart(this.workerPid).then((startedAt) => {
|
|
@@ -225,6 +367,7 @@ export class ManagedDshRunPort implements DshPlannerRunPort {
|
|
|
225
367
|
if (this.closePromise) return this.closePromise
|
|
226
368
|
this.closePromise = (async () => {
|
|
227
369
|
const started = performance.now()
|
|
370
|
+
this.stopWatching()
|
|
228
371
|
this.failPending(new Error('managed DSH run port closed'))
|
|
229
372
|
this.handle.terminate()
|
|
230
373
|
// Observe worker failure even if range observation rejects or times out
|