@opengeni/worker-bundle 0.23.4-canary.0 → 0.24.2-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +7 -0
- package/dist/activities/agent-turn/model-usage.d.ts +5 -0
- package/dist/activities/agent-turn/quiescence.d.ts +2 -0
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +1 -0
- package/dist/activities/agent-turn/turn-context.d.ts +2 -1
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/capabilities.d.ts +1 -0
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/{activities-control-SGU2WEVB.js → activities-control-ZWH7J4KZ.js} +102 -94
- package/dist/activities-control-ZWH7J4KZ.js.map +1 -0
- package/dist/{activities-turn-7EAQXW6Q.js → activities-turn-I4OPLQLS.js} +613 -284
- package/dist/activities-turn-I4OPLQLS.js.map +1 -0
- package/dist/{chunk-TJIPCHMP.js → chunk-65JEWP2Q.js} +105 -6
- package/dist/chunk-65JEWP2Q.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-S2QVRVJW.js → chunk-RQNH6CGR.js} +28 -41
- package/dist/chunk-RQNH6CGR.js.map +1 -0
- package/dist/index.js +16 -3
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +33 -7
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/package.json +17 -17
- package/src/activities/agent-turn/admission.ts +19 -0
- package/src/activities/agent-turn/claim.ts +9 -1
- package/src/activities/agent-turn/compaction-prep.ts +6 -1
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/model-usage.ts +33 -4
- package/src/activities/agent-turn/quiescence.ts +11 -2
- package/src/activities/agent-turn/run.ts +74 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +258 -135
- package/src/activities/agent-turn/tool-environment.ts +41 -7
- package/src/activities/agent-turn/turn-context.ts +4 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/api-integrations.ts +11 -1
- package/src/activities/capabilities.ts +57 -0
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +1 -0
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/scheduled-tasks.ts +111 -98
- package/src/index.ts +15 -0
- package/src/observability-metrics.ts +152 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/dist/activities-control-SGU2WEVB.js.map +0 -1
- package/dist/activities-turn-7EAQXW6Q.js.map +0 -1
- package/dist/chunk-S2QVRVJW.js.map +0 -1
- package/dist/chunk-TJIPCHMP.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/concurrency.ts","../src/sandbox-reaper-contract.ts","../src/sandbox-reaper-timeout.ts"],"sourcesContent":["import type { Settings } from \"@opengeni/config\";\nimport type { Observability } from \"@opengeni/observability\";\nimport { readFileSync } from \"node:fs\";\nimport type {\n ActivitySlotInfo,\n CustomSlotSupplier,\n SlotMarkUsedContext,\n SlotPermit,\n SlotReleaseContext,\n SlotReserveContext,\n WorkerOptions,\n WorkerTuner,\n} from \"@temporalio/worker\";\n\nconst MIB = 1024 * 1024;\n\n/**\n * Default fixed/HPA turn-worker admission contract. The configured fixed-mode\n * ceiling may be lower, but every live Temporal permit reserves the complete\n * hard budget until the runAgentTurn activity promise physically completes.\n */\nexport const TURN_WORKER_MAX_CONCURRENT_TURNS = 16;\nexport const TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN = 100 * MIB;\nexport const TURN_WORKER_NATIVE_HEADROOM_BYTES = 512 * MIB;\n\nexport type TurnWorkerConcurrencySettings = Pick<\n Settings,\n | \"turnWorkerConcurrencyMode\"\n | \"turnWorkerMaxConcurrentTurns\"\n | \"turnWorkerTargetCpuUsage\"\n | \"turnWorkerTargetMemoryUsage\"\n>;\n\nexport type TurnWorkerConcurrencyOptions = Pick<\n WorkerOptions,\n \"maxConcurrentActivityTaskExecutions\" | \"tuner\"\n>;\n\n/**\n * Fixed profiles keep a per-process ceiling. Resource-aware profiles retain a\n * hard per-process maximum while Temporal meters new activity admission against\n * cgroup/host CPU and memory. The ramp lets usage become visible before a burst\n * can reserve the entire ceiling.\n */\nexport function turnWorkerConcurrencyOptions(\n settings: TurnWorkerConcurrencySettings,\n): TurnWorkerConcurrencyOptions {\n if (settings.turnWorkerConcurrencyMode === \"fixed\") {\n return {\n maxConcurrentActivityTaskExecutions: settings.turnWorkerMaxConcurrentTurns,\n };\n }\n\n return {\n tuner: {\n tunerOptions: {\n targetCpuUsage: settings.turnWorkerTargetCpuUsage,\n targetMemoryUsage: settings.turnWorkerTargetMemoryUsage,\n },\n activityTaskSlotOptions: {\n minimumSlots: 1,\n maximumSlots: settings.turnWorkerMaxConcurrentTurns,\n rampThrottle: \"250ms\",\n },\n },\n };\n}\n\nexport type TurnWorkerConcurrencyPlan = {\n options: TurnWorkerConcurrencyOptions;\n admission: MemoryAwareTurnSlotSupplier | null;\n};\n\n/**\n * Build the runtime concurrency policy for a turn worker.\n *\n * Fixed/HPA deployments use the hard-reservation supplier below. The explicit\n * non-HA single-machine profile keeps Temporal's native CPU/memory resource\n * tuner because it controls whole-machine load rather than a pod cgroup; that\n * profile documents that its configured maximum is not a per-turn reservation.\n */\nexport function createTurnWorkerConcurrencyPlan(\n settings: TurnWorkerConcurrencySettings,\n options: Omit<TurnAdmissionOptions, \"maximumTurns\"> = {},\n): TurnWorkerConcurrencyPlan {\n if (settings.turnWorkerConcurrencyMode === \"resource-based\") {\n return {\n options: turnWorkerConcurrencyOptions(settings),\n admission: null,\n };\n }\n\n const fixed = createTurnWorkerTuner({\n ...options,\n maximumTurns: settings.turnWorkerMaxConcurrentTurns,\n });\n return {\n options: { tuner: fixed.tuner },\n admission: fixed.admission,\n };\n}\n\nexport function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {\n concurrencyMode: \"fixed\" | \"resource-based\";\n maxConcurrentTurns: number;\n targetCpuUsage: number | null;\n targetMemoryUsage: number | null;\n} {\n const resourceBased = settings.turnWorkerConcurrencyMode === \"resource-based\";\n return {\n concurrencyMode: settings.turnWorkerConcurrencyMode,\n maxConcurrentTurns: settings.turnWorkerMaxConcurrentTurns,\n targetCpuUsage: resourceBased ? settings.turnWorkerTargetCpuUsage : null,\n targetMemoryUsage: resourceBased ? settings.turnWorkerTargetMemoryUsage : null,\n };\n}\n\nexport const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;\nexport const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;\n\nconst CGROUP_MEMORY_FILES = [\n {\n limit: \"/sys/fs/cgroup/memory.max\",\n current: \"/sys/fs/cgroup/memory.current\",\n },\n {\n limit: \"/sys/fs/cgroup/memory/memory.limit_in_bytes\",\n current: \"/sys/fs/cgroup/memory/memory.usage_in_bytes\",\n },\n] as const;\n\n// cgroup v1 uses enormous sentinels for \"unlimited\". Values this large are\n// host capacity rather than a meaningful container admission boundary.\nconst UNLIMITED_CGROUP_BYTES = 1n << 60n;\n\nexport type CgroupMemorySnapshot = {\n currentBytes: number;\n limitBytes: number | null;\n source: \"cgroup-v1\" | \"cgroup-v2\" | \"process\";\n};\n\nexport type TurnAdmissionSnapshot = {\n baselineBytes: number;\n currentBytes: number;\n limitBytes: number | null;\n hardBytesPerTurn: number;\n nativeHeadroomBytes: number;\n maximumTurns: number;\n memoryBoundCapacity: number;\n reservedSlots: number;\n usedSlots: number;\n availableSlots: number;\n};\n\nexport type TurnAdmissionOptions = {\n maximumTurns?: number;\n hardBytesPerTurn?: number;\n nativeHeadroomBytes?: number;\n memorySnapshot?: () => CgroupMemorySnapshot;\n retryDelayMs?: number;\n observability?: Observability;\n};\n\ntype TurnSlotPermit = SlotPermit & {\n readonly owner: symbol;\n used: boolean;\n};\n\n/**\n * Temporal custom activity-slot supplier for turn workers.\n *\n * A permit is required before Temporal polls a task. Reservation is allowed\n * only when both independent safety checks pass:\n *\n * baseline + reserved-after × hard-turn-budget + native headroom <= limit\n * current + reserved-after × hard-turn-budget + native headroom <= limit\n *\n * The first check enforces the release contract even when current RSS happens\n * to be low. The second accounts for native/retained memory observed after\n * startup and keeps the complete hard budget reserved for every live permit.\n * A used permit is not evidence that its allocation is already visible in the\n * cgroup, and unrelated native growth cannot safely be attributed to a turn.\n * Kubernetes OOM is never an admission mechanism.\n */\nexport class MemoryAwareTurnSlotSupplier implements CustomSlotSupplier<ActivitySlotInfo> {\n readonly type = \"custom\" as const;\n\n private readonly owner = Symbol(\"opengeni-turn-admission\");\n private readonly permits = new Set<TurnSlotPermit>();\n private baselineBytes: number;\n private readonly maximumTurns: number;\n private readonly hardBytesPerTurn: number;\n private readonly nativeHeadroomBytes: number;\n private readonly memorySnapshot: () => CgroupMemorySnapshot;\n private readonly retryDelayMs: number;\n private readonly observability: Observability | undefined;\n\n constructor(options: TurnAdmissionOptions = {}) {\n this.maximumTurns = positiveInteger(\n \"maximumTurns\",\n options.maximumTurns ?? TURN_WORKER_MAX_CONCURRENT_TURNS,\n );\n this.hardBytesPerTurn = positiveInteger(\n \"hardBytesPerTurn\",\n options.hardBytesPerTurn ?? TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN,\n );\n this.nativeHeadroomBytes = nonnegativeInteger(\n \"nativeHeadroomBytes\",\n options.nativeHeadroomBytes ?? TURN_WORKER_NATIVE_HEADROOM_BYTES,\n );\n this.retryDelayMs = positiveInteger(\"retryDelayMs\", options.retryDelayMs ?? 100);\n this.memorySnapshot = options.memorySnapshot ?? readCgroupMemorySnapshot;\n this.observability = options.observability;\n const initial = this.memorySnapshot();\n this.baselineBytes = initial.currentBytes;\n if (initial.limitBytes !== null && this.memoryBoundCapacity(initial.limitBytes) < 1) {\n throw new Error(\n \"Turn worker memory limit cannot safely admit one turn: \" +\n `baseline=${this.baselineBytes} hardTurn=${this.hardBytesPerTurn} ` +\n `nativeHeadroom=${this.nativeHeadroomBytes} limit=${initial.limitBytes}`,\n );\n }\n this.refreshMetrics(initial);\n }\n\n async reserveSlot(_ctx: SlotReserveContext, abortSignal: AbortSignal): Promise<SlotPermit> {\n for (;;) {\n if (abortSignal.aborted) throw abortError();\n const permit = this.reserveIfSafe();\n if (permit) return permit;\n await abortableDelay(this.retryDelayMs, abortSignal);\n }\n }\n\n tryReserveSlot(_ctx: SlotReserveContext): SlotPermit | null {\n return this.reserveIfSafe();\n }\n\n markSlotUsed(ctx: SlotMarkUsedContext<ActivitySlotInfo>): void {\n const permit = this.ownedPermit(ctx.permit);\n permit.used = true;\n this.refreshMetrics();\n }\n\n releaseSlot(ctx: SlotReleaseContext<ActivitySlotInfo>): void {\n const permit = this.ownedPermit(ctx.permit);\n this.permits.delete(permit);\n this.refreshMetrics();\n }\n\n snapshot(memory = this.memorySnapshot()): TurnAdmissionSnapshot {\n const usedSlots = this.usedSlots();\n const reservedSlots = this.permits.size;\n const availableSlots = this.availableSlots(memory);\n // Never report less capacity than work already admitted. The dynamic\n // denominator is exactly reserved + what a new poll could reserve now.\n const memoryBoundCapacity = reservedSlots + availableSlots;\n return {\n baselineBytes: this.baselineBytes,\n currentBytes: memory.currentBytes,\n limitBytes: memory.limitBytes,\n hardBytesPerTurn: this.hardBytesPerTurn,\n nativeHeadroomBytes: this.nativeHeadroomBytes,\n maximumTurns: this.maximumTurns,\n memoryBoundCapacity,\n reservedSlots,\n usedSlots,\n availableSlots,\n };\n }\n\n /**\n * Finalize the admission baseline after Temporal/native construction.\n * Worker.create can retain memory after the constructor's provisional\n * sample. That initialization delta must remain in the baseline for every\n * later projection rather than competing with turn reservations.\n */\n finalizeStartupBaseline(): void {\n const memory = this.memorySnapshot();\n if (this.permits.size !== 0) {\n throw new Error(\"Turn worker startup admission check ran after slot reservation\");\n }\n const finalizedBaselineBytes = memory.currentBytes;\n const finalizedCapacity =\n memory.limitBytes === null\n ? this.maximumTurns\n : this.memoryBoundCapacity(memory.limitBytes, finalizedBaselineBytes);\n if (memory.limitBytes !== null && finalizedCapacity < 1) {\n throw new Error(\n \"Turn worker memory limit cannot safely admit one turn after worker initialization: \" +\n `provisionalBaseline=${this.baselineBytes} finalizedBaseline=${finalizedBaselineBytes} ` +\n `hardTurn=${this.hardBytesPerTurn} nativeHeadroom=${this.nativeHeadroomBytes} ` +\n `limit=${String(memory.limitBytes)}`,\n );\n }\n this.baselineBytes = finalizedBaselineBytes;\n this.refreshMetrics(memory);\n }\n\n private reserveIfSafe(): TurnSlotPermit | null {\n const memory = this.memorySnapshot();\n if (this.permits.size >= this.maximumTurns || !this.memoryAllowsAnother(memory)) {\n this.refreshMetrics(memory);\n return null;\n }\n const permit: TurnSlotPermit = { owner: this.owner, used: false };\n this.permits.add(permit);\n this.refreshMetrics(memory);\n return permit;\n }\n\n private memoryAllowsAnother(memory: CgroupMemorySnapshot): boolean {\n return this.availableSlots(memory) > 0;\n }\n\n private availableSlots(memory: CgroupMemorySnapshot): number {\n const densityRemaining = this.maximumTurns - this.permits.size;\n if (memory.limitBytes === null) return Math.max(0, densityRemaining);\n\n const baselineRemaining = this.memoryBoundCapacity(memory.limitBytes) - this.permits.size;\n const observedRemaining =\n Math.floor(\n (memory.limitBytes - memory.currentBytes - this.nativeHeadroomBytes) /\n this.hardBytesPerTurn,\n ) - this.permits.size;\n return Math.max(0, Math.min(densityRemaining, baselineRemaining, observedRemaining));\n }\n\n private memoryBoundCapacity(limitBytes: number, baselineBytes = this.baselineBytes): number {\n return Math.max(\n 0,\n Math.floor((limitBytes - baselineBytes - this.nativeHeadroomBytes) / this.hardBytesPerTurn),\n );\n }\n\n private usedSlots(): number {\n let used = 0;\n for (const permit of this.permits) {\n if (permit.used) used += 1;\n }\n return used;\n }\n\n private ownedPermit(permit: SlotPermit): TurnSlotPermit {\n const candidate = permit as Partial<TurnSlotPermit>;\n if (candidate.owner !== this.owner || !this.permits.has(permit as TurnSlotPermit)) {\n throw new Error(\"Temporal returned a turn slot permit owned by another supplier\");\n }\n return permit as TurnSlotPermit;\n }\n\n private refreshMetrics(memory = this.memorySnapshot()): void {\n if (!this.observability) return;\n const value = this.snapshot(memory);\n const set = (name: string, help: string, metricValue: number) =>\n this.observability?.setGauge({ name, help, value: metricValue });\n set(\n \"opengeni_turn_slots_capacity\",\n \"Turn slots this worker can safely support after the hard memory contract.\",\n value.memoryBoundCapacity,\n );\n set(\n \"opengeni_turn_slots_reserved\",\n \"Temporal turn activity slots reserved for polling or execution in this worker.\",\n value.reservedSlots,\n );\n set(\n \"opengeni_turn_slots_used\",\n \"Temporal turn activity slots currently executing in this worker.\",\n value.usedSlots,\n );\n set(\n \"opengeni_turn_slots_available\",\n \"Memory-safe turn activity slots still available to reserve in this worker.\",\n value.availableSlots,\n );\n set(\n \"opengeni_turn_slot_saturation_ratio\",\n \"Fraction of this worker's memory-safe turn capacity currently executing.\",\n value.memoryBoundCapacity === 0 ? 1 : value.usedSlots / value.memoryBoundCapacity,\n );\n set(\n \"opengeni_turn_admission_memory_baseline_bytes\",\n \"Turn-worker cgroup memory after Temporal/native worker initialization.\",\n value.baselineBytes,\n );\n set(\n \"opengeni_turn_admission_memory_current_bytes\",\n \"Current cgroup memory used for turn admission decisions.\",\n value.currentBytes,\n );\n set(\n \"opengeni_turn_admission_memory_limit_bytes\",\n \"Finite cgroup memory limit used for turn admission, or zero when unlimited.\",\n value.limitBytes ?? 0,\n );\n set(\n \"opengeni_turn_admission_hard_bytes_per_turn\",\n \"Hard incremental memory budget reserved for each admitted turn.\",\n value.hardBytesPerTurn,\n );\n set(\n \"opengeni_turn_admission_native_headroom_bytes\",\n \"Cgroup memory retained for runtime, native, and GC headroom.\",\n value.nativeHeadroomBytes,\n );\n }\n}\n\nexport function createTurnWorkerTuner(options: TurnAdmissionOptions = {}): {\n tuner: WorkerTuner;\n admission: MemoryAwareTurnSlotSupplier;\n} {\n const admission = new MemoryAwareTurnSlotSupplier(options);\n return {\n admission,\n tuner: {\n workflowTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n activityTaskSlotSupplier: admission,\n localActivityTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n nexusTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n },\n };\n}\n\nexport function readCgroupMemorySnapshot(): CgroupMemorySnapshot {\n for (const [index, files] of CGROUP_MEMORY_FILES.entries()) {\n const limitRaw = readCgroupText(files.limit);\n const currentRaw = readCgroupText(files.current);\n if (limitRaw === null && currentRaw === null) continue;\n if (limitRaw === null || currentRaw === null) {\n throw new Error(\n `Incomplete cgroup memory controller: limit=${files.limit} current=${files.current}`,\n );\n }\n return {\n currentBytes: parseCgroupCurrentValue(currentRaw),\n limitBytes: parseCgroupLimitValue(limitRaw),\n source: index === 0 ? \"cgroup-v2\" : \"cgroup-v1\",\n };\n }\n return {\n currentBytes: process.memoryUsage().rss,\n limitBytes: null,\n source: \"process\",\n };\n}\n\nexport function parseCgroupLimitValue(value: string): number | null {\n if (value.trim() === \"max\") return null;\n const parsed = parseCgroupBigInt(value);\n if (parsed === null || parsed <= 0n) {\n throw new Error(`Malformed finite cgroup memory limit: ${JSON.stringify(value.trim())}`);\n }\n if (parsed >= UNLIMITED_CGROUP_BYTES) return null;\n const number = Number(parsed);\n if (!Number.isSafeInteger(number)) {\n throw new Error(\n `Finite cgroup memory limit exceeds JavaScript's safe integer range: ${parsed}`,\n );\n }\n return number;\n}\n\nexport function parseCgroupCurrentValue(value: string): number {\n const parsed = parseCgroupBigInt(value);\n if (parsed === null || parsed < 0n) {\n throw new Error(`Malformed cgroup current memory value: ${JSON.stringify(value.trim())}`);\n }\n const number = Number(parsed);\n if (!Number.isSafeInteger(number)) {\n throw new Error(`Cgroup current memory exceeds JavaScript's safe integer range: ${parsed}`);\n }\n return number;\n}\n\nfunction parseCgroupBigInt(value: string): bigint | null {\n try {\n return BigInt(value.trim());\n } catch {\n return null;\n }\n}\n\nfunction readCgroupText(path: string): string | null {\n try {\n return readFileSync(path, \"utf8\");\n } catch (error) {\n if ((error as NodeJS.ErrnoException).code === \"ENOENT\") return null;\n throw new Error(\n `Unable to read present cgroup memory controller file ${path}: ${\n error instanceof Error ? error.message : String(error)\n }`,\n { cause: error },\n );\n }\n}\n\nfunction positiveInteger(name: string, value: number): number {\n if (!Number.isSafeInteger(value) || value <= 0) {\n throw new Error(`${name} must be a positive integer`);\n }\n return value;\n}\n\nfunction nonnegativeInteger(name: string, value: number): number {\n if (!Number.isSafeInteger(value) || value < 0) {\n throw new Error(`${name} must be a non-negative integer`);\n }\n return value;\n}\n\nfunction abortError(): DOMException {\n return new DOMException(\"Turn slot reservation aborted\", \"AbortError\");\n}\n\nfunction abortableDelay(milliseconds: number, signal: AbortSignal): Promise<void> {\n return new Promise((resolve, reject) => {\n if (signal.aborted) {\n reject(abortError());\n return;\n }\n const timer = setTimeout(() => {\n signal.removeEventListener(\"abort\", onAbort);\n resolve();\n }, milliseconds);\n timer.unref?.();\n const onAbort = () => {\n clearTimeout(timer);\n reject(abortError());\n };\n signal.addEventListener(\"abort\", onAbort, { once: true });\n });\n}\n\n// Session workflows finish at idle boundaries and replay from durable history.\n// Bound sticky VM state explicitly: the SDK otherwise derives this from the\n// host heap limit (roughly 600 cached workflows/GiB), which is far above this\n// process's actual concurrent workflow-task capacity and permanently retains\n// idle workflow state in every control replica.\nexport const CONTROL_WORKER_MAX_CACHED_WORKFLOWS = 64;\n","export const SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS = 5 * 60_000;\nexport const SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS = 5 * 60_000;\nexport const SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS = 65 * 60_000;\nexport const SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS = 60_000;\nexport const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 20_000;\nexport const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS = 5_000;\n// Match the DB inventory's fair 500-row bound so one sweep never discovers a\n// due box and then needlessly defers it for another schedule period. Five\n// hundred small StartChild commands remain far below Temporal history limits;\n// actual provider I/O stays bounded by worker activity concurrency.\nexport const SANDBOX_REAPER_CHILD_DISPATCH_LIMIT = 500;\nexport const SANDBOX_REAPER_V2_WORKFLOW_ID = \"opengeni-sandbox-lease-reaper-v2\";\nexport const SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID = \"opengeni-sandbox-lease-maintenance-v1\";\n\n// A protocol-specific queue is the rolling-deploy compatibility boundary for\n// the split reaper activities. An old control worker polls only the base queue,\n// so it can replay already-started legacy workflows but can never receive a new\n// activity type that its binary does not register.\nexport const SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX = \"-sandbox-lifecycle-v1\";\n\nexport function sandboxLifecycleTaskQueue(baseTaskQueue: string): string {\n return baseTaskQueue.endsWith(SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX)\n ? baseTaskQueue\n : `${baseTaskQueue}${SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX}`;\n}\n\nexport type SandboxDrainTimeoutClass = \"fast\" | \"extended\";\n\nexport type SandboxDrainTarget = {\n workspaceId: string;\n sandboxGroupId: string;\n instanceId: string | null;\n leaseEpoch: number;\n};\n\nexport type SandboxDrainActivityInput = {\n target: SandboxDrainTarget;\n timeoutClass: SandboxDrainTimeoutClass;\n /** Provider operation budget frozen when the durable child is created. A\n * rolling config change cannot make an existing fast child retry forever. */\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n /** Stable logical teardown operation; every Temporal activity retry keeps it\n * while receiving a distinct exact capture-attempt id. */\n operationId: string;\n};\n\nexport type SandboxDrainWorkflowInput = Omit<SandboxDrainActivityInput, \"operationId\">;\n\nexport type SandboxLeaseSweepMaintenanceInput = {\n examined: number;\n started: number;\n alreadyRunning: number;\n startFailed: number;\n rotationsRequested: number;\n};\n\nexport function sandboxDrainTimeoutClass(captureTimeoutMs: number): SandboxDrainTimeoutClass {\n return captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS <\n SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS\n ? \"fast\"\n : \"extended\";\n}\n\nexport function sandboxDrainActivityTimeoutMs(timeoutClass: SandboxDrainTimeoutClass): number {\n return timeoutClass === \"fast\"\n ? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS\n : SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS;\n}\n\nexport function sandboxDrainWorkflowId(target: SandboxDrainTarget): string {\n return `sandbox-drain:${target.workspaceId}:${target.sandboxGroupId}:${target.leaseEpoch}`;\n}\n","import {\n SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,\n SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,\n sandboxArchiveCaptureTimeoutMs,\n type Settings,\n} from \"@opengeni/config\";\nimport {\n SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS,\n sandboxDrainActivityTimeoutMs,\n sandboxDrainTimeoutClass,\n type SandboxDrainTimeoutClass,\n} from \"./sandbox-reaper-contract\";\n\nconst STRICT_TIMEOUT_FENCE_MS = 1;\n\nexport function sandboxDrainTiming(settings: Pick<Settings, \"sandboxSnapshotTimeoutMs\">): {\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n timeoutClass: SandboxDrainTimeoutClass;\n activityTimeoutMs: number;\n} {\n const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);\n const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);\n return {\n snapshotTimeoutMs: settings.sandboxSnapshotTimeoutMs,\n captureTimeoutMs,\n timeoutClass,\n activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),\n };\n}\n\nexport function assertSandboxReaperActivityTimeout(\n settings: Pick<Settings, \"sandboxSnapshotTimeoutMs\">,\n): void {\n const timing = sandboxDrainTiming(settings);\n if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {\n throw new Error(\n `Sandbox provider snapshot timeout (${timing.snapshotTimeoutMs}) must be strictly less than ` +\n `its durable capture claim (${timing.captureTimeoutMs}).`,\n );\n }\n if (\n timing.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS + STRICT_TIMEOUT_FENCE_MS >=\n timing.activityTimeoutMs\n ) {\n throw new Error(\n `Sandbox ${timing.timeoutClass} drain activity timeout (${timing.activityTimeoutMs}) must ` +\n `strictly exceed one durable archive capture and settlement margin ` +\n `(${timing.captureTimeoutMs} + ${SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS} milliseconds).`,\n );\n }\n}\n\nexport function assertSandboxDrainInputTiming(input: {\n timeoutClass: SandboxDrainTimeoutClass;\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n}): void {\n if (\n !Number.isSafeInteger(input.snapshotTimeoutMs) ||\n input.snapshotTimeoutMs <= 0 ||\n input.snapshotTimeoutMs > SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS ||\n !Number.isSafeInteger(input.captureTimeoutMs) ||\n input.captureTimeoutMs <= 0 ||\n input.captureTimeoutMs > SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS\n ) {\n throw new Error(\"Sandbox drain timeout values are invalid\");\n }\n const inputActivityTimeoutMs = sandboxDrainActivityTimeoutMs(input.timeoutClass);\n if (\n sandboxDrainTimeoutClass(input.captureTimeoutMs) !== input.timeoutClass ||\n input.snapshotTimeoutMs >= input.captureTimeoutMs ||\n input.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS >= inputActivityTimeoutMs\n ) {\n throw new Error(\n `Sandbox drain timing is inconsistent with the ${input.timeoutClass} activity budget`,\n );\n }\n}\n"],"mappings":";AAEA,SAAS,oBAAoB;AAY7B,IAAM,MAAM,OAAO;AAOZ,IAAM,mCAAmC;AACzC,IAAM,yCAAyC,MAAM;AACrD,IAAM,oCAAoC,MAAM;AAqBhD,SAAS,6BACd,UAC8B;AAC9B,MAAI,SAAS,8BAA8B,SAAS;AAClD,WAAO;AAAA,MACL,qCAAqC,SAAS;AAAA,IAChD;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,MACL,cAAc;AAAA,QACZ,gBAAgB,SAAS;AAAA,QACzB,mBAAmB,SAAS;AAAA,MAC9B;AAAA,MACA,yBAAyB;AAAA,QACvB,cAAc;AAAA,QACd,cAAc,SAAS;AAAA,QACvB,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAeO,SAAS,gCACd,UACA,UAAsD,CAAC,GAC5B;AAC3B,MAAI,SAAS,8BAA8B,kBAAkB;AAC3D,WAAO;AAAA,MACL,SAAS,6BAA6B,QAAQ;AAAA,MAC9C,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,QAAQ,sBAAsB;AAAA,IAClC,GAAG;AAAA,IACH,cAAc,SAAS;AAAA,EACzB,CAAC;AACD,SAAO;AAAA,IACL,SAAS,EAAE,OAAO,MAAM,MAAM;AAAA,IAC9B,WAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,+BAA+B,UAK7C;AACA,QAAM,gBAAgB,SAAS,8BAA8B;AAC7D,SAAO;AAAA,IACL,iBAAiB,SAAS;AAAA,IAC1B,oBAAoB,SAAS;AAAA,IAC7B,gBAAgB,gBAAgB,SAAS,2BAA2B;AAAA,IACpE,mBAAmB,gBAAgB,SAAS,8BAA8B;AAAA,EAC5E;AACF;AAEO,IAAM,2CAA2C;AACjD,IAAM,+CAA+C;AAE5D,IAAM,sBAAsB;AAAA,EAC1B;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AAAA,EACA;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AACF;AAIA,IAAM,yBAAyB,MAAM;AAmD9B,IAAM,8BAAN,MAAkF;AAAA,EAC9E,OAAO;AAAA,EAEC,QAAQ,OAAO,yBAAyB;AAAA,EACxC,UAAU,oBAAI,IAAoB;AAAA,EAC3C;AAAA,EACS;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAAgC,CAAC,GAAG;AAC9C,SAAK,eAAe;AAAA,MAClB;AAAA,MACA,QAAQ,gBAAgB;AAAA,IAC1B;AACA,SAAK,mBAAmB;AAAA,MACtB;AAAA,MACA,QAAQ,oBAAoB;AAAA,IAC9B;AACA,SAAK,sBAAsB;AAAA,MACzB;AAAA,MACA,QAAQ,uBAAuB;AAAA,IACjC;AACA,SAAK,eAAe,gBAAgB,gBAAgB,QAAQ,gBAAgB,GAAG;AAC/E,SAAK,iBAAiB,QAAQ,kBAAkB;AAChD,SAAK,gBAAgB,QAAQ;AAC7B,UAAM,UAAU,KAAK,eAAe;AACpC,SAAK,gBAAgB,QAAQ;AAC7B,QAAI,QAAQ,eAAe,QAAQ,KAAK,oBAAoB,QAAQ,UAAU,IAAI,GAAG;AACnF,YAAM,IAAI;AAAA,QACR,mEACc,KAAK,aAAa,aAAa,KAAK,gBAAgB,mBAC9C,KAAK,mBAAmB,UAAU,QAAQ,UAAU;AAAA,MAC1E;AAAA,IACF;AACA,SAAK,eAAe,OAAO;AAAA,EAC7B;AAAA,EAEA,MAAM,YAAY,MAA0B,aAA+C;AACzF,eAAS;AACP,UAAI,YAAY,QAAS,OAAM,WAAW;AAC1C,YAAM,SAAS,KAAK,cAAc;AAClC,UAAI,OAAQ,QAAO;AACnB,YAAM,eAAe,KAAK,cAAc,WAAW;AAAA,IACrD;AAAA,EACF;AAAA,EAEA,eAAe,MAA6C;AAC1D,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,aAAa,KAAkD;AAC7D,UAAM,SAAS,KAAK,YAAY,IAAI,MAAM;AAC1C,WAAO,OAAO;AACd,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,YAAY,KAAiD;AAC3D,UAAM,SAAS,KAAK,YAAY,IAAI,MAAM;AAC1C,SAAK,QAAQ,OAAO,MAAM;AAC1B,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,SAAS,SAAS,KAAK,eAAe,GAA0B;AAC9D,UAAM,YAAY,KAAK,UAAU;AACjC,UAAM,gBAAgB,KAAK,QAAQ;AACnC,UAAM,iBAAiB,KAAK,eAAe,MAAM;AAGjD,UAAM,sBAAsB,gBAAgB;AAC5C,WAAO;AAAA,MACL,eAAe,KAAK;AAAA,MACpB,cAAc,OAAO;AAAA,MACrB,YAAY,OAAO;AAAA,MACnB,kBAAkB,KAAK;AAAA,MACvB,qBAAqB,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,0BAAgC;AAC9B,UAAM,SAAS,KAAK,eAAe;AACnC,QAAI,KAAK,QAAQ,SAAS,GAAG;AAC3B,YAAM,IAAI,MAAM,gEAAgE;AAAA,IAClF;AACA,UAAM,yBAAyB,OAAO;AACtC,UAAM,oBACJ,OAAO,eAAe,OAClB,KAAK,eACL,KAAK,oBAAoB,OAAO,YAAY,sBAAsB;AACxE,QAAI,OAAO,eAAe,QAAQ,oBAAoB,GAAG;AACvD,YAAM,IAAI;AAAA,QACR,0GACyB,KAAK,aAAa,sBAAsB,sBAAsB,aACzE,KAAK,gBAAgB,mBAAmB,KAAK,mBAAmB,UACnE,OAAO,OAAO,UAAU,CAAC;AAAA,MACtC;AAAA,IACF;AACA,SAAK,gBAAgB;AACrB,SAAK,eAAe,MAAM;AAAA,EAC5B;AAAA,EAEQ,gBAAuC;AAC7C,UAAM,SAAS,KAAK,eAAe;AACnC,QAAI,KAAK,QAAQ,QAAQ,KAAK,gBAAgB,CAAC,KAAK,oBAAoB,MAAM,GAAG;AAC/E,WAAK,eAAe,MAAM;AAC1B,aAAO;AAAA,IACT;AACA,UAAM,SAAyB,EAAE,OAAO,KAAK,OAAO,MAAM,MAAM;AAChE,SAAK,QAAQ,IAAI,MAAM;AACvB,SAAK,eAAe,MAAM;AAC1B,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,QAAuC;AACjE,WAAO,KAAK,eAAe,MAAM,IAAI;AAAA,EACvC;AAAA,EAEQ,eAAe,QAAsC;AAC3D,UAAM,mBAAmB,KAAK,eAAe,KAAK,QAAQ;AAC1D,QAAI,OAAO,eAAe,KAAM,QAAO,KAAK,IAAI,GAAG,gBAAgB;AAEnE,UAAM,oBAAoB,KAAK,oBAAoB,OAAO,UAAU,IAAI,KAAK,QAAQ;AACrF,UAAM,oBACJ,KAAK;AAAA,OACF,OAAO,aAAa,OAAO,eAAe,KAAK,uBAC9C,KAAK;AAAA,IACT,IAAI,KAAK,QAAQ;AACnB,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,kBAAkB,mBAAmB,iBAAiB,CAAC;AAAA,EACrF;AAAA,EAEQ,oBAAoB,YAAoB,gBAAgB,KAAK,eAAuB;AAC1F,WAAO,KAAK;AAAA,MACV;AAAA,MACA,KAAK,OAAO,aAAa,gBAAgB,KAAK,uBAAuB,KAAK,gBAAgB;AAAA,IAC5F;AAAA,EACF;AAAA,EAEQ,YAAoB;AAC1B,QAAI,OAAO;AACX,eAAW,UAAU,KAAK,SAAS;AACjC,UAAI,OAAO,KAAM,SAAQ;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,YAAY,QAAoC;AACtD,UAAM,YAAY;AAClB,QAAI,UAAU,UAAU,KAAK,SAAS,CAAC,KAAK,QAAQ,IAAI,MAAwB,GAAG;AACjF,YAAM,IAAI,MAAM,gEAAgE;AAAA,IAClF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,eAAe,SAAS,KAAK,eAAe,GAAS;AAC3D,QAAI,CAAC,KAAK,cAAe;AACzB,UAAM,QAAQ,KAAK,SAAS,MAAM;AAClC,UAAM,MAAM,CAAC,MAAc,MAAc,gBACvC,KAAK,eAAe,SAAS,EAAE,MAAM,MAAM,OAAO,YAAY,CAAC;AACjE;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM,wBAAwB,IAAI,IAAI,MAAM,YAAY,MAAM;AAAA,IAChE;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM,cAAc;AAAA,IACtB;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB,UAAgC,CAAC,GAGrE;AACA,QAAM,YAAY,IAAI,4BAA4B,OAAO;AACzD,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,0BAA0B,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,MAC5D,0BAA0B;AAAA,MAC1B,+BAA+B,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,MACjE,uBAAuB,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,IAC3D;AAAA,EACF;AACF;AAEO,SAAS,2BAAiD;AAC/D,aAAW,CAAC,OAAO,KAAK,KAAK,oBAAoB,QAAQ,GAAG;AAC1D,UAAM,WAAW,eAAe,MAAM,KAAK;AAC3C,UAAM,aAAa,eAAe,MAAM,OAAO;AAC/C,QAAI,aAAa,QAAQ,eAAe,KAAM;AAC9C,QAAI,aAAa,QAAQ,eAAe,MAAM;AAC5C,YAAM,IAAI;AAAA,QACR,8CAA8C,MAAM,KAAK,YAAY,MAAM,OAAO;AAAA,MACpF;AAAA,IACF;AACA,WAAO;AAAA,MACL,cAAc,wBAAwB,UAAU;AAAA,MAChD,YAAY,sBAAsB,QAAQ;AAAA,MAC1C,QAAQ,UAAU,IAAI,cAAc;AAAA,IACtC;AAAA,EACF;AACA,SAAO;AAAA,IACL,cAAc,QAAQ,YAAY,EAAE;AAAA,IACpC,YAAY;AAAA,IACZ,QAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBAAsB,OAA8B;AAClE,MAAI,MAAM,KAAK,MAAM,MAAO,QAAO;AACnC,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,WAAW,QAAQ,UAAU,IAAI;AACnC,UAAM,IAAI,MAAM,yCAAyC,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC,EAAE;AAAA,EACzF;AACA,MAAI,UAAU,uBAAwB,QAAO;AAC7C,QAAM,SAAS,OAAO,MAAM;AAC5B,MAAI,CAAC,OAAO,cAAc,MAAM,GAAG;AACjC,UAAM,IAAI;AAAA,MACR,uEAAuE,MAAM;AAAA,IAC/E;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,wBAAwB,OAAuB;AAC7D,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,WAAW,QAAQ,SAAS,IAAI;AAClC,UAAM,IAAI,MAAM,0CAA0C,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC,EAAE;AAAA,EAC1F;AACA,QAAM,SAAS,OAAO,MAAM;AAC5B,MAAI,CAAC,OAAO,cAAc,MAAM,GAAG;AACjC,UAAM,IAAI,MAAM,kEAAkE,MAAM,EAAE;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,OAA8B;AACvD,MAAI;AACF,WAAO,OAAO,MAAM,KAAK,CAAC;AAAA,EAC5B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,eAAe,MAA6B;AACnD,MAAI;AACF,WAAO,aAAa,MAAM,MAAM;AAAA,EAClC,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU,QAAO;AAC/D,UAAM,IAAI;AAAA,MACR,wDAAwD,IAAI,KAC1D,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CACvD;AAAA,MACA,EAAE,OAAO,MAAM;AAAA,IACjB;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,MAAc,OAAuB;AAC5D,MAAI,CAAC,OAAO,cAAc,KAAK,KAAK,SAAS,GAAG;AAC9C,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACtD;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,MAAc,OAAuB;AAC/D,MAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG;AAC7C,UAAM,IAAI,MAAM,GAAG,IAAI,iCAAiC;AAAA,EAC1D;AACA,SAAO;AACT;AAEA,SAAS,aAA2B;AAClC,SAAO,IAAI,aAAa,iCAAiC,YAAY;AACvE;AAEA,SAAS,eAAe,cAAsB,QAAoC;AAChF,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,QAAI,OAAO,SAAS;AAClB,aAAO,WAAW,CAAC;AACnB;AAAA,IACF;AACA,UAAM,QAAQ,WAAW,MAAM;AAC7B,aAAO,oBAAoB,SAAS,OAAO;AAC3C,cAAQ;AAAA,IACV,GAAG,YAAY;AACf,UAAM,QAAQ;AACd,UAAM,UAAU,MAAM;AACpB,mBAAa,KAAK;AAClB,aAAO,WAAW,CAAC;AAAA,IACrB;AACA,WAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,EAC1D,CAAC;AACH;AAOO,IAAM,sCAAsC;;;AC5hB5C,IAAM,0CAA0C,IAAI;AACpD,IAAM,yCAAyC,IAAI;AACnD,IAAM,6CAA6C,KAAK;AACxD,IAAM,qCAAqC;AAE3C,IAAM,gDAAgD;AAKtD,IAAM,sCAAsC;AAC5C,IAAM,gCAAgC;AAOtC,IAAM,sCAAsC;AAE5C,SAAS,0BAA0B,eAA+B;AACvE,SAAO,cAAc,SAAS,mCAAmC,IAC7D,gBACA,GAAG,aAAa,GAAG,mCAAmC;AAC5D;AAiCO,SAAS,yBAAyB,kBAAoD;AAC3F,SAAO,mBAAmB,qCACxB,yCACE,SACA;AACN;AAEO,SAAS,8BAA8B,cAAgD;AAC5F,SAAO,iBAAiB,SACpB,yCACA;AACN;;;ACpEA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAQP,IAAM,0BAA0B;AAEzB,SAAS,mBAAmB,UAKjC;AACA,QAAM,mBAAmB,+BAA+B,QAAQ;AAChE,QAAM,eAAe,yBAAyB,gBAAgB;AAC9D,SAAO;AAAA,IACL,mBAAmB,SAAS;AAAA,IAC5B;AAAA,IACA;AAAA,IACA,mBAAmB,8BAA8B,YAAY;AAAA,EAC/D;AACF;AAEO,SAAS,mCACd,UACM;AACN,QAAM,SAAS,mBAAmB,QAAQ;AAC1C,MAAI,OAAO,qBAAqB,OAAO,kBAAkB;AACvD,UAAM,IAAI;AAAA,MACR,sCAAsC,OAAO,iBAAiB,2DAC9B,OAAO,gBAAgB;AAAA,IACzD;AAAA,EACF;AACA,MACE,OAAO,mBAAmB,qCAAqC,2BAC/D,OAAO,mBACP;AACA,UAAM,IAAI;AAAA,MACR,WAAW,OAAO,YAAY,4BAA4B,OAAO,iBAAiB,6EAE5E,OAAO,gBAAgB,MAAM,kCAAkC;AAAA,IACvE;AAAA,EACF;AACF;AAEO,SAAS,8BAA8B,OAIrC;AACP,MACE,CAAC,OAAO,cAAc,MAAM,iBAAiB,KAC7C,MAAM,qBAAqB,KAC3B,MAAM,oBAAoB,mCAC1B,CAAC,OAAO,cAAc,MAAM,gBAAgB,KAC5C,MAAM,oBAAoB,KAC1B,MAAM,mBAAmB,wCACzB;AACA,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AACA,QAAM,yBAAyB,8BAA8B,MAAM,YAAY;AAC/E,MACE,yBAAyB,MAAM,gBAAgB,MAAM,MAAM,gBAC3D,MAAM,qBAAqB,MAAM,oBACjC,MAAM,mBAAmB,sCAAsC,wBAC/D;AACA,UAAM,IAAI;AAAA,MACR,iDAAiD,MAAM,YAAY;AAAA,IACrE;AAAA,EACF;AACF;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../src/concurrency.ts","../src/sandbox-reaper-contract.ts","../src/sandbox-reaper-timeout.ts"],"sourcesContent":["import type { Settings } from \"@opengeni/config\";\nimport type { Observability } from \"@opengeni/observability\";\nimport { readFileSync } from \"node:fs\";\nimport type {\n ActivitySlotInfo,\n CustomSlotSupplier,\n SlotMarkUsedContext,\n SlotPermit,\n SlotReleaseContext,\n SlotReserveContext,\n WorkerOptions,\n WorkerTuner,\n} from \"@temporalio/worker\";\n\nconst MIB = 1024 * 1024;\n\n/**\n * Default fixed/HPA turn-worker admission contract. The configured fixed-mode\n * ceiling may be lower, but every live Temporal permit reserves the complete\n * hard budget until the runAgentTurn activity promise physically completes.\n */\nexport const TURN_WORKER_MAX_CONCURRENT_TURNS = 16;\nexport const TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN = 100 * MIB;\nexport const TURN_WORKER_NATIVE_HEADROOM_BYTES = 512 * MIB;\n\nexport type TurnWorkerConcurrencySettings = Pick<\n Settings,\n | \"turnWorkerConcurrencyMode\"\n | \"turnWorkerMaxConcurrentTurns\"\n | \"turnWorkerTargetCpuUsage\"\n | \"turnWorkerTargetMemoryUsage\"\n>;\n\nexport type TurnWorkerConcurrencyOptions = Pick<\n WorkerOptions,\n \"maxConcurrentActivityTaskExecutions\" | \"tuner\"\n>;\n\n/**\n * Fixed profiles keep a per-process ceiling. Resource-aware profiles retain a\n * hard per-process maximum while Temporal meters new activity admission against\n * cgroup/host CPU and memory. The ramp lets usage become visible before a burst\n * can reserve the entire ceiling.\n */\nexport function turnWorkerConcurrencyOptions(\n settings: TurnWorkerConcurrencySettings,\n): TurnWorkerConcurrencyOptions {\n if (settings.turnWorkerConcurrencyMode === \"fixed\") {\n return {\n maxConcurrentActivityTaskExecutions: settings.turnWorkerMaxConcurrentTurns,\n };\n }\n\n return {\n tuner: {\n tunerOptions: {\n targetCpuUsage: settings.turnWorkerTargetCpuUsage,\n targetMemoryUsage: settings.turnWorkerTargetMemoryUsage,\n },\n activityTaskSlotOptions: {\n minimumSlots: 1,\n maximumSlots: settings.turnWorkerMaxConcurrentTurns,\n rampThrottle: \"250ms\",\n },\n },\n };\n}\n\nexport type TurnWorkerConcurrencyPlan = {\n options: TurnWorkerConcurrencyOptions;\n admission: MemoryAwareTurnSlotSupplier | null;\n};\n\n/**\n * Build the runtime concurrency policy for a turn worker.\n *\n * Fixed/HPA deployments use the hard-reservation supplier below. The explicit\n * non-HA single-machine profile keeps Temporal's native CPU/memory resource\n * tuner because it controls whole-machine load rather than a pod cgroup; that\n * profile documents that its configured maximum is not a per-turn reservation.\n */\nexport function createTurnWorkerConcurrencyPlan(\n settings: TurnWorkerConcurrencySettings,\n options: Omit<TurnAdmissionOptions, \"maximumTurns\"> = {},\n): TurnWorkerConcurrencyPlan {\n if (settings.turnWorkerConcurrencyMode === \"resource-based\") {\n return {\n options: turnWorkerConcurrencyOptions(settings),\n admission: null,\n };\n }\n\n const fixed = createTurnWorkerTuner({\n ...options,\n maximumTurns: settings.turnWorkerMaxConcurrentTurns,\n });\n return {\n options: { tuner: fixed.tuner },\n admission: fixed.admission,\n };\n}\n\nexport function turnWorkerConcurrencyLogFields(settings: TurnWorkerConcurrencySettings): {\n concurrencyMode: \"fixed\" | \"resource-based\";\n maxConcurrentTurns: number;\n targetCpuUsage: number | null;\n targetMemoryUsage: number | null;\n} {\n const resourceBased = settings.turnWorkerConcurrencyMode === \"resource-based\";\n return {\n concurrencyMode: settings.turnWorkerConcurrencyMode,\n maxConcurrentTurns: settings.turnWorkerMaxConcurrentTurns,\n targetCpuUsage: resourceBased ? settings.turnWorkerTargetCpuUsage : null,\n targetMemoryUsage: resourceBased ? settings.turnWorkerTargetMemoryUsage : null,\n };\n}\n\nexport const CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES = 32;\nexport const CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS = 40;\n\nconst CGROUP_MEMORY_FILES = [\n {\n limit: \"/sys/fs/cgroup/memory.max\",\n current: \"/sys/fs/cgroup/memory.current\",\n },\n {\n limit: \"/sys/fs/cgroup/memory/memory.limit_in_bytes\",\n current: \"/sys/fs/cgroup/memory/memory.usage_in_bytes\",\n },\n] as const;\n\n// cgroup v1 uses enormous sentinels for \"unlimited\". Values this large are\n// host capacity rather than a meaningful container admission boundary.\nconst UNLIMITED_CGROUP_BYTES = 1n << 60n;\n\nexport type CgroupMemorySnapshot = {\n currentBytes: number;\n limitBytes: number | null;\n source: \"cgroup-v1\" | \"cgroup-v2\" | \"process\";\n};\n\nexport type TurnAdmissionSnapshot = {\n baselineBytes: number;\n currentBytes: number;\n limitBytes: number | null;\n hardBytesPerTurn: number;\n nativeHeadroomBytes: number;\n maximumTurns: number;\n memoryBoundCapacity: number;\n reservedSlots: number;\n usedSlots: number;\n availableSlots: number;\n};\n\nexport type TurnAdmissionOptions = {\n maximumTurns?: number;\n hardBytesPerTurn?: number;\n nativeHeadroomBytes?: number;\n memorySnapshot?: () => CgroupMemorySnapshot;\n retryDelayMs?: number;\n observability?: Observability;\n};\n\ntype TurnSlotPermit = SlotPermit & {\n readonly owner: symbol;\n used: boolean;\n};\n\n/**\n * Temporal custom activity-slot supplier for turn workers.\n *\n * A permit is required before Temporal polls a task. Reservation is allowed\n * only when both independent safety checks pass:\n *\n * baseline + reserved-after × hard-turn-budget + native headroom <= limit\n * current + reserved-after × hard-turn-budget + native headroom <= limit\n *\n * The first check enforces the release contract even when current RSS happens\n * to be low. The second accounts for native/retained memory observed after\n * startup and keeps the complete hard budget reserved for every live permit.\n * A used permit is not evidence that its allocation is already visible in the\n * cgroup, and unrelated native growth cannot safely be attributed to a turn.\n * Kubernetes OOM is never an admission mechanism.\n */\nexport class MemoryAwareTurnSlotSupplier implements CustomSlotSupplier<ActivitySlotInfo> {\n readonly type = \"custom\" as const;\n\n private readonly owner = Symbol(\"opengeni-turn-admission\");\n private readonly permits = new Set<TurnSlotPermit>();\n private baselineBytes: number;\n private readonly maximumTurns: number;\n private readonly hardBytesPerTurn: number;\n private readonly nativeHeadroomBytes: number;\n private readonly memorySnapshot: () => CgroupMemorySnapshot;\n private readonly retryDelayMs: number;\n private readonly observability: Observability | undefined;\n\n constructor(options: TurnAdmissionOptions = {}) {\n this.maximumTurns = positiveInteger(\n \"maximumTurns\",\n options.maximumTurns ?? TURN_WORKER_MAX_CONCURRENT_TURNS,\n );\n this.hardBytesPerTurn = positiveInteger(\n \"hardBytesPerTurn\",\n options.hardBytesPerTurn ?? TURN_WORKER_HARD_MEMORY_BYTES_PER_TURN,\n );\n this.nativeHeadroomBytes = nonnegativeInteger(\n \"nativeHeadroomBytes\",\n options.nativeHeadroomBytes ?? TURN_WORKER_NATIVE_HEADROOM_BYTES,\n );\n this.retryDelayMs = positiveInteger(\"retryDelayMs\", options.retryDelayMs ?? 100);\n this.memorySnapshot = options.memorySnapshot ?? readCgroupMemorySnapshot;\n this.observability = options.observability;\n const initial = this.memorySnapshot();\n this.baselineBytes = initial.currentBytes;\n if (initial.limitBytes !== null && this.memoryBoundCapacity(initial.limitBytes) < 1) {\n throw new Error(\n \"Turn worker memory limit cannot safely admit one turn: \" +\n `baseline=${this.baselineBytes} hardTurn=${this.hardBytesPerTurn} ` +\n `nativeHeadroom=${this.nativeHeadroomBytes} limit=${initial.limitBytes}`,\n );\n }\n this.refreshMetrics(initial);\n }\n\n async reserveSlot(_ctx: SlotReserveContext, abortSignal: AbortSignal): Promise<SlotPermit> {\n for (;;) {\n if (abortSignal.aborted) throw abortError();\n const permit = this.reserveIfSafe();\n if (permit) return permit;\n await abortableDelay(this.retryDelayMs, abortSignal);\n }\n }\n\n tryReserveSlot(_ctx: SlotReserveContext): SlotPermit | null {\n return this.reserveIfSafe();\n }\n\n markSlotUsed(ctx: SlotMarkUsedContext<ActivitySlotInfo>): void {\n const permit = this.ownedPermit(ctx.permit);\n permit.used = true;\n this.refreshMetrics();\n }\n\n releaseSlot(ctx: SlotReleaseContext<ActivitySlotInfo>): void {\n const permit = this.ownedPermit(ctx.permit);\n this.permits.delete(permit);\n this.refreshMetrics();\n }\n\n snapshot(memory = this.memorySnapshot()): TurnAdmissionSnapshot {\n const usedSlots = this.usedSlots();\n const reservedSlots = this.permits.size;\n const availableSlots = this.availableSlots(memory);\n // Never report less capacity than work already admitted. The dynamic\n // denominator is exactly reserved + what a new poll could reserve now.\n const memoryBoundCapacity = reservedSlots + availableSlots;\n return {\n baselineBytes: this.baselineBytes,\n currentBytes: memory.currentBytes,\n limitBytes: memory.limitBytes,\n hardBytesPerTurn: this.hardBytesPerTurn,\n nativeHeadroomBytes: this.nativeHeadroomBytes,\n maximumTurns: this.maximumTurns,\n memoryBoundCapacity,\n reservedSlots,\n usedSlots,\n availableSlots,\n };\n }\n\n /**\n * Finalize the admission baseline after Temporal/native construction.\n * Worker.create can retain memory after the constructor's provisional\n * sample. That initialization delta must remain in the baseline for every\n * later projection rather than competing with turn reservations.\n */\n finalizeStartupBaseline(): void {\n const memory = this.memorySnapshot();\n if (this.permits.size !== 0) {\n throw new Error(\"Turn worker startup admission check ran after slot reservation\");\n }\n const finalizedBaselineBytes = memory.currentBytes;\n const finalizedCapacity =\n memory.limitBytes === null\n ? this.maximumTurns\n : this.memoryBoundCapacity(memory.limitBytes, finalizedBaselineBytes);\n if (memory.limitBytes !== null && finalizedCapacity < 1) {\n throw new Error(\n \"Turn worker memory limit cannot safely admit one turn after worker initialization: \" +\n `provisionalBaseline=${this.baselineBytes} finalizedBaseline=${finalizedBaselineBytes} ` +\n `hardTurn=${this.hardBytesPerTurn} nativeHeadroom=${this.nativeHeadroomBytes} ` +\n `limit=${String(memory.limitBytes)}`,\n );\n }\n this.baselineBytes = finalizedBaselineBytes;\n this.refreshMetrics(memory);\n }\n\n private reserveIfSafe(): TurnSlotPermit | null {\n const memory = this.memorySnapshot();\n if (this.permits.size >= this.maximumTurns || !this.memoryAllowsAnother(memory)) {\n this.refreshMetrics(memory);\n return null;\n }\n const permit: TurnSlotPermit = { owner: this.owner, used: false };\n this.permits.add(permit);\n this.refreshMetrics(memory);\n return permit;\n }\n\n private memoryAllowsAnother(memory: CgroupMemorySnapshot): boolean {\n return this.availableSlots(memory) > 0;\n }\n\n private availableSlots(memory: CgroupMemorySnapshot): number {\n const densityRemaining = this.maximumTurns - this.permits.size;\n if (memory.limitBytes === null) return Math.max(0, densityRemaining);\n\n const baselineRemaining = this.memoryBoundCapacity(memory.limitBytes) - this.permits.size;\n const observedRemaining =\n Math.floor(\n (memory.limitBytes - memory.currentBytes - this.nativeHeadroomBytes) /\n this.hardBytesPerTurn,\n ) - this.permits.size;\n return Math.max(0, Math.min(densityRemaining, baselineRemaining, observedRemaining));\n }\n\n private memoryBoundCapacity(limitBytes: number, baselineBytes = this.baselineBytes): number {\n return Math.max(\n 0,\n Math.floor((limitBytes - baselineBytes - this.nativeHeadroomBytes) / this.hardBytesPerTurn),\n );\n }\n\n private usedSlots(): number {\n let used = 0;\n for (const permit of this.permits) {\n if (permit.used) used += 1;\n }\n return used;\n }\n\n private ownedPermit(permit: SlotPermit): TurnSlotPermit {\n const candidate = permit as Partial<TurnSlotPermit>;\n if (candidate.owner !== this.owner || !this.permits.has(permit as TurnSlotPermit)) {\n throw new Error(\"Temporal returned a turn slot permit owned by another supplier\");\n }\n return permit as TurnSlotPermit;\n }\n\n private refreshMetrics(memory = this.memorySnapshot()): void {\n if (!this.observability) return;\n const value = this.snapshot(memory);\n const set = (name: string, help: string, metricValue: number) =>\n this.observability?.setGauge({ name, help, value: metricValue });\n set(\n \"opengeni_turn_slots_capacity\",\n \"Turn slots this worker can safely support after the hard memory contract.\",\n value.memoryBoundCapacity,\n );\n set(\n \"opengeni_turn_slots_reserved\",\n \"Temporal turn activity slots reserved for polling or execution in this worker.\",\n value.reservedSlots,\n );\n set(\n \"opengeni_turn_slots_used\",\n \"Temporal turn activity slots currently executing in this worker.\",\n value.usedSlots,\n );\n set(\n \"opengeni_turn_slots_available\",\n \"Memory-safe turn activity slots still available to reserve in this worker.\",\n value.availableSlots,\n );\n set(\n \"opengeni_turn_slot_saturation_ratio\",\n \"Fraction of this worker's memory-safe turn capacity currently executing.\",\n value.memoryBoundCapacity === 0 ? 1 : value.usedSlots / value.memoryBoundCapacity,\n );\n set(\n \"opengeni_turn_admission_memory_baseline_bytes\",\n \"Turn-worker cgroup memory after Temporal/native worker initialization.\",\n value.baselineBytes,\n );\n set(\n \"opengeni_turn_admission_memory_current_bytes\",\n \"Current cgroup memory used for turn admission decisions.\",\n value.currentBytes,\n );\n set(\n \"opengeni_turn_admission_memory_limit_bytes\",\n \"Finite cgroup memory limit used for turn admission, or zero when unlimited.\",\n value.limitBytes ?? 0,\n );\n set(\n \"opengeni_turn_admission_hard_bytes_per_turn\",\n \"Hard incremental memory budget reserved for each admitted turn.\",\n value.hardBytesPerTurn,\n );\n set(\n \"opengeni_turn_admission_native_headroom_bytes\",\n \"Cgroup memory retained for runtime, native, and GC headroom.\",\n value.nativeHeadroomBytes,\n );\n }\n}\n\nexport function createTurnWorkerTuner(options: TurnAdmissionOptions = {}): {\n tuner: WorkerTuner;\n admission: MemoryAwareTurnSlotSupplier;\n} {\n const admission = new MemoryAwareTurnSlotSupplier(options);\n return {\n admission,\n tuner: {\n workflowTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n activityTaskSlotSupplier: admission,\n localActivityTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n nexusTaskSlotSupplier: { type: \"fixed-size\", numSlots: 1 },\n },\n };\n}\n\nexport function readCgroupMemorySnapshot(): CgroupMemorySnapshot {\n for (const [index, files] of CGROUP_MEMORY_FILES.entries()) {\n const limitRaw = readCgroupText(files.limit);\n const currentRaw = readCgroupText(files.current);\n if (limitRaw === null && currentRaw === null) continue;\n if (limitRaw === null || currentRaw === null) {\n throw new Error(\n `Incomplete cgroup memory controller: limit=${files.limit} current=${files.current}`,\n );\n }\n return {\n currentBytes: parseCgroupCurrentValue(currentRaw),\n limitBytes: parseCgroupLimitValue(limitRaw),\n source: index === 0 ? \"cgroup-v2\" : \"cgroup-v1\",\n };\n }\n return {\n currentBytes: process.memoryUsage().rss,\n limitBytes: null,\n source: \"process\",\n };\n}\n\nexport function parseCgroupLimitValue(value: string): number | null {\n if (value.trim() === \"max\") return null;\n const parsed = parseCgroupBigInt(value);\n if (parsed === null || parsed <= 0n) {\n throw new Error(`Malformed finite cgroup memory limit: ${JSON.stringify(value.trim())}`);\n }\n if (parsed >= UNLIMITED_CGROUP_BYTES) return null;\n const number = Number(parsed);\n if (!Number.isSafeInteger(number)) {\n throw new Error(\n `Finite cgroup memory limit exceeds JavaScript's safe integer range: ${parsed}`,\n );\n }\n return number;\n}\n\nexport function parseCgroupCurrentValue(value: string): number {\n const parsed = parseCgroupBigInt(value);\n if (parsed === null || parsed < 0n) {\n throw new Error(`Malformed cgroup current memory value: ${JSON.stringify(value.trim())}`);\n }\n const number = Number(parsed);\n if (!Number.isSafeInteger(number)) {\n throw new Error(`Cgroup current memory exceeds JavaScript's safe integer range: ${parsed}`);\n }\n return number;\n}\n\nfunction parseCgroupBigInt(value: string): bigint | null {\n try {\n return BigInt(value.trim());\n } catch {\n return null;\n }\n}\n\nfunction readCgroupText(path: string): string | null {\n try {\n return readFileSync(path, \"utf8\");\n } catch (error) {\n if ((error as NodeJS.ErrnoException).code === \"ENOENT\") return null;\n throw new Error(\n `Unable to read present cgroup memory controller file ${path}: ${\n error instanceof Error ? error.message : String(error)\n }`,\n { cause: error },\n );\n }\n}\n\nfunction positiveInteger(name: string, value: number): number {\n if (!Number.isSafeInteger(value) || value <= 0) {\n throw new Error(`${name} must be a positive integer`);\n }\n return value;\n}\n\nfunction nonnegativeInteger(name: string, value: number): number {\n if (!Number.isSafeInteger(value) || value < 0) {\n throw new Error(`${name} must be a non-negative integer`);\n }\n return value;\n}\n\nfunction abortError(): DOMException {\n return new DOMException(\"Turn slot reservation aborted\", \"AbortError\");\n}\n\nfunction abortableDelay(milliseconds: number, signal: AbortSignal): Promise<void> {\n return new Promise((resolve, reject) => {\n if (signal.aborted) {\n reject(abortError());\n return;\n }\n const timer = setTimeout(() => {\n signal.removeEventListener(\"abort\", onAbort);\n resolve();\n }, milliseconds);\n timer.unref?.();\n const onAbort = () => {\n clearTimeout(timer);\n reject(abortError());\n };\n signal.addEventListener(\"abort\", onAbort, { once: true });\n });\n}\n\n// Session workflows finish at idle boundaries and replay from durable history.\n// Bound sticky VM state explicitly: the SDK otherwise derives this from the\n// host heap limit (roughly 600 cached workflows/GiB), which is far above this\n// process's actual concurrent workflow-task capacity and permanently retains\n// idle workflow state in every control replica.\nexport const CONTROL_WORKER_MAX_CACHED_WORKFLOWS = 64;\n","export const SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS = 5 * 60_000;\nexport const SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS = 5 * 60_000;\nexport const SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS = 65 * 60_000;\nexport const SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS = 60_000;\nexport const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 20_000;\nexport const SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS = 5_000;\n// Match the DB inventory's fair 500-row bound so one sweep never discovers a\n// due box and then needlessly defers it for another schedule period. Five\n// hundred small StartChild commands remain far below Temporal history limits;\n// actual provider I/O stays bounded by worker activity concurrency.\nexport const SANDBOX_REAPER_CHILD_DISPATCH_LIMIT = 500;\nexport const SANDBOX_REAPER_V2_WORKFLOW_ID = \"opengeni-sandbox-lease-reaper-v2\";\nexport const SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID = \"opengeni-sandbox-lease-maintenance-v1\";\n\n// A protocol-specific queue is the rolling-deploy compatibility boundary for\n// the split reaper activities. An old control worker polls only the base queue,\n// so it can replay already-started legacy workflows but can never receive a new\n// activity type that its binary does not register.\nexport const SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX = \"-sandbox-lifecycle-v1\";\n\nexport function sandboxLifecycleTaskQueue(baseTaskQueue: string): string {\n return baseTaskQueue.endsWith(SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX)\n ? baseTaskQueue\n : `${baseTaskQueue}${SANDBOX_LIFECYCLE_TASK_QUEUE_SUFFIX}`;\n}\n\nexport type SandboxDrainTimeoutClass = \"fast\" | \"extended\";\n\nexport type SandboxDrainTarget = {\n workspaceId: string;\n sandboxGroupId: string;\n instanceId: string | null;\n leaseEpoch: number;\n};\n\nexport type SandboxDrainActivityInput = {\n target: SandboxDrainTarget;\n timeoutClass: SandboxDrainTimeoutClass;\n /** Provider operation budget frozen when the durable child is created. A\n * rolling config change cannot make an existing fast child retry forever. */\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n /** Stable logical teardown operation; every Temporal activity retry keeps it\n * while receiving a distinct exact capture-attempt id. */\n operationId: string;\n};\n\nexport type SandboxDrainWorkflowInput = Omit<SandboxDrainActivityInput, \"operationId\">;\n\nexport type SandboxLeaseSweepMaintenanceInput = {\n examined: number;\n started: number;\n alreadyRunning: number;\n startFailed: number;\n rotationsRequested: number;\n};\n\nexport function sandboxDrainTimeoutClass(captureTimeoutMs: number): SandboxDrainTimeoutClass {\n return captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS <\n SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS\n ? \"fast\"\n : \"extended\";\n}\n\nexport function sandboxDrainActivityTimeoutMs(timeoutClass: SandboxDrainTimeoutClass): number {\n return timeoutClass === \"fast\"\n ? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS\n : SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS;\n}\n\nexport function sandboxDrainWorkflowId(target: SandboxDrainTarget): string {\n return `sandbox-drain:${target.workspaceId}:${target.sandboxGroupId}:${target.leaseEpoch}`;\n}\n","import {\n SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,\n SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS,\n effectiveSandboxDrainSnapshotTimeoutMs,\n sandboxArchiveCaptureTimeoutMs,\n type Settings,\n} from \"@opengeni/config\";\nimport {\n SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS,\n sandboxDrainActivityTimeoutMs,\n sandboxDrainTimeoutClass,\n type SandboxDrainTimeoutClass,\n} from \"./sandbox-reaper-contract\";\n\nconst STRICT_TIMEOUT_FENCE_MS = 1;\n\nexport function sandboxDrainTiming(\n settings: Pick<Settings, \"sandboxSnapshotTimeoutMs\" | \"sandboxDrainSnapshotTimeoutMs\">,\n): {\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n timeoutClass: SandboxDrainTimeoutClass;\n activityTimeoutMs: number;\n} {\n const snapshotTimeoutMs = effectiveSandboxDrainSnapshotTimeoutMs(settings);\n const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs({\n sandboxSnapshotTimeoutMs: snapshotTimeoutMs,\n });\n const timeoutClass = sandboxDrainTimeoutClass(captureTimeoutMs);\n return {\n snapshotTimeoutMs,\n captureTimeoutMs,\n timeoutClass,\n activityTimeoutMs: sandboxDrainActivityTimeoutMs(timeoutClass),\n };\n}\n\nexport function assertSandboxReaperActivityTimeout(\n settings: Pick<Settings, \"sandboxSnapshotTimeoutMs\" | \"sandboxDrainSnapshotTimeoutMs\">,\n): void {\n const timing = sandboxDrainTiming(settings);\n if (timing.snapshotTimeoutMs >= timing.captureTimeoutMs) {\n throw new Error(\n `Sandbox provider snapshot timeout (${timing.snapshotTimeoutMs}) must be strictly less than ` +\n `its durable capture claim (${timing.captureTimeoutMs}).`,\n );\n }\n if (\n timing.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS + STRICT_TIMEOUT_FENCE_MS >=\n timing.activityTimeoutMs\n ) {\n throw new Error(\n `Sandbox ${timing.timeoutClass} drain activity timeout (${timing.activityTimeoutMs}) must ` +\n `strictly exceed one durable archive capture and settlement margin ` +\n `(${timing.captureTimeoutMs} + ${SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS} milliseconds).`,\n );\n }\n}\n\nexport function assertSandboxDrainInputTiming(input: {\n timeoutClass: SandboxDrainTimeoutClass;\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n}): void {\n if (\n !Number.isSafeInteger(input.snapshotTimeoutMs) ||\n input.snapshotTimeoutMs <= 0 ||\n input.snapshotTimeoutMs > SANDBOX_SNAPSHOT_MAX_TIMEOUT_MS ||\n !Number.isSafeInteger(input.captureTimeoutMs) ||\n input.captureTimeoutMs <= 0 ||\n input.captureTimeoutMs > SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS\n ) {\n throw new Error(\"Sandbox drain timeout values are invalid\");\n }\n const inputActivityTimeoutMs = sandboxDrainActivityTimeoutMs(input.timeoutClass);\n if (\n sandboxDrainTimeoutClass(input.captureTimeoutMs) !== input.timeoutClass ||\n input.snapshotTimeoutMs >= input.captureTimeoutMs ||\n input.captureTimeoutMs + SANDBOX_DRAIN_SETTLEMENT_MARGIN_MS >= inputActivityTimeoutMs\n ) {\n throw new Error(\n `Sandbox drain timing is inconsistent with the ${input.timeoutClass} activity budget`,\n );\n }\n}\n"],"mappings":";AAEA,SAAS,oBAAoB;AAY7B,IAAM,MAAM,OAAO;AAOZ,IAAM,mCAAmC;AACzC,IAAM,yCAAyC,MAAM;AACrD,IAAM,oCAAoC,MAAM;AAqBhD,SAAS,6BACd,UAC8B;AAC9B,MAAI,SAAS,8BAA8B,SAAS;AAClD,WAAO;AAAA,MACL,qCAAqC,SAAS;AAAA,IAChD;AAAA,EACF;AAEA,SAAO;AAAA,IACL,OAAO;AAAA,MACL,cAAc;AAAA,QACZ,gBAAgB,SAAS;AAAA,QACzB,mBAAmB,SAAS;AAAA,MAC9B;AAAA,MACA,yBAAyB;AAAA,QACvB,cAAc;AAAA,QACd,cAAc,SAAS;AAAA,QACvB,cAAc;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;AAeO,SAAS,gCACd,UACA,UAAsD,CAAC,GAC5B;AAC3B,MAAI,SAAS,8BAA8B,kBAAkB;AAC3D,WAAO;AAAA,MACL,SAAS,6BAA6B,QAAQ;AAAA,MAC9C,WAAW;AAAA,IACb;AAAA,EACF;AAEA,QAAM,QAAQ,sBAAsB;AAAA,IAClC,GAAG;AAAA,IACH,cAAc,SAAS;AAAA,EACzB,CAAC;AACD,SAAO;AAAA,IACL,SAAS,EAAE,OAAO,MAAM,MAAM;AAAA,IAC9B,WAAW,MAAM;AAAA,EACnB;AACF;AAEO,SAAS,+BAA+B,UAK7C;AACA,QAAM,gBAAgB,SAAS,8BAA8B;AAC7D,SAAO;AAAA,IACL,iBAAiB,SAAS;AAAA,IAC1B,oBAAoB,SAAS;AAAA,IAC7B,gBAAgB,gBAAgB,SAAS,2BAA2B;AAAA,IACpE,mBAAmB,gBAAgB,SAAS,8BAA8B;AAAA,EAC5E;AACF;AAEO,IAAM,2CAA2C;AACjD,IAAM,+CAA+C;AAE5D,IAAM,sBAAsB;AAAA,EAC1B;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AAAA,EACA;AAAA,IACE,OAAO;AAAA,IACP,SAAS;AAAA,EACX;AACF;AAIA,IAAM,yBAAyB,MAAM;AAmD9B,IAAM,8BAAN,MAAkF;AAAA,EAC9E,OAAO;AAAA,EAEC,QAAQ,OAAO,yBAAyB;AAAA,EACxC,UAAU,oBAAI,IAAoB;AAAA,EAC3C;AAAA,EACS;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAAgC,CAAC,GAAG;AAC9C,SAAK,eAAe;AAAA,MAClB;AAAA,MACA,QAAQ,gBAAgB;AAAA,IAC1B;AACA,SAAK,mBAAmB;AAAA,MACtB;AAAA,MACA,QAAQ,oBAAoB;AAAA,IAC9B;AACA,SAAK,sBAAsB;AAAA,MACzB;AAAA,MACA,QAAQ,uBAAuB;AAAA,IACjC;AACA,SAAK,eAAe,gBAAgB,gBAAgB,QAAQ,gBAAgB,GAAG;AAC/E,SAAK,iBAAiB,QAAQ,kBAAkB;AAChD,SAAK,gBAAgB,QAAQ;AAC7B,UAAM,UAAU,KAAK,eAAe;AACpC,SAAK,gBAAgB,QAAQ;AAC7B,QAAI,QAAQ,eAAe,QAAQ,KAAK,oBAAoB,QAAQ,UAAU,IAAI,GAAG;AACnF,YAAM,IAAI;AAAA,QACR,mEACc,KAAK,aAAa,aAAa,KAAK,gBAAgB,mBAC9C,KAAK,mBAAmB,UAAU,QAAQ,UAAU;AAAA,MAC1E;AAAA,IACF;AACA,SAAK,eAAe,OAAO;AAAA,EAC7B;AAAA,EAEA,MAAM,YAAY,MAA0B,aAA+C;AACzF,eAAS;AACP,UAAI,YAAY,QAAS,OAAM,WAAW;AAC1C,YAAM,SAAS,KAAK,cAAc;AAClC,UAAI,OAAQ,QAAO;AACnB,YAAM,eAAe,KAAK,cAAc,WAAW;AAAA,IACrD;AAAA,EACF;AAAA,EAEA,eAAe,MAA6C;AAC1D,WAAO,KAAK,cAAc;AAAA,EAC5B;AAAA,EAEA,aAAa,KAAkD;AAC7D,UAAM,SAAS,KAAK,YAAY,IAAI,MAAM;AAC1C,WAAO,OAAO;AACd,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,YAAY,KAAiD;AAC3D,UAAM,SAAS,KAAK,YAAY,IAAI,MAAM;AAC1C,SAAK,QAAQ,OAAO,MAAM;AAC1B,SAAK,eAAe;AAAA,EACtB;AAAA,EAEA,SAAS,SAAS,KAAK,eAAe,GAA0B;AAC9D,UAAM,YAAY,KAAK,UAAU;AACjC,UAAM,gBAAgB,KAAK,QAAQ;AACnC,UAAM,iBAAiB,KAAK,eAAe,MAAM;AAGjD,UAAM,sBAAsB,gBAAgB;AAC5C,WAAO;AAAA,MACL,eAAe,KAAK;AAAA,MACpB,cAAc,OAAO;AAAA,MACrB,YAAY,OAAO;AAAA,MACnB,kBAAkB,KAAK;AAAA,MACvB,qBAAqB,KAAK;AAAA,MAC1B,cAAc,KAAK;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQA,0BAAgC;AAC9B,UAAM,SAAS,KAAK,eAAe;AACnC,QAAI,KAAK,QAAQ,SAAS,GAAG;AAC3B,YAAM,IAAI,MAAM,gEAAgE;AAAA,IAClF;AACA,UAAM,yBAAyB,OAAO;AACtC,UAAM,oBACJ,OAAO,eAAe,OAClB,KAAK,eACL,KAAK,oBAAoB,OAAO,YAAY,sBAAsB;AACxE,QAAI,OAAO,eAAe,QAAQ,oBAAoB,GAAG;AACvD,YAAM,IAAI;AAAA,QACR,0GACyB,KAAK,aAAa,sBAAsB,sBAAsB,aACzE,KAAK,gBAAgB,mBAAmB,KAAK,mBAAmB,UACnE,OAAO,OAAO,UAAU,CAAC;AAAA,MACtC;AAAA,IACF;AACA,SAAK,gBAAgB;AACrB,SAAK,eAAe,MAAM;AAAA,EAC5B;AAAA,EAEQ,gBAAuC;AAC7C,UAAM,SAAS,KAAK,eAAe;AACnC,QAAI,KAAK,QAAQ,QAAQ,KAAK,gBAAgB,CAAC,KAAK,oBAAoB,MAAM,GAAG;AAC/E,WAAK,eAAe,MAAM;AAC1B,aAAO;AAAA,IACT;AACA,UAAM,SAAyB,EAAE,OAAO,KAAK,OAAO,MAAM,MAAM;AAChE,SAAK,QAAQ,IAAI,MAAM;AACvB,SAAK,eAAe,MAAM;AAC1B,WAAO;AAAA,EACT;AAAA,EAEQ,oBAAoB,QAAuC;AACjE,WAAO,KAAK,eAAe,MAAM,IAAI;AAAA,EACvC;AAAA,EAEQ,eAAe,QAAsC;AAC3D,UAAM,mBAAmB,KAAK,eAAe,KAAK,QAAQ;AAC1D,QAAI,OAAO,eAAe,KAAM,QAAO,KAAK,IAAI,GAAG,gBAAgB;AAEnE,UAAM,oBAAoB,KAAK,oBAAoB,OAAO,UAAU,IAAI,KAAK,QAAQ;AACrF,UAAM,oBACJ,KAAK;AAAA,OACF,OAAO,aAAa,OAAO,eAAe,KAAK,uBAC9C,KAAK;AAAA,IACT,IAAI,KAAK,QAAQ;AACnB,WAAO,KAAK,IAAI,GAAG,KAAK,IAAI,kBAAkB,mBAAmB,iBAAiB,CAAC;AAAA,EACrF;AAAA,EAEQ,oBAAoB,YAAoB,gBAAgB,KAAK,eAAuB;AAC1F,WAAO,KAAK;AAAA,MACV;AAAA,MACA,KAAK,OAAO,aAAa,gBAAgB,KAAK,uBAAuB,KAAK,gBAAgB;AAAA,IAC5F;AAAA,EACF;AAAA,EAEQ,YAAoB;AAC1B,QAAI,OAAO;AACX,eAAW,UAAU,KAAK,SAAS;AACjC,UAAI,OAAO,KAAM,SAAQ;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,YAAY,QAAoC;AACtD,UAAM,YAAY;AAClB,QAAI,UAAU,UAAU,KAAK,SAAS,CAAC,KAAK,QAAQ,IAAI,MAAwB,GAAG;AACjF,YAAM,IAAI,MAAM,gEAAgE;AAAA,IAClF;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,eAAe,SAAS,KAAK,eAAe,GAAS;AAC3D,QAAI,CAAC,KAAK,cAAe;AACzB,UAAM,QAAQ,KAAK,SAAS,MAAM;AAClC,UAAM,MAAM,CAAC,MAAc,MAAc,gBACvC,KAAK,eAAe,SAAS,EAAE,MAAM,MAAM,OAAO,YAAY,CAAC;AACjE;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM,wBAAwB,IAAI,IAAI,MAAM,YAAY,MAAM;AAAA,IAChE;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM,cAAc;AAAA,IACtB;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AAAA,EACF;AACF;AAEO,SAAS,sBAAsB,UAAgC,CAAC,GAGrE;AACA,QAAM,YAAY,IAAI,4BAA4B,OAAO;AACzD,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,MACL,0BAA0B,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,MAC5D,0BAA0B;AAAA,MAC1B,+BAA+B,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,MACjE,uBAAuB,EAAE,MAAM,cAAc,UAAU,EAAE;AAAA,IAC3D;AAAA,EACF;AACF;AAEO,SAAS,2BAAiD;AAC/D,aAAW,CAAC,OAAO,KAAK,KAAK,oBAAoB,QAAQ,GAAG;AAC1D,UAAM,WAAW,eAAe,MAAM,KAAK;AAC3C,UAAM,aAAa,eAAe,MAAM,OAAO;AAC/C,QAAI,aAAa,QAAQ,eAAe,KAAM;AAC9C,QAAI,aAAa,QAAQ,eAAe,MAAM;AAC5C,YAAM,IAAI;AAAA,QACR,8CAA8C,MAAM,KAAK,YAAY,MAAM,OAAO;AAAA,MACpF;AAAA,IACF;AACA,WAAO;AAAA,MACL,cAAc,wBAAwB,UAAU;AAAA,MAChD,YAAY,sBAAsB,QAAQ;AAAA,MAC1C,QAAQ,UAAU,IAAI,cAAc;AAAA,IACtC;AAAA,EACF;AACA,SAAO;AAAA,IACL,cAAc,QAAQ,YAAY,EAAE;AAAA,IACpC,YAAY;AAAA,IACZ,QAAQ;AAAA,EACV;AACF;AAEO,SAAS,sBAAsB,OAA8B;AAClE,MAAI,MAAM,KAAK,MAAM,MAAO,QAAO;AACnC,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,WAAW,QAAQ,UAAU,IAAI;AACnC,UAAM,IAAI,MAAM,yCAAyC,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC,EAAE;AAAA,EACzF;AACA,MAAI,UAAU,uBAAwB,QAAO;AAC7C,QAAM,SAAS,OAAO,MAAM;AAC5B,MAAI,CAAC,OAAO,cAAc,MAAM,GAAG;AACjC,UAAM,IAAI;AAAA,MACR,uEAAuE,MAAM;AAAA,IAC/E;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,wBAAwB,OAAuB;AAC7D,QAAM,SAAS,kBAAkB,KAAK;AACtC,MAAI,WAAW,QAAQ,SAAS,IAAI;AAClC,UAAM,IAAI,MAAM,0CAA0C,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC,EAAE;AAAA,EAC1F;AACA,QAAM,SAAS,OAAO,MAAM;AAC5B,MAAI,CAAC,OAAO,cAAc,MAAM,GAAG;AACjC,UAAM,IAAI,MAAM,kEAAkE,MAAM,EAAE;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,OAA8B;AACvD,MAAI;AACF,WAAO,OAAO,MAAM,KAAK,CAAC;AAAA,EAC5B,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,eAAe,MAA6B;AACnD,MAAI;AACF,WAAO,aAAa,MAAM,MAAM;AAAA,EAClC,SAAS,OAAO;AACd,QAAK,MAAgC,SAAS,SAAU,QAAO;AAC/D,UAAM,IAAI;AAAA,MACR,wDAAwD,IAAI,KAC1D,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CACvD;AAAA,MACA,EAAE,OAAO,MAAM;AAAA,IACjB;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,MAAc,OAAuB;AAC5D,MAAI,CAAC,OAAO,cAAc,KAAK,KAAK,SAAS,GAAG;AAC9C,UAAM,IAAI,MAAM,GAAG,IAAI,6BAA6B;AAAA,EACtD;AACA,SAAO;AACT;AAEA,SAAS,mBAAmB,MAAc,OAAuB;AAC/D,MAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG;AAC7C,UAAM,IAAI,MAAM,GAAG,IAAI,iCAAiC;AAAA,EAC1D;AACA,SAAO;AACT;AAEA,SAAS,aAA2B;AAClC,SAAO,IAAI,aAAa,iCAAiC,YAAY;AACvE;AAEA,SAAS,eAAe,cAAsB,QAAoC;AAChF,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,QAAI,OAAO,SAAS;AAClB,aAAO,WAAW,CAAC;AACnB;AAAA,IACF;AACA,UAAM,QAAQ,WAAW,MAAM;AAC7B,aAAO,oBAAoB,SAAS,OAAO;AAC3C,cAAQ;AAAA,IACV,GAAG,YAAY;AACf,UAAM,QAAQ;AACd,UAAM,UAAU,MAAM;AACpB,mBAAa,KAAK;AAClB,aAAO,WAAW,CAAC;AAAA,IACrB;AACA,WAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAAA,EAC1D,CAAC;AACH;AAOO,IAAM,sCAAsC;;;AC5hB5C,IAAM,0CAA0C,IAAI;AACpD,IAAM,yCAAyC,IAAI;AACnD,IAAM,6CAA6C,KAAK;AACxD,IAAM,qCAAqC;AAE3C,IAAM,gDAAgD;AAKtD,IAAM,sCAAsC;AAC5C,IAAM,gCAAgC;AAOtC,IAAM,sCAAsC;AAE5C,SAAS,0BAA0B,eAA+B;AACvE,SAAO,cAAc,SAAS,mCAAmC,IAC7D,gBACA,GAAG,aAAa,GAAG,mCAAmC;AAC5D;AAiCO,SAAS,yBAAyB,kBAAoD;AAC3F,SAAO,mBAAmB,qCACxB,yCACE,SACA;AACN;AAEO,SAAS,8BAA8B,cAAgD;AAC5F,SAAO,iBAAiB,SACpB,yCACA;AACN;;;ACpEA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAQP,IAAM,0BAA0B;AAEzB,SAAS,mBACd,UAMA;AACA,QAAM,oBAAoB,uCAAuC,QAAQ;AACzE,QAAM,mBAAmB,+BAA+B;AAAA,IACtD,0BAA0B;AAAA,EAC5B,CAAC;AACD,QAAM,eAAe,yBAAyB,gBAAgB;AAC9D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA,mBAAmB,8BAA8B,YAAY;AAAA,EAC/D;AACF;AAEO,SAAS,mCACd,UACM;AACN,QAAM,SAAS,mBAAmB,QAAQ;AAC1C,MAAI,OAAO,qBAAqB,OAAO,kBAAkB;AACvD,UAAM,IAAI;AAAA,MACR,sCAAsC,OAAO,iBAAiB,2DAC9B,OAAO,gBAAgB;AAAA,IACzD;AAAA,EACF;AACA,MACE,OAAO,mBAAmB,qCAAqC,2BAC/D,OAAO,mBACP;AACA,UAAM,IAAI;AAAA,MACR,WAAW,OAAO,YAAY,4BAA4B,OAAO,iBAAiB,6EAE5E,OAAO,gBAAgB,MAAM,kCAAkC;AAAA,IACvE;AAAA,EACF;AACF;AAEO,SAAS,8BAA8B,OAIrC;AACP,MACE,CAAC,OAAO,cAAc,MAAM,iBAAiB,KAC7C,MAAM,qBAAqB,KAC3B,MAAM,oBAAoB,mCAC1B,CAAC,OAAO,cAAc,MAAM,gBAAgB,KAC5C,MAAM,oBAAoB,KAC1B,MAAM,mBAAmB,wCACzB;AACA,UAAM,IAAI,MAAM,0CAA0C;AAAA,EAC5D;AACA,QAAM,yBAAyB,8BAA8B,MAAM,YAAY;AAC/E,MACE,yBAAyB,MAAM,gBAAgB,MAAM,MAAM,gBAC3D,MAAM,qBAAqB,MAAM,oBACjC,MAAM,mBAAmB,sCAAsC,wBAC/D;AACA,UAAM,IAAI;AAAA,MACR,iDAAiD,MAAM,YAAY;AAAA,IACrE;AAAA,EACF;AACF;","names":[]}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
observabilityEventLogger
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-65JEWP2Q.js";
|
|
4
4
|
|
|
5
5
|
// src/activity-services.ts
|
|
6
6
|
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
@@ -1167,6 +1167,7 @@ function goalContinuationPrompt(_goal, _autoContinuation, _cap, options = {}) {
|
|
|
1167
1167
|
"Waiting on child sessions or external events:",
|
|
1168
1168
|
"- When the next progress depends on child sessions you spawned or on an external event, do not sleep, loop, or poll sessions_list/session_get/session_events to wait for it.",
|
|
1169
1169
|
"- Re-check sessions_list or session_get once; if the work is still in flight, call opengeni__goal_wait with a concrete reason and a deadline (untilSeconds), then end your turn immediately. You will be woken by a child result, a message, a human prompt, or at the deadline, and this goal stays active.",
|
|
1170
|
+
"- If the immediately preceding user-facing update already reported this same unchanged wait, do not restate it or produce another equivalent final answer. Call opengeni__goal_wait and end the turn. Report only material new state or a newly discovered blocker.",
|
|
1170
1171
|
"- A hold is for child/external progress only. If you are blocked on a human decision, use opengeni__goal_pause under the blocked audit below instead.",
|
|
1171
1172
|
""
|
|
1172
1173
|
] : [];
|
|
@@ -2724,49 +2725,35 @@ function legacyNativeArchiveFromEnvelope(envelope) {
|
|
|
2724
2725
|
existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
|
|
2725
2726
|
);
|
|
2726
2727
|
}
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
const reinstated = await reinstateTurnLeaseHolder(db, {
|
|
2733
|
-
accountId: ids.accountId,
|
|
2734
|
-
workspaceId: ids.workspaceId,
|
|
2735
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
2736
|
-
sessionId: ids.sessionId,
|
|
2737
|
-
attemptId: ids.attemptId,
|
|
2738
|
-
expectedEpoch: expected.leaseEpoch,
|
|
2739
|
-
expectedInstanceId: expected.instanceId,
|
|
2740
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
2728
|
+
function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
|
|
2729
|
+
let continuationRegistered = false;
|
|
2730
|
+
let resolvePhysicalSettlement = () => void 0;
|
|
2731
|
+
const physicalSettlement = new Promise((resolve) => {
|
|
2732
|
+
resolvePhysicalSettlement = resolve;
|
|
2741
2733
|
});
|
|
2742
|
-
|
|
2743
|
-
const message = "sandbox rotation checkpoint reinstated the live turn holder";
|
|
2744
|
-
const fields = {
|
|
2745
|
-
workspaceId: ids.workspaceId,
|
|
2746
|
-
sessionId: ids.sessionId,
|
|
2747
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
2748
|
-
leaseEpoch: expected.leaseEpoch,
|
|
2749
|
-
liveness: reinstated.lease.liveness
|
|
2750
|
-
};
|
|
2751
|
-
if (services.observability) services.observability.warn(message, fields);
|
|
2752
|
-
else console.warn(message, fields);
|
|
2753
|
-
}
|
|
2754
|
-
if (reinstated.status !== "present" && reinstated.status !== "reinstated") {
|
|
2755
|
-
return { checkpointed: false, holder: reinstated.status, lease: null };
|
|
2756
|
-
}
|
|
2757
|
-
await maybePersistWarmWorkspaceSnapshot(
|
|
2734
|
+
const result = persistWarmWorkspaceSnapshot(
|
|
2758
2735
|
services,
|
|
2759
2736
|
ids,
|
|
2760
2737
|
session,
|
|
2761
|
-
|
|
2738
|
+
leaseEpoch,
|
|
2762
2739
|
signal,
|
|
2763
|
-
|
|
2740
|
+
force,
|
|
2741
|
+
(continuation) => {
|
|
2742
|
+
continuationRegistered = true;
|
|
2743
|
+
void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
|
|
2744
|
+
}
|
|
2745
|
+
);
|
|
2746
|
+
void result.then(
|
|
2747
|
+
() => {
|
|
2748
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
2749
|
+
},
|
|
2750
|
+
() => {
|
|
2751
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
2752
|
+
}
|
|
2764
2753
|
);
|
|
2765
|
-
|
|
2766
|
-
const checkpointed = lease?.leaseEpoch === expected.leaseEpoch && lease.instanceId === expected.instanceId && lease.rotationRequestedAt !== null && lease.archiveComplete && signal?.aborted !== true;
|
|
2767
|
-
return { checkpointed, holder: reinstated.status, lease };
|
|
2754
|
+
return Object.assign(result, { settled: physicalSettlement });
|
|
2768
2755
|
}
|
|
2769
|
-
async function
|
|
2756
|
+
async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force, registerOwnedContinuation) {
|
|
2770
2757
|
const { db, settings } = services;
|
|
2771
2758
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
2772
2759
|
if (intervalMs <= 0 && !force) {
|
|
@@ -2978,6 +2965,7 @@ async function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEp
|
|
|
2978
2965
|
return { kind: "settled", persisted: false };
|
|
2979
2966
|
}
|
|
2980
2967
|
);
|
|
2968
|
+
registerOwnedContinuation(settled.then(() => void 0));
|
|
2981
2969
|
let cancelListener;
|
|
2982
2970
|
const outcome = await Promise.race([
|
|
2983
2971
|
settled,
|
|
@@ -3993,10 +3981,10 @@ function createTurnSandboxProvisioner(establish, options = {}) {
|
|
|
3993
3981
|
|
|
3994
3982
|
// src/activities/agent-turn/quiescence.ts
|
|
3995
3983
|
function shouldRunTurnEndWorkspacePersistence(input) {
|
|
3996
|
-
return input.activityStatus !== "cancelled" && !input.cancellationRequested;
|
|
3984
|
+
return input.activityStatus !== "cancelled" && !input.cancellationRequested && input.deadlineRotationRequested !== true;
|
|
3997
3985
|
}
|
|
3998
3986
|
function shouldStartPeriodicWorkspaceSnapshot(input) {
|
|
3999
|
-
return input.firstProviderRequestStarted && !input.snapshotInFlight && !input.turnEndCaptureInProgress;
|
|
3987
|
+
return input.firstProviderRequestStarted && !input.snapshotInFlight && !input.turnEndCaptureInProgress && !input.deadlineRotationRequested;
|
|
4000
3988
|
}
|
|
4001
3989
|
function assertPhysicalToolQuiescenceForCancellation(input) {
|
|
4002
3990
|
if (!input.acknowledgeQuiescence || input.physicalToolQuiescenceConfirmed) return;
|
|
@@ -4939,7 +4927,6 @@ export {
|
|
|
4939
4927
|
validateIncidentTelemetrySystemUpdateAuthority,
|
|
4940
4928
|
sandboxLeaseHolderIdForAttempt,
|
|
4941
4929
|
waitForWarmSnapshot,
|
|
4942
|
-
persistSandboxDeadlineRotationCheckpoint,
|
|
4943
4930
|
maybePersistWarmWorkspaceSnapshot,
|
|
4944
4931
|
resumeBoxForTurn,
|
|
4945
4932
|
SandboxDeadlineRotationError,
|
|
@@ -4994,4 +4981,4 @@ export {
|
|
|
4994
4981
|
codexUsageLimitFailurePayload,
|
|
4995
4982
|
CODEX_USAGE_LIMIT_MAX_RESUME_MS
|
|
4996
4983
|
};
|
|
4997
|
-
//# sourceMappingURL=chunk-
|
|
4984
|
+
//# sourceMappingURL=chunk-RQNH6CGR.js.map
|