@opengeni/worker-bundle 3.1.2 → 3.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/retained-process-diagnostics.ts","../src/retained-process-retry.ts","../src/activities/sandbox-lease.ts","../src/activities/browser-deadline-checkpoint.ts"],"sourcesContent":["import {\n databaseFailureCode,\n nestedPostgresSqlState,\n safeDatabaseErrorFacts,\n SandboxWorkspaceMutationFencedError,\n type SandboxRetainedProcess,\n} from \"@opengeni/db\";\nimport type { Observability } from \"@opengeni/observability\";\n\n/** Protected cause locations and schema facts; public logs receive only the diagnostic key. */\nexport function warnRetainedProcessProofFailure(\n observability: Pick<Observability, \"recordFailureDiagnostic\" | \"warn\">,\n error: unknown,\n process: Pick<\n SandboxRetainedProcess,\n \"id\" | \"sessionId\" | \"ownerAttemptId\" | \"ownerTurnId\" | \"reconcileAttempts\"\n >,\n): void {\n let correlationId: string | undefined;\n try {\n const sqlState = nestedPostgresSqlState(error);\n const constraint = safeDatabaseErrorFacts(error).constraint;\n correlationId = observability.recordFailureDiagnostic({\n code:\n error instanceof SandboxWorkspaceMutationFencedError\n ? \"retained_process_fenced\"\n : sqlState\n ? databaseFailureCode(sqlState)\n : \"retained_process_proof_failed\",\n stage: \"sandbox_retained_processes.proof\",\n error,\n processId: process.id,\n sessionId: process.sessionId,\n ...(process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {}),\n ...(process.ownerTurnId ? { turnId: process.ownerTurnId } : {}),\n attempts: process.reconcileAttempts,\n sqlState,\n ...(constraint ? { constraint } : {}),\n });\n } catch {\n // A diagnostic/export failure must not change reconciliation or release its fence.\n }\n try {\n observability.warn(\"sandbox reaper: retained-process proof checkpoint failed\", {\n ...(correlationId ? { correlationId } : {}),\n });\n } catch {\n // Keep the original failure behavior even when the public sink fails.\n }\n}\n","import type { LeaseSnapshot, SandboxRetainedProcess } from \"@opengeni/db\";\n\n/** Observation backoff must leave the exact provider's rotation window usable.\n * This changes only the next probe time, never cancellation or settlement proof. */\nexport function retainedProcessDeadlineRetryMs(\n process: Pick<\n SandboxRetainedProcess,\n \"leaseId\" | \"leaseEpoch\" | \"providerBackend\" | \"providerInstanceId\" | \"routeTargetId\"\n >,\n lease: Pick<\n LeaseSnapshot,\n \"id\" | \"leaseEpoch\" | \"backend\" | \"instanceId\" | \"providerDeadlineAt\"\n > | null,\n settings: { sandboxRotationLeadMs: number; sandboxLeaseReaperPeriodMs: number },\n retryAfterMs: number,\n now = Date.now(),\n): number {\n if (\n process.providerBackend !== \"modal\" ||\n process.routeTargetId !== null ||\n !lease ||\n lease.id !== process.leaseId ||\n lease.leaseEpoch !== process.leaseEpoch ||\n lease.backend !== process.providerBackend ||\n lease.instanceId !== process.providerInstanceId ||\n !lease.providerDeadlineAt\n )\n return retryAfterMs;\n\n const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;\n if (!Number.isFinite(untilRotation)) return retryAfterMs;\n return Math.min(\n retryAfterMs,\n Math.max(1_000, settings.sandboxLeaseReaperPeriodMs, untilRotation),\n );\n}\n","// apps/worker/src/activities/sandbox-lease.ts — the sole global lease inventory,\n// GC and cost-stop activity set (P1.3 / OD-3).\n//\n// One scheduled workflow inventories at most the DB reaper's fair 500-row batch,\n// then starts one durable child per exact (workspace, group, epoch). Each child\n// claims, snapshots and terminates only its own provider box. Temporal's control\n// worker bounds real provider concurrency; a slow or dead provider cannot hold\n// the global scan or any sibling drain. Activity heartbeats make worker loss\n// visible promptly, while the DB capture id remains the authoritative teardown\n// fence through provider termination and the final draining->cold commit.\n//\n// The legacy `reapSandboxLeases` composite remains only as a direct/embedded test\n// harness and executes admitted drains concurrently. There is no per-session\n// timer, viewer activity, owner task queue, or provider-specific lifecycle path\n// in the normal drain state machine.\nimport { warnDrainSnapshotFailure } from \"../sandbox-snapshot-diagnostics\";\nimport { maybePersistWarmWorkspaceSnapshot } from \"../sandbox-resume\";\nimport { publishDurableSessionEvents } from \"../session-event-fanout\";\nimport { warnRetainedProcessProofFailure } from \"../retained-process-diagnostics\";\nimport { retainedProcessDeadlineRetryMs } from \"../retained-process-retry\";\n\nimport { createHash, randomUUID } from \"node:crypto\";\nimport { requestRetainedProcessDeadlineCancellation } from \"@opengeni/db/retained-provider-commands\";\nimport type { ProviderCommandPersistence, ProviderCommandSession } from \"@opengeni/runtime\";\nimport { Context } from \"@temporalio/activity\";\nimport { OpLostReason, OpState, type OpStatus } from \"@opengeni/agent-proto\";\nimport {\n accrueWarmSeconds,\n retainedProviderCommandPersistence,\n adoptLegacyModalCheckpointArtifact,\n confirmDrainCold,\n markWarmLeaseInstanceLost,\n appendSessionEventToSandboxGroup,\n bindRetainedProcessProviderIdentity,\n claimWorkspaceArchiveCapture,\n leaseMayHaveUntrackedWriters,\n releaseWorkspaceArchiveCapture,\n listIdleCheckpointCandidates,\n recordIdleCheckpointAttempt,\n claimSandboxCheckpointArtifactsForGc,\n claimTerminalRetainedProcesses,\n getEnrollment,\n countActiveRetainedProcessesByOwnerState,\n countSandboxCheckpointArtifactsByState,\n countExpiredDrainingSandboxLeases,\n countQueuedTurns,\n countSandboxLeasesByLiveness,\n deferRetainedProcessReconciliation,\n forceDrainOverLimitViewerOnlyBoxes,\n listCreditBalancesByAccount,\n countActiveUsers,\n readCreditGrantTotals,\n readManagedAuthNewSignupsSwitch,\n readVerifiedSignupTrialSwitch,\n listLegacyModalCheckpointSlots,\n listLiveModalSandboxLeaseAttributions,\n listPendingModalProviderCreates,\n claimModalProviderCreateRecovery,\n recordRecoveredModalProviderCreate,\n markWarmBillingStopCutoff,\n listMeterableWarmLeases,\n countInteractionOnlyWarmLeasesByIdle,\n listSandboxViewerForceDrainWorkspaceIds,\n markSandboxCheckpointArtifactDeletePending,\n persistDrainSnapshot,\n pruneDeletedSandboxCheckpointArtifacts,\n registerSandboxCheckpointArtifact,\n recordRetainedProcessReconciliationProof,\n replaceWorkspaceArchiveCaptureAfterProof,\n listEndedEpochWorkspaceBlockers,\n listEndedEpochSandboxSetupUncertainties,\n reconcileCompletedSandboxSetup,\n readLease,\n settleEndedEpochWorkspaceBlockers,\n readWorkspaceArchiveCapturePreflight,\n reapExpiredSessionListSnapshots,\n reapStaleLeaseHoldersGlobal,\n requestDueSandboxRotationsGlobal,\n readSandboxRotationBacklog,\n readSandboxCheckpointStaleness,\n readRecentSandboxRecoveryObservations,\n workspaceArchiveCaptureDeadlineElapsed,\n retainedProcessReconciliationProof,\n retainedProcessSettlementIdentity,\n recoverManagedSessionBackgroundCommand,\n settleClaimedConnectedMachineBackgroundCommand,\n settleSandboxCheckpointArtifactGc,\n rlsContextForWorkspace,\n settleRetainedProcess,\n type MeterableWarmLease,\n type ReapDrainable,\n type RetainedProcessProviderProof,\n type SandboxRetainedProcess,\n type LeaseSnapshot,\n} from \"@opengeni/db\";\nimport {\n claimConnectedCommandOutputReleases,\n claimConnectedMachineSessionBackgroundCommands,\n deferConnectedMachineBackgroundCommandReconciliation,\n recordConnectedMachineBackgroundCommandProof,\n recordConnectedCommandOutputConsumption,\n settleConnectedCommandOutputReleaseClaim,\n type ConnectedMachineBackgroundCommandClaim,\n type ConnectedMachineBackgroundCommandProof,\n} from \"@opengeni/db/session-background-commands\";\nimport {\n SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,\n sandboxArchiveCaptureTimeoutMs,\n sandboxDeadlineMandatoryCaptureLeadMs,\n sandboxWarmRateMicrosPerSecond,\n} from \"@opengeni/config\";\nimport { appendSessionCommandOutput } from \"@opengeni/db/session-command-output\";\nimport { captureConnectedCommandOutput } from \"../sandbox-routing\";\nimport { sandboxLeaseTelemetryKey } from \"@opengeni/observability\";\nimport {\n // Normal drain teardown builds the client and resumes the envelope directly:\n // a live box gets its /workspace persisted before termination, while a gone\n // box is typed as provider loss. Stale capture reconciliation separately uses\n // establishSandboxSessionFromEnvelope in strict resume-only mode so it can\n // reuse the runtime's provider-ready proof without ever cold-restoring.\n captureWorkspaceArchiveForStorage,\n disposeWorkspaceArchive,\n type VerifiedWorkspaceArchivePayload,\n type VerifiedHostWorkspaceArchive,\n cancelSelfhostedOp,\n assertConsistentSandboxProviderIdentity,\n createSandboxClientForBackend,\n deleteModalCheckpointSnapshot,\n deserializeSandboxSessionStateEnvelope,\n establishSandboxSessionFromEnvelope,\n inspectModalSandboxLifecycle,\n isExecSessionLostBanner,\n isProviderSandboxNotFoundError,\n isProviderSandboxGoneDuringRoutedOperation,\n NatsControlRpc,\n NatsOpStreamTransport,\n OpStreamExecClient,\n defaultSelfhostedRetryClock,\n stripExecBanner,\n parseExecBannerExitCode,\n prepareProviderForTeardownAfterCapture,\n providerWorkspaceCapturePolicy,\n providerWorkspaceCaptureIsPointInTime,\n resolveModalCheckpointProviderBindingForLiveSandbox,\n resolveModalCheckpointProviderBinding,\n findModalProviderCreateReceipt,\n resolveModalCheckpointProviderBindingForSession,\n querySelfhostedOp,\n resumeExactSandboxSession,\n sandboxBackendForSdkBackendId,\n sandboxProviderInstanceIdFromEnvelope,\n sandboxCommandExitCode,\n sandboxCommandStillRunning,\n SelfhostedControlError,\n subjectFor,\n modalAppHasRunningSandboxes,\n sweepModalOrphanSandboxes,\n terminateManagedSandboxSession,\n terminateModalSandboxById,\n verifySandboxExecReadiness,\n type DockerHostWorkspaceRelease,\n type ModalOrphanSweepTermination,\n type ModalCheckpointProviderBinding,\n type ProviderWorkspaceCapturePolicy,\n type WorkspaceArchiveDescriptor,\n type ControlRpc,\n type NatsRequestConnection,\n} from \"@opengeni/runtime/sandbox\";\nimport {\n SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,\n SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,\n type SandboxDrainActivityInput,\n type SandboxLeaseSweepMaintenanceInput,\n sandboxDrainTimeoutClass,\n sandboxIdleCheckpointWorkflowId,\n type SandboxIdleCheckpointActivityInput,\n type SandboxIdleCheckpointPlan,\n} from \"../sandbox-reaper-contract\";\nimport { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from \"../concurrency\";\nimport { browserCheckpointPrerequisites } from \"./browser-deadline-checkpoint\";\nimport { assertSandboxDrainInputTiming, sandboxDrainTiming } from \"../sandbox-reaper-timeout\";\nimport type { ControlActivityServices as ActivityServices } from \"./types\";\nimport { reconcilePendingParentSystemUpdates } from \"./parent-wake\";\nimport {\n recordActiveUserGauges,\n recordCreditBalanceGauges,\n recordCreditGrantGauges,\n recordCreditMicros,\n recordExpiredDrainingSandboxLeaseGauges,\n recordOpenSandboxKubernetesInventoryGauges,\n recordRetainedProcessInventoryGauges,\n recordRetainedProcessReconciliation,\n recordSandboxCheckpointArtifactGauges,\n recordSandboxCheckpointArtifactOutcome,\n recordSandboxDeadlineRotationsRequested,\n recordSandboxInventoryProjectionFailure,\n recordSandboxInventoryProjectionSuccess,\n type RetainedProcessReconciliationOutcome,\n type SandboxInventoryProjectionDomain,\n recordSandboxLeaseGauges,\n recordModalSandboxInventoryGauges,\n recordInteractionOnlyLeaseGauges,\n recordSandboxOrphansTerminated,\n recordSandboxCommandContainment,\n recordSandboxIdleInteractionRelease,\n recordSandboxProviderMissingBeforeCapture,\n recordSandboxDockerWorkspaceRelease,\n recordSandboxRecoveryObservationGauges,\n recordSandboxRotationBacklogGauges,\n recordSandboxCheckpointStalenessGauges,\n recordTurnsQueuedGauge,\n recordVerifiedSignupTrialDeploymentFlagGauge,\n recordManagedAuthNewSignupsSwitchGauge,\n recordVerifiedSignupTrialSwitchGauge,\n runtimeMetricsHooksForObservability,\n} from \"../observability-metrics\";\nimport {\n inspectOpenSandboxKubernetesInventory,\n type OpenSandboxKubernetesInventory,\n} from \"../opensandbox-kubernetes-inventory\";\nimport type { ObjectStorage } from \"@opengeni/storage\";\nimport {\n persistWorkspaceArchiveCandidate,\n putVersion1TarArchiveOrInline,\n} from \"../sandbox-archive-storage\";\n\nexport { sandboxLeaseTelemetryKey } from \"@opengeni/observability\";\n\nexport type ReapSandboxLeasesResult = {\n /** Stale viewer holders + warming-death rows the sweep touched is folded into\n * the DB pass; here we report the terminate outcomes. */\n examined: number;\n /** Boxes whose provider terminate fired AND whose lease went cold. */\n terminated: number;\n /** Drainable rows skipped because the CAS no longer held (re-armed / newer\n * epoch / already drained by a concurrent sweep) — provider stop() did NOT\n * fire. */\n skipped: number;\n /** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */\n metered: number;\n /** Holderless boxes accelerated to draining by the optional warm cap. */\n forceDrained: number;\n /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */\n modalOrphansTerminated: number;\n};\n\ntype CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;\n\ntype DrainSandboxClient = {\n backendId: string;\n resume?: (state: unknown) => Promise<unknown>;\n deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;\n delete?: (state: unknown) => Promise<void>;\n attachWorkspaceForDrain?: (\n state: unknown,\n assertCurrentCapture: () => Promise<{ archivePublished: boolean }>,\n ) => Promise<PersistableSession>;\n};\n\n/**\n * The CAS-write the terminate seam calls to PERSIST the captured /workspace\n * archive onto the lease under the epoch fence, BEFORE the box is terminated\n * (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over\n * db/accountId/row and delegates to persistDrainSnapshot. Returns:\n * `wrote:false` means the CAS missed (re-armed / newer epoch / vanished). The\n * box is wanted again, so the seam MUST NOT terminate it. Provider-native\n * deletion is owned independently by artifact-ledger GC.\n *\n * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for\n * backends with no persistWorkspace — ensures a re-arm during the snapshot window\n * aborts the terminate before client.delete()).\n */\nexport type PersistArchiveFn = (\n archiveBase64: string | VerifiedHostWorkspaceArchive | null,\n archiveMetadata?: WorkspaceArchiveDescriptor,\n providerSession?: unknown,\n providerBinding?: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBindingForSession>\n > | null,\n) => Promise<{\n wrote: boolean;\n archiveRevision?: string | null;\n}>;\n\nexport type ProviderTerminationOutcome = {\n /** false means an epoch/refcount CAS proved the box was re-armed, so it was\n * deliberately left running and the cold commit must not execute. */\n terminated: boolean;\n /** True only when a definitive provider NotFound proved the cloud box had\n * already vanished before this drain could capture its workspace. */\n providerMissingBeforeCapture: boolean;\n};\n\nexport type DrainCaptureDisposition = \"capture_required\" | \"archive_published\";\n\n/** The provider-terminate seam. Production wires the real resume-by-id ->\n * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->\n * provider stop() (`terminateProviderBox`); a unit test injects a spy so the\n * drain/CAS logic is exercised against a real DB without a live provider box.\n * Test seams may return the legacy boolean shorthand. Production returns the\n * typed outcome so the cold commit can distinguish a clean stop from definitive\n * provider disappearance before capture. */\nexport type TerminateBoxFn = (\n settings: ActivityServices[\"settings\"],\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,\n observability: ActivityServices[\"observability\"],\n persistArchive: PersistArchiveFn,\n providerCaptureRequestId?: string,\n captureDisposition?: DrainCaptureDisposition,\n capturePolicy?: ProviderWorkspaceCapturePolicy | null,\n diskBackedArchives?: boolean,\n releaseFailedCapture?: () => Promise<void>,\n workspaceId?: string,\n beforeProviderStop?: () => Promise<void>,\n beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,\n) => Promise<boolean | ProviderTerminationOutcome>;\n\nexport type SweepModalOrphansFn = (\n settings: ActivityServices[\"settings\"],\n db: ActivityServices[\"db\"],\n observability: ActivityServices[\"observability\"],\n) => Promise<number>;\n\nexport type InspectOpenSandboxKubernetesInventoryFn = (input: {\n namespace: string;\n}) => Promise<OpenSandboxKubernetesInventory>;\n\nexport type SandboxLeaseActivityOptions = {\n /** Override the provider terminate (tests spy this; defaults to the real\n * resume-by-id + provider stop()). */\n terminateBox?: TerminateBoxFn;\n /** Override the provider-side Modal orphan sweep (tests spy this; defaults to\n * Modal list+tag comparison when Modal is configured). */\n sweepModalOrphans?: SweepModalOrphansFn;\n /** Only provider discovery is injected; durable recovery remains real. */\n findModalProviderCreateReceipt?: typeof findModalProviderCreateReceipt;\n /** Override the read-only bounded OpenSandbox Kubernetes projection. */\n inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;\n /** Override only the read-only provider process probe. The canonical DB\n * settlement remains real in tests and production. */\n probeRetainedProcess?: RetainedProcessProbeFn;\n /** Observe a historical Modal box after its lease row has advanced to a\n * successor identity. Lifecycle-only; a live box remains ambiguous. */\n inspectHistoricalModalSandbox?: HistoricalModalSandboxLifecycleProbeFn;\n /** Override the read-only exact-instance readiness probe used before the\n * reaper settles blockers for a provider that has definitively vanished. */\n probeDrainableProvider?: DrainableProviderProbeFn;\n /** Override only the resume-by-id of the box an idle checkpoint captures.\n * Returns the live session, or null when the provider no longer has it. The\n * capture claim, publication and release remain real. */\n resumeIdleCheckpointSession?: ResumeIdleCheckpointSessionFn;\n /** Override only the provider client factory of the default drain. The\n * claim, capture, publication and teardown ordering remain real. */\n createClientForBackend?: CreateSandboxClientForBackendFn;\n};\n\nexport type ResumeIdleCheckpointSessionFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n) => Promise<unknown | null>;\n\nexport type RetainedProcessProbeResult =\n | { status: \"proved\"; proof: RetainedProcessProviderProof }\n | {\n status: \"binding\";\n providerBindingKey: string;\n providerBinding: ModalCheckpointProviderBinding;\n }\n | {\n status: \"deferred\";\n reason:\n | \"identity_mismatch\"\n | \"resume_state_missing\"\n | \"backend_unsupported\"\n | \"provider_running\"\n | \"provider_unknown\"\n | \"provider_timeout\"\n | \"provider_error\"\n | \"provider_binding_missing\"\n | \"provider_binding_mismatch\"\n | \"process_observation_unavailable\";\n /** A running command whose last read still returned a large backlog. */\n outputBacklog?: boolean;\n };\n\nexport type RetainedProcessProbeFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n process: SandboxRetainedProcess,\n mode?: \"observe\" | \"cancel\",\n captureOutput?: (\n result: unknown,\n chunkId: string,\n stream?: \"stdout\" | \"stderr\",\n streamFidelity?: \"separate\" | \"merged\",\n ) => Promise<void>,\n providerPersistence?: ProviderCommandPersistence,\n drainBudget?: RetainedProcessDrainBudget,\n) => Promise<RetainedProcessProbeResult>;\n\n/** Shared wall-clock bound on output draining within one reconciliation sweep. */\nexport type RetainedProcessDrainBudget = { until: number };\n\nexport type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;\n\nexport type DrainableProviderProbeFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n) => Promise<\"ready\" | \"missing\">;\n\nexport const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;\nexport const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;\nexport const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;\n/** Bounds on reading a command's output backlog: per claim, and across one\n * reconciliation sweep so the reaper activity stays well inside its timeout.\n * Each read returns up to one provider page per stream. */\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 10_000;\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 60_000;\n/** A live process whose read returned at least this much output left a backlog\n * behind it. A server or watcher that only trickles output stays on ordinary\n * backoff; a process the provider reports exited is always read to EOF. */\nexport const RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;\nexport const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;\nexport const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;\nexport const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;\nexport const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;\n\nexport class SandboxProviderCaptureTimeoutError extends Error {\n readonly name = \"SandboxProviderCaptureTimeoutError\";\n\n constructor(\n public readonly sandboxGroupId: string,\n public readonly backend: string,\n public readonly timeoutMs: number,\n public readonly leaseEpoch: number,\n public readonly instanceId: string,\n ) {\n super(\n `sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} ` +\n `workspace capture exceeded ${timeoutMs}ms; ` +\n \"the exact durable claim remains fenced while its provider response settles\",\n );\n }\n}\n\nexport async function awaitProviderCaptureWithLatePublication<T>(input: {\n capture: Promise<T>;\n timeoutMs: number;\n timeoutError: SandboxProviderCaptureTimeoutError;\n publishLate: (value: T) => Promise<void>;\n observeLateFailure: (error: unknown) => void;\n observeLatePublicationFailure: (error: unknown) => void;\n /** Called once the detached late path has fully settled (after publication\n * or failure handling), never on the in-budget path. */\n onLateSettled?: () => void;\n}): Promise<T> {\n let timeout: ReturnType<typeof setTimeout> | undefined;\n const timeoutError = input.timeoutError;\n try {\n return await Promise.race([\n input.capture,\n new Promise<never>((_, reject) => {\n timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);\n }),\n ]);\n } catch (error) {\n if (error !== timeoutError) throw error;\n // Promise cancellation is not a generic SandboxSession contract. Keep the\n // exact failed activity's result path alive in-process, but permit only the\n // durable archive publication: never provider teardown or a cold commit from\n // an activity Temporal has already rejected. A replaced claim fences this\n // callback; Modal candidates then flow to artifact-ledger GC.\n const settled = () => {\n try {\n input.onLateSettled?.();\n } catch {\n // Settlement bookkeeping cannot surface as an unhandled rejection.\n }\n };\n void input.capture\n .then(input.publishLate, input.observeLateFailure)\n .catch(input.observeLatePublicationFailure)\n .then(settled, settled);\n throw error;\n } finally {\n if (timeout) clearTimeout(timeout);\n }\n}\n\nfunction startSandboxReaperHeartbeat(details: Record<string, string | number | null>): () => void {\n let context: Context;\n try {\n context = Context.current();\n } catch {\n // Direct unit/embedded harnesses intentionally run without Temporal.\n return () => undefined;\n }\n const heartbeat = (): void => {\n try {\n context.heartbeat({ ...details, heartbeatAt: new Date().toISOString() });\n } catch {\n // Heartbeat can synchronously surface Temporal cancellation. This timer\n // must never turn that into an uncaught process-level exception while an\n // already-issued provider capture/teardown is settling under its durable\n // DB claim. The activity result/cancellation boundary remains Temporal's;\n // claim recovery remains the database's.\n }\n };\n heartbeat();\n const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);\n if (\"unref\" in timer && typeof timer.unref === \"function\") timer.unref();\n return () => clearInterval(timer);\n}\n\nexport type SandboxLeaseSweepPlan = {\n drainable: ReapDrainable[];\n timeoutClass: SandboxDrainActivityInput[\"timeoutClass\"];\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n /** Deadline rotations admitted immediately before inventory, so zero-holder\n * boxes enter the same sweep instead of paying another schedule period. */\n rotationsRequested: number;\n};\n\nexport type SandboxIdleCheckpointActivityResult = {\n status: \"checkpointed\" | \"skipped\";\n reason?: string;\n};\n\n/** Children per sweep; the next sweep continues with the least recently tried\n * boxes (each box is tried at most once per snapshot interval). Half the\n * lifecycle worker's activity slots, so a burst of captures never queues the\n * drains that stop idle boxes behind it. */\nconst SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(\n 1,\n Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2),\n);\n\nexport type SandboxLeaseSweepMaintenanceResult = {\n metered: number;\n forceDrained: number;\n rotationsRequested: number;\n modalOrphansTerminated: number;\n};\n\nexport type SandboxDrainActivityResult = { status: \"terminated\" | \"skipped\" };\n\ntype SandboxDrainCaptureAttempt = {\n operationId: string;\n captureId: string;\n attempt: number;\n /** Durable child input, frozen when the workflow was created. Never\n * re-derive this from a later deployment's config/formula. */\n captureTimeoutMs: number;\n};\n\nclass SandboxDrainRecoveryDeferredError extends Error {\n readonly name = \"SandboxDrainRecoveryDeferredError\";\n}\n\n/**\n * Drain captures still physically running in this worker, keyed by exact lease\n * epoch and provider instance. A capture that outlives its attempt's budget\n * keeps running and may publish late through its claim's provider-request\n * lineage. A parallel-read provider (Docker, for one) also lets the next\n * attempt replace that unpublished claim at once and start its own full read,\n * so a capture slower than the retry interval used to gain one more concurrent\n * read and host spool per retry. This keeps one physical capture per lease\n * epoch and instance in each worker; the durable claim still decides which\n * archive publishes, and a capture started by another worker (or one that died\n * with its process) is handled by the ordinary claim replacement. An entry\n * older than the longest durable capture window no longer defers, so a\n * provider call that never settles cannot pin the lease to this worker forever.\n */\nconst settlingDrainCaptures = new Map<string, { startedAtMs: number }>();\n\ntype DrainCaptureIdentity = {\n sandboxGroupId: string;\n leaseEpoch: number;\n instanceId: string | null;\n};\n\nfunction drainCaptureKey(lease: DrainCaptureIdentity & { instanceId: string }): string {\n return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);\n}\n\n/** True while this worker still runs (or publishes, or removes the spool of)\n * a drain capture of this exact lease epoch and instance. */\nfunction sandboxDrainCaptureSettlingInProcess(\n lease: DrainCaptureIdentity,\n nowMs = Date.now(),\n): boolean {\n if (!lease.instanceId) return false;\n const entry = settlingDrainCaptures.get(\n drainCaptureKey({ ...lease, instanceId: lease.instanceId }),\n );\n return entry !== undefined && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;\n}\n\nfunction trackSettlingDrainCapture(lease: DrainCaptureIdentity): () => void {\n if (!lease.instanceId) return () => undefined;\n const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });\n const entry = { startedAtMs: Date.now() };\n settlingDrainCaptures.set(key, entry);\n return () => {\n if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);\n };\n}\n\n/** Stable UUID per logical operation + Temporal attempt. Attempt one uses the\n * operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.\n * This makes the DB receipt idempotent even if an activity delivery is replayed. */\nexport function sandboxDrainCaptureId(operationId: string, attempt: number): string {\n if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {\n throw new Error(\"Sandbox drain operation id is invalid\");\n }\n if (!Number.isSafeInteger(attempt) || attempt <= 0) {\n throw new Error(\"Sandbox drain activity attempt is invalid\");\n }\n if (attempt === 1) return operationId.toLowerCase();\n const bytes = createHash(\"sha256\")\n .update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`)\n .digest()\n .subarray(0, 16);\n bytes[6] = (bytes[6]! & 0x0f) | 0x50;\n bytes[8] = (bytes[8]! & 0x3f) | 0x80;\n const hex = bytes.toString(\"hex\");\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;\n}\n\nfunction sandboxDrainActivityAttempt(): number {\n try {\n const attempt = Context.current().info.attempt;\n return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;\n } catch {\n return 1;\n }\n}\n\nexport function createSandboxLeaseActivities(\n services: () => Promise<ActivityServices>,\n options: SandboxLeaseActivityOptions = {},\n) {\n const terminateBox: TerminateBoxFn =\n (options.terminateBox\n ? async (...args: Parameters<TerminateBoxFn>) => {\n // The injected test seam represents the stop itself; the real\n // implementation invokes this only after capture has succeeded.\n await args[10]?.();\n return options.terminateBox!(...args);\n }\n : null) ??\n (async (\n settings,\n lease,\n observability,\n persistArchive,\n providerCaptureRequestId,\n captureDisposition,\n capturePolicy,\n diskBackedArchives,\n releaseFailedCapture,\n workspaceId,\n beforeProviderStop,\n beforeWorkspaceCapture,\n ) =>\n await terminateProviderBox(\n settings,\n lease,\n observability,\n persistArchive,\n options.createClientForBackend ?? createSandboxClientForBackend,\n terminateModalSandboxById,\n providerCaptureRequestId,\n captureDisposition,\n capturePolicy,\n diskBackedArchives,\n releaseFailedCapture,\n workspaceId,\n beforeProviderStop,\n beforeWorkspaceCapture,\n ));\n const sweepModalOrphans: SweepModalOrphansFn =\n options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;\n const inspectOpenSandboxInventory =\n options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;\n const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;\n const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;\n const resumeIdleCheckpointSession =\n options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;\n /** Embedded idle checkpoints still running, by exact lease epoch. */\n const embeddedIdleCheckpoints = new Set<string>();\n const endedEpochProbeBackoff: EndedEpochProbeBackoff = new Map();\n async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {\n const stopHeartbeat = startSandboxReaperHeartbeat({ phase: \"prepare\" });\n try {\n const { db, settings, observability, objectStorage } = await services();\n const timing = sandboxDrainTiming(settings);\n const commandContainment = {\n // Browsers and desktops follow the general idle grace: unused for it,\n // they stop holding the box (saved browsers resume on demand).\n // Never under a minute, so a visible live view's 30 s heartbeat\n // always lands inside the window.\n idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 60_000),\n idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),\n onIdleInteractionRelease: (\n outcome: Parameters<typeof recordSandboxIdleInteractionRelease>[1],\n ) => recordSandboxIdleInteractionRelease(observability, outcome),\n idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,\n deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),\n onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>\n recordSandboxCommandContainment(observability, outcome),\n onCommandContainmentError: (error: unknown) => {\n observability.warn(\"sandbox reaper: retained command containment inspection failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n },\n };\n if (!settings.sandboxOwnershipEnabled) {\n // Turns skip leases when the flag is off, but Computer/Browser attach\n // still acquire them. Leaving drainable rows untouched strands Desktop\n // behind rotation_in_progress forever. Inventory and drain those rows;\n // do not request NEW deadline rotations or meter warm time.\n const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {\n ...commandContainment,\n viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,\n turnHolderTtlMs: settings.sandboxLeaseTtlMs,\n interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);\n return {\n drainable,\n timeoutClass: timing.timeoutClass,\n snapshotTimeoutMs: timing.snapshotTimeoutMs,\n captureTimeoutMs: timing.captureTimeoutMs,\n rotationsRequested: 0,\n };\n }\n\n // Admit finite-lifetime rotations immediately before inventory. The DB\n // function is bounded and provider-I/O-free; a due zero-holder box becomes\n // draining there and is therefore dispatched in THIS sweep, eliminating\n // the old extra schedule-period availability gap. Failure is isolated: the\n // inventory must still dispatch every row that was already drainable.\n const rotationsRequested = await requestDueSandboxRotationsGlobal(\n db,\n settings.sandboxRotationLeadMs,\n settings.sandboxRotationBatchSize,\n ).catch((error) => {\n observability.warn(\"sandbox reaper: provider-deadline rotation request failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n });\n recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);\n\n // Keep the remaining dispatch-critical path to one bounded DB inventory.\n // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges\n // run only after every drainable box has its own durable child.\n const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {\n ...commandContainment,\n viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,\n // Dead-worker turn holders: a live holder is touched every 10s from the\n // moment it is registered (resumeBoxForTurn's holder-liveness loop covers\n // the whole warmup — waitForWarm/establish/display-stack — and the turn\n // heartbeat covers the run), so NO live path is ever silent for more than\n // one tick. The ordinary warm lease TTL is therefore already a generous\n // dead-worker horizon. Canonical holders whose turn is durably closed are\n // removed immediately by the DB function; no path-duration guess belongs\n // in this lifecycle contract.\n turnHolderTtlMs: settings.sandboxLeaseTtlMs,\n interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);\n if (drainableInventory.length > drainable.length) {\n observability.info(\"sandbox reaper: drain inventory continues next sweep\", {\n admitted: drainable.length,\n deferred: drainableInventory.length - drainable.length,\n });\n }\n return {\n drainable,\n timeoutClass: timing.timeoutClass,\n snapshotTimeoutMs: timing.snapshotTimeoutMs,\n captureTimeoutMs: timing.captureTimeoutMs,\n rotationsRequested,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Inventory for the idle checkpoint sweep. DB only: warm boxes no turn\n * holds, kept warm by viewers, interactions or running background commands,\n * with writes no capture may cover yet and neither a capture attempt nor an\n * idle checkpoint attempt within the snapshot interval.\n * Least recently tried first; the next sweep continues past the batch. */\n async function listIdleSandboxCheckpoints(): Promise<SandboxIdleCheckpointPlan> {\n const stopHeartbeat = startSandboxReaperHeartbeat({ phase: \"idle_checkpoint_inventory\" });\n try {\n const { db, settings } = await services();\n // The idle capture claims the warm checkpoint budget, not the drain's;\n // freeze the matching activity class with each child.\n const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));\n if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {\n return { targets: [], timeoutClass };\n }\n const rows = await listIdleCheckpointCandidates(db, {\n limit: SANDBOX_IDLE_CHECKPOINT_BATCH,\n intervalMs: settings.sandboxSnapshotIntervalMs,\n });\n return {\n targets: rows.map(({ accountId: _accountId, ...target }) => target),\n timeoutClass,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Checkpoint one warm box between turns. Reuses the turn's warm checkpoint\n * path with an idle owner: the exact capture claim requires that only\n * viewers, interactions and active background commands hold the\n * box and that no other request is open, then the native Modal snapshot runs\n * around them and is published one generation behind the workspace. */\n async function checkpointIdleSandboxLease(\n input: SandboxIdleCheckpointActivityInput,\n ): Promise<SandboxIdleCheckpointActivityResult> {\n const target = input.target;\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"idle_checkpoint\",\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n leaseEpoch: target.leaseEpoch,\n instanceId: target.instanceId,\n });\n try {\n const { db, settings, observability, objectStorage } = await services();\n const fields = {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),\n leaseEpoch: target.leaseEpoch,\n };\n const skipped = (reason: string): SandboxIdleCheckpointActivityResult => {\n observability.info(\"sandbox reaper: idle checkpoint skipped\", { ...fields, reason });\n return { status: \"skipped\", reason };\n };\n if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {\n return skipped(\"disabled\");\n }\n const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);\n if (\n !lease ||\n lease.liveness !== \"warm\" ||\n lease.leaseEpoch !== target.leaseEpoch ||\n lease.instanceId !== target.instanceId\n ) {\n return skipped(\"lease_fenced\");\n }\n // A turn's own checkpoints cover a box it holds; a box nothing holds\n // belongs to the idle drain.\n if (lease.turnHolders > 0) return skipped(\"turn_held\");\n if (lease.refcount === 0) return skipped(\"drain_owned\");\n const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);\n // Counted as tried whatever happens next, so a refused or failing box\n // waits a full interval instead of taking a batch slot every sweep.\n const stamped = await recordIdleCheckpointAttempt(db, {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n expectedEpoch: target.leaseEpoch,\n expectedInstanceId: target.instanceId,\n });\n if (!stamped) return skipped(\"lease_fenced\");\n if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {\n return skipped(\"capture_policy\");\n }\n if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped(\"clean\");\n if (lease.archiveCapture) {\n const tookOver = await releaseWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n captureId: lease.archiveCapture.id,\n expectedEpoch: target.leaseEpoch,\n expectedInstanceId: target.instanceId,\n onlyExpiredWarmTakeover: true,\n });\n if (!tookOver) return skipped(\"capture_in_progress\");\n observability.warn(\"sandbox reaper: released an expired warm capture claim\", {\n ...fields,\n captureId: lease.archiveCapture.id,\n });\n }\n let session: unknown;\n try {\n session = await resumeIdleCheckpointSession(settings, lease);\n } catch (error) {\n observability.warn(\"sandbox reaper: idle checkpoint resume failed\", {\n ...fields,\n error: error instanceof Error ? error.message : String(error),\n });\n return skipped(\"provider_error\");\n }\n if (session === null) return skipped(\"provider_missing\");\n const capture = maybePersistWarmWorkspaceSnapshot(\n { db, settings, objectStorage, observability },\n {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n idleCheckpoint: true,\n },\n session,\n target.leaseEpoch,\n );\n const persisted = await capture;\n // Return only after the provider capture settled and its exact claim\n // was released, so the activity never outlives its own gate unobserved.\n await capture.settled;\n if (!persisted) return skipped(\"not_captured\");\n observability.info(\"sandbox reaper: idle checkpoint published\", fields);\n return { status: \"checkpointed\" };\n } finally {\n stopHeartbeat();\n }\n }\n\n async function drainSandboxLease(\n input: SandboxDrainActivityInput,\n ): Promise<SandboxDrainActivityResult> {\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"drain\",\n workspaceId: input.target.workspaceId,\n sandboxGroupId: input.target.sandboxGroupId,\n leaseEpoch: input.target.leaseEpoch,\n instanceId: input.target.instanceId,\n });\n try {\n const { db, settings, observability, objectStorage, bus } = await services();\n assertSandboxDrainInputTiming(input);\n const drainSettings =\n settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs\n ? settings\n : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };\n const row = input.target;\n const activityAttempt = sandboxDrainActivityAttempt();\n const captureAttempt: SandboxDrainCaptureAttempt = {\n operationId: input.operationId,\n captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),\n attempt: activityAttempt,\n captureTimeoutMs: input.captureTimeoutMs,\n };\n try {\n const drainedCold = await terminateDrainableBox(\n db,\n drainSettings,\n row,\n observability,\n terminateBox,\n probeDrainableProvider,\n captureAttempt,\n objectStorage,\n bus,\n );\n return { status: drainedCold ? \"terminated\" : \"skipped\" };\n } catch (error) {\n const lease = await readLease(db, row.workspaceId, row.sandboxGroupId).catch(() => null);\n const details = {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n leaseEpoch: row.leaseEpoch,\n backend: lease?.backend ?? \"unknown\",\n instanceId: lease?.instanceId ?? row.instanceId,\n captureId: lease?.archiveCapture?.id ?? null,\n captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,\n captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,\n timeoutClass: input.timeoutClass,\n error: error instanceof Error ? error.message : String(error),\n };\n if (error instanceof SandboxDrainRecoveryDeferredError) {\n observability.info(\"sandbox reaper: per-sandbox recovery deferred safely\", details);\n } else {\n observability.warn(\"sandbox reaper: per-sandbox drain failed\", details);\n }\n throw error;\n }\n } finally {\n stopHeartbeat();\n }\n }\n\n async function maintainSandboxLeaseSweep(\n input: SandboxLeaseSweepMaintenanceInput,\n ): Promise<SandboxLeaseSweepMaintenanceResult> {\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"maintain\",\n examined: input.examined,\n started: input.started,\n });\n try {\n const service = await services();\n const { db, settings, observability } = service;\n const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(\n (error) => {\n observability.warn(\"system-update outbox reconciliation failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { claimed: 0, delivered: 0, failed: 1 };\n },\n );\n const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(\n (error) => {\n observability.warn(\"session-list snapshot reconciliation failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n },\n );\n if (expiredSessionListSnapshots > 0) {\n observability.info(\"expired session-list snapshots reaped\", {\n deleted: expiredSessionListSnapshots,\n });\n }\n if (parentUpdates.claimed > 0) {\n observability.info(\"system-update outbox reconciled\", parentUpdates);\n }\n\n let metered = 0;\n let forceDrained = 0;\n const rotationsRequested = input.rotationsRequested;\n let modalOrphansTerminated = 0;\n\n // Connected Machine commands are not managed-sandbox lease state. Their\n // immutable launch locators must reconcile even when managed ownership is\n // disabled, otherwise an exact runner exit/loss proof can remain stranded.\n await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);\n await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);\n\n // Disabling new lease ownership cannot strand a previously dispatched\n // operation. This only attributes a positive receipt; normal draining\n // already runs in both ownership modes.\n try {\n await reconcileModalProviderCreates(\n db,\n settings,\n observability,\n options.findModalProviderCreateReceipt,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal create recovery inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n\n if (!settings.sandboxOwnershipEnabled) {\n // Inventory remains useful while provider ownership is intentionally\n // disabled, and this scheduled activity is its single projection owner.\n await refreshQueueLeaseAndCreditGauges(\n db,\n settings,\n observability,\n inspectOpenSandboxInventory,\n );\n return {\n metered,\n forceDrained,\n rotationsRequested,\n modalOrphansTerminated,\n };\n }\n\n // Warm metering and cost-stop run after current drain children launch.\n // Freshly force-drained billing rows are admission-fenced immediately and\n // receive their own provider child on the next bounded schedule tick.\n const meterResult = await accrueWarmTick(db, settings, observability);\n metered = meterResult.accrued;\n const forceDrainWorkspaceIds = new Set<string>();\n if (\n settings.sandboxWarmBillingMode === \"credits\" ||\n settings.sandboxMaxWarmSecondsPerWorkspace > 0\n ) {\n for (const workspaceId of meterResult.workspaceIds) {\n forceDrainWorkspaceIds.add(workspaceId);\n }\n }\n try {\n for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {\n forceDrainWorkspaceIds.add(workspaceId);\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: viewer force-drain workspace read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n forceDrained = await forceDrainOverLimitWorkspaces(\n db,\n settings,\n forceDrainWorkspaceIds,\n observability,\n );\n\n // Terminal-owner process reconciliation is ancillary to lease dispatch.\n // Exact provider proof is durably checkpointed before settlement; every\n // ambiguous outcome remains claimed/deferred for another sweep.\n await reconcileTerminalRetainedProcesses(\n db,\n settings,\n observability,\n probeRetainedProcess,\n options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,\n service.bus,\n );\n\n // A request or PTY left on a Modal box whose lease epoch has ended can\n // never progress once that box is gone; settle it on exact proof.\n await settleEndedEpochBlockers(\n db,\n settings,\n observability,\n options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,\n endedEpochProbeBackoff,\n );\n\n try {\n modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);\n recordSandboxOrphansTerminated(observability, modalOrphansTerminated);\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal orphan sweep failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n\n const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(\n (error) => {\n observability.warn(\"sandbox reaper: checkpoint artifact GC failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { claimed: 0, deleted: 0, failed: 0 };\n },\n );\n\n await refreshQueueLeaseAndCreditGauges(\n db,\n settings,\n observability,\n inspectOpenSandboxInventory,\n );\n\n if (\n input.examined > 0 ||\n metered > 0 ||\n forceDrained > 0 ||\n modalOrphansTerminated > 0 ||\n rotationsRequested > 0 ||\n checkpointGc.claimed > 0\n ) {\n observability.info(\"sandbox reaper dispatched\", {\n drainable: input.examined,\n childrenStarted: input.started,\n childrenAlreadyRunning: input.alreadyRunning,\n childrenStartFailed: input.startFailed,\n metered,\n forceDrained,\n modalOrphansTerminated,\n rotationsRequested,\n checkpointArtifactsClaimed: checkpointGc.claimed,\n checkpointArtifactsDeleted: checkpointGc.deleted,\n checkpointArtifactsFailed: checkpointGc.failed,\n });\n }\n\n return {\n metered,\n forceDrained,\n rotationsRequested,\n modalOrphansTerminated,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Direct/embedded compatibility harness. Production uses the split Temporal\n * workflow above. The production activity is also the rolling-deploy bridge:\n * its workflow command stays byte-for-byte compatible while this activity\n * starts V2 through the injected client edge. Old workers may still execute\n * the bounded composite during rollout; exact DB claims keep both paths safe. */\n async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {\n const service = await services();\n if (service.startSandboxReaperWorkflow) {\n const dispatch = await service.startSandboxReaperWorkflow();\n service.observability.info(\"sandbox reaper routed to versioned workflow\", {\n dispatch,\n });\n return {\n examined: 0,\n terminated: 0,\n skipped: 0,\n metered: 0,\n forceDrained: 0,\n modalOrphansTerminated: 0,\n };\n }\n\n const plan = await prepareSandboxLeaseSweep();\n const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);\n const outcomes = await Promise.allSettled(\n admitted.map((target) =>\n drainSandboxLease({\n target,\n timeoutClass: plan.timeoutClass,\n snapshotTimeoutMs: plan.snapshotTimeoutMs,\n captureTimeoutMs: plan.captureTimeoutMs,\n operationId: randomUUID(),\n }),\n ),\n );\n // The embedded path has no idle checkpoint sweep workflow; run the same\n // inventory and per-box activity. Captures run detached so a provider\n // snapshot never holds up this sweep's drains; a box whose capture is\n // still running is not started twice. Failure defers to the next sweep.\n try {\n const { targets, timeoutClass } = await listIdleSandboxCheckpoints();\n for (const target of targets) {\n const key = sandboxIdleCheckpointWorkflowId(target);\n if (embeddedIdleCheckpoints.has(key)) continue;\n embeddedIdleCheckpoints.add(key);\n void checkpointIdleSandboxLease({ target, timeoutClass })\n .catch((error: unknown) => {\n service.observability.warn(\"sandbox reaper: idle checkpoint failed\", {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),\n error: error instanceof Error ? error.message : String(error),\n });\n })\n .finally(() => embeddedIdleCheckpoints.delete(key));\n }\n } catch (error) {\n service.observability.warn(\"sandbox reaper: idle checkpoint inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n const terminated = outcomes.filter(\n (outcome) => outcome.status === \"fulfilled\" && outcome.value.status === \"terminated\",\n ).length;\n const skipped = outcomes.length - terminated;\n const maintenance = await maintainSandboxLeaseSweep({\n examined: admitted.length,\n started: admitted.length,\n alreadyRunning: 0,\n startFailed: 0,\n rotationsRequested: plan.rotationsRequested,\n });\n return {\n examined: admitted.length,\n terminated,\n skipped,\n metered: maintenance.metered,\n forceDrained: maintenance.forceDrained,\n modalOrphansTerminated: maintenance.modalOrphansTerminated,\n };\n }\n\n return {\n prepareSandboxLeaseSweep,\n drainSandboxLease,\n listIdleSandboxCheckpoints,\n checkpointIdleSandboxLease,\n maintainSandboxLeaseSweep,\n reapSandboxLeases,\n };\n}\n\n/** Resume the exact warm box by id for an idle checkpoint, with no cold\n * restore and no replacement. Returns null when the provider no longer has\n * it; the ordinary loss paths own that box. The session is not closed: the\n * box keeps running its commands. */\nexport async function resumeIdleCheckpointSessionAtProvider(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n): Promise<unknown | null> {\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n if (backend !== \"modal\" || !lease.instanceId || !lease.resumeState) {\n throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);\n }\n if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {\n throw new Error(\"idle checkpoint lease envelope does not name its provider instance\");\n }\n const client = createSandboxClientForBackend(backend as never, settings) as\n | RetainedProcessProbeClient\n | undefined;\n if (!client?.resume) {\n throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);\n }\n try {\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n const resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n lease.instanceId,\n );\n if (resumedState === undefined) {\n throw new Error(`sandbox backend ${backend} returned no resumable provider state`);\n }\n const resumed = await withRetainedProcessProbeTimeout(\n resumeExactSandboxSession(client, backend, resumedState, lease.instanceId),\n );\n return resumed.session;\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n throw new Error(\"idle checkpoint provider resume timed out\", { cause: error });\n }\n if (isProviderSandboxNotFoundError(client.backendId, error)) return null;\n throw error;\n }\n}\n\nconst CHECKPOINT_GC_LIMIT = 50;\nconst CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;\n/** The checkpoint artifact GC claim TTL, which the reaper's work check previews. */\nexport const SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;\nconst CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;\nconst CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;\nconst SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;\n\ntype ConnectedCommandReconciliationOutcome =\n | \"claim_failed\"\n | \"provider_running\"\n | \"provider_offline\"\n | \"provider_error\"\n | \"proof_exited\"\n | \"proof_lost\"\n | \"proof_checkpoint_failed\"\n | \"settled_exited\"\n | \"settled_lost\"\n | \"output_unavailable\"\n | \"output_ack_published\"\n | \"output_released\"\n | \"settlement_failed\"\n | \"defer_failed\";\n\nfunction recordConnectedCommandReconciliation(\n observability: ActivityServices[\"observability\"],\n outcome: ConnectedCommandReconciliationOutcome,\n): void {\n observability.incrementCounter({\n name: \"opengeni_connected_background_command_reconciliation_total\",\n help: \"Exact-locator Connected Machine background-command reconciliation outcomes.\",\n labels: { outcome },\n });\n}\n\n/** Why a connected command's runner can never answer for it again. */\ntype ConnectedRunnerRetirement =\n | \"enrollment_revoked\"\n | \"connection_replaced\"\n | \"connection_stopped\";\ntype ConnectedRunnerLookups = Map<string, Promise<ConnectedRunnerRetirement | null>>;\n\n/** Offline reasons after which the runner process has exited for good. */\nconst CONNECTED_RUNNER_STOPPED_REASONS = new Set([\n \"GOING_OFFLINE_REASON_USER_STOP\",\n \"GOING_OFFLINE_REASON_UPDATE\",\n \"GOING_OFFLINE_REASON_HOST_SHUTDOWN\",\n]);\n\n/**\n * A connected command's operation lives in the memory of the runner process\n * that launched it and is addressed through that process's connection\n * instance. No responder on that exact address is not proof by itself: the\n * machine may be offline or reconnecting. It is proof once the enrollment\n * also retires that process, by the same rules the running-command claim\n * applies: the enrollment was revoked, a different runner process now holds\n * the connection, or the runner went offline by stopping, updating or shutting\n * down. Anything else, including a failed read, keeps the existing retry. One\n * read per runner address per sweep.\n */\nasync function connectedCommandRunnerRetirement(\n db: ActivityServices[\"db\"],\n claim: { controlWorkspaceId: string; enrollmentId: string; connectionInstanceId: string },\n error: unknown,\n lookups: ConnectedRunnerLookups,\n): Promise<ConnectedRunnerRetirement | null> {\n if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {\n return null;\n }\n const key = JSON.stringify([\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ]);\n let lookup = lookups.get(key);\n if (!lookup) {\n lookup = (async (): Promise<ConnectedRunnerRetirement | null> => {\n const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);\n if (!enrollment) return null;\n if (enrollment.status === \"revoked\") return \"enrollment_revoked\";\n if (enrollment.connectionInstanceId) {\n return enrollment.connectionInstanceId === claim.connectionInstanceId\n ? null\n : \"connection_replaced\";\n }\n return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? \"\")\n ? \"connection_stopped\"\n : null;\n })().catch(() => null);\n lookups.set(key, lookup);\n }\n return await lookup;\n}\n\nasync function forEachWithConcurrency<T>(\n values: readonly T[],\n concurrency: number,\n visit: (value: T, index: number) => Promise<void>,\n): Promise<void> {\n if (values.length === 0) return;\n const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));\n let nextIndex = 0;\n await Promise.all(\n Array.from({ length: workerCount }, async () => {\n while (nextIndex < values.length) {\n const index = nextIndex;\n nextIndex += 1;\n await visit(values[index]!, index);\n }\n }),\n );\n}\n\nasync function gcSandboxCheckpointArtifacts(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<{ claimed: number; deleted: number; failed: number }> {\n // Explicit OPENGENI_MODAL_TOKEN_* settings are optional: the SDK also supports\n // an operator-provisioned Modal profile. Durable artifacts must not become\n // immortal merely because credentials arrive through that supported path.\n const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(\n (error) => {\n observability.warn(\"sandbox reaper: legacy Modal checkpoint discovery failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n },\n );\n if (adopted > 0) {\n recordSandboxCheckpointArtifactOutcome(observability, \"legacy_adopted\", adopted);\n observability.info(\"sandbox reaper: adopted legacy Modal checkpoint receipts\", { adopted });\n }\n const claimId = crypto.randomUUID();\n const claims = await claimSandboxCheckpointArtifactsForGc(db, {\n claimId,\n limit: CHECKPOINT_GC_LIMIT,\n claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS,\n });\n recordSandboxCheckpointArtifactOutcome(observability, \"claimed\", claims.length);\n let deleted = 0;\n let failed = 0;\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n try {\n if (claim.providerBackend !== \"modal\") {\n throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);\n }\n await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);\n const settled = await settleSandboxCheckpointArtifactGc(db, {\n artifactId: claim.id,\n claimId,\n deleted: true,\n retryAfterMs: 1_000,\n });\n if (settled) {\n deleted += 1;\n recordSandboxCheckpointArtifactOutcome(observability, \"deleted\");\n }\n } catch (error) {\n failed += 1;\n recordSandboxCheckpointArtifactOutcome(observability, \"delete_failed\");\n const retryAfterMs = Math.min(\n 6 * 60 * 60_000,\n 5_000 * 2 ** Math.min(claim.deleteAttempts, 17),\n );\n await settleSandboxCheckpointArtifactGc(db, {\n artifactId: claim.id,\n claimId,\n deleted: false,\n error: error instanceof Error ? error.message : String(error),\n retryAfterMs,\n }).catch(() => false);\n observability.warn(\"sandbox reaper: checkpoint artifact delete failed\", {\n artifactId: claim.id,\n providerBackend: claim.providerBackend,\n objectKind: claim.objectKind,\n attempt: claim.deleteAttempts,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n const pruned = await pruneDeletedSandboxCheckpointArtifacts(\n db,\n CHECKPOINT_TOMBSTONE_RETENTION_MS,\n CHECKPOINT_TOMBSTONE_PRUNE_LIMIT,\n );\n recordSandboxCheckpointArtifactOutcome(observability, \"tombstone_pruned\", pruned);\n return { claimed: claims.length, deleted, failed };\n}\n\nasync function adoptLegacyModalCheckpointReceipts(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n const slots = await listLegacyModalCheckpointSlots(db, 25);\n if (slots.length === 0) return 0;\n const bindings = new Map<\n string,\n Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>\n >();\n let adopted = 0;\n await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {\n try {\n let binding = bindings.get(slot.instanceId);\n if (!binding) {\n binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);\n bindings.set(slot.instanceId, binding);\n }\n const resolvedBinding = await binding;\n if (\n await adoptLegacyModalCheckpointArtifact(db, {\n accountId: slot.accountId,\n workspaceId: slot.workspaceId,\n sandboxGroupId: slot.sandboxGroupId,\n leaseId: slot.leaseId,\n leaseEpoch: slot.leaseEpoch,\n workspaceGeneration: slot.workspaceGeneration,\n slot: slot.slot,\n archiveBase64: slot.archiveBase64,\n descriptor: slot.descriptor,\n providerBindingKey: resolvedBinding.key,\n providerBinding: resolvedBinding.binding,\n })\n ) {\n adopted += 1;\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: legacy Modal checkpoint adoption failed\", {\n workspaceId: slot.workspaceId,\n sandboxGroupId: slot.sandboxGroupId,\n slot: slot.slot,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n return adopted;\n}\n\n/** Reconcile commands only against the immutable physical locator copied at\n * adoption. A successor enrollment connection is intentionally irrelevant: no\n * query or cancellation is ever rebound to it. */\nexport async function reconcileConnectedMachineBackgroundCommands(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n bus: ActivityServices[\"bus\"],\n controlRpcOverride?: ControlRpc,\n): Promise<void> {\n const dueBefore = new Date();\n const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;\n const offlineConnections = new Set<string>();\n const firstConnectionProbes = new Map<string, Promise<void>>();\n // Drain the fixed due frontier with the existing concurrency. Yield to the\n // remaining maintenance work each period; retries belong to a later sweep.\n do {\n const claimId = crypto.randomUUID();\n let claims: ConnectedMachineBackgroundCommandClaim[];\n try {\n claims = await claimConnectedMachineSessionBackgroundCommands(db, {\n claimId,\n dueBefore,\n limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,\n claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: connected-command claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n\n const controlRpc =\n controlRpcOverride ??\n new NatsControlRpc(\n async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),\n );\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n let proof = claim.proof;\n let outputReplay: Awaited<ReturnType<OpStreamExecClient[\"readExisting\"]>> | null = null;\n const connectionKey = JSON.stringify([\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ]);\n if (!proof) {\n const firstProbe = firstConnectionProbes.get(connectionKey);\n if (firstProbe) await firstProbe;\n if (offlineConnections.has(connectionKey)) {\n await deferConnectedCommandClaim(db, settings, observability, claim, \"provider_offline\");\n return;\n }\n // Share only the initial connection observation, never another command's result.\n let finishFirstProbe: (() => void) | undefined;\n if (!firstProbe) {\n firstConnectionProbes.set(\n connectionKey,\n new Promise<void>((resolve) => {\n finishFirstProbe = resolve;\n }),\n );\n }\n try {\n proof = await probeConnectedMachineBackgroundCommand(\n claim,\n controlRpc,\n Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n ),\n );\n // Replay currently retained frames for running as well as completed\n // commands. This is attach-only and leaves final ACK/observation alone.\n if (proof?.outcome !== \"lost\") {\n if (!bus.getOpStreamConnection)\n throw new Error(\"Connected command output transport unavailable\");\n const client = new OpStreamExecClient({\n workspaceId: claim.controlWorkspaceId,\n agentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n epoch: 0,\n controlRpc,\n rpcSubject: subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ),\n transport: new NatsOpStreamTransport(\n async () => bus.getOpStreamConnection?.() ?? null,\n ),\n controlTimeoutMs: Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n ),\n retryClock: defaultSelfhostedRetryClock,\n journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },\n });\n outputReplay = await replayConnectedCommandOutput(\n client,\n claim.opId,\n Boolean(proof),\n async (frames) => {\n await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);\n },\n );\n }\n if (!proof) {\n await deferConnectedCommandClaim(\n db,\n settings,\n observability,\n claim,\n \"provider_running\",\n );\n return;\n }\n } catch (error) {\n const offline =\n error instanceof SelfhostedControlError &&\n (error.agentOffline || error.reason === \"agent_reconnecting\" || error.draining);\n if (offline) offlineConnections.add(connectionKey);\n if (!offline) {\n observability.warn(\"sandbox reaper: connected-command provider probe failed\", {\n commandId: claim.commandId,\n enrollmentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n await deferConnectedCommandClaim(\n db,\n settings,\n observability,\n claim,\n offline ? \"provider_offline\" : \"provider_error\",\n );\n return;\n } finally {\n finishFirstProbe?.();\n }\n\n try {\n await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });\n recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"proof_checkpoint_failed\");\n observability.warn(\"sandbox reaper: connected-command proof checkpoint failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n }\n\n try {\n const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });\n if (!settlement.settled) {\n throw new Error(\"Connected command settlement lost its exact claim or proof\");\n }\n const receipt =\n outputReplay?.outputReceipt ??\n (outputReplay?.status === \"running\" ? outputReplay.terminal?.outputReceipt : undefined);\n const terminal =\n outputReplay?.status === \"completed\" ? outputReplay : outputReplay?.terminal;\n if (proof.outcome === \"exited\" && receipt && terminal) {\n await recordConnectedCommandOutputConsumption(db, {\n ...claim,\n receipt,\n exitCode: terminal.outcome.response.exitCode,\n });\n }\n if (settlement.events.length > 0) {\n try {\n await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);\n } catch (error) {\n observability.warn(\"sandbox reaper: connected-command event fanout failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"settlement_failed\");\n observability.warn(\"sandbox reaper: connected-command settlement failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n await deferConnectedCommandClaim(db, settings, observability, claim, \"settlement_failed\");\n }\n });\n if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;\n } while (Date.now() < deadline);\n}\n\nexport async function replayConnectedCommandOutput(\n client: Pick<OpStreamExecClient, \"readExisting\">,\n opId: string,\n terminalKnown: boolean,\n capture: Parameters<OpStreamExecClient[\"readExisting\"]>[2],\n): Promise<Awaited<ReturnType<OpStreamExecClient[\"readExisting\"]>>> {\n // A completed operation can need several bounded reads to replay its retained\n // frames. Keep the reader's integrity checkpoint until the exit is verified;\n // replacing it after each partial read would restart forever at frame one.\n let capturedThrough = 0n;\n while (true) {\n const before = capturedThrough;\n const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, capture);\n if (!terminalKnown || replay.status === \"completed\" || replay.terminal) return replay;\n // Quiet jobs can retain many heartbeat frames (or an incomplete UTF-8\n // chunk). These are real replay progress even when capture receives no\n // stdout/stderr. Use the reader's verified, contiguous protocol frontier.\n capturedThrough = BigInt(replay.replaySequence);\n // This is a finite terminal drain, not a poller for a running command.\n // Failed persistence, transport/integrity errors, and no-progress reads\n // return control to normal reconciliation without licensing settlement.\n if (capturedThrough === before) {\n throw new Error(\"Connected command terminal output replay has not reached its exit frontier\");\n }\n }\n}\n\n/** Drain the durable terminal-output obligation independently of the launching\n * attempt or model completion notice. Existing terminal rows receive no inferred\n * receipt: they must replay and capture all retained bytes before any final ACK. */\nexport async function reconcileConnectedCommandOutputReleases(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n bus: ActivityServices[\"bus\"],\n controlRpcOverride?: ControlRpc,\n): Promise<void> {\n const dueBefore = new Date();\n const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;\n const controlRpc =\n controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());\n const runnerLookups: ConnectedRunnerLookups = new Map();\n do {\n let claims: Awaited<ReturnType<typeof claimConnectedCommandOutputReleases>>;\n try {\n claims = await claimConnectedCommandOutputReleases(db, {\n claimId: crypto.randomUUID(),\n dueBefore,\n limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,\n claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: connected-command output claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n let receipt = claim.receipt;\n try {\n if (!bus.getOpStreamConnection)\n throw new Error(\"Connected command output transport unavailable\");\n const rpcSubject = subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n );\n const controlTimeoutMs = Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n const client = new OpStreamExecClient({\n workspaceId: claim.controlWorkspaceId,\n agentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n epoch: 0,\n controlRpc,\n rpcSubject,\n transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),\n controlTimeoutMs,\n retryClock: defaultSelfhostedRetryClock,\n journal: {\n attachGeneration: () => String(Date.now()),\n persistSettled: () => {\n throw new Error(\"Background output release requires its database receipt\");\n },\n },\n });\n if (!receipt) {\n const status = await querySelfhostedOp({\n controlRpc,\n rpcSubject,\n opId: claim.opId,\n controlTimeoutMs,\n });\n if (status.state === OpState.OP_STATE_LOST) {\n if (\n !(await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: \"unavailable\",\n retryAfterMs: 0,\n }))\n )\n throw new Error(\"Connected command output loss no longer owns its exact claim\");\n recordConnectedCommandReconciliation(observability, \"output_unavailable\");\n return;\n }\n if (status.state !== OpState.OP_STATE_COMPLETE)\n throw new Error(\"Terminal command output returned a live operation\");\n const replay = await replayConnectedCommandOutput(\n client,\n claim.opId,\n true,\n async (frames) =>\n captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames),\n );\n const terminal = replay.status === \"completed\" ? replay : replay.terminal;\n receipt =\n replay.outputReceipt ??\n (replay.status === \"running\" ? replay.terminal?.outputReceipt : undefined) ??\n null;\n if (!terminal || !receipt)\n throw new Error(\"Connected command output replay has no complete custody receipt\");\n await recordConnectedCommandOutputConsumption(db, {\n ...claim,\n receipt,\n exitCode: terminal.outcome.response.exitCode,\n });\n }\n const outcome = await client.releaseCapturedOutput(claim.opId, receipt);\n if (\n !(await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome,\n retryAfterMs: outcome === \"published\" ? settings.sandboxLeaseReaperPeriodMs : 0,\n }))\n )\n throw new Error(\"Connected command output release no longer owns its exact claim\");\n recordConnectedCommandReconciliation(\n observability,\n outcome === \"published\" ? \"output_ack_published\" : \"output_released\",\n );\n } catch (error) {\n const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);\n // A retired runner retains nothing: captured output is released, and\n // output it never handed over is recorded as missing.\n if (\n retired &&\n (await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: receipt ? \"not_retained\" : \"unavailable\",\n retryAfterMs: 0,\n }).catch(() => false))\n ) {\n recordConnectedCommandReconciliation(\n observability,\n receipt ? \"output_released\" : \"output_unavailable\",\n );\n return;\n }\n await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: \"retry\",\n retryAfterMs: Math.min(\n 5 * 60_000,\n Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) *\n 2 ** Math.min(4, claim.reconcileAttempts - 1),\n ),\n }).catch(() => false);\n observability.warn(\"sandbox reaper: connected-command output release deferred\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;\n } while (Date.now() < deadline);\n}\n\nexport async function probeConnectedMachineBackgroundCommand(\n claim: ConnectedMachineBackgroundCommandClaim,\n controlRpc: ControlRpc,\n controlTimeoutMs: number,\n observedAt = new Date(),\n): Promise<ConnectedMachineBackgroundCommandProof | null> {\n const input = {\n controlRpc,\n rpcSubject: subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ),\n opId: claim.opId,\n controlTimeoutMs,\n };\n const status =\n claim.state === \"stopping\" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);\n return connectedCommandProofFromStatus(status, observedAt);\n}\n\nexport function connectedCommandProofFromStatus(\n status: OpStatus,\n observedAt: Date,\n): ConnectedMachineBackgroundCommandProof | null {\n switch (status.state) {\n case OpState.OP_STATE_ACCEPTED:\n case OpState.OP_STATE_RUNNING:\n return null;\n case OpState.OP_STATE_COMPLETE: {\n const exit = status.exit;\n if (!exit) {\n throw new Error(\"Connected command completed without an exit record\");\n }\n const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, \"_\").slice(0, 128);\n const reason = failureCode\n ? `op_failure_${failureCode}`\n : exit.cancelled\n ? \"op_cancelled\"\n : exit.timedOut\n ? \"op_timed_out\"\n : \"op_exit\";\n return {\n outcome: \"exited\",\n exitCode: exit.exitCode,\n reason,\n ...(failureCode\n ? {\n failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },\n }\n : {}),\n observedAt,\n };\n }\n case OpState.OP_STATE_LOST:\n return {\n outcome: \"lost\",\n exitCode: null,\n reason:\n status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED\n ? \"op_lost_evicted\"\n : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED\n ? \"op_lost_agent_restarted\"\n : \"op_lost_unspecified\",\n observedAt,\n };\n default:\n throw new Error(`Connected command returned invalid op state ${status.state}`);\n }\n}\n\nasync function deferConnectedCommandClaim(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n claim: ConnectedMachineBackgroundCommandClaim,\n outcome: \"provider_running\" | \"provider_offline\" | \"provider_error\" | \"settlement_failed\",\n): Promise<void> {\n const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));\n const retryAfterMs = Math.min(\n 5 * 60_000,\n Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,\n );\n try {\n if (\n !(await deferConnectedMachineBackgroundCommandReconciliation(db, {\n claim,\n outcome,\n retryAfterMs,\n }))\n ) {\n throw new Error(\"Connected command defer lost its exact claim\");\n }\n recordConnectedCommandReconciliation(observability, outcome);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"defer_failed\");\n observability.warn(\"sandbox reaper: connected-command defer failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\n/** Tuples listed per sweep (random order, the next sweep lists again) and the\n * share of the reaper tick the probes may use, so the pass never crowds out\n * the orphan sweep and GC after it. A box the provider cannot yet prove gone is\n * re-probed on an exponential backoff instead of on every tick. */\nconst ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;\nconst ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 60_000;\nconst ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 60_000;\nconst ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 60_000;\n\n/** Per worker process: when each historical box may be probed again. */\nexport type EndedEpochProbeBackoff = Map<string, { failures: number; nextProbeAtMs: number }>;\n\n/**\n * Settle open requests still recorded on a Modal box whose lease\n * epoch has ended (a legacy loss, or a cold commit that left them). Lease\n * succession alone proves nothing about the old box, so each exact tuple is\n * settled only after its historical sandbox is observed terminated. NotFound\n * is ambiguous here (deletion or a rotated Modal credential workspace) and,\n * like a running or unreachable box, keeps the rows for a later probe (with\n * backoff) or an operator.\n *\n * The same observation also ends a turn's wait on sandbox setup whose start\n * outcome is unknown: once the exact box all its setup ran on is terminated,\n * nothing it may have started can still run, so the turn's setup reruns its\n * idempotent preparation (`reconcileCompletedSandboxSetup`). A start that\n * failed before any process was recorded otherwise had nothing to exit and\n * stayed recovering forever.\n */\nasync function settleEndedEpochBlockers(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,\n backoff: EndedEpochProbeBackoff,\n): Promise<void> {\n // Same scope as the Modal orphan sweep: no Modal API calls from workers of\n // a deployment that never configured Modal.\n if (settings.sandboxBackend !== \"modal\" && !settings.modalTokenId && !settings.modalTokenSecret) {\n return;\n }\n const startedAtMs = Date.now();\n // One observation per box: several tuples and turns can name the same box.\n const observed = new Map<string, string>();\n const kept: Record<string, number> = {};\n /** The box's status, or null when it is backing off or the budget is spent. */\n async function observe(\n instanceId: string,\n details: Record<string, unknown>,\n ): Promise<string | null> {\n const known = observed.get(instanceId);\n if (known !== undefined) return known;\n const nowMs = Date.now();\n if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;\n if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;\n let status: string;\n try {\n status = (\n await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(settings, instanceId, null),\n )\n ).status;\n } catch (error) {\n status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"error\";\n observability.warn(\"sandbox reaper: ended-epoch box probe failed\", {\n ...details,\n error:\n error === RETAINED_PROCESS_PROBE_TIMEOUT\n ? \"provider_timeout\"\n : error instanceof Error\n ? error.message\n : String(error),\n });\n }\n observed.set(instanceId, status);\n if (status === \"terminated\") {\n backoff.delete(instanceId);\n } else {\n deferEndedEpochProbe(backoff, instanceId);\n kept[status] = (kept[status] ?? 0) + 1;\n }\n return status;\n }\n\n let tuples: Awaited<ReturnType<typeof listEndedEpochWorkspaceBlockers>>;\n try {\n tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch blocker inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n tuples = [];\n }\n for (const tuple of tuples) {\n const instanceId = tuple.lostInstanceId;\n const details = {\n workspaceId: tuple.workspaceId,\n sandboxGroupId: tuple.sandboxGroupId,\n lostEpoch: tuple.lostEpoch,\n instanceId,\n };\n try {\n const status = await observe(instanceId, details);\n if (status !== \"terminated\") continue;\n const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);\n if (settled) {\n observability.info(\"sandbox reaper: settled blockers of an ended lease epoch\", {\n ...details,\n ...settled,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch blocker settlement failed\", {\n ...details,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n let setups: Awaited<ReturnType<typeof listEndedEpochSandboxSetupUncertainties>>;\n try {\n setups = await listEndedEpochSandboxSetupUncertainties(db, {\n limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT,\n });\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch setup inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n setups = [];\n }\n for (const setup of setups) {\n const instanceId = setup.box.providerInstanceId;\n // A turn whose reconciliation is refused on a terminated box (for example\n // an ancestor is paused) backs off by turn, so it never re-probes per tick.\n const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;\n if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;\n const details = {\n workspaceId: setup.workspaceId,\n sessionId: setup.sessionId,\n turnId: setup.turnId,\n sandboxGroupId: setup.sandboxGroupId,\n lostEpoch: setup.box.leaseEpoch,\n instanceId,\n };\n try {\n const status = await observe(instanceId, details);\n if (status !== \"terminated\") continue;\n const { reconciled } = await reconcileCompletedSandboxSetup(db, {\n accountId: setup.accountId,\n workspaceId: setup.workspaceId,\n sessionId: setup.sessionId,\n turnId: setup.turnId,\n attemptId: setup.attemptId,\n terminatedBox: setup.box,\n });\n if (reconciled) {\n backoff.delete(turnKey);\n observability.info(\n \"sandbox reaper: resumed a turn whose unknown setup ran on a terminated box\",\n details,\n );\n } else {\n deferEndedEpochProbe(backoff, turnKey);\n }\n } catch (error) {\n deferEndedEpochProbe(backoff, turnKey);\n observability.warn(\"sandbox reaper: ended-epoch setup reconciliation failed\", {\n ...details,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n if (Object.keys(kept).length) {\n observability.info(\"sandbox reaper: ended-epoch blockers kept without terminal proof\", kept);\n }\n}\n\nfunction deferEndedEpochProbe(backoff: EndedEpochProbeBackoff, key: string): void {\n const failures = (backoff.get(key)?.failures ?? 0) + 1;\n backoff.set(key, {\n failures,\n nextProbeAtMs:\n Date.now() +\n Math.min(\n ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,\n ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16),\n ),\n });\n}\n\nasync function reconcileTerminalRetainedProcesses(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n probe: RetainedProcessProbeFn,\n inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,\n bus: ActivityServices[\"bus\"],\n): Promise<void> {\n const claimId = crypto.randomUUID();\n let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;\n try {\n claims = await claimTerminalRetainedProcesses(db, {\n claimId,\n limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,\n claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: retained-process claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n\n const drainBudget: RetainedProcessDrainBudget = {\n until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,\n };\n // Boxes already retired as lost in this batch: their other commands were\n // settled by that same transaction and need no probe of their own.\n const lostBoxes = new Set<string>();\n for (const claim of claims) {\n let process = claim.process;\n if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))\n continue;\n const expected = retainedProcessSettlementIdentity(process);\n let proof = retainedProcessReconciliationProof(process);\n // The exact current box itself answered NotFound to this probe: that is\n // loss proof for the whole box, not just this command.\n let wholeBoxGone = false;\n const processScope = {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n };\n // Recover a yield whose observation failed before normal receipt adoption.\n // Do this before provider I/O: an unavailable observer must not fence every\n // subsequent turn. Physical retention and unknown outcome stay unchanged.\n if (\n process.ownerActorKind === \"turn\" &&\n process.providerBackend === \"modal\" &&\n !proof &&\n !claim.ownerState.startsWith(\"background_\")\n ) {\n try {\n const command = await recoverManagedSessionBackgroundCommand(db, {\n ...processScope,\n expected,\n reconciliationClaimId: claim.claimId,\n });\n if (command) {\n claim.ownerState = `background_${command.state}`;\n recordRetainedProcessReconciliation(observability, \"background_recovered\");\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command background recovery failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n const storedCommandPersistence = retainedProviderCommandPersistence(\n db,\n processScope,\n bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : undefined,\n );\n const commandPersistence = {\n ...storedCommandPersistence,\n requestCancellation: async (reason: \"provider_deadline\" | \"explicit_stop\") => {\n if (reason === \"provider_deadline\") {\n await requestRetainedProcessDeadlineCancellation(db, processScope);\n if (!(await storedCommandPersistence.cancellationRequested()))\n throw new Error(\n \"Supervised deadline cancellation no longer owns its original rotating lease\",\n );\n } else await storedCommandPersistence.requestCancellation(reason);\n },\n };\n if (proof && process.providerBackend === \"modal\") {\n try {\n const command = await commandPersistence.load();\n if (\n command?.kind === \"modal-router-v1\" &&\n command.supervision &&\n (!(await commandPersistence.loadSupervisionReceipt()) ||\n ![command.streams.stdout, command.streams.stderr].every(\n (stream) => stream.eof && stream.exitCode === 0,\n ))\n )\n proof = null;\n } catch {\n // An earlier terminal observation cannot make missing control/output\n // proof unreachable. Reprobe the original binding; DB remains the gate.\n proof = null;\n }\n }\n if (!proof) {\n let observation: RetainedProcessProbeResult | null = null;\n\n // Legacy rows created before provider bindings existed are adoptable only\n // when the exact historical sandbox resolves in the configured Modal\n // namespace. NotFound is ambiguous here: it can mean either deletion or\n // credential rotation, so it never creates ownership or loss proof.\n if (\n process.providerBackend === \"modal\" &&\n process.routeTargetId === null &&\n !process.providerBindingKey\n ) {\n try {\n const lifecycle = await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),\n );\n if (lifecycle.status === \"not_found\") {\n observation = {\n status: \"deferred\",\n reason: \"provider_binding_missing\",\n };\n } else {\n process = await bindRetainedProcessProviderIdentity(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n providerBindingKey: lifecycle.providerBindingKey,\n providerBinding: lifecycle.providerBinding,\n });\n recordRetainedProcessReconciliation(observability, \"provider_binding_adopted\");\n if (lifecycle.status === \"terminated\") {\n observation = {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_terminated\",\n },\n };\n }\n }\n } catch (error) {\n observation = {\n status: \"deferred\",\n reason:\n error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n\n if (\n observation?.status === \"deferred\" &&\n process.providerBackend === \"modal\" &&\n !process.providerBindingKey &&\n !process.deadlineCancellationRequestedAt\n ) {\n // An unbound historical command cannot be signalled safely. Still\n // record deadline intent when its lease matches: a failed observer\n // must not block capture until the provider expires.\n try {\n await requestRetainedProcessDeadlineCancellation(db, processScope);\n } catch (error) {\n observability.warn(\"sandbox reaper: legacy deadline cancellation intent failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n if (observation === null) {\n const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(\n () => null,\n );\n const currentLeaseIdentityMatches =\n Boolean(lease) &&\n lease!.id === process.leaseId &&\n lease!.leaseEpoch === process.leaseEpoch &&\n lease!.backend === process.providerBackend &&\n lease!.instanceId === process.providerInstanceId &&\n process.routeTargetId === null;\n if (!currentLeaseIdentityMatches) {\n // The lease may already be cold or belong to a successor. Its current\n // resume envelope cannot interrogate the historical process, but a\n // bound process can safely inspect the exact old Modal sandbox id.\n if (process.providerBackend !== \"modal\" || process.routeTargetId !== null) {\n observation = { status: \"deferred\", reason: \"identity_mismatch\" };\n } else {\n try {\n const lifecycle = await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(\n settings,\n process.providerInstanceId,\n process.providerBindingKey,\n ),\n );\n observation =\n lifecycle.status === \"not_found\"\n ? {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n }\n : lifecycle.status === \"terminated\"\n ? {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_terminated\",\n },\n }\n : { status: \"deferred\", reason: \"identity_mismatch\" };\n } catch (error) {\n observation = {\n status: \"deferred\",\n reason:\n error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n } else {\n try {\n const command = await commandPersistence.load();\n const supervised = command?.kind === \"modal-router-v1\" && Boolean(command.supervision);\n const supervisionMetric = (outcome: string) =>\n observability.incrementCounter({\n name: \"opengeni_command_supervision_total\",\n help: \"Native retained command supervision reconciliation outcomes.\",\n labels: { outcome },\n });\n if (\n supervised &&\n (lease!.rotationReason === \"provider_deadline\" ||\n claim.ownerState === \"background_stopping\")\n ) {\n await commandPersistence.requestCancellation(\n lease!.rotationReason === \"provider_deadline\"\n ? \"provider_deadline\"\n : \"explicit_stop\",\n );\n supervisionMetric(\"cancellation_intent\");\n }\n const legacyDeadlineStop =\n lease!.rotationReason === \"provider_deadline\" &&\n !supervised &&\n !process.deadlineCancellationRequestedAt;\n const probeOutcome = await probe(\n settings,\n lease!,\n process,\n (claim.ownerState === \"background_stopping\" &&\n !process.deadlineCancellationRequestedAt) ||\n legacyDeadlineStop\n ? \"cancel\"\n : \"observe\",\n async (result, chunkId, stream, streamFidelity) => {\n if (\n typeof result !== \"string\" ||\n (!stream && isExecSessionLostBanner(result, process.providerSessionId))\n )\n return;\n const events = await appendSessionCommandOutput(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n commandId: process.id,\n chunkId,\n stream: stream ?? \"stdout\",\n streamFidelity: streamFidelity ?? (stream ? \"separate\" : \"merged\"),\n chunk: stream ? result : stripExecBanner(result),\n });\n if (events.length && bus)\n await bus\n .publish(process.workspaceId, process.sessionId, events)\n .catch(() => undefined);\n },\n commandPersistence,\n drainBudget,\n ).then(\n (value) => ({ ok: true as const, value }),\n (error: unknown) => ({ ok: false as const, error }),\n );\n // Legacy PTY input reserves Ctrl-C before the cancellation fence.\n // Non-PTY has no command signal in the Modal SDK: probe it without\n // injecting Ctrl-C as data, then fence and save after the grace.\n if (\n legacyDeadlineStop &&\n !(await requestRetainedProcessDeadlineCancellation(db, processScope))\n )\n throw new Error(\"Deadline cancellation no longer owns its rotating lease\");\n if (!probeOutcome.ok) throw probeOutcome.error;\n observation = probeOutcome.value;\n wholeBoxGone =\n observation.status === \"proved\" &&\n observation.proof.outcome === \"lost\" &&\n observation.proof.reason === \"provider_instance_not_found\";\n if (supervised) {\n supervisionMetric(\n (await commandPersistence.loadSupervisionReceipt())\n ? \"proof_retained\"\n : \"proof_missing\",\n );\n if (observation.status === \"deferred\")\n supervisionMetric(\n observation.reason === \"provider_running\"\n ? \"checkpoint_blocked\"\n : \"provider_failure\",\n );\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-process provider probe failed\", {\n processId: process.id,\n providerBackend: process.providerBackend,\n error: error instanceof Error ? error.message : String(error),\n });\n observation = { status: \"deferred\", reason: \"provider_error\" };\n }\n }\n }\n if (observation.status === \"binding\") {\n try {\n process = await bindRetainedProcessProviderIdentity(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n providerBindingKey: observation.providerBindingKey,\n providerBinding: observation.providerBinding,\n });\n recordRetainedProcessReconciliation(observability, \"provider_binding_adopted\");\n observation = { status: \"deferred\", reason: \"provider_running\" };\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-process binding adoption failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n observation = { status: \"deferred\", reason: \"provider_error\" };\n }\n }\n if (observation.status === \"deferred\") {\n await deferRetainedProcessClaim(\n db,\n settings,\n observability,\n process,\n expected,\n claim.claimId,\n observation.reason,\n observation.outputBacklog === true,\n );\n continue;\n }\n try {\n process = await recordRetainedProcessReconciliationProof(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n proof: observation.proof,\n });\n proof = observation.proof;\n recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"proof_checkpoint_failed\");\n warnRetainedProcessProofFailure(observability, error, process);\n continue;\n }\n }\n\n if (wholeBoxGone) {\n // Settle every blocker of the exact lost box in one transaction: all its\n // commands, open requests, PTYs and process holders, then cold the lease\n // and wake waiters. Settling one command per probe took a 20-per-sweep\n // queue 11 minutes for 35 commands in staging session 5040c525 and kept\n // the session waiting on a box that was already gone.\n // Commands of this box that a closed turn never adopted get their\n // session background record first, exactly as their own claim would\n // have, so the agent hears about them when the box settles below.\n const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;\n for (const other of claims) {\n const sibling = other.process;\n if (\n other === claim ||\n `${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey ||\n sibling.ownerActorKind !== \"turn\" ||\n sibling.providerBackend !== \"modal\" ||\n other.ownerState.startsWith(\"background_\") ||\n retainedProcessReconciliationProof(sibling)\n )\n continue;\n try {\n await recoverManagedSessionBackgroundCommand(db, {\n accountId: sibling.accountId,\n workspaceId: sibling.workspaceId,\n sessionId: sibling.sessionId,\n processId: sibling.id,\n expected: retainedProcessSettlementIdentity(sibling),\n reconciliationClaimId: other.claimId,\n });\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command background recovery failed\", {\n processId: sibling.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n try {\n const marked = await markWarmLeaseInstanceLost(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sandboxGroupId: process.sandboxGroupId,\n expectedEpoch: process.leaseEpoch,\n expectedInstanceId: process.providerInstanceId,\n expectedBackend: process.providerBackend,\n });\n if (marked.status === \"marked\") {\n lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);\n recordRetainedProcessReconciliation(observability, \"provider_lost_whole_box\");\n if (marked.backgroundCommandEvents?.length) {\n await publishDurableSessionEvents(\n bus,\n process.workspaceId,\n marked.backgroundCommandEvents,\n (error) => {\n observability.warn(\"sandbox reaper: lost-box command event fanout failed\", {\n sandboxGroupId: process.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n },\n );\n }\n continue;\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: whole-box loss settlement failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n // Stale (a draining drain or another observer owns the box): fall back\n // to this command's own exact settlement below.\n }\n try {\n const settlement = await settleRetainedProcess(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n reconciliationClaimId: claim.claimId,\n outcome: proof.outcome,\n exitCode: proof.exitCode,\n reason: proof.reason,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n if (settlement.process.state === \"active\") {\n throw new Error(\"Retained-process reconciliation returned an active durable process\");\n }\n if (settlement.backgroundCommandEvents.length > 0) {\n try {\n await bus.publish(\n process.workspaceId,\n process.sessionId,\n settlement.backgroundCommandEvents,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command event fanout failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"settlement_failed\");\n observability.warn(\"sandbox reaper: retained-process settlement failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n await deferRetainedProcessClaim(\n db,\n settings,\n observability,\n process,\n expected,\n claim.claimId,\n \"settlement_failed\",\n );\n }\n }\n}\n\nasync function deferRetainedProcessClaim(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n process: SandboxRetainedProcess,\n expected: ReturnType<typeof retainedProcessSettlementIdentity>,\n claimId: string,\n outcome: RetainedProcessDeferralOutcome,\n outputBacklog = false,\n): Promise<void> {\n const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);\n try {\n // A healthy long-lived command can already be on a five-minute backoff\n // when rotation becomes due. Wake it at the lead boundary, then at the\n // reaper cadence, so observation does not consume the capture window.\n const lease =\n process.providerBackend === \"modal\" && process.routeTargetId === null\n ? await readLease(db, process.workspaceId, process.sandboxGroupId)\n : null;\n await deferRetainedProcessReconciliation(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId,\n outcome: deferral.durableOutcome,\n retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs),\n });\n recordRetainedProcessReconciliation(observability, deferral.metricOutcome);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"defer_failed\");\n observability.warn(\"sandbox reaper: retained-process defer failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\ntype RetainedProcessDeferralOutcome =\n | Extract<RetainedProcessProbeResult, { status: \"deferred\" }>[\"reason\"]\n | \"settlement_failed\";\n\nexport function retainedProcessReconciliationDeferral(\n settings: Pick<ActivityServices[\"settings\"], \"sandboxLeaseReaperPeriodMs\">,\n process: Pick<SandboxRetainedProcess, \"reconcileAttempts\">,\n outcome: RetainedProcessDeferralOutcome,\n outputBacklog = false,\n): {\n durableOutcome: string;\n metricOutcome: RetainedProcessReconciliationOutcome;\n retryAfterMs: number;\n quarantined: boolean;\n} {\n const bindingQuarantine =\n process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS &&\n (outcome === \"provider_binding_missing\" ||\n outcome === \"provider_binding_mismatch\" ||\n outcome === \"process_observation_unavailable\");\n if (bindingQuarantine) {\n return {\n durableOutcome: `quarantined_${outcome}`,\n metricOutcome:\n outcome === \"provider_binding_missing\"\n ? \"quarantined_binding_missing\"\n : outcome === \"provider_binding_mismatch\"\n ? \"quarantined_binding_mismatch\"\n : \"quarantined_process_observation_unavailable\",\n retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,\n quarantined: true,\n };\n }\n const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000);\n // A backlog still being drained is progress, not an unhealthy provider:\n // come back at the reaper cadence, without backoff.\n const exponent =\n outcome === \"provider_running\" && outputBacklog\n ? 0\n : Math.min(4, Math.max(0, process.reconcileAttempts - 1));\n const retryAfterMs = Math.min(5 * 60_000, baseRetryMs * 2 ** exponent);\n return {\n durableOutcome: outcome,\n metricOutcome: outcome,\n retryAfterMs,\n quarantined: false,\n };\n}\n\ntype RetainedProcessProbeClient = {\n backendId: string;\n resume?: (state: unknown) => Promise<unknown>;\n deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;\n};\n\ntype RetainedProcessProbeSession = ProviderCommandSession & {\n acknowledgeCommandOutput?: (result: string) => Promise<void>;\n writeStdin?: (args: {\n sessionId: number;\n chars: string;\n yieldTimeMs: number;\n maxOutputTokens: number;\n }) => Promise<unknown>;\n};\n\nconst RETAINED_PROCESS_PROBE_TIMEOUT = Symbol(\"retained-process-provider-probe-timeout\");\n\n/** A command's exit is reported only after both output streams reach EOF, so a\n * finished command with a large backlog looks running until it is read. While a\n * backlog remains, keep reading within the per-claim and per-sweep budgets, so\n * it settles in this claim instead of one page per backoff step. Every page,\n * including `first`, is captured here. */\nexport async function drainRetainedCommandBacklog(\n session: RetainedProcessProbeSession,\n providerSessionId: number,\n first: unknown,\n capturePage: (value: unknown) => Promise<void>,\n budget?: RetainedProcessDrainBudget,\n): Promise<RetainedProcessProbeResult> {\n // Measure before capture: capturing a byte-offset page consumes its receipt.\n let backlog = retainedBacklogRemains(session, first);\n await capturePage(first);\n let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);\n if (typeof session.writeStdin !== \"function\") return observation;\n const until = Math.min(\n Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,\n budget?.until ?? Number.POSITIVE_INFINITY,\n );\n for (\n let reads = 0;\n backlog &&\n observation.status === \"deferred\" &&\n observation.reason === \"provider_running\" &&\n reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS &&\n Date.now() < until;\n reads++\n ) {\n let next: unknown;\n try {\n next = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: providerSessionId,\n chars: \"\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n } catch {\n // The observation already made stays authoritative; the next sweep\n // classifies any provider failure through the ordinary path.\n break;\n }\n backlog = retainedBacklogRemains(session, next);\n await capturePage(next);\n observation = classifyRetainedProcessPollResult(next, providerSessionId, session);\n }\n return backlog && observation.status === \"deferred\" && observation.reason === \"provider_running\"\n ? { ...observation, outputBacklog: true }\n : observation;\n}\n\n/** Whether a byte-offset page leaves a backlog worth reading now: the provider\n * reports the process exited but its output is not yet read to EOF, or a live\n * process produced a large page. Must run before the page is captured, which\n * consumes its receipt. Supervised commands are left to their proof path. */\nfunction retainedBacklogRemains(session: RetainedProcessProbeSession, result: unknown): boolean {\n const page = session.getProviderCommandOutput?.(result);\n if (!page?.expected || page.command.kind !== \"modal-router-v1\") return false;\n if (page.command.supervision || page.exitCode !== null) return false;\n if (page.providerExited) return true;\n const { command, expected } = page;\n return ([\"stdout\", \"stderr\"] as const).some(\n (stream) =>\n command.streams[stream].byteOffset - expected.streams[stream].byteOffset >=\n RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,\n );\n}\n\nasync function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {\n let timer: ReturnType<typeof setTimeout> | undefined;\n try {\n return await Promise.race([\n promise,\n new Promise<never>((_resolve, reject) => {\n timer = setTimeout(\n () => reject(RETAINED_PROCESS_PROBE_TIMEOUT),\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n timer.unref?.();\n }),\n ]);\n } finally {\n if (timer) clearTimeout(timer);\n }\n}\n\n/** Consumed SDK output cannot be polled again. Retain the receipt across\n * persistence retries in this worker; a worker crash remains unreplayable. */\nconst pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();\n\nexport async function captureRetainedProbeOutput(\n processId: string,\n result: unknown,\n capture: (result: unknown, chunkId: string) => Promise<void>,\n): Promise<void> {\n const pending = pendingRetainedProbeOutput.get(processId) ?? {\n result,\n chunkId: crypto.randomUUID(),\n };\n pendingRetainedProbeOutput.set(processId, pending);\n await capture(pending.result, pending.chunkId);\n pendingRetainedProbeOutput.delete(processId);\n}\n\nexport async function probeRetainedProcessAtProvider(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n process: SandboxRetainedProcess,\n mode: \"observe\" | \"cancel\" = \"observe\",\n captureOutput?: (\n result: unknown,\n chunkId: string,\n stream?: \"stdout\" | \"stderr\",\n streamFidelity?: \"separate\" | \"merged\",\n ) => Promise<void>,\n providerPersistence?: ProviderCommandPersistence,\n drainBudget?: RetainedProcessDrainBudget,\n): Promise<RetainedProcessProbeResult> {\n if (\n lease.id !== process.leaseId ||\n lease.sandboxGroupId !== process.sandboxGroupId ||\n lease.leaseEpoch !== process.leaseEpoch ||\n lease.backend !== process.providerBackend ||\n lease.instanceId !== process.providerInstanceId ||\n !lease.resumeBackendId ||\n sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend ||\n process.routeTargetId !== null\n ) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n if (!lease.resumeState) {\n return { status: \"deferred\", reason: \"resume_state_missing\" };\n }\n const pending = pendingRetainedProbeOutput.get(process.id);\n if (pending) {\n if (!captureOutput) throw new Error(\"Pending command output requires its persistence callback\");\n await captureRetainedProbeOutput(process.id, pending.result, captureOutput);\n const pendingObservation = classifyRetainedProcessPollResult(\n pending.result,\n process.providerSessionId,\n );\n // Capture of a previous page must not consume the only deadline stop\n // attempt. Terminal proof needs no further provider call.\n if (mode !== \"cancel\" || pendingObservation.status === \"proved\") return pendingObservation;\n }\n const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;\n if (\n envelopeBackend !== undefined &&\n (typeof envelopeBackend !== \"string\" ||\n sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)\n ) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n\n let configuredModalBinding: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBinding>\n > | null = null;\n if (process.providerBackend === \"modal\" && process.providerBindingKey) {\n try {\n configuredModalBinding = await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBinding(settings),\n );\n } catch (error) {\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n if (configuredModalBinding.key !== process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_mismatch\" };\n }\n }\n\n let client: RetainedProcessProbeClient | undefined;\n try {\n client = createSandboxClientForBackend(process.providerBackend as never, settings) as\n | RetainedProcessProbeClient\n | undefined;\n } catch {\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (\n !client ||\n sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend ||\n !client.resume\n ) {\n return { status: \"deferred\", reason: \"backend_unsupported\" };\n }\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n let resumedState: unknown;\n try {\n resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n process.providerInstanceId,\n );\n } catch {\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (resumedState === undefined) {\n return { status: \"deferred\", reason: \"resume_state_missing\" };\n }\n\n let session: RetainedProcessProbeSession;\n try {\n const resumed = await withRetainedProcessProbeTimeout(\n resumeExactSandboxSession(\n client,\n process.providerBackend,\n resumedState,\n process.providerInstanceId,\n ),\n );\n session = resumed.session as RetainedProcessProbeSession;\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n return { status: \"deferred\", reason: \"provider_timeout\" };\n }\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (typeof session.writeStdin !== \"function\") {\n return { status: \"deferred\", reason: \"backend_unsupported\" };\n }\n let legacyPty = false;\n if (process.providerBackend === \"modal\") {\n let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;\n try {\n liveBinding = await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBindingForSession(settings, session),\n );\n } catch (error) {\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n if (!process.providerBindingKey) {\n return {\n status: \"binding\",\n providerBindingKey: liveBinding.key,\n providerBinding: liveBinding.binding,\n };\n }\n if (\n liveBinding.key !== process.providerBindingKey ||\n configuredModalBinding?.key !== process.providerBindingKey\n ) {\n return { status: \"deferred\", reason: \"provider_binding_mismatch\" };\n }\n // Legacy SDK-local handles cannot be reconstructed. New commands carry\n // an opaque provider locator retained outside the sandbox's authority.\n const command = await providerPersistence?.load();\n if (!command || !providerPersistence || !session.bindProviderCommand)\n return { status: \"deferred\", reason: \"process_observation_unavailable\" };\n session.bindProviderCommand(process.providerSessionId, command, providerPersistence);\n legacyPty = command.pty === true;\n }\n const capturePage = async (value: unknown): Promise<void> => {\n if (!captureOutput) {\n if (session.getProviderCommandOutput?.(value))\n throw new Error(\"Provider command output requires durable capture before settlement\");\n return;\n }\n const page = session.getProviderCommandOutput?.(value);\n if (page?.command.kind === \"modal-router-v1\" && typeof value === \"string\") {\n if (!session.captureCommandOutput || !(await session.captureCommandOutput(value)))\n throw new Error(\"Byte-offset output requires atomic capture before settlement\");\n return;\n }\n if (page) {\n for (const chunk of page.chunks)\n if (chunk.text)\n await captureOutput(chunk.text, chunk.chunkId, chunk.stream, page.streamFidelity);\n } else {\n await captureRetainedProbeOutput(process.id, value, captureOutput);\n }\n if (typeof value === \"string\") await session.acknowledgeCommandOutput?.(value);\n };\n\n let result: unknown;\n try {\n const supervisedCancelled =\n mode === \"cancel\" || lease.rotationReason === \"provider_deadline\"\n ? ((await session.cancelSupervisedCommand?.(\n process.providerSessionId,\n lease.rotationReason === \"provider_deadline\" ? \"provider_deadline\" : \"explicit_stop\",\n )) ?? false)\n : false;\n result = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: process.providerSessionId,\n // Ctrl-C is a signal only through a PTY. For a non-PTY command it is\n // ordinary stdin data and can corrupt the command's input.\n chars:\n mode === \"cancel\" &&\n !supervisedCancelled &&\n (process.providerBackend !== \"modal\" || legacyPty)\n ? \"\\u0003\"\n : \"\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n return { status: \"deferred\", reason: \"provider_timeout\" };\n }\n if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n // Cancellation keeps draining with empty reads too: a stopped command's exit\n // is also reported only after its unread output reaches EOF.\n const observation = await drainRetainedCommandBacklog(\n session,\n process.providerSessionId,\n result,\n capturePage,\n drainBudget,\n );\n if (\n observation.status === \"deferred\" &&\n observation.reason === \"provider_running\" &&\n lease.rotationRequestedAt !== null &&\n mode !== \"cancel\" &&\n !process.cancellationRequestedAt &&\n legacyPty &&\n !(await providerPersistence\n ?.load()\n .then((command) => command?.kind === \"modal-router-v1\" && command.supervision))\n ) {\n // A background PTY cannot outlive the finite provider box. Interrupt only\n // this exact durable provider session after rotation admission is fenced;\n // settlement still requires an exit/loss banner on this or a later sweep.\n try {\n const interrupted = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: process.providerSessionId,\n chars: \"\\u0003\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n await capturePage(interrupted);\n return classifyRetainedProcessPollResult(interrupted, process.providerSessionId, session);\n } catch (error) {\n if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n return observation;\n}\n\n/** Classify only exact SDK control banners. Arbitrary output, malformed text,\n * and a running marker are observations to retry; none is physical exit proof. */\nexport function classifyRetainedProcessPollResult(\n result: unknown,\n providerSessionId: number,\n source?: object,\n): RetainedProcessProbeResult {\n if (typeof result !== \"string\") {\n return { status: \"deferred\", reason: \"provider_unknown\" };\n }\n if (isExecSessionLostBanner(result, providerSessionId, source)) {\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_session_lost_banner\",\n },\n };\n }\n const exitCode = parseExecBannerExitCode(result);\n if (exitCode !== null) {\n return {\n status: \"proved\",\n proof: { outcome: \"exited\", exitCode, reason: \"provider_exit_banner\" },\n };\n }\n return {\n status: \"deferred\",\n reason: result.includes(\"Process running with session ID\")\n ? \"provider_running\"\n : \"provider_unknown\",\n };\n}\n\n/**\n * The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM\n * viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those\n * meter on the turn heartbeat). Returns the count accrued + the distinct\n * workspaces touched (so the force-drain pass only checks workspaces that have a\n * live warm box, not the whole fleet). Per-row best-effort: one row's metering\n * error must not abort the sweep.\n */\nasync function accrueWarmTick(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<{ accrued: number; workspaceIds: Set<string> }> {\n const workspaceIds = new Set<string>();\n let accrued = 0;\n let leases: MeterableWarmLease[] = [];\n try {\n leases = await listMeterableWarmLeases(db);\n } catch (error) {\n observability.warn(\"sandbox reaper: warm-lease read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { accrued, workspaceIds };\n }\n for (const lease of leases) {\n workspaceIds.add(lease.workspaceId);\n }\n await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {\n try {\n const rate =\n settings.sandboxWarmBillingMode === \"usage_only\"\n ? 0\n : sandboxWarmRateMicrosPerSecond(settings, lease.backend);\n const result = await accrueWarmSeconds(db, {\n accountId: lease.accountId,\n workspaceId: lease.workspaceId,\n sandboxGroupId: lease.sandboxGroupId,\n expectedEpoch: lease.leaseEpoch,\n warmRateMicrosPerSecond: rate,\n billingMode: settings.sandboxWarmBillingMode,\n subjectId: lease.sandboxGroupId,\n });\n if (result.accrued) {\n accrued += 1;\n }\n if (result.costMicros > 0 && settings.sandboxWarmBillingMode === \"credits\") {\n recordCreditMicros(observability, \"usage\", result.costMicros);\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: warm-seconds accrual failed for lease\", {\n workspaceId: lease.workspaceId,\n sandboxGroupId: lease.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n return { accrued, workspaceIds };\n}\n\n/**\n * The reaper maintenance pass's read-only projections, for a reaper tick that\n * started no workflow because there was no work. Same gauges, same process:\n * the control worker both runs the cadence and polls the lifecycle queue.\n */\nexport async function refreshIdleSandboxReaperProjections(input: {\n db: ActivityServices[\"db\"];\n settings: ActivityServices[\"settings\"];\n observability: ActivityServices[\"observability\"];\n inspectOpenSandboxInventory?: InspectOpenSandboxKubernetesInventoryFn;\n}): Promise<void> {\n await refreshQueueLeaseAndCreditGauges(\n input.db,\n input.settings,\n input.observability,\n input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory,\n );\n}\n\n/**\n * Whether the Modal orphan sweep could act: the deployment uses Modal and its\n * app has an unfinished sandbox. Called only once the database checks found no\n * warm lease, so an empty listing is the sweep's complete, empty inventory and\n * is projected exactly as the sweep would project it.\n */\nexport async function modalOrphanSweepMayHaveWork(input: {\n settings: ActivityServices[\"settings\"];\n observability: ActivityServices[\"observability\"];\n hasRunningSandboxes?: (settings: ActivityServices[\"settings\"]) => Promise<boolean>;\n}): Promise<boolean> {\n const { settings, observability } = input;\n if (!modalOrphanSweepConfigured(settings)) return false;\n if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;\n recordModalSandboxInventoryGauges(observability, {\n running: 0,\n unleased: 0,\n liveLeaseInstancesMissing: 0,\n });\n recordSandboxInventoryProjectionSuccess(observability, \"modal_provider\");\n return false;\n}\n\nfunction modalOrphanSweepConfigured(settings: ActivityServices[\"settings\"]): boolean {\n return (\n settings.sandboxBackend === \"modal\" || !!settings.modalTokenId || !!settings.modalTokenSecret\n );\n}\n\nasync function refreshQueueLeaseAndCreditGauges(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n inspectOpenSandboxInventory: InspectOpenSandboxKubernetesInventoryFn,\n): Promise<void> {\n await Promise.all([\n (async () => {\n try {\n recordTurnsQueuedGauge(observability, await countQueuedTurns(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: queued-turn gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n refreshSandboxInventoryGauge(observability, \"leases\", \"sandbox-lease\", async () => {\n recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));\n }),\n refreshSandboxInventoryGauge(\n observability,\n \"checkpoint_artifacts\",\n \"checkpoint-artifact\",\n async () => {\n recordSandboxCheckpointArtifactGauges(\n observability,\n await countSandboxCheckpointArtifactsByState(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"recovery_observations\",\n \"recovery-observations\",\n async () => {\n recordSandboxRecoveryObservationGauges(\n observability,\n await readRecentSandboxRecoveryObservations(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"rotation_backlog\",\n \"rotation-backlog\",\n async () => {\n recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"checkpoint_staleness\",\n \"checkpoint-staleness\",\n async () => {\n recordSandboxCheckpointStalenessGauges(\n observability,\n await readSandboxCheckpointStaleness(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"retained_processes\",\n \"retained-process\",\n async () => {\n recordRetainedProcessInventoryGauges(\n observability,\n await countActiveRetainedProcessesByOwnerState(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"interaction_idle\",\n \"interaction-only-lease\",\n async () => {\n recordInteractionOnlyLeaseGauges(\n observability,\n await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db)),\n );\n },\n ),\n refreshSandboxInventoryGauge(observability, \"expired_drains\", \"expired-draining\", async () => {\n recordExpiredDrainingSandboxLeaseGauges(\n observability,\n await countExpiredDrainingSandboxLeases(db),\n );\n }),\n ...(settings.sandboxBackend === \"opensandbox\" &&\n settings.openSandboxKubernetesInventoryNamespace\n ? [\n refreshSandboxInventoryGauge(\n observability,\n \"opensandbox_kubernetes\",\n \"OpenSandbox Kubernetes\",\n async () => {\n recordOpenSandboxKubernetesInventoryGauges(\n observability,\n await inspectOpenSandboxInventory({\n namespace: settings.openSandboxKubernetesInventoryNamespace!,\n }),\n );\n },\n ),\n ]\n : []),\n (async () => {\n try {\n recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: credit-balance gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n recordActiveUserGauges(observability, await countActiveUsers(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: active-user gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n recordCreditGrantGauges(observability, await readCreditGrantTotals(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: credit-grant gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n recordVerifiedSignupTrialDeploymentFlagGauge(\n observability,\n settings.verifiedSignupTrialCreditsEnabled,\n );\n try {\n const trialSwitch = await readVerifiedSignupTrialSwitch(db);\n recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);\n } catch (error) {\n observability.warn(\"sandbox reaper: trial-credit switch gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);\n recordManagedAuthNewSignupsSwitchGauge(\n observability,\n signupsSwitch?.signupsEnabled !== false,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: new-signups switch gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n ]);\n}\n\nasync function refreshSandboxInventoryGauge(\n observability: ActivityServices[\"observability\"],\n domain: SandboxInventoryProjectionDomain,\n diagnosticName: string,\n refresh: () => Promise<void>,\n): Promise<void> {\n try {\n await refresh();\n recordSandboxInventoryProjectionSuccess(observability, domain);\n } catch (error) {\n recordSandboxInventoryProjectionFailure(observability, domain);\n observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\n/**\n * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a\n * live warm box, under the usage lock: if it is over its optional warm cap,\n * accelerate only holderless boxes to draining\n * (guarded turn_holders=0 — a paying turn is never killed). Returns the count of\n * viewer-only boxes force-drained. Per-workspace best-effort.\n */\nasync function forceDrainOverLimitWorkspaces(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n workspaceIds: Set<string>,\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n const enforceBalance = false;\n const cap = settings.sandboxMaxWarmSecondsPerWorkspace;\n let forceDrained = 0;\n await forEachWithConcurrency(\n [...workspaceIds],\n SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,\n async (workspaceId) => {\n try {\n const result = await forceDrainOverLimitViewerOnlyBoxes(db, {\n workspaceId,\n enforceBalance,\n maxWarmSecondsPerWorkspace: cap,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n if (result.overLimit && result.drained.length > 0) {\n forceDrained += result.drained.length;\n observability.info(\"sandbox reaper: force-drained viewer-only boxes (over limit)\", {\n workspaceId,\n reason: result.reason,\n drained: result.drained.length,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: force-drain check failed for workspace\", {\n workspaceId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n },\n );\n return forceDrained;\n}\n\nexport async function reconcileModalProviderCreates(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n discover: typeof findModalProviderCreateReceipt = findModalProviderCreateReceipt,\n): Promise<void> {\n if (settings.sandboxBackend !== \"modal\" && !settings.modalTokenId && !settings.modalTokenSecret)\n return;\n const candidates = await listPendingModalProviderCreates(db);\n await forEachWithConcurrency(candidates, 2, async (candidate) => {\n try {\n const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);\n const identity = { ...candidate, accountId };\n const attempt = await claimModalProviderCreateRecovery(db, identity);\n if (!attempt) return;\n const instanceId = await discover(settings, attempt);\n if (!instanceId) return; // Absence never settles an unknown dispatch.\n if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {\n observability.info(\"sandbox reaper: recovered Modal create receipt\", {\n ...candidate,\n operationId: attempt.operationId,\n instanceId,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal create remains fenced pending discovery\", {\n ...candidate,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n}\n\nexport function modalOrphanTerminationStillEligible(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n candidate: ModalOrphanSweepTermination,\n): boolean {\n // The provider may have accepted a create whose reply has not arrived.\n // Before attribution, neither its ID nor its separately-applied tags are\n // available here. Postpone orphan deletion while any such owner exists;\n // age/absence cannot distinguish its provider from an abandoned one.\n if (latest.some((lease) => lease.liveness === \"warming\" && lease.instanceId === null)) {\n return false;\n }\n return modalOrphanCandidateStillUnowned(latest, candidate);\n}\n\n/** Live leases still WARM on an instance the complete listing did not show.\n * A draining lease legitimately loses its box mid-drain (capture, terminate,\n * then cold), and warming has not published a box yet, so only a warm lease\n * whose exact box is gone is a zombie. */\nexport function countWarmModalLeasesMissingInstance(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n missingInstanceIds: string[],\n): number {\n return missingInstanceIds.filter((instanceId) =>\n latest.some((lease) => lease.liveness === \"warm\" && lease.instanceId === instanceId),\n ).length;\n}\n\n/** Whether no live lease owns the candidate box by exact instance or by its\n * attribution tags. Unlike termination eligibility this ignores the pending\n * create postponement: a postponed orphan is still running without a lease. */\nexport function modalOrphanCandidateStillUnowned(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n candidate: ModalOrphanSweepTermination,\n): boolean {\n if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {\n return false;\n }\n\n const leaseId = candidate.tags.opengeni_lease_id;\n const workspaceId = candidate.tags.opengeni_workspace_id;\n const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;\n if (!leaseId || !workspaceId || !sandboxGroupId) {\n return true;\n }\n const activeAttribution = latest.find(\n (lease) =>\n lease.leaseId === leaseId &&\n lease.workspaceId === workspaceId &&\n lease.sandboxGroupId === sandboxGroupId,\n );\n return Boolean(\n !activeAttribution ||\n (activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),\n );\n}\n\nasync function sweepModalOrphansForConfiguredBackend(\n settings: ActivityServices[\"settings\"],\n db: ActivityServices[\"db\"],\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n // Keep the provider-side sweep tightly scoped to deployments that have a Modal\n // app path configured. Local/docker-only workers should not attempt Modal API\n // calls just because modalAppName has a default.\n if (!modalOrphanSweepConfigured(settings)) {\n return 0;\n }\n let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;\n try {\n liveLeases = await listLiveModalSandboxLeaseAttributions(db);\n } catch (error) {\n observability.warn(\"sandbox reaper: live Modal lease attribution read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n }\n\n const result = await sweepModalOrphanSandboxes(settings, liveLeases, {\n revalidateTermination: async (candidate) => {\n const latest = await listLiveModalSandboxLeaseAttributions(db);\n return modalOrphanTerminationStillEligible(latest, candidate);\n },\n });\n // Only a complete listing is an inventory; a pass cut short by the\n // termination budget leaves the previous projection to age out as stale.\n // Re-read durable ownership after the listing so a lease that started or\n // finished while the app was being listed is not reported as a leak.\n if (result.inventory.complete) {\n try {\n const latest = await listLiveModalSandboxLeaseAttributions(db);\n recordModalSandboxInventoryGauges(observability, {\n running: result.inventory.running,\n unleased: result.inventory.unterminated.filter((candidate) =>\n modalOrphanCandidateStillUnowned(latest, candidate),\n ).length,\n liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(\n latest,\n result.inventory.missingLiveLeaseInstanceIds,\n ),\n });\n recordSandboxInventoryProjectionSuccess(observability, \"modal_provider\");\n } catch (error) {\n recordSandboxInventoryProjectionFailure(observability, \"modal_provider\");\n observability.warn(\"sandbox reaper: Modal inventory ownership re-read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n for (const terminated of result.terminated) {\n observability.warn(\"sandbox reaper: terminated Modal orphan sandbox\", {\n sandboxId: terminated.sandboxId,\n reason: terminated.reason,\n tags: JSON.stringify(terminated.tags),\n });\n }\n if (result.examined > 0) {\n observability.info(\"sandbox reaper: Modal orphan sweep completed\", {\n examined: result.examined,\n terminated: result.terminated.length,\n skipped: result.skipped,\n appName: settings.modalAppName,\n });\n }\n return result.terminated.length;\n}\n\n/**\n * A capture claim whose owning worker disappeared must not fence a workspace\n * forever. The drain reaper reaches this seam only after every holder is gone.\n * It resumes the exact attributed provider without replacement and proves that\n * a no-op command can execute. This distinguishes typed provider loss from a\n * still-addressable instance; it does not prove that an old capture RPC has\n * stopped. The caller applies the provider's durable takeover policy and the\n * claim deadline. Ambiguity preserves the old claim and therefore the files.\n */\nasync function probeDrainableProviderReadiness(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n): Promise<\"ready\" | \"missing\"> {\n if (!lease.instanceId || !lease.resumeState) {\n throw new Error(\"Expired workspace capture has no resumable provider identity\");\n }\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;\n try {\n established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {\n sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,\n recovery: \"resume-only\",\n backendOverride: backend as never,\n });\n } catch (error) {\n if (isProviderSandboxNotFoundError(backend, error)) return \"missing\";\n throw error;\n }\n if (established.instanceId !== lease.instanceId) {\n throw new Error(\"Expired workspace capture resumed a different provider instance\");\n }\n try {\n if (established.backendId === \"modal\") {\n await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);\n return \"ready\";\n }\n const session = established.session as {\n exec?: (args: {\n cmd: string;\n yieldTimeMs?: number;\n maxOutputTokens?: number;\n }) => Promise<unknown>;\n execCommand?: (args: {\n cmd: string;\n yieldTimeMs?: number;\n maxOutputTokens?: number;\n }) => Promise<unknown>;\n };\n const run = session.exec ?? session.execCommand;\n if (!run) throw new Error(\"Expired workspace capture provider has no readiness command\");\n let timer: ReturnType<typeof setTimeout> | undefined;\n const result = await Promise.race([\n run.call(session, {\n cmd: \"true\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 1_000,\n }),\n new Promise<never>((_, reject) => {\n timer = setTimeout(\n () => reject(new Error(\"Expired workspace capture readiness probe timed out\")),\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n }),\n ]).finally(() => {\n if (timer) clearTimeout(timer);\n });\n if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {\n throw new Error(\"Expired workspace capture provider is not command-ready\");\n }\n return \"ready\";\n } catch (error) {\n if (isProviderSandboxNotFoundError(backend, error)) return \"missing\";\n throw error;\n }\n}\n\nexport type SandboxDrainReason =\n | \"idle\"\n | \"provider_deadline\"\n | \"operator\"\n | \"teardown_claim\"\n | \"rotation_requested\";\n\n/** The bounded drain reason carried on `sandbox.box.terminated`: the lease's\n * rotation reason when a rotation was requested, otherwise the ordinary idle\n * drain. A requested rotation without a recorded reason (legacy rows) is\n * reported as `rotation_requested` rather than guessed. */\nexport function sandboxDrainReason(\n lease: Pick<LeaseSnapshot, \"rotationRequestedAt\" | \"rotationReason\">,\n): SandboxDrainReason {\n if (lease.rotationRequestedAt === null) return \"idle\";\n return lease.rotationReason ?? \"rotation_requested\";\n}\n\n/**\n * Terminate one drainable box by id, then CAS its lease draining->cold under the\n * epoch fence. Returns true when the lease went cold (the box is ours to stop\n * and was stopped), false when a concurrent sweep / re-arm / newer epoch means\n * we must NOT stop it (provider terminate is skipped).\n *\n * The ordering is deliberately receipt-gated on BOTH ends:\n * - BEFORE provider I/O: re-read the exact draining epoch and acquire/retain\n * its durable capture/teardown claim. Before that claim an arrival may\n * re-arm for availability; afterward every arrival waits and cannot race\n * provider termination.\n * - AFTER provider terminate: confirmDrainCold commits only the exact current\n * capture id together with draining->cold. A miss means a recovery attempt\n * or epoch already replaced this callback; it says nothing about provider\n * liveness, so only that durable successor may reconcile and commit cold.\n */\nasync function terminateDrainableBox(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n row: ReapDrainable,\n observability: ActivityServices[\"observability\"],\n terminateBox: TerminateBoxFn,\n probeDrainableProvider: DrainableProviderProbeFn,\n attempt: SandboxDrainCaptureAttempt,\n objectStorage: ObjectStorage | null,\n bus: ActivityServices[\"bus\"] | null = null,\n): Promise<boolean> {\n // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep\n // returns no account_id; the workspace->account map is the bootstrap read).\n const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);\n\n // Re-read the lease for the resume envelope AND the pre-terminate CAS guard.\n const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);\n if (!lease) {\n // Lease vanished (cold-reaped by a concurrent sweep). Nothing to stop.\n return false;\n }\n if (\n lease.liveness !== \"draining\" ||\n (lease.refcount !== 0 && !lease.unobservableCommandDrainIds?.length) ||\n lease.leaseEpoch !== row.leaseEpoch\n ) {\n // Re-armed (warm again) / a newer epoch / already drained by a concurrent\n // sweep. Skip — provider stop() must fire ONLY past the drain grace at\n // refcount=0 on the box we observed, never while a turn or viewer holds it.\n return false;\n }\n\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n const managedProvider = backend !== \"none\" && backend !== \"selfhosted\";\n const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);\n if (managedProvider && !capturePolicy) {\n throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);\n }\n if (capturePolicy?.liveInstance === \"replaced\") {\n throw new Error(\n `sandbox backend ${backend} workspace capture would replace the fenced live instance`,\n );\n }\n\n const captureTimeoutMs = attempt.captureTimeoutMs;\n if (\n managedProvider &&\n !lease.archiveCapture?.publishedAt &&\n sandboxDrainCaptureSettlingInProcess(lease)\n ) {\n // An earlier attempt's capture is still reading, publishing late, or\n // removing its spool in this worker. Leave its claim untouched and do not\n // start a second physical capture of the same workspace; a later retry\n // tears down from its published archive or, if it failed, captures anew.\n // An already published archive still resumes teardown immediately.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`,\n );\n }\n let captureClaim:\n | NonNullable<\n Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: \"claimed\" }>\n >[\"claim\"]\n | null = null;\n let providerMissingBeforeCapture = false;\n let captureDisposition: DrainCaptureDisposition = \"capture_required\";\n if (lease.instanceId && managedProvider) {\n if (lease.archiveCapture) {\n const priorCapture = lease.archiveCapture;\n const instanceId = lease.instanceId;\n if (priorCapture.publishedAt) {\n // Publication is an irreversible durable phase transition. The archive\n // already represents this exact fenced generation, so a crash/retry\n // resumes teardown immediately: no deadline wait, provider probe, claim\n // replacement, or duplicate snapshot request.\n captureClaim = {\n ...priorCapture,\n leaseId: lease.id,\n leaseEpoch: lease.leaseEpoch,\n instanceId,\n archiveGeneration: lease.archiveGeneration,\n archiveComplete: lease.archiveComplete,\n };\n captureDisposition = \"archive_published\";\n } else {\n if (\n priorCapture.operationId === attempt.operationId &&\n priorCapture.attempt >= attempt.attempt\n ) {\n // Never let an accepted duplicate/stale delivery complete the durable\n // child while its exact claim still fences the lease. Throwing asks\n // Temporal for a strictly newer attempt; an actually stale task token is\n // rejected by Temporal and cannot affect the workflow either way.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`,\n );\n }\n const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: priorCapture.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: instanceId,\n });\n const unpublishedWarmingProvider =\n backend === \"modal\" &&\n !sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) &&\n lease.recovery.provider.status === \"creating\" &&\n lease.recovery.provider.instanceId === instanceId &&\n lease.recovery.workspace.status === \"not_ready\";\n // Temporal knows that the prior activity attempt is no longer accepted,\n // but it cannot generically cancel provider I/O. Immediate replacement is\n // legal only when BOTH the durable claim and the current adapter declare\n // takeover safety: either the same idempotent request, or an independently\n // repeatable read-only capture. A readiness probe alone never proves an\n // unrelated exclusive snapshot RPC has settled.\n const replace = async () =>\n await replaceWorkspaceArchiveCaptureAfterProof(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n priorCaptureId: priorCapture.id,\n captureId: attempt.captureId,\n operationId: attempt.operationId,\n attempt: attempt.attempt,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: instanceId,\n captureTimeoutMs,\n });\n\n // If no durable complete archive exists, replacement is safe only when\n // this exact provider can be recaptured. A typed missing provider before\n // the old deadline may mean the dead worker still holds the sole archive\n // bytes in memory, so preserve its callback until that bounded window.\n const currentTakeoverSafe =\n capturePolicy !== null && capturePolicy.takeover !== \"exclusive\";\n if (!lease.resumeState) {\n if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`,\n );\n }\n captureClaim = replacement;\n } else {\n if (!deadlineExpired) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,\n );\n }\n if (!unpublishedWarmingProvider) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`,\n );\n }\n // Legacy/unrelated owner on a never-published Modal warming box. Its\n // deadline elapsed and no user-visible workspace ever existed. Fence\n // the old callback before the by-id termination below.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`,\n );\n }\n captureClaim = replacement;\n }\n } else if (priorCapture.takeoverSafe && currentTakeoverSafe) {\n // The stable lineage reuses the exact external idempotency key where\n // supported. Parallel-read adapters may instead recapture independently;\n // both cases preserve the addressed instance and cannot overlap writes.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`,\n );\n }\n captureClaim = replacement;\n } else {\n if (!deadlineExpired) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,\n );\n }\n const providerState = await probeDrainableProvider(settings, lease);\n if (providerState === \"missing\") {\n // Take over on the DB clock before committing loss. This fences an\n // old teardown callback. Its provider-request lineage remains\n // durable so a verified late archive can still repair the cold row.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`,\n );\n }\n providerMissingBeforeCapture = true;\n captureClaim = replacement;\n } else {\n // An exclusive provider operation can outlive both its Temporal\n // activity and its diagnostic deadline. A live command channel says\n // nothing about whether that operation is still reading/pausing the\n // workspace. Keep the durable fence; an operator/provider-specific\n // receipt is required to resolve this deliberately fail-closed case.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`,\n );\n }\n }\n }\n } else {\n const claimed = await claimWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: attempt.captureId,\n operationId: attempt.operationId,\n attempt: attempt.attempt,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n liveness: \"draining\",\n captureTimeoutMs,\n minIntervalMs: 0,\n pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),\n deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),\n providerReplaySafe: capturePolicy?.takeover === \"same_request\",\n takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== \"exclusive\",\n });\n if (claimed.status === \"claimed\") {\n captureClaim = claimed.claim;\n } else if (claimed.status === \"mutation_in_progress\") {\n // An admission is normally authoritative proof that a provider\n // operation may still be running. It cannot, however, pin a draining\n // lease forever after the exact provider has disappeared. Probe the\n // attributed instance without replacement. Only typed NotFound permits\n // the cold commit to reject the exact stale admissions; a live or\n // ambiguous provider remains fenced for a later sweep.\n const providerState = await probeDrainableProvider(settings, lease);\n if (providerState !== \"missing\") return false;\n providerMissingBeforeCapture = true;\n } else if (claimed.status === \"capture_in_progress\") {\n // A legacy activity or a racing delivery installed the durable claim.\n // Its owner may still complete; this child remains the repair owner if\n // it does not. Completing here would recreate a schedule-sized gap.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`,\n );\n } else {\n return false;\n }\n }\n }\n\n // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has\n // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).\n // Same exact lease/capture guard as confirmDrainCold. A stale capture owner or\n // newer epoch writes ZERO rows → wrote:false → the seam leaves the box RUNNING\n // and skips the cold-commit below.\n // `persisted` tracks whether a real archive landed on the lease this drain —\n // the durable sandbox.box.terminated event below carries it, so a \"terminated\n // with NOTHING persisted\" (box already dead at drain) is visible in the DB.\n let persisted = captureDisposition === \"archive_published\";\n let archiveRevision: string | null =\n captureDisposition === \"archive_published\"\n ? (lease.recovery.archive.current?.revision ?? null)\n : null;\n const persistArchive: PersistArchiveFn = async (\n archiveBase64: string | VerifiedHostWorkspaceArchive | null,\n archiveMetadata?: WorkspaceArchiveDescriptor,\n _providerSession?: unknown,\n providerBinding?: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBindingForSession>\n > | null,\n ) => {\n // A live persistable provider must own the exact durable capture claim.\n // Without it we cannot prove that these bytes came from a provider that was\n // admission-fenced for the full pause/capture interval.\n if (!lease.instanceId || !captureClaim) {\n return { wrote: false, archiveRevision: null };\n }\n const archiveMetrics = runtimeMetricsHooksForObservability(observability);\n let checkpointArtifactId: string | null = null;\n if (\n typeof archiveBase64 === \"string\" &&\n archiveBase64 &&\n archiveMetadata?.version === 2 &&\n (archiveMetadata.provider === \"modal_snapshot_filesystem\" ||\n archiveMetadata.provider === \"modal_snapshot_directory\")\n ) {\n if (!providerBinding) {\n throw new Error(\"Modal native snapshot has no exact session provider identity\");\n }\n const candidate = await registerSandboxCheckpointArtifact(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n sourceLeaseId: lease.id,\n sourceLeaseEpoch: row.leaseEpoch,\n sourceInstanceId: lease.instanceId,\n sourceWorkspaceGeneration: captureClaim.workspaceGeneration,\n providerBindingKey: providerBinding.key,\n providerBinding: providerBinding.binding,\n workspaceArchive: archiveBase64,\n workspaceArchiveMeta: archiveMetadata,\n });\n checkpointArtifactId = candidate.id;\n }\n const baseInput = {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedLeaseId: lease.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n expectedWorkspaceGeneration: captureClaim.workspaceGeneration,\n captureId: captureClaim.id,\n providerRequestId: captureClaim.providerRequestId,\n };\n let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;\n if (archiveBase64 === null) {\n result = await persistDrainSnapshot(db, {\n ...baseInput,\n workspaceArchive: null,\n });\n } else {\n if (!archiveMetadata) {\n throw new Error(\"Sandbox snapshot publication requires a verified archive descriptor\");\n }\n const published = await putVersion1TarArchiveOrInline({\n backend: lease.backend,\n objectStorage,\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n archive:\n typeof archiveBase64 === \"object\"\n ? archiveBase64\n : {\n bytes: Buffer.from(archiveBase64, \"base64\"),\n descriptor: archiveMetadata,\n base64: archiveBase64,\n },\n metrics: archiveMetrics,\n });\n result = await persistWorkspaceArchiveCandidate({\n objectStorage,\n ...(published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {}),\n persist: () =>\n persistDrainSnapshot(db, {\n ...baseInput,\n workspaceArchiveMeta: archiveMetadata,\n ...published,\n ...(checkpointArtifactId ? { checkpointArtifactId } : {}),\n }),\n });\n }\n if (!result.wrote && checkpointArtifactId) {\n await markSandboxCheckpointArtifactDeletePending(db, {\n accountId,\n workspaceId: row.workspaceId,\n artifactId: checkpointArtifactId,\n reason: \"drain_snapshot_publication_fenced\",\n }).catch(() => undefined);\n }\n if (result.wrote && archiveBase64 !== null) {\n persisted = true;\n archiveRevision = result.archiveRevision;\n }\n return result;\n };\n\n // Resume-by-id -> verified capture -> persist-onto-lease -> provider\n // terminate. Persisting atomically hands any displaced provider snapshot to\n // the provider-bound artifact ledger; its global GC worker owns deletion.\n // Definitive NotFound before capture is a typed loss, not ordinary teardown\n // success: confirmDrainCold preserves any prior archive or marks recovery\n // unrecoverable when none exists. A genuine capture/resume failure leaves the\n // lease draining for a later sweep (NEVER terminate a box whose files we\n // could not capture). A persist CAS miss means the box was re-armed and left\n // running, so the cold commit is skipped.\n // A published capture retry skips persistence. Revalidate legacy containment\n // even on that path: stale enrollment is not native-supervisor exit proof.\n // Typed provider absence is independently fenced by the canonical loss commit.\n if (\n !providerMissingBeforeCapture &&\n backend === \"modal\" &&\n lease.instanceId &&\n !(await readWorkspaceArchiveCapturePreflight(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n liveness: \"draining\",\n }))\n ) {\n return false;\n }\n // Stamp the charge horizon after verified capture and immediately before\n // provider stop. Capture can fail and release its claim, allowing re-arm.\n // A retry after stop success cannot charge post-termination wall time.\n let stopCutoffMarked = false;\n const beforeProviderStop = async (): Promise<void> => {\n const cutoff = await markWarmBillingStopCutoff(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n });\n if (!cutoff) throw new Error(\"sandbox stop cutoff was fenced before provider termination\");\n stopCutoffMarked = true;\n };\n const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture\n ? { terminated: true, providerMissingBeforeCapture: true }\n : await terminateBox(\n settings,\n lease,\n observability,\n persistArchive,\n captureClaim?.providerRequestId ?? attempt.operationId,\n captureDisposition,\n capturePolicy,\n Boolean(objectStorage),\n async () => {\n if (!captureClaim || !lease.instanceId) return;\n await releaseWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: captureClaim.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n });\n },\n row.workspaceId,\n beforeProviderStop,\n backend === \"docker\"\n ? async () => {\n if (!captureClaim || !lease.instanceId)\n throw new Error(\"Docker drain has no current claimed capture\");\n return await verifyDockerDrainCaptureFence(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n leaseId: lease.id,\n leaseEpoch: row.leaseEpoch,\n instanceId: lease.instanceId,\n workspaceGeneration: captureClaim.workspaceGeneration,\n captureId: captureClaim.id,\n providerRequestId: captureClaim.providerRequestId,\n });\n }\n : undefined,\n );\n const terminated = typeof termination === \"boolean\" ? termination : termination.terminated;\n if (!terminated) {\n return false;\n }\n\n // Self-hosted/none and an unassigned lease have no provider stop callback;\n // finish their usage at logical detach without touching customer compute.\n // A managed instance that was stopped without its callback must fail closed.\n const providerMissingDuringStop =\n providerMissingBeforeCapture ||\n (typeof termination !== \"boolean\" && termination.providerMissingBeforeCapture);\n if (!stopCutoffMarked && !providerMissingDuringStop) {\n if (managedProvider && lease.instanceId) {\n throw new Error(\"managed sandbox stopped without a warm usage cutoff\");\n }\n await beforeProviderStop();\n }\n\n // Stop was confirmed but the lease is not cold yet. This final epoch-fenced\n // tick settles the interval since the last heartbeat/reaper sweep, including\n // a box drained before its first periodic tick. If settlement fails, keep\n // the draining receipt for retry instead of silently losing the charge.\n const finalRate =\n settings.sandboxWarmBillingMode === \"usage_only\"\n ? 0\n : sandboxWarmRateMicrosPerSecond(settings, backend);\n if (!providerMissingDuringStop) {\n const finalTick = await accrueWarmSeconds(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n warmRateMicrosPerSecond: finalRate,\n billingMode: settings.sandboxWarmBillingMode,\n finalDrain: true,\n });\n if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === \"credits\") {\n recordCreditMicros(observability, \"usage\", finalTick.costMicros);\n }\n }\n\n const providerMissing =\n providerMissingBeforeCapture ||\n (typeof termination === \"boolean\" ? false : termination.providerMissingBeforeCapture);\n // The authoritative commit clears the durable capture/teardown claim together\n // with draining->cold. Until this succeeds, arrivals remain fenced by that\n // exact claim; a timestamp or a failed provider call can never reopen a box\n // while termination may still be in flight.\n const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(\n db,\n {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),\n providerMissingBeforeCapture: providerMissing,\n idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,\n // Mirrors terminateProviderBox: a selfhosted machine is never stopped.\n providerStopped:\n managedProvider && lease.backend !== \"selfhosted\" && lease.resumeBackendId !== \"selfhosted\",\n },\n );\n // The command terminal events and agent inputs are already durable in the\n // cold commit; this is only best-effort live fanout.\n await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {\n observability.warn(\"sandbox reaper: contained command event fanout failed\", {\n sandboxGroupId: row.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n });\n if (wentCold) {\n // Only the exact successful cold commit counts provider loss. A missing\n // probe, a stale capture, a failed commit, or a retried child is not another\n // observed loss. Keep this outside the best-effort session event writer.\n // An unpublished warming replacement (its creator died before warm\n // publication) held no workspace; its absence is not a capture loss.\n if (providerMissing && !unpublishedProviderLost) {\n recordSandboxProviderMissingBeforeCapture(observability, backend);\n }\n if (lease.unobservableCommandDrainIds?.length) {\n recordSandboxCommandContainment(\n observability,\n providerMissing ? \"provider_missing\" : \"contained\",\n );\n }\n // Durable termination record (sandbox-file-persistence observability): who\n // ended this box and whether its /workspace was captured first, appended to\n // every session sharing the group's box. Best-effort: attribution must\n // never affect the drain outcome.\n try {\n await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {\n type: \"sandbox.box.terminated\",\n payload: {\n actor: \"reaper\",\n persisted,\n archiveRevision,\n instanceId: lease.instanceId,\n providerMissingBeforeCapture: providerMissing,\n // Why the box drained: the lease's bounded rotation reason when one\n // was requested (provider deadline, operator/image conflict, teardown\n // claim), otherwise the ordinary idle drain. Additive diagnostic\n // fact; no identifiers beyond the ones already carried above.\n drainReason: sandboxDrainReason(lease),\n },\n });\n } catch (eventError) {\n observability.warn(\n \"sandbox reaper: box-terminated event write failed (drain outcome unaffected)\",\n {\n sandboxGroupId: row.sandboxGroupId,\n leaseEpoch: row.leaseEpoch,\n backend: lease.backend,\n instanceId: lease.instanceId,\n error: eventError instanceof Error ? eventError.message : String(eventError),\n },\n );\n }\n }\n return wentCold;\n}\n\n// The persist-capable slice of a live provider session. persistWorkspace()\n// snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar\n// archive) and returns the archive bytes. The session also carries the Modal SDK\n// client used to bind the snapshot receipt to the exact provider namespace.\ntype PersistableSession = {\n state?: { workspacePersistence?: unknown };\n persistWorkspace?: (options?: { requestId: string }) => Promise<Uint8Array | undefined>;\n runPreStopHooks?: () => Promise<void>;\n preStop?: (options?: { reason?: string }) => Promise<void>;\n stop?: (options?: { reason?: string }) => Promise<void>;\n shutdown?: (options?: { reason?: string }) => Promise<void>;\n delete?: (options?: { reason?: string }) => Promise<void>;\n close?: () => Promise<void>;\n};\n\nfunction providerDrainLogIdentity(\n lease: LeaseSnapshot,\n backend: string,\n): {\n leaseId: string;\n sandboxGroupId: string;\n leaseEpoch: number;\n backend: string;\n instanceId: string | null;\n} {\n return {\n leaseId: lease.id,\n sandboxGroupId: lease.sandboxGroupId,\n leaseEpoch: lease.leaseEpoch,\n backend,\n instanceId: lease.instanceId,\n };\n}\n\n/**\n * Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the\n * lease under the epoch fence, hand the superseded snapshot to durable GC, THEN\n * terminate. Resumes DIRECTLY (no cold-restore fallback) so a gone box is a clean\n * no-op, never a wasteful create-then-kill:\n * - build the client for the lease's backend;\n * - deserialize + resume the envelope (warm reattach by id, R4-safe);\n * - session.persistWorkspace() -> capture the /workspace archive;\n * - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark\n * the displaced artifact delete-pending;\n * - the provider-bound global GC deletes that object asynchronously;\n * - terminate the live handle through the standard Agents SDK lifecycle.\n *\n * Returns true when the box was terminated (or was already gone), false when the\n * persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.\n *\n * Failure discipline (NEVER lose files, NEVER leak a box):\n * - resume NotFound -> the box is already down; SUCCESS (return true), nothing\n * to persist, the caller colds the lease.\n * - a transient/auth/network resume failure on a box that may be ALIVE -> throw\n * (the caller skips; the lease stays draining for the next sweep).\n * - a persistWorkspace failure (snapshot timeout / provider error) -> throw\n * BEFORE any terminate, so the box is NOT torn down with un-captured files;\n * the next sweep retries, and the provider idle-timeout is the ultimate\n * backstop. We never terminate a box whose snapshot we could not capture.\n * A provider-backed row with instance_id but no envelope is never guessed away:\n * Modal has an explicit by-id rescue adapter above; every other backend stays\n * draining until its adapter can prove termination. A truly instance-less or\n * `none`-backed row is a no-op.\n */\n/** Internal server pre-read fence; not a grant to arbitrary callers. The\n * owning drain already derived account/workspace/group authority and claimed\n * capture through the canonical DB lifecycle. No claim is fabricated here. */\nexport async function verifyDockerDrainCaptureFence(\n db: ActivityServices[\"db\"],\n input: {\n accountId: string;\n workspaceId: string;\n sandboxGroupId: string;\n leaseId: string;\n leaseEpoch: number;\n instanceId: string;\n workspaceGeneration: number;\n captureId: string;\n providerRequestId: string;\n },\n readers = { readLease, readWorkspaceArchiveCapturePreflight },\n): Promise<{ archivePublished: boolean }> {\n const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);\n if (\n !current ||\n current.id !== input.leaseId ||\n current.liveness !== \"draining\" ||\n current.leaseEpoch !== input.leaseEpoch ||\n current.instanceId !== input.instanceId ||\n current.backend !== \"docker\" ||\n current.workspaceGeneration !== input.workspaceGeneration ||\n current.archiveCapture?.id !== input.captureId ||\n current.archiveCapture.providerRequestId !== input.providerRequestId ||\n current.archiveCapture.workspaceGeneration !== input.workspaceGeneration\n )\n throw new Error(\"Docker drain capture lease/epoch/claim changed\");\n const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {\n accountId: input.accountId,\n workspaceId: input.workspaceId,\n sandboxGroupId: input.sandboxGroupId,\n expectedEpoch: input.leaseEpoch,\n expectedInstanceId: input.instanceId,\n liveness: \"draining\",\n });\n if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)\n throw new Error(\"Docker drain capture has an active writer or changed generation\");\n return {\n archivePublished:\n current.archiveCapture.publishedAt instanceof Date &&\n Number.isFinite(current.archiveCapture.publishedAt.getTime()) &&\n current.archiveComplete === true,\n };\n}\n\nfunction observeDockerWorkspaceRelease(\n observability: ActivityServices[\"observability\"],\n session: unknown,\n logIdentity: ReturnType<typeof providerDrainLogIdentity>,\n): void {\n const release = (session as { hostWorkspaceRelease?: DockerHostWorkspaceRelease | null })\n .hostWorkspaceRelease;\n if (!release) return;\n recordSandboxDockerWorkspaceRelease(observability, release.status);\n if (release.status === \"retained\") {\n observability.warn(\n \"sandbox reaper: Docker drain kept its host workspace directory after teardown\",\n { ...logIdentity, reason: release.reason },\n );\n }\n}\n\nexport async function terminateProviderBox(\n settings: ActivityServices[\"settings\"],\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,\n observability: ActivityServices[\"observability\"],\n persistArchive: PersistArchiveFn,\n createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,\n terminateModalById: typeof terminateModalSandboxById = terminateModalSandboxById,\n providerCaptureRequestId?: string,\n captureDisposition: DrainCaptureDisposition = \"capture_required\",\n claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,\n diskBackedArchives = false,\n releaseFailedCapture?: () => Promise<void>,\n workspaceId?: string,\n beforeProviderStop?: () => Promise<void>,\n beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,\n): Promise<ProviderTerminationOutcome> {\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n const logIdentity = providerDrainLogIdentity(lease, backend);\n const warnCaptureFailure = (message: string, error: unknown): void =>\n warnDrainSnapshotFailure(observability, message, error, {\n sandboxGroupId: lease.sandboxGroupId,\n leaseEpoch: lease.leaseEpoch,\n ...(workspaceId ? { workspaceId } : {}),\n });\n const releaseAfterCaptureFailure = async (): Promise<void> => {\n try {\n await releaseFailedCapture?.();\n } catch (error) {\n warnCaptureFailure(\"sandbox reaper: failed capture claim release failed\", error);\n }\n };\n // 'none' / no backend -> nothing to terminate.\n if (!backend || backend === \"none\") {\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop\n // a selfhosted box. The \"box\" is a user's PHYSICAL machine — you cannot kill it,\n // and a delete()/kill() reaching the agent would be catastrophic. The reaper\n // DRAINS the lease to cold only (refcount→0 → draining → this returns true so\n // the caller's confirmDrainCold flips draining→cold) but issues NO provider stop:\n // no client is built, no resume, no persistWorkspace (the machine IS the\n // persistence — nothing to snapshot), no delete/kill. The session simply detaches;\n // the machine stays up under the agent's own process lifetime (§23.0). Checked on\n // BOTH the lease backend and the resume envelope's backend so neither path leaks.\n if (\n backend === \"selfhosted\" ||\n lease.backend === \"selfhosted\" ||\n lease.resumeBackendId === \"selfhosted\"\n ) {\n observability.info(\n \"sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)\",\n {\n ...logIdentity,\n },\n );\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n const durableCapturePolicy =\n claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);\n if (!durableCapturePolicy) {\n throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);\n }\n if (durableCapturePolicy.liveInstance !== \"preserved\") {\n throw new Error(\n `sandbox backend ${backend} workspace capture would replace the fenced live instance`,\n );\n }\n\n const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);\n // `instance_id` is the lease's authoritative provider address. A recovery\n // envelope without it is archive/config state, never permission to target a\n // provider inferred only from stale JSON. Conversely, a provider identity in\n // the envelope with no authoritative lease identity is inconsistent state:\n // fail closed instead of silently leaving a live provider behind while the\n // caller commits the lease cold.\n if (!lease.instanceId) {\n if (lease.providerCreateAttempt?.instanceId === null) {\n throw new Error(\n \"provider_create_outcome_unknown: cannot infer termination from missing instance ID\",\n );\n }\n if (persistedInstanceId) {\n throw new Error(\n `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,\n );\n }\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {\n throw new Error(\n `sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`,\n );\n }\n\n // A warming-death row can have a provider instance id recorded immediately\n // after create() but no resumable envelope yet. For Modal, instance_id is\n // enough to terminate directly; CAS-check first so a re-arm during the sweep\n // leaves the box running.\n const isUnpublishedWarmingProvider =\n backend === \"modal\" &&\n !persistedInstanceId &&\n lease.recovery.provider.status === \"creating\" &&\n lease.recovery.provider.instanceId === lease.instanceId &&\n lease.recovery.workspace.status === \"not_ready\";\n if (isUnpublishedWarmingProvider) {\n const { wrote } = await persistArchive(null);\n if (!wrote) {\n observability.info(\n \"sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING\",\n {\n ...logIdentity,\n },\n );\n return { terminated: false, providerMissingBeforeCapture: false };\n }\n try {\n await beforeProviderStop?.();\n await terminateModalById(\n settings,\n lease.instanceId,\n undefined,\n lease.providerCreateAttempt?.providerBindingKey,\n );\n } catch (error) {\n if (!isProviderSandboxNotFoundError(\"modal\", error)) {\n throw error;\n }\n observability.info(\"sandbox reaper: Modal sandbox already gone during direct terminate\", {\n ...logIdentity,\n });\n }\n // This branch is a warming-death box recorded before any resumable envelope\n // was published. It was never a ready workspace, so a missing instance does\n // not imply loss of a previously exposed workspace revision.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // resume_state is the folded group box-envelope (the provider sessionState the\n // box was last persisted as). An attributed provider with no envelope may\n // still be live. Only Modal exposes a verified by-id rescue adapter above;\n // never cold another backend merely because generic resume is impossible.\n if (!persistedInstanceId || !lease.resumeState) {\n throw new Error(\n `sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`,\n );\n }\n\n const client = createClientForBackend(backend as never, settings) as\n | DrainSandboxClient\n | undefined;\n if (!client) {\n // 'none' backend resolved to no client.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is\n // already gone; success.\n let session: PersistableSession | undefined;\n let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;\n try {\n if (!client.deserializeSessionState || (backend !== \"docker\" && !client.resume)) {\n // A cloud backend that cannot prove provider state is not safely\n // terminable: treating this as success would cold the lease while the\n // provider may still be live. Leave it draining for a later retry.\n throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);\n }\n // resume_state is the lease ENVELOPE: `{ backendId, sessionState: {\n // providerState: { sandboxId, ... }, manifest, ... } }` (the shape\n // serializeEstablishedSandboxEnvelope folds onto the lease). The provider\n // payload deserializeSandboxSessionStateEnvelope re-hydrates is the INNER\n // `sessionState` — pass the WHOLE envelope and it reads `state.providerState`\n // at the top level, finds nothing (providerState is nested under\n // `sessionState`), drops `sandboxId`, and `client.resume(state)` throws\n // \"requires a persisted sandboxId\". This is exactly what the working\n // resume-by-id paths (establishSandboxSessionFromEnvelope) avoid: they unwrap\n // `envelope.sessionState` first. Mirror that. (`?? lease.resumeState` keeps a\n // legacy flat envelope — providerState at the top — resumable too.)\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n const resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n lease.instanceId,\n );\n if (resumedState === undefined) {\n throw new Error(`sandbox backend ${backend} returned no resumable provider state`);\n }\n if (backend === \"docker\") {\n if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {\n throw new Error(\n \"Docker drain requires a creation-free attachment and current capture fence\",\n );\n }\n await beforeWorkspaceCapture();\n session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);\n sessionState = resumedState;\n } else {\n const resumed = await resumeExactSandboxSession(\n client,\n backend,\n resumedState,\n lease.instanceId,\n );\n session = resumed.session as PersistableSession;\n sessionState = resumed.sessionState;\n }\n } catch (error) {\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: drainable box already gone before workspace capture\", {\n ...logIdentity,\n });\n return {\n terminated: true,\n providerMissingBeforeCapture: captureDisposition !== \"archive_published\",\n };\n }\n // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays\n // draining for the next sweep) — never cold a box we could not prove is gone.\n throw error;\n }\n\n const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);\n if (\n !liveCapturePolicy ||\n liveCapturePolicy.takeover !== durableCapturePolicy.takeover ||\n liveCapturePolicy.strategy !== durableCapturePolicy.strategy ||\n liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance\n ) {\n throw new Error(\n `sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`,\n );\n }\n\n // PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to\n // snapshot must re-throw BEFORE any terminate so files are never lost — the next\n // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the\n // box raced gone between resume and persist) is success: nothing to persist.\n let verifiedArchive: VerifiedWorkspaceArchivePayload | undefined;\n const sessionWorkspacePersistence = (\n session as { state?: { workspacePersistence?: unknown } } | undefined\n )?.state?.workspacePersistence;\n const checkpointBinding =\n captureDisposition === \"capture_required\" &&\n backend === \"modal\" &&\n (sessionWorkspacePersistence === \"snapshot_filesystem\" ||\n sessionWorkspacePersistence === \"snapshot_directory\")\n ? await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBindingForSession(settings, session),\n )\n : null;\n // Physical capture bookkeeping for this worker: released once the capture\n // and its spool are settled, or by the late path that inherits them.\n let releaseSettlingCapture: () => void = () => undefined;\n try {\n if (captureDisposition !== \"archive_published\" && session?.persistWorkspace) {\n releaseSettlingCapture = trackSettlingDrainCapture(lease);\n // The budget bounds only this attempt's wait, never the capture: a slow\n // but finite capture keeps reading and publishes late, and the retry\n // that finds it still settling here defers instead of starting another.\n const capture = captureWorkspaceArchiveForStorage(\n session,\n Date.now(),\n {\n requestId: providerCaptureRequestId ?? randomUUID(),\n strategy: liveCapturePolicy.strategy,\n },\n diskBackedArchives,\n );\n verifiedArchive = await awaitProviderCaptureWithLatePublication({\n capture,\n timeoutMs: settings.sandboxSnapshotTimeoutMs,\n timeoutError: new SandboxProviderCaptureTimeoutError(\n lease.sandboxGroupId,\n backend,\n settings.sandboxSnapshotTimeoutMs,\n lease.leaseEpoch,\n lease.instanceId,\n ),\n publishLate: async (archive) => {\n try {\n const result = await persistArchive(\n archive.kind === \"host_spool\" ? archive : archive.base64,\n archive.descriptor,\n session,\n checkpointBinding,\n );\n observability.info(\n result.wrote\n ? \"sandbox reaper: late workspace capture published under durable claim\"\n : \"sandbox reaper: late workspace capture fenced by successor\",\n {\n ...logIdentity,\n },\n );\n } catch (error) {\n warnCaptureFailure(\"sandbox reaper: late workspace capture publication failed\", error);\n } finally {\n await disposeWorkspaceArchive(archive);\n }\n },\n observeLatePublicationFailure: (error) => {\n warnCaptureFailure(\"sandbox reaper: late capture publication cleanup failed\", error);\n },\n observeLateFailure: (error) => {\n // Capture has now rejected and this timed-out path can never reach\n // provider teardown. Release only our unpublished claim so a waiting\n // turn can re-arm the intact live box; stale callbacks are DB-fenced.\n void releaseAfterCaptureFailure();\n warnCaptureFailure(\"sandbox reaper: timed-out provider capture later failed\", error);\n },\n onLateSettled: () => releaseSettlingCapture(),\n });\n }\n } catch (error) {\n // Every rejection except the caller's own timeout means the capture has\n // settled; a timed-out capture is released by its late path instead.\n if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: box gone during workspace capture\", {\n ...logIdentity,\n });\n return { terminated: true, providerMissingBeforeCapture: true };\n }\n warnCaptureFailure(\n \"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)\",\n error,\n );\n // An unresolved timeout is not proof of capture failure: retain the fence\n // until its provider promise settles. A settled failure cannot enter the\n // teardown path below, so its unpublished claim may safely be released.\n if (!(error instanceof SandboxProviderCaptureTimeoutError)) {\n await releaseAfterCaptureFailure();\n }\n // NEVER terminate a box whose snapshot we could not capture.\n throw error;\n }\n\n // Fold the captured archive onto the lease under the exact capture fence, then\n // GC the superseded snapshot. A CAS miss means this callback no longer owns\n // teardown — DO NOT terminate (the caller skips confirmDrainCold).\n //\n // Ownership guard for the no-archive path: even when persistWorkspace returns\n // no bytes, require the exact capture receipt before delete(). The null-archive\n // path performs that CAS without changing archive bytes; wrote:false aborts.\n if (captureDisposition === \"archive_published\") {\n observability.info(\"sandbox reaper: resuming teardown from durable archive publication\", {\n ...logIdentity,\n });\n } else if (verifiedArchive) {\n let wrote: boolean;\n try {\n ({ wrote } = await persistArchive(\n verifiedArchive.kind === \"host_spool\" ? verifiedArchive : verifiedArchive.base64,\n verifiedArchive.descriptor,\n session,\n checkpointBinding,\n ));\n } finally {\n try {\n await disposeWorkspaceArchive(verifiedArchive);\n } finally {\n releaseSettlingCapture();\n }\n }\n if (!wrote) {\n observability.info(\n \"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)\",\n {\n ...logIdentity,\n },\n );\n return { terminated: false, providerMissingBeforeCapture: false };\n }\n } else {\n // A resumable cloud box that produces no verified archive cannot be safely\n // deleted: it may contain work newer than the lease's last durable revision.\n // Keep the lease draining and retry instead of knowingly discarding bytes.\n throw new Error(\n `sandbox backend ${backend} produced no verified workspace archive during drain`,\n );\n }\n\n // Provider terminate through the declared Agents SDK lifecycle. A terminate\n // that fails because the box is already gone is success.\n try {\n if (!session || !sessionState) {\n throw new Error(`sandbox backend ${backend} returned no terminable session state`);\n }\n prepareProviderForTeardownAfterCapture(backend, session);\n await beforeProviderStop?.();\n await terminateManagedSandboxSession(client, sessionState, session);\n if (backend === \"docker\") observeDockerWorkspaceRelease(observability, session, logIdentity);\n return { terminated: true, providerMissingBeforeCapture: false };\n } catch (error) {\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: provider already gone during terminate\", {\n ...logIdentity,\n });\n // Capture was already durably folded above, so this is not a\n // missing-before-capture outcome.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n throw error;\n }\n}\n","import { Context } from \"@temporalio/activity\";\nimport {\n environmentsEncryptionKeyBytes,\n resolveFirstPartyDelegationSecret,\n type Settings,\n} from \"@opengeni/config\";\nimport { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from \"@opengeni/contracts\";\nimport {\n browserDeadlineCheckpoint,\n clearSuspendedBrowserSessionController,\n commitBrowserSessionSuspension,\n dispatchBrowserSessionOperation,\n failBrowserSessionSuspension,\n listBrowserDeadlineCheckpoints,\n readLease,\n releaseLeaseHolder,\n withRlsContext,\n type BrowserDeadlineCheckpointTarget,\n} from \"@opengeni/db\";\nimport { deriveBrowserControllerAdminToken } from \"@opengeni/interaction/browser-controller-authority\";\nimport {\n browserStateArtifactAad,\n browserStateManifestDigest,\n browserStateObjectKey,\n deriveBrowserStateDataKey,\n wrapBrowserStateDataKey,\n} from \"@opengeni/interaction/browser-state-authority\";\nimport {\n BrowserControlClient,\n BrowserControlRequestError,\n createSandboxClientForBackend,\n deserializeSandboxSessionStateEnvelope,\n resumeExactSandboxSession,\n type BrowserControlPlacementSession,\n} from \"@opengeni/runtime\";\nimport type { ControlActivityServices } from \"./types\";\n\nconst UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 60_000;\n\nfunction heartbeat(): void {\n try {\n Context.current().heartbeat();\n } catch {\n // Direct embedded calls have no Temporal context. In-flight controller\n // operations still settle under their exact durable database claim.\n }\n}\n\nasync function withActivityHeartbeat<T>(run: () => Promise<T>): Promise<T> {\n heartbeat();\n const timer = setInterval(heartbeat, 5_000);\n try {\n return await run();\n } finally {\n clearInterval(timer);\n }\n}\n\ntype CheckpointController = Pick<BrowserControlClient, \"captureState\" | \"endSession\">;\ntype CheckpointPlacement = {\n target: BrowserDeadlineCheckpointTarget;\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>;\n settings: Settings;\n controllerRoot: string;\n};\n\nasync function resumeCheckpointController({\n target,\n lease,\n settings,\n controllerRoot,\n}: CheckpointPlacement): Promise<CheckpointController> {\n // This edge cannot create a box or route through a user's current machine.\n const provider = createSandboxClientForBackend(\"modal\", settings) as Parameters<\n typeof deserializeSandboxSessionStateEnvelope\n >[0];\n const envelope = lease.resumeState as { sessionState?: unknown };\n const state = await deserializeSandboxSessionStateEnvelope(\n provider,\n envelope.sessionState ?? lease.resumeState,\n target.instanceId,\n );\n const resumed = await resumeExactSandboxSession(provider, \"modal\", state, target.instanceId);\n return new BrowserControlClient(resumed.session as BrowserControlPlacementSession, {\n adminToken: deriveBrowserControllerAdminToken({\n rootSecret: controllerRoot,\n accountId: target.accountId,\n workspaceId: target.workspaceId,\n placement: { kind: \"sandbox_group\", sandboxGroupId: target.sandboxGroupId },\n placementInstanceId: target.instanceId,\n }),\n });\n}\n\n/** Whether this worker can save browser profiles: ownership, object storage,\n * the controller delegation secret and the environments encryption key. */\nexport function browserCheckpointPrerequisites(\n settings: Settings,\n objectStorage: ControlActivityServices[\"objectStorage\"],\n): boolean {\n if (\n !settings.sandboxOwnershipEnabled ||\n !objectStorage ||\n !resolveFirstPartyDelegationSecret(settings)\n )\n return false;\n const root = environmentsEncryptionKeyBytes(settings);\n if (!root) return false;\n root.fill(0);\n return true;\n}\n\nexport function createBrowserDeadlineCheckpointActivities(\n services: () => Promise<ControlActivityServices>,\n connectController: (\n placement: CheckpointPlacement,\n ) => Promise<CheckpointController> = resumeCheckpointController,\n) {\n return {\n async listDueBrowserCheckpoints() {\n return await withActivityHeartbeat(async () => {\n const { db, settings, objectStorage } = await services();\n if (!browserCheckpointPrerequisites(settings, objectStorage)) return [];\n return await listBrowserDeadlineCheckpoints(db);\n });\n },\n async checkpointBrowserBeforeDeadline(target: BrowserDeadlineCheckpointTarget) {\n return await withActivityHeartbeat(async () => {\n const { db, settings, objectStorage } = await services();\n const rootKey = environmentsEncryptionKeyBytes(settings);\n const controllerRoot = resolveFirstPartyDelegationSecret(settings);\n if (!rootKey || !objectStorage || !controllerRoot) {\n rootKey?.fill(0);\n throw new Error(\"browser deadline checkpoint storage or encryption is unavailable\");\n }\n let dataKey: Buffer | null = null;\n let aad: Buffer | null = null;\n let timer: ReturnType<typeof setInterval> | null = null;\n try {\n // Only the reaper's locked idle decision prepares an idle save; this\n // activity continues it. A deadline save prepares here.\n const claim = await browserDeadlineCheckpoint(db, target, {\n prepare: target.reason !== \"idle\",\n touch: true,\n });\n if (!claim) return { status: \"skipped\" as const };\n let pulseRunning = false;\n let authorityLost = false;\n const pulse = async () => {\n if (pulseRunning) return;\n pulseRunning = true;\n try {\n if (!(await browserDeadlineCheckpoint(db, target, { touch: true })))\n authorityLost = true;\n } finally {\n pulseRunning = false;\n }\n };\n timer = setInterval(() => {\n void pulse().catch(() => undefined);\n }, 5_000);\n const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);\n if (\n !lease ||\n lease.instanceId !== target.instanceId ||\n lease.leaseEpoch !== target.leaseEpoch ||\n !lease.resumeState\n ) {\n throw new Error(\"browser deadline checkpoint placement changed\");\n }\n const placement = {\n kind: \"sandbox_group\" as const,\n sandboxGroupId: target.sandboxGroupId,\n };\n const client = await connectController({ target, lease, settings, controllerRoot });\n if (authorityLost || !(await browserDeadlineCheckpoint(db, target, { touch: true }))) {\n throw new Error(\"browser deadline checkpoint authority changed\");\n }\n if (claim.state !== \"completed\") {\n const scope = { ...target, operationId: claim.operationId };\n const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);\n dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });\n aad = browserStateArtifactAad({ ...scope, objectKey });\n await dispatchBrowserSessionOperation(db, {\n ...scope,\n deadlineTarget: target,\n stateUpload: {\n objectKey,\n cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS),\n },\n });\n const signed = await objectStorage.createPutUrl({\n key: objectKey,\n contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,\n audience: \"sandbox\",\n });\n const receipt = await client\n .captureState({\n ...scope,\n objectKey,\n afterCapture: \"stop\",\n dataKey,\n aad,\n upload: {\n url: signed.url,\n requiredHeaders: signed.requiredHeaders,\n expiresAt: signed.expiresAt.toISOString(),\n },\n })\n .catch(async (error: unknown) => {\n if (\n error instanceof BrowserControlRequestError &&\n (error.error.code === \"outcome_unknown\" || !error.error.retryable)\n ) {\n await failBrowserSessionSuspension(db, { ...scope, error: error.error });\n }\n throw error;\n });\n if (authorityLost) throw new Error(\"browser deadline checkpoint authority changed\");\n const portable = receipt.manifest.profileCrypto !== \"platform_bound\";\n await commitBrowserSessionSuspension(db, {\n ...scope,\n deadlineTarget: target,\n artifact: {\n kind: \"chromium_profile\",\n format: receipt.format,\n objectKey,\n artifactDigest: receipt.artifactDigest,\n contentDigest: receipt.contentDigest,\n manifestDigest: browserStateManifestDigest(receipt.manifest),\n sizeBytes: receipt.sizeBytes,\n encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {\n ...scope,\n objectKey,\n artifactDigest: receipt.artifactDigest,\n contentDigest: receipt.contentDigest,\n }),\n materialization: {\n portability: portable ? \"portable\" : \"placement_bound\",\n reason: portable\n ? null\n : \"Profile encryption depends on the source operating-system credential store.\",\n platform: receipt.manifest.platform,\n architecture: receipt.manifest.architecture,\n engine: receipt.manifest.engine,\n engineVersion: receipt.manifest.engineVersion,\n driverId: receipt.manifest.driverId,\n driverSchemaVersion: receipt.manifest.driverSchemaVersion,\n profileCrypto: receipt.manifest.profileCrypto,\n providerId: null,\n placement: portable ? null : placement,\n },\n },\n });\n }\n // Only remove the local profile after the encrypted artifact and its\n // authority committed. A crash resumes this cleanup, never capture.\n await client.endSession(target, { removeState: true }).catch((error: unknown) => {\n if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;\n });\n await withRlsContext(\n db,\n target,\n async (tx) => {\n if ((await browserDeadlineCheckpoint(tx, target))?.state !== \"completed\") {\n throw new Error(\"browser deadline checkpoint cleanup authority changed\");\n }\n await clearSuspendedBrowserSessionController(tx, {\n ...target,\n expectedControllerGeneration: target.controllerGeneration,\n });\n // One transaction prevents a human resume from acquiring a new\n // same-browser holder between clearing and releasing the old one.\n await releaseLeaseHolder(tx, {\n ...target,\n kind: \"interaction\",\n holderId: `browser-session:${target.browserSessionId}`,\n // The idle window already elapsed before this save began, so\n // a box the saved browser was last to hold drains now.\n idleGraceMs: target.reason === \"idle\" ? 0 : settings.sandboxIdleGraceMs,\n preserveHoldersChangedAt: target.reason === \"idle\",\n });\n },\n undefined,\n \"none\",\n );\n return { status: \"suspended\" as const };\n } finally {\n if (timer) clearInterval(timer);\n dataKey?.fill(0);\n aad?.fill(0);\n rootKey.fill(0);\n }\n });\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAIA,SAAS,gCACd,eACA,OACA,SAIM;AACN,MAAI;AACJ,MAAI;AACF,UAAM,WAAW,uBAAuB,KAAK;AAC7C,UAAM,aAAa,uBAAuB,KAAK,EAAE;AACjD,oBAAgB,cAAc,wBAAwB;AAAA,MACpD,MACE,iBAAiB,sCACb,4BACA,WACE,oBAAoB,QAAQ,IAC5B;AAAA,MACR,OAAO;AAAA,MACP;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,GAAI,QAAQ,iBAAiB,EAAE,WAAW,QAAQ,eAAe,IAAI,CAAC;AAAA,MACtE,GAAI,QAAQ,cAAc,EAAE,QAAQ,QAAQ,YAAY,IAAI,CAAC;AAAA,MAC7D,UAAU,QAAQ;AAAA,MAClB;AAAA,MACA,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AAAA,EACH,QAAQ;AAAA,EAER;AACA,MAAI;AACF,kBAAc,KAAK,4DAA4D;AAAA,MAC7E,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,QAAQ;AAAA,EAER;AACF;;;AC7CO,SAAS,+BACd,SAIA,OAIA,UACA,cACA,MAAM,KAAK,IAAI,GACP;AACR,MACE,QAAQ,oBAAoB,WAC5B,QAAQ,kBAAkB,QAC1B,CAAC,SACD,MAAM,OAAO,QAAQ,WACrB,MAAM,eAAe,QAAQ,cAC7B,MAAM,YAAY,QAAQ,mBAC1B,MAAM,eAAe,QAAQ,sBAC7B,CAAC,MAAM;AAEP,WAAO;AAET,QAAM,gBAAgB,MAAM,mBAAmB,QAAQ,IAAI,SAAS,wBAAwB;AAC5F,MAAI,CAAC,OAAO,SAAS,aAAa,EAAG,QAAO;AAC5C,SAAO,KAAK;AAAA,IACV;AAAA,IACA,KAAK,IAAI,KAAO,SAAS,4BAA4B,aAAa;AAAA,EACpE;AACF;;;ACdA,SAAS,YAAY,kBAAkB;AACvC,SAAS,kDAAkD;AAE3D,SAAS,WAAAA,gBAAe;AACxB,SAAS,cAAc,eAA8B;AACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,kCAAkC;AAE3C,SAAS,gCAAgC;AACzC;AAAA,EAME;AAAA,EACA;AAAA,EAGA;AAAA,EACA;AAAA,EACA,iCAAAC;AAAA,EACA;AAAA,EACA,0CAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,6BAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAQK;;;ACvKP,SAAS,eAAe;AACxB;AAAA,EACE;AAAA,EACA;AAAA,OAEK;AACP,SAAS,2CAA2C;AACpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,yCAAyC;AAClD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAGP,IAAM,0BAA0B,KAAK,KAAK;AAE1C,SAAS,YAAkB;AACzB,MAAI;AACF,YAAQ,QAAQ,EAAE,UAAU;AAAA,EAC9B,QAAQ;AAAA,EAGR;AACF;AAEA,eAAe,sBAAyB,KAAmC;AACzE,YAAU;AACV,QAAM,QAAQ,YAAY,WAAW,GAAK;AAC1C,MAAI;AACF,WAAO,MAAM,IAAI;AAAA,EACnB,UAAE;AACA,kBAAc,KAAK;AAAA,EACrB;AACF;AAUA,eAAe,2BAA2B;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAAuD;AAErD,QAAM,WAAW,8BAA8B,SAAS,QAAQ;AAGhE,QAAM,WAAW,MAAM;AACvB,QAAM,QAAQ,MAAM;AAAA,IAClB;AAAA,IACA,SAAS,gBAAgB,MAAM;AAAA,IAC/B,OAAO;AAAA,EACT;AACA,QAAM,UAAU,MAAM,0BAA0B,UAAU,SAAS,OAAO,OAAO,UAAU;AAC3F,SAAO,IAAI,qBAAqB,QAAQ,SAA2C;AAAA,IACjF,YAAY,kCAAkC;AAAA,MAC5C,YAAY;AAAA,MACZ,WAAW,OAAO;AAAA,MAClB,aAAa,OAAO;AAAA,MACpB,WAAW,EAAE,MAAM,iBAAiB,gBAAgB,OAAO,eAAe;AAAA,MAC1E,qBAAqB,OAAO;AAAA,IAC9B,CAAC;AAAA,EACH,CAAC;AACH;AAIO,SAAS,+BACd,UACA,eACS;AACT,MACE,CAAC,SAAS,2BACV,CAAC,iBACD,CAAC,kCAAkC,QAAQ;AAE3C,WAAO;AACT,QAAM,OAAO,+BAA+B,QAAQ;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,OAAK,KAAK,CAAC;AACX,SAAO;AACT;AAEO,SAAS,0CACd,UACA,oBAEqC,4BACrC;AACA,SAAO;AAAA,IACL,MAAM,4BAA4B;AAChC,aAAO,MAAM,sBAAsB,YAAY;AAC7C,cAAM,EAAE,IAAI,UAAU,cAAc,IAAI,MAAM,SAAS;AACvD,YAAI,CAAC,+BAA+B,UAAU,aAAa,EAAG,QAAO,CAAC;AACtE,eAAO,MAAM,+BAA+B,EAAE;AAAA,MAChD,CAAC;AAAA,IACH;AAAA,IACA,MAAM,gCAAgC,QAAyC;AAC7E,aAAO,MAAM,sBAAsB,YAAY;AAC7C,cAAM,EAAE,IAAI,UAAU,cAAc,IAAI,MAAM,SAAS;AACvD,cAAM,UAAU,+BAA+B,QAAQ;AACvD,cAAM,iBAAiB,kCAAkC,QAAQ;AACjE,YAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,gBAAgB;AACjD,mBAAS,KAAK,CAAC;AACf,gBAAM,IAAI,MAAM,kEAAkE;AAAA,QACpF;AACA,YAAI,UAAyB;AAC7B,YAAI,MAAqB;AACzB,YAAI,QAA+C;AACnD,YAAI;AAGF,gBAAM,QAAQ,MAAM,0BAA0B,IAAI,QAAQ;AAAA,YACxD,SAAS,OAAO,WAAW;AAAA,YAC3B,OAAO;AAAA,UACT,CAAC;AACD,cAAI,CAAC,MAAO,QAAO,EAAE,QAAQ,UAAmB;AAChD,cAAI,eAAe;AACnB,cAAI,gBAAgB;AACpB,gBAAM,QAAQ,YAAY;AACxB,gBAAI,aAAc;AAClB,2BAAe;AACf,gBAAI;AACF,kBAAI,CAAE,MAAM,0BAA0B,IAAI,QAAQ,EAAE,OAAO,KAAK,CAAC;AAC/D,gCAAgB;AAAA,YACpB,UAAE;AACA,6BAAe;AAAA,YACjB;AAAA,UACF;AACA,kBAAQ,YAAY,MAAM;AACxB,iBAAK,MAAM,EAAE,MAAM,MAAM,MAAS;AAAA,UACpC,GAAG,GAAK;AACR,gBAAM,QAAQ,MAAM,UAAU,IAAI,OAAO,aAAa,OAAO,cAAc;AAC3E,cACE,CAAC,SACD,MAAM,eAAe,OAAO,cAC5B,MAAM,eAAe,OAAO,cAC5B,CAAC,MAAM,aACP;AACA,kBAAM,IAAI,MAAM,+CAA+C;AAAA,UACjE;AACA,gBAAM,YAAY;AAAA,YAChB,MAAM;AAAA,YACN,gBAAgB,OAAO;AAAA,UACzB;AACA,gBAAM,SAAS,MAAM,kBAAkB,EAAE,QAAQ,OAAO,UAAU,eAAe,CAAC;AAClF,cAAI,iBAAiB,CAAE,MAAM,0BAA0B,IAAI,QAAQ,EAAE,OAAO,KAAK,CAAC,GAAI;AACpF,kBAAM,IAAI,MAAM,+CAA+C;AAAA,UACjE;AACA,cAAI,MAAM,UAAU,aAAa;AAC/B,kBAAM,QAAQ,EAAE,GAAG,QAAQ,aAAa,MAAM,YAAY;AAC1D,kBAAM,YAAY,sBAAsB,OAAO,aAAa,MAAM,WAAW;AAC7E,sBAAU,0BAA0B,SAAS,EAAE,GAAG,OAAO,UAAU,CAAC;AACpE,kBAAM,wBAAwB,EAAE,GAAG,OAAO,UAAU,CAAC;AACrD,kBAAM,gCAAgC,IAAI;AAAA,cACxC,GAAG;AAAA,cACH,gBAAgB;AAAA,cAChB,aAAa;AAAA,gBACX;AAAA,gBACA,cAAc,IAAI,KAAK,KAAK,IAAI,IAAI,uBAAuB;AAAA,cAC7D;AAAA,YACF,CAAC;AACD,kBAAM,SAAS,MAAM,cAAc,aAAa;AAAA,cAC9C,KAAK;AAAA,cACL,aAAa;AAAA,cACb,UAAU;AAAA,YACZ,CAAC;AACD,kBAAM,UAAU,MAAM,OACnB,aAAa;AAAA,cACZ,GAAG;AAAA,cACH;AAAA,cACA,cAAc;AAAA,cACd;AAAA,cACA;AAAA,cACA,QAAQ;AAAA,gBACN,KAAK,OAAO;AAAA,gBACZ,iBAAiB,OAAO;AAAA,gBACxB,WAAW,OAAO,UAAU,YAAY;AAAA,cAC1C;AAAA,YACF,CAAC,EACA,MAAM,OAAO,UAAmB;AAC/B,kBACE,iBAAiB,+BAChB,MAAM,MAAM,SAAS,qBAAqB,CAAC,MAAM,MAAM,YACxD;AACA,sBAAM,6BAA6B,IAAI,EAAE,GAAG,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,cACzE;AACA,oBAAM;AAAA,YACR,CAAC;AACH,gBAAI,cAAe,OAAM,IAAI,MAAM,+CAA+C;AAClF,kBAAM,WAAW,QAAQ,SAAS,kBAAkB;AACpD,kBAAM,+BAA+B,IAAI;AAAA,cACvC,GAAG;AAAA,cACH,gBAAgB;AAAA,cAChB,UAAU;AAAA,gBACR,MAAM;AAAA,gBACN,QAAQ,QAAQ;AAAA,gBAChB;AAAA,gBACA,gBAAgB,QAAQ;AAAA,gBACxB,eAAe,QAAQ;AAAA,gBACvB,gBAAgB,2BAA2B,QAAQ,QAAQ;AAAA,gBAC3D,WAAW,QAAQ;AAAA,gBACnB,kBAAkB,wBAAwB,SAAS,SAAS;AAAA,kBAC1D,GAAG;AAAA,kBACH;AAAA,kBACA,gBAAgB,QAAQ;AAAA,kBACxB,eAAe,QAAQ;AAAA,gBACzB,CAAC;AAAA,gBACD,iBAAiB;AAAA,kBACf,aAAa,WAAW,aAAa;AAAA,kBACrC,QAAQ,WACJ,OACA;AAAA,kBACJ,UAAU,QAAQ,SAAS;AAAA,kBAC3B,cAAc,QAAQ,SAAS;AAAA,kBAC/B,QAAQ,QAAQ,SAAS;AAAA,kBACzB,eAAe,QAAQ,SAAS;AAAA,kBAChC,UAAU,QAAQ,SAAS;AAAA,kBAC3B,qBAAqB,QAAQ,SAAS;AAAA,kBACtC,eAAe,QAAQ,SAAS;AAAA,kBAChC,YAAY;AAAA,kBACZ,WAAW,WAAW,OAAO;AAAA,gBAC/B;AAAA,cACF;AAAA,YACF,CAAC;AAAA,UACH;AAGA,gBAAM,OAAO,WAAW,QAAQ,EAAE,aAAa,KAAK,CAAC,EAAE,MAAM,CAAC,UAAmB;AAC/E,gBAAI,EAAE,iBAAiB,8BAA8B,MAAM,WAAW,KAAM,OAAM;AAAA,UACpF,CAAC;AACD,gBAAM;AAAA,YACJ;AAAA,YACA;AAAA,YACA,OAAO,OAAO;AACZ,mBAAK,MAAM,0BAA0B,IAAI,MAAM,IAAI,UAAU,aAAa;AACxE,sBAAM,IAAI,MAAM,uDAAuD;AAAA,cACzE;AACA,oBAAM,uCAAuC,IAAI;AAAA,gBAC/C,GAAG;AAAA,gBACH,8BAA8B,OAAO;AAAA,cACvC,CAAC;AAGD,oBAAM,mBAAmB,IAAI;AAAA,gBAC3B,GAAG;AAAA,gBACH,MAAM;AAAA,gBACN,UAAU,mBAAmB,OAAO,gBAAgB;AAAA;AAAA;AAAA,gBAGpD,aAAa,OAAO,WAAW,SAAS,IAAI,SAAS;AAAA,gBACrD,0BAA0B,OAAO,WAAW;AAAA,cAC9C,CAAC;AAAA,YACH;AAAA,YACA;AAAA,YACA;AAAA,UACF;AACA,iBAAO,EAAE,QAAQ,YAAqB;AAAA,QACxC,UAAE;AACA,cAAI,MAAO,eAAc,KAAK;AAC9B,mBAAS,KAAK,CAAC;AACf,eAAK,KAAK,CAAC;AACX,kBAAQ,KAAK,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ADtEA,SAAS,4BAAAC,iCAAgC;AAwLlC,IAAM,wCAAwC;AAC9C,IAAM,+CAA+C,IAAI;AACzD,IAAM,6CAA6C;AAInD,IAAM,0CAA0C;AAChD,IAAM,0CAA0C;AAChD,IAAM,gDAAgD;AAItD,IAAM,wCAAwC,MAAM;AACpD,IAAM,qDAAqD;AAC3D,IAAM,+CAA+C,KAAK,KAAK;AAC/D,IAAM,yCAAyC;AAC/C,IAAM,gDAAgD,IAAI;AAE1D,IAAM,qCAAN,cAAiD,MAAM;AAAA,EAG5D,YACkB,gBACA,SACA,WACA,YACA,YAChB;AACA;AAAA,MACE,WAAW,cAAc,UAAU,UAAU,IAAI,OAAO,aAAa,UAAU,+BAC/C,SAAS;AAAA,IAE3C;AAVgB;AACA;AACA;AACA;AACA;AAAA,EAOlB;AAAA,EAdS,OAAO;AAelB;AAEA,eAAsB,wCAA2C,OAUlD;AACb,MAAI;AACJ,QAAM,eAAe,MAAM;AAC3B,MAAI;AACF,WAAO,MAAM,QAAQ,KAAK;AAAA,MACxB,MAAM;AAAA,MACN,IAAI,QAAe,CAAC,GAAG,WAAW;AAChC,kBAAU,WAAW,MAAM,OAAO,YAAY,GAAG,MAAM,SAAS;AAAA,MAClE,CAAC;AAAA,IACH,CAAC;AAAA,EACH,SAAS,OAAO;AACd,QAAI,UAAU,aAAc,OAAM;AAMlC,UAAM,UAAU,MAAM;AACpB,UAAI;AACF,cAAM,gBAAgB;AAAA,MACxB,QAAQ;AAAA,MAER;AAAA,IACF;AACA,SAAK,MAAM,QACR,KAAK,MAAM,aAAa,MAAM,kBAAkB,EAChD,MAAM,MAAM,6BAA6B,EACzC,KAAK,SAAS,OAAO;AACxB,UAAM;AAAA,EACR,UAAE;AACA,QAAI,QAAS,cAAa,OAAO;AAAA,EACnC;AACF;AAEA,SAAS,4BAA4B,SAA6D;AAChG,MAAI;AACJ,MAAI;AACF,cAAUC,SAAQ,QAAQ;AAAA,EAC5B,QAAQ;AAEN,WAAO,MAAM;AAAA,EACf;AACA,QAAMC,aAAY,MAAY;AAC5B,QAAI;AACF,cAAQ,UAAU,EAAE,GAAG,SAAS,cAAa,oBAAI,KAAK,GAAE,YAAY,EAAE,CAAC;AAAA,IACzE,QAAQ;AAAA,IAMR;AAAA,EACF;AACA,EAAAA,WAAU;AACV,QAAM,QAAQ,YAAYA,YAAW,6CAA6C;AAClF,MAAI,WAAW,SAAS,OAAO,MAAM,UAAU,WAAY,OAAM,MAAM;AACvE,SAAO,MAAM,cAAc,KAAK;AAClC;AAqBA,IAAM,gCAAgC,KAAK;AAAA,EACzC;AAAA,EACA,KAAK,MAAM,2CAA2C,CAAC;AACzD;AAoBA,IAAM,oCAAN,cAAgD,MAAM;AAAA,EAC3C,OAAO;AAClB;AAgBA,IAAM,wBAAwB,oBAAI,IAAqC;AAQvE,SAAS,gBAAgB,OAA8D;AACrF,SAAO,KAAK,UAAU,CAAC,MAAM,gBAAgB,MAAM,YAAY,MAAM,UAAU,CAAC;AAClF;AAIA,SAAS,qCACP,OACA,QAAQ,KAAK,IAAI,GACR;AACT,MAAI,CAAC,MAAM,WAAY,QAAO;AAC9B,QAAM,QAAQ,sBAAsB;AAAA,IAClC,gBAAgB,EAAE,GAAG,OAAO,YAAY,MAAM,WAAW,CAAC;AAAA,EAC5D;AACA,SAAO,UAAU,UAAa,QAAQ,MAAM,cAAc;AAC5D;AAEA,SAAS,0BAA0B,OAAyC;AAC1E,MAAI,CAAC,MAAM,WAAY,QAAO,MAAM;AACpC,QAAM,MAAM,gBAAgB,EAAE,GAAG,OAAO,YAAY,MAAM,WAAW,CAAC;AACtE,QAAM,QAAQ,EAAE,aAAa,KAAK,IAAI,EAAE;AACxC,wBAAsB,IAAI,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,sBAAsB,IAAI,GAAG,MAAM,MAAO,uBAAsB,OAAO,GAAG;AAAA,EAChF;AACF;AAKO,SAAS,sBAAsB,aAAqB,SAAyB;AAClF,MAAI,CAAC,kEAAkE,KAAK,WAAW,GAAG;AACxF,UAAM,IAAI,MAAM,uCAAuC;AAAA,EACzD;AACA,MAAI,CAAC,OAAO,cAAc,OAAO,KAAK,WAAW,GAAG;AAClD,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,MAAI,YAAY,EAAG,QAAO,YAAY,YAAY;AAClD,QAAM,QAAQ,WAAW,QAAQ,EAC9B,OAAO,0BAA0B,YAAY,YAAY,CAAC,IAAI,OAAO,EAAE,EACvE,OAAO,EACP,SAAS,GAAG,EAAE;AACjB,QAAM,CAAC,IAAK,MAAM,CAAC,IAAK,KAAQ;AAChC,QAAM,CAAC,IAAK,MAAM,CAAC,IAAK,KAAQ;AAChC,QAAM,MAAM,MAAM,SAAS,KAAK;AAChC,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AAC1G;AAEA,SAAS,8BAAsC;AAC7C,MAAI;AACF,UAAM,UAAUD,SAAQ,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,cAAc,OAAO,KAAK,UAAU,IAAI,UAAU;AAAA,EAClE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,6BACd,UACA,UAAuC,CAAC,GACxC;AACA,QAAM,gBACH,QAAQ,eACL,UAAU,SAAqC;AAG7C,UAAM,KAAK,EAAE,IAAI;AACjB,WAAO,QAAQ,aAAc,GAAG,IAAI;AAAA,EACtC,IACA,UACH,OACC,UACA,OACA,eACA,gBACA,0BACA,oBACA,eACA,oBACA,sBACA,aACA,oBACA,2BAEA,MAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,0BAA0BE;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACJ,QAAM,oBACJ,QAAQ,qBAAqB;AAC/B,QAAM,8BACJ,QAAQ,yCAAyC;AACnD,QAAM,uBAAuB,QAAQ,wBAAwB;AAC7D,QAAM,yBAAyB,QAAQ,0BAA0B;AACjE,QAAM,8BACJ,QAAQ,+BAA+B;AAEzC,QAAM,0BAA0B,oBAAI,IAAY;AAChD,QAAM,yBAAiD,oBAAI,IAAI;AAC/D,iBAAe,2BAA2D;AACxE,UAAM,gBAAgB,4BAA4B,EAAE,OAAO,UAAU,CAAC;AACtE,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,cAAc,IAAI,MAAM,SAAS;AACtE,YAAM,SAAS,mBAAmB,QAAQ;AAC1C,YAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA,QAKzB,0BAA0B,KAAK,IAAI,SAAS,oBAAoB,GAAM;AAAA,QACtE,wBAAwB,+BAA+B,UAAU,aAAa;AAAA,QAC9E,0BAA0B,CACxB,YACG,oCAAoC,eAAe,OAAO;AAAA,QAC/D,0BAA0B,SAAS;AAAA,QACnC,gCAAgC,sCAAsC,QAAQ;AAAA,QAC9E,sBAAsB,CAAC,YACrB,gCAAgC,eAAe,OAAO;AAAA,QACxD,2BAA2B,CAAC,UAAmB;AAC7C,wBAAc,KAAK,kEAAkE;AAAA,YACnF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,UAAI,CAAC,SAAS,yBAAyB;AAKrC,cAAMC,sBAAqB,MAAM,4BAA4B,IAAI;AAAA,UAC/D,GAAG;AAAA,UACH,mBAAmB,SAAS;AAAA,UAC5B,iBAAiB,SAAS;AAAA,UAC1B,wBAAwB,SAAS;AAAA,UACjC,aAAa,SAAS;AAAA,QACxB,CAAC;AACD,cAAMC,aAAYD,oBAAmB,MAAM,GAAG,mCAAmC;AACjF,eAAO;AAAA,UACL,WAAAC;AAAA,UACA,cAAc,OAAO;AAAA,UACrB,mBAAmB,OAAO;AAAA,UAC1B,kBAAkB,OAAO;AAAA,UACzB,oBAAoB;AAAA,QACtB;AAAA,MACF;AAOA,YAAM,qBAAqB,MAAM;AAAA,QAC/B;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,MACX,EAAE,MAAM,CAAC,UAAU;AACjB,sBAAc,KAAK,6DAA6D;AAAA,UAC9E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,eAAO;AAAA,MACT,CAAC;AACD,8CAAwC,eAAe,kBAAkB;AAKzE,YAAM,qBAAqB,MAAM,4BAA4B,IAAI;AAAA,QAC/D,GAAG;AAAA,QACH,mBAAmB,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAS5B,iBAAiB,SAAS;AAAA,QAC1B,wBAAwB,SAAS;AAAA,QACjC,aAAa,SAAS;AAAA,MACxB,CAAC;AACD,YAAM,YAAY,mBAAmB,MAAM,GAAG,mCAAmC;AACjF,UAAI,mBAAmB,SAAS,UAAU,QAAQ;AAChD,sBAAc,KAAK,wDAAwD;AAAA,UACzE,UAAU,UAAU;AAAA,UACpB,UAAU,mBAAmB,SAAS,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AACA,aAAO;AAAA,QACL;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B,kBAAkB,OAAO;AAAA,QACzB;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,6BAAiE;AAC9E,UAAM,gBAAgB,4BAA4B,EAAE,OAAO,4BAA4B,CAAC;AACxF,QAAI;AACF,YAAM,EAAE,IAAI,SAAS,IAAI,MAAM,SAAS;AAGxC,YAAM,eAAe,yBAAyB,+BAA+B,QAAQ,CAAC;AACtF,UAAI,CAAC,SAAS,2BAA2B,SAAS,6BAA6B,GAAG;AAChF,eAAO,EAAE,SAAS,CAAC,GAAG,aAAa;AAAA,MACrC;AACA,YAAM,OAAO,MAAM,6BAA6B,IAAI;AAAA,QAClD,OAAO;AAAA,QACP,YAAY,SAAS;AAAA,MACvB,CAAC;AACD,aAAO;AAAA,QACL,SAAS,KAAK,IAAI,CAAC,EAAE,WAAW,YAAY,GAAG,OAAO,MAAM,MAAM;AAAA,QAClE;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,2BACb,OAC8C;AAC9C,UAAM,SAAS,MAAM;AACrB,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,aAAa,OAAO;AAAA,MACpB,gBAAgB,OAAO;AAAA,MACvB,YAAY,OAAO;AAAA,MACnB,YAAY,OAAO;AAAA,IACrB,CAAC;AACD,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,cAAc,IAAI,MAAM,SAAS;AACtE,YAAM,SAAS;AAAA,QACb,iBAAiB,yBAAyB,OAAO,aAAa,OAAO,cAAc;AAAA,QACnF,YAAY,OAAO;AAAA,MACrB;AACA,YAAM,UAAU,CAAC,WAAwD;AACvE,sBAAc,KAAK,2CAA2C,EAAE,GAAG,QAAQ,OAAO,CAAC;AACnF,eAAO,EAAE,QAAQ,WAAW,OAAO;AAAA,MACrC;AACA,UAAI,CAAC,SAAS,2BAA2B,SAAS,6BAA6B,GAAG;AAChF,eAAO,QAAQ,UAAU;AAAA,MAC3B;AACA,YAAM,QAAQ,MAAMC,WAAU,IAAI,OAAO,aAAa,OAAO,cAAc;AAC3E,UACE,CAAC,SACD,MAAM,aAAa,UACnB,MAAM,eAAe,OAAO,cAC5B,MAAM,eAAe,OAAO,YAC5B;AACA,eAAO,QAAQ,cAAc;AAAA,MAC/B;AAGA,UAAI,MAAM,cAAc,EAAG,QAAO,QAAQ,WAAW;AACrD,UAAI,MAAM,aAAa,EAAG,QAAO,QAAQ,aAAa;AACtD,YAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,OAAO,WAAW;AAGzE,YAAM,UAAU,MAAM,4BAA4B,IAAI;AAAA,QACpD;AAAA,QACA,aAAa,OAAO;AAAA,QACpB,gBAAgB,OAAO;AAAA,QACvB,eAAe,OAAO;AAAA,QACtB,oBAAoB,OAAO;AAAA,MAC7B,CAAC;AACD,UAAI,CAAC,QAAS,QAAO,QAAQ,cAAc;AAC3C,UAAI,CAAC,sCAAsC,MAAM,SAAS,MAAM,WAAW,GAAG;AAC5E,eAAO,QAAQ,gBAAgB;AAAA,MACjC;AACA,UAAI,MAAM,mBAAmB,CAAC,6BAA6B,KAAK,EAAG,QAAO,QAAQ,OAAO;AACzF,UAAI,MAAM,gBAAgB;AACxB,cAAM,WAAW,MAAM,+BAA+B,IAAI;AAAA,UACxD;AAAA,UACA,aAAa,OAAO;AAAA,UACpB,gBAAgB,OAAO;AAAA,UACvB,WAAW,MAAM,eAAe;AAAA,UAChC,eAAe,OAAO;AAAA,UACtB,oBAAoB,OAAO;AAAA,UAC3B,yBAAyB;AAAA,QAC3B,CAAC;AACD,YAAI,CAAC,SAAU,QAAO,QAAQ,qBAAqB;AACnD,sBAAc,KAAK,0DAA0D;AAAA,UAC3E,GAAG;AAAA,UACH,WAAW,MAAM,eAAe;AAAA,QAClC,CAAC;AAAA,MACH;AACA,UAAI;AACJ,UAAI;AACF,kBAAU,MAAM,4BAA4B,UAAU,KAAK;AAAA,MAC7D,SAAS,OAAO;AACd,sBAAc,KAAK,iDAAiD;AAAA,UAClE,GAAG;AAAA,UACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,eAAO,QAAQ,gBAAgB;AAAA,MACjC;AACA,UAAI,YAAY,KAAM,QAAO,QAAQ,kBAAkB;AACvD,YAAM,UAAU;AAAA,QACd,EAAE,IAAI,UAAU,eAAe,cAAc;AAAA,QAC7C;AAAA,UACE;AAAA,UACA,aAAa,OAAO;AAAA,UACpB,gBAAgB,OAAO;AAAA,UACvB,gBAAgB;AAAA,QAClB;AAAA,QACA;AAAA,QACA,OAAO;AAAA,MACT;AACA,YAAM,YAAY,MAAM;AAGxB,YAAM,QAAQ;AACd,UAAI,CAAC,UAAW,QAAO,QAAQ,cAAc;AAC7C,oBAAc,KAAK,6CAA6C,MAAM;AACtE,aAAO,EAAE,QAAQ,eAAe;AAAA,IAClC,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAEA,iBAAe,kBACb,OACqC;AACrC,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,aAAa,MAAM,OAAO;AAAA,MAC1B,gBAAgB,MAAM,OAAO;AAAA,MAC7B,YAAY,MAAM,OAAO;AAAA,MACzB,YAAY,MAAM,OAAO;AAAA,IAC3B,CAAC;AACD,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,eAAe,IAAI,IAAI,MAAM,SAAS;AAC3E,oCAA8B,KAAK;AACnC,YAAM,gBACJ,SAAS,6BAA6B,MAAM,oBACxC,WACA,EAAE,GAAG,UAAU,0BAA0B,MAAM,kBAAkB;AACvE,YAAM,MAAM,MAAM;AAClB,YAAM,kBAAkB,4BAA4B;AACpD,YAAM,iBAA6C;AAAA,QACjD,aAAa,MAAM;AAAA,QACnB,WAAW,sBAAsB,MAAM,aAAa,eAAe;AAAA,QACnE,SAAS;AAAA,QACT,kBAAkB,MAAM;AAAA,MAC1B;AACA,UAAI;AACF,cAAM,cAAc,MAAM;AAAA,UACxB;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,eAAO,EAAE,QAAQ,cAAc,eAAe,UAAU;AAAA,MAC1D,SAAS,OAAO;AACd,cAAM,QAAQ,MAAMA,WAAU,IAAI,IAAI,aAAa,IAAI,cAAc,EAAE,MAAM,MAAM,IAAI;AACvF,cAAM,UAAU;AAAA,UACd,iBAAiB,yBAAyB,IAAI,aAAa,IAAI,cAAc;AAAA,UAC7E,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,YAAY,IAAI;AAAA,UAChB,SAAS,OAAO,WAAW;AAAA,UAC3B,YAAY,OAAO,cAAc,IAAI;AAAA,UACrC,WAAW,OAAO,gBAAgB,MAAM;AAAA,UACxC,oBAAoB,OAAO,gBAAgB,eAAe,MAAM;AAAA,UAChE,gBAAgB,OAAO,gBAAgB,WAAW;AAAA,UAClD,cAAc,MAAM;AAAA,UACpB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D;AACA,YAAI,iBAAiB,mCAAmC;AACtD,wBAAc,KAAK,wDAAwD,OAAO;AAAA,QACpF,OAAO;AACL,wBAAc,KAAK,4CAA4C,OAAO;AAAA,QACxE;AACA,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAEA,iBAAe,0BACb,OAC6C;AAC7C,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,UAAU,MAAM;AAAA,MAChB,SAAS,MAAM;AAAA,IACjB,CAAC;AACD,QAAI;AACF,YAAM,UAAU,MAAM,SAAS;AAC/B,YAAM,EAAE,IAAI,UAAU,cAAc,IAAI;AACxC,YAAM,gBAAgB,MAAM,oCAAoC,SAAS,GAAG,EAAE;AAAA,QAC5E,CAAC,UAAU;AACT,wBAAc,KAAK,8CAA8C;AAAA,YAC/D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,EAAE;AAAA,QAC/C;AAAA,MACF;AACA,YAAM,8BAA8B,MAAM,gCAAgC,IAAI,GAAG,EAAE;AAAA,QACjF,CAAC,UAAU;AACT,wBAAc,KAAK,+CAA+C;AAAA,YAChE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,8BAA8B,GAAG;AACnC,sBAAc,KAAK,yCAAyC;AAAA,UAC1D,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AACA,UAAI,cAAc,UAAU,GAAG;AAC7B,sBAAc,KAAK,mCAAmC,aAAa;AAAA,MACrE;AAEA,UAAI,UAAU;AACd,UAAI,eAAe;AACnB,YAAM,qBAAqB,MAAM;AACjC,UAAI,yBAAyB;AAK7B,YAAM,4CAA4C,IAAI,UAAU,eAAe,QAAQ,GAAG;AAC1F,YAAM,wCAAwC,IAAI,UAAU,eAAe,QAAQ,GAAG;AAKtF,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,QACV;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,0DAA0D;AAAA,UAC3E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAEA,UAAI,CAAC,SAAS,yBAAyB;AAGrC,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAKA,YAAM,cAAc,MAAM,eAAe,IAAI,UAAU,aAAa;AACpE,gBAAU,YAAY;AACtB,YAAM,yBAAyB,oBAAI,IAAY;AAC/C,UACE,SAAS,2BAA2B,aACpC,SAAS,oCAAoC,GAC7C;AACA,mBAAW,eAAe,YAAY,cAAc;AAClD,iCAAuB,IAAI,WAAW;AAAA,QACxC;AAAA,MACF;AACA,UAAI;AACF,mBAAW,eAAe,MAAM,wCAAwC,EAAE,GAAG;AAC3E,iCAAuB,IAAI,WAAW;AAAA,QACxC;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AACA,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAKA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,iCAAiC;AAAA,QACzC,QAAQ;AAAA,MACV;AAIA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,iCAAiC;AAAA,QACzC;AAAA,MACF;AAEA,UAAI;AACF,iCAAyB,MAAM,kBAAkB,UAAU,IAAI,aAAa;AAC5E,uCAA+B,eAAe,sBAAsB;AAAA,MACtE,SAAS,OAAO;AACd,sBAAc,KAAK,6CAA6C;AAAA,UAC9D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAEA,YAAM,eAAe,MAAM,6BAA6B,IAAI,UAAU,aAAa,EAAE;AAAA,QACnF,CAAC,UAAU;AACT,wBAAc,KAAK,iDAAiD;AAAA,YAClE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,EAAE;AAAA,QAC7C;AAAA,MACF;AAEA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAEA,UACE,MAAM,WAAW,KACjB,UAAU,KACV,eAAe,KACf,yBAAyB,KACzB,qBAAqB,KACrB,aAAa,UAAU,GACvB;AACA,sBAAc,KAAK,6BAA6B;AAAA,UAC9C,WAAW,MAAM;AAAA,UACjB,iBAAiB,MAAM;AAAA,UACvB,wBAAwB,MAAM;AAAA,UAC9B,qBAAqB,MAAM;AAAA,UAC3B;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,4BAA4B,aAAa;AAAA,UACzC,4BAA4B,aAAa;AAAA,UACzC,2BAA2B,aAAa;AAAA,QAC1C,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,oBAAsD;AACnE,UAAM,UAAU,MAAM,SAAS;AAC/B,QAAI,QAAQ,4BAA4B;AACtC,YAAM,WAAW,MAAM,QAAQ,2BAA2B;AAC1D,cAAQ,cAAc,KAAK,+CAA+C;AAAA,QACxE;AAAA,MACF,CAAC;AACD,aAAO;AAAA,QACL,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,SAAS;AAAA,QACT,SAAS;AAAA,QACT,cAAc;AAAA,QACd,wBAAwB;AAAA,MAC1B;AAAA,IACF;AAEA,UAAM,OAAO,MAAM,yBAAyB;AAC5C,UAAM,WAAW,KAAK,UAAU,MAAM,GAAG,wCAAwC;AACjF,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,SAAS;AAAA,QAAI,CAAC,WACZ,kBAAkB;AAAA,UAChB;AAAA,UACA,cAAc,KAAK;AAAA,UACnB,mBAAmB,KAAK;AAAA,UACxB,kBAAkB,KAAK;AAAA,UACvB,aAAa,WAAW;AAAA,QAC1B,CAAC;AAAA,MACH;AAAA,IACF;AAKA,QAAI;AACF,YAAM,EAAE,SAAS,aAAa,IAAI,MAAM,2BAA2B;AACnE,iBAAW,UAAU,SAAS;AAC5B,cAAM,MAAM,gCAAgC,MAAM;AAClD,YAAI,wBAAwB,IAAI,GAAG,EAAG;AACtC,gCAAwB,IAAI,GAAG;AAC/B,aAAK,2BAA2B,EAAE,QAAQ,aAAa,CAAC,EACrD,MAAM,CAAC,UAAmB;AACzB,kBAAQ,cAAc,KAAK,0CAA0C;AAAA,YACnE,iBAAiB,yBAAyB,OAAO,aAAa,OAAO,cAAc;AAAA,YACnF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH,CAAC,EACA,QAAQ,MAAM,wBAAwB,OAAO,GAAG,CAAC;AAAA,MACtD;AAAA,IACF,SAAS,OAAO;AACd,cAAQ,cAAc,KAAK,oDAAoD;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AACA,UAAM,aAAa,SAAS;AAAA,MAC1B,CAAC,YAAY,QAAQ,WAAW,eAAe,QAAQ,MAAM,WAAW;AAAA,IAC1E,EAAE;AACF,UAAM,UAAU,SAAS,SAAS;AAClC,UAAM,cAAc,MAAM,0BAA0B;AAAA,MAClD,UAAU,SAAS;AAAA,MACnB,SAAS,SAAS;AAAA,MAClB,gBAAgB;AAAA,MAChB,aAAa;AAAA,MACb,oBAAoB,KAAK;AAAA,IAC3B,CAAC;AACD,WAAO;AAAA,MACL,UAAU,SAAS;AAAA,MACnB;AAAA,MACA;AAAA,MACA,SAAS,YAAY;AAAA,MACrB,cAAc,YAAY;AAAA,MAC1B,wBAAwB,YAAY;AAAA,IACtC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAMA,eAAsB,sCACpB,UACA,OACyB;AACzB,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,MAAI,YAAY,WAAW,CAAC,MAAM,cAAc,CAAC,MAAM,aAAa;AAClE,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,sCAAsC,MAAM,WAAW,MAAM,MAAM,YAAY;AACjF,UAAM,IAAI,MAAM,oEAAoE;AAAA,EACtF;AACA,QAAM,SAASH,+BAA8B,SAAkB,QAAQ;AAGvE,MAAI,CAAC,QAAQ,QAAQ;AACnB,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI;AACF,UAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,UAAM,eAAe,MAAMI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA,QAAI,iBAAiB,QAAW;AAC9B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,UAAM,UAAU,MAAM;AAAA,MACpBC,2BAA0B,QAAQ,SAAS,cAAc,MAAM,UAAU;AAAA,IAC3E;AACA,WAAO,QAAQ;AAAA,EACjB,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,YAAM,IAAI,MAAM,6CAA6C,EAAE,OAAO,MAAM,CAAC;AAAA,IAC/E;AACA,QAAI,+BAA+B,OAAO,WAAW,KAAK,EAAG,QAAO;AACpE,UAAM;AAAA,EACR;AACF;AAEA,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B,KAAK;AAEjC,IAAM,qCAAqC;AAClD,IAAM,oCAAoC,KAAK,KAAK,KAAK;AACzD,IAAM,mCAAmC;AACzC,IAAM,uCAAuC;AAkB7C,SAAS,qCACP,eACA,SACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAUA,IAAM,mCAAmC,oBAAI,IAAI;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAaD,eAAe,iCACb,IACA,OACA,OACA,SAC2C;AAC3C,MAAI,EAAE,iBAAiB,2BAA2B,CAAC,MAAM,gBAAgB,CAAC,MAAM,WAAW;AACzF,WAAO;AAAA,EACT;AACA,QAAM,MAAM,KAAK,UAAU;AAAA,IACzB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACD,MAAI,SAAS,QAAQ,IAAI,GAAG;AAC5B,MAAI,CAAC,QAAQ;AACX,cAAU,YAAuD;AAC/D,YAAM,aAAa,MAAM,cAAc,IAAI,MAAM,oBAAoB,MAAM,YAAY;AACvF,UAAI,CAAC,WAAY,QAAO;AACxB,UAAI,WAAW,WAAW,UAAW,QAAO;AAC5C,UAAI,WAAW,sBAAsB;AACnC,eAAO,WAAW,yBAAyB,MAAM,uBAC7C,OACA;AAAA,MACN;AACA,aAAO,iCAAiC,IAAI,WAAW,qBAAqB,EAAE,IAC1E,uBACA;AAAA,IACN,GAAG,EAAE,MAAM,MAAM,IAAI;AACrB,YAAQ,IAAI,KAAK,MAAM;AAAA,EACzB;AACA,SAAO,MAAM;AACf;AAEA,eAAe,uBACb,QACA,aACA,OACe;AACf,MAAI,OAAO,WAAW,EAAG;AACzB,QAAM,cAAc,KAAK,IAAI,OAAO,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,WAAW,CAAC,CAAC;AAChF,MAAI,YAAY;AAChB,QAAM,QAAQ;AAAA,IACZ,MAAM,KAAK,EAAE,QAAQ,YAAY,GAAG,YAAY;AAC9C,aAAO,YAAY,OAAO,QAAQ;AAChC,cAAM,QAAQ;AACd,qBAAa;AACb,cAAM,MAAM,OAAO,KAAK,GAAI,KAAK;AAAA,MACnC;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,eAAe,6BACb,IACA,UACA,eAC+D;AAI/D,QAAM,UAAU,MAAM,mCAAmC,IAAI,UAAU,aAAa,EAAE;AAAA,IACpF,CAAC,UAAU;AACT,oBAAc,KAAK,4DAA4D;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACA,MAAI,UAAU,GAAG;AACf,2CAAuC,eAAe,kBAAkB,OAAO;AAC/E,kBAAc,KAAK,4DAA4D,EAAE,QAAQ,CAAC;AAAA,EAC5F;AACA,QAAM,UAAU,OAAO,WAAW;AAClC,QAAM,SAAS,MAAM,qCAAqC,IAAI;AAAA,IAC5D;AAAA,IACA,OAAO;AAAA,IACP,YAAY;AAAA,EACd,CAAC;AACD,yCAAuC,eAAe,WAAW,OAAO,MAAM;AAC9E,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,QAAI;AACF,UAAI,MAAM,oBAAoB,SAAS;AACrC,cAAM,IAAI,MAAM,mCAAmC,MAAM,eAAe,EAAE;AAAA,MAC5E;AACA,YAAM,8BAA8B,UAAU,MAAM,oBAAoB,MAAM,QAAQ;AACtF,YAAM,UAAU,MAAM,kCAAkC,IAAI;AAAA,QAC1D,YAAY,MAAM;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,cAAc;AAAA,MAChB,CAAC;AACD,UAAI,SAAS;AACX,mBAAW;AACX,+CAAuC,eAAe,SAAS;AAAA,MACjE;AAAA,IACF,SAAS,OAAO;AACd,gBAAU;AACV,6CAAuC,eAAe,eAAe;AACrE,YAAM,eAAe,KAAK;AAAA,QACxB,IAAI,KAAK;AAAA,QACT,MAAQ,KAAK,KAAK,IAAI,MAAM,gBAAgB,EAAE;AAAA,MAChD;AACA,YAAM,kCAAkC,IAAI;AAAA,QAC1C,YAAY,MAAM;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC5D;AAAA,MACF,CAAC,EAAE,MAAM,MAAM,KAAK;AACpB,oBAAc,KAAK,qDAAqD;AAAA,QACtE,YAAY,MAAM;AAAA,QAClB,iBAAiB,MAAM;AAAA,QACvB,YAAY,MAAM;AAAA,QAClB,SAAS,MAAM;AAAA,QACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,yCAAuC,eAAe,oBAAoB,MAAM;AAChF,SAAO,EAAE,SAAS,OAAO,QAAQ,SAAS,OAAO;AACnD;AAEA,eAAe,mCACb,IACA,UACA,eACiB;AACjB,QAAM,QAAQ,MAAM,+BAA+B,IAAI,EAAE;AACzD,MAAI,MAAM,WAAW,EAAG,QAAO;AAC/B,QAAM,WAAW,oBAAI,IAGnB;AACF,MAAI,UAAU;AACd,QAAM,uBAAuB,OAAO,sCAAsC,OAAO,SAAS;AACxF,QAAI;AACF,UAAI,UAAU,SAAS,IAAI,KAAK,UAAU;AAC1C,UAAI,CAAC,SAAS;AACZ,kBAAU,oDAAoD,UAAU,KAAK,UAAU;AACvF,iBAAS,IAAI,KAAK,YAAY,OAAO;AAAA,MACvC;AACA,YAAM,kBAAkB,MAAM;AAC9B,UACE,MAAM,mCAAmC,IAAI;AAAA,QAC3C,WAAW,KAAK;AAAA,QAChB,aAAa,KAAK;AAAA,QAClB,gBAAgB,KAAK;AAAA,QACrB,SAAS,KAAK;AAAA,QACd,YAAY,KAAK;AAAA,QACjB,qBAAqB,KAAK;AAAA,QAC1B,MAAM,KAAK;AAAA,QACX,eAAe,KAAK;AAAA,QACpB,YAAY,KAAK;AAAA,QACjB,oBAAoB,gBAAgB;AAAA,QACpC,iBAAiB,gBAAgB;AAAA,MACnC,CAAC,GACD;AACA,mBAAW;AAAA,MACb;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,2DAA2D;AAAA,QAC5E,aAAa,KAAK;AAAA,QAClB,gBAAgB,KAAK;AAAA,QACrB,MAAM,KAAK;AAAA,QACX,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAKA,eAAsB,4CACpB,IACA,UACA,eACA,KACA,oBACe;AACf,QAAM,YAAY,oBAAI,KAAK;AAC3B,QAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AACvC,QAAM,qBAAqB,oBAAI,IAAY;AAC3C,QAAM,wBAAwB,oBAAI,IAA2B;AAG7D,KAAG;AACD,UAAM,UAAU,OAAO,WAAW;AAClC,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,+CAA+C,IAAI;AAAA,QAChE;AAAA,QACA;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH,SAAS,OAAO;AACd,2CAAqC,eAAe,cAAc;AAClE,oBAAc,KAAK,kDAAkD;AAAA,QACnE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD;AAAA,IACF;AAEA,UAAM,aACJ,sBACA,IAAI;AAAA,MACF,YAAmD,IAAI,qBAAqB;AAAA,IAC9E;AACF,UAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,UAAI,QAAQ,MAAM;AAClB,UAAI,eAA+E;AACnF,YAAM,gBAAgB,KAAK,UAAU;AAAA,QACnC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,MAAM;AAAA,MACR,CAAC;AACD,UAAI,CAAC,OAAO;AACV,cAAM,aAAa,sBAAsB,IAAI,aAAa;AAC1D,YAAI,WAAY,OAAM;AACtB,YAAI,mBAAmB,IAAI,aAAa,GAAG;AACzC,gBAAM,2BAA2B,IAAI,UAAU,eAAe,OAAO,kBAAkB;AACvF;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,CAAC,YAAY;AACf,gCAAsB;AAAA,YACpB;AAAA,YACA,IAAI,QAAc,CAAC,YAAY;AAC7B,iCAAmB;AAAA,YACrB,CAAC;AAAA,UACH;AAAA,QACF;AACA,YAAI;AACF,kBAAQ,MAAM;AAAA,YACZ;AAAA,YACA;AAAA,YACA,KAAK;AAAA,cACH,SAAS;AAAA,cACT;AAAA,YACF;AAAA,UACF;AAGA,cAAI,OAAO,YAAY,QAAQ;AAC7B,gBAAI,CAAC,IAAI;AACP,oBAAM,IAAI,MAAM,gDAAgD;AAClE,kBAAM,SAAS,IAAI,mBAAmB;AAAA,cACpC,aAAa,MAAM;AAAA,cACnB,SAAS,MAAM;AAAA,cACf,sBAAsB,MAAM;AAAA,cAC5B,OAAO;AAAA,cACP;AAAA,cACA,YAAY;AAAA,gBACV,MAAM;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,WAAW,IAAI;AAAA,gBACb,YAAY,IAAI,wBAAwB,KAAK;AAAA,cAC/C;AAAA,cACA,kBAAkB,KAAK;AAAA,gBACrB,SAAS;AAAA,gBACT;AAAA,cACF;AAAA,cACA,YAAY;AAAA,cACZ,SAAS,EAAE,kBAAkB,MAAM,OAAO,KAAK,IAAI,CAAC,GAAG,gBAAgB,MAAM;AAAA,cAAC,EAAE;AAAA,YAClF,CAAC;AACD,2BAAe,MAAM;AAAA,cACnB;AAAA,cACA,MAAM;AAAA,cACN,QAAQ,KAAK;AAAA,cACb,OAAO,WAAW;AAChB,sBAAM,8BAA8B,IAAI,OAAO,GAAG,EAAE,MAAM,WAAW,MAAM;AAAA,cAC7E;AAAA,YACF;AAAA,UACF;AACA,cAAI,CAAC,OAAO;AACV,kBAAM;AAAA,cACJ;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,gBAAM,UACJ,iBAAiB,2BAChB,MAAM,gBAAgB,MAAM,WAAW,wBAAwB,MAAM;AACxE,cAAI,QAAS,oBAAmB,IAAI,aAAa;AACjD,cAAI,CAAC,SAAS;AACZ,0BAAc,KAAK,2DAA2D;AAAA,cAC5E,WAAW,MAAM;AAAA,cACjB,cAAc,MAAM;AAAA,cACpB,sBAAsB,MAAM;AAAA,cAC5B,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AAAA,UACH;AACA,gBAAM;AAAA,YACJ;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,UAAU,qBAAqB;AAAA,UACjC;AACA;AAAA,QACF,UAAE;AACA,6BAAmB;AAAA,QACrB;AAEA,YAAI;AACF,gBAAM,6CAA6C,IAAI,EAAE,OAAO,MAAM,CAAC;AACvE,+CAAqC,eAAe,SAAS,MAAM,OAAO,EAAE;AAAA,QAC9E,SAAS,OAAO;AACd,+CAAqC,eAAe,yBAAyB;AAC7E,wBAAc,KAAK,6DAA6D;AAAA,YAC9E,WAAW,MAAM;AAAA,YACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD;AAAA,QACF;AAAA,MACF;AAEA,UAAI;AACF,cAAM,aAAa,MAAM,+CAA+C,IAAI,EAAE,MAAM,CAAC;AACrF,YAAI,CAAC,WAAW,SAAS;AACvB,gBAAM,IAAI,MAAM,4DAA4D;AAAA,QAC9E;AACA,cAAM,UACJ,cAAc,kBACb,cAAc,WAAW,YAAY,aAAa,UAAU,gBAAgB;AAC/E,cAAM,WACJ,cAAc,WAAW,cAAc,eAAe,cAAc;AACtE,YAAI,MAAM,YAAY,YAAY,WAAW,UAAU;AACrD,gBAAM,wCAAwC,IAAI;AAAA,YAChD,GAAG;AAAA,YACH;AAAA,YACA,UAAU,SAAS,QAAQ,SAAS;AAAA,UACtC,CAAC;AAAA,QACH;AACA,YAAI,WAAW,OAAO,SAAS,GAAG;AAChC,cAAI;AACF,kBAAM,IAAI,QAAQ,MAAM,aAAa,MAAM,WAAW,WAAW,MAAM;AAAA,UACzE,SAAS,OAAO;AACd,0BAAc,KAAK,yDAAyD;AAAA,cAC1E,WAAW,MAAM;AAAA,cACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AAAA,UACH;AAAA,QACF;AACA,6CAAqC,eAAe,WAAW,MAAM,OAAO,EAAE;AAAA,MAChF,SAAS,OAAO;AACd,6CAAqC,eAAe,mBAAmB;AACvE,sBAAc,KAAK,uDAAuD;AAAA,UACxE,WAAW,MAAM;AAAA,UACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,cAAM,2BAA2B,IAAI,UAAU,eAAe,OAAO,mBAAmB;AAAA,MAC1F;AAAA,IACF,CAAC;AACD,QAAI,OAAO,SAAS,uCAAwC;AAAA,EAC9D,SAAS,KAAK,IAAI,IAAI;AACxB;AAEA,eAAsB,6BACpB,QACA,MACA,eACA,SACkE;AAIlE,MAAI,kBAAkB;AACtB,SAAO,MAAM;AACX,UAAM,SAAS;AACf,UAAM,SAAS,MAAM,OAAO,aAAa,MAAM,gBAAgB,MAAQ,KAAK,OAAO;AACnF,QAAI,CAAC,iBAAiB,OAAO,WAAW,eAAe,OAAO,SAAU,QAAO;AAI/E,sBAAkB,OAAO,OAAO,cAAc;AAI9C,QAAI,oBAAoB,QAAQ;AAC9B,YAAM,IAAI,MAAM,4EAA4E;AAAA,IAC9F;AAAA,EACF;AACF;AAKA,eAAsB,wCACpB,IACA,UACA,eACA,KACA,oBACe;AACf,QAAM,YAAY,oBAAI,KAAK;AAC3B,QAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AACvC,QAAM,aACJ,sBAAsB,IAAI,eAAe,YAAY,IAAI,qBAAqB,CAAC;AACjF,QAAM,gBAAwC,oBAAI,IAAI;AACtD,KAAG;AACD,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,oCAAoC,IAAI;AAAA,QACrD,SAAS,OAAO,WAAW;AAAA,QAC3B;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH,SAAS,OAAO;AACd,2CAAqC,eAAe,cAAc;AAClE,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD;AAAA,IACF;AACA,UAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,UAAI,UAAU,MAAM;AACpB,UAAI;AACF,YAAI,CAAC,IAAI;AACP,gBAAM,IAAI,MAAM,gDAAgD;AAClE,cAAM,aAAa;AAAA,UACjB,MAAM;AAAA,UACN,MAAM;AAAA,UACN,MAAM;AAAA,QACR;AACA,cAAM,mBAAmB,KAAK;AAAA,UAC5B,SAAS;AAAA,UACT;AAAA,QACF;AACA,cAAM,SAAS,IAAI,mBAAmB;AAAA,UACpC,aAAa,MAAM;AAAA,UACnB,SAAS,MAAM;AAAA,UACf,sBAAsB,MAAM;AAAA,UAC5B,OAAO;AAAA,UACP;AAAA,UACA;AAAA,UACA,WAAW,IAAI,sBAAsB,YAAY,IAAI,wBAAwB,KAAK,IAAI;AAAA,UACtF;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,YACP,kBAAkB,MAAM,OAAO,KAAK,IAAI,CAAC;AAAA,YACzC,gBAAgB,MAAM;AACpB,oBAAM,IAAI,MAAM,yDAAyD;AAAA,YAC3E;AAAA,UACF;AAAA,QACF,CAAC;AACD,YAAI,CAAC,SAAS;AACZ,gBAAM,SAAS,MAAM,kBAAkB;AAAA,YACrC;AAAA,YACA;AAAA,YACA,MAAM,MAAM;AAAA,YACZ;AAAA,UACF,CAAC;AACD,cAAI,OAAO,UAAU,QAAQ,eAAe;AAC1C,gBACE,CAAE,MAAM,yCAAyC,IAAI;AAAA,cACnD;AAAA,cACA,SAAS;AAAA,cACT,cAAc;AAAA,YAChB,CAAC;AAED,oBAAM,IAAI,MAAM,8DAA8D;AAChF,iDAAqC,eAAe,oBAAoB;AACxE;AAAA,UACF;AACA,cAAI,OAAO,UAAU,QAAQ;AAC3B,kBAAM,IAAI,MAAM,mDAAmD;AACrE,gBAAM,SAAS,MAAM;AAAA,YACnB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA,OAAO,WACL,8BAA8B,IAAI,OAAO,GAAG,EAAE,MAAM,WAAW,MAAM;AAAA,UACzE;AACA,gBAAM,WAAW,OAAO,WAAW,cAAc,SAAS,OAAO;AACjE,oBACE,OAAO,kBACN,OAAO,WAAW,YAAY,OAAO,UAAU,gBAAgB,WAChE;AACF,cAAI,CAAC,YAAY,CAAC;AAChB,kBAAM,IAAI,MAAM,iEAAiE;AACnF,gBAAM,wCAAwC,IAAI;AAAA,YAChD,GAAG;AAAA,YACH;AAAA,YACA,UAAU,SAAS,QAAQ,SAAS;AAAA,UACtC,CAAC;AAAA,QACH;AACA,cAAM,UAAU,MAAM,OAAO,sBAAsB,MAAM,MAAM,OAAO;AACtE,YACE,CAAE,MAAM,yCAAyC,IAAI;AAAA,UACnD;AAAA,UACA;AAAA,UACA,cAAc,YAAY,cAAc,SAAS,6BAA6B;AAAA,QAChF,CAAC;AAED,gBAAM,IAAI,MAAM,iEAAiE;AACnF;AAAA,UACE;AAAA,UACA,YAAY,cAAc,yBAAyB;AAAA,QACrD;AAAA,MACF,SAAS,OAAO;AACd,cAAM,UAAU,MAAM,iCAAiC,IAAI,OAAO,OAAO,aAAa;AAGtF,YACE,WACC,MAAM,yCAAyC,IAAI;AAAA,UAClD;AAAA,UACA,SAAS,UAAU,iBAAiB;AAAA,UACpC,cAAc;AAAA,QAChB,CAAC,EAAE,MAAM,MAAM,KAAK,GACpB;AACA;AAAA,YACE;AAAA,YACA,UAAU,oBAAoB;AAAA,UAChC;AACA;AAAA,QACF;AACA,cAAM,yCAAyC,IAAI;AAAA,UACjD;AAAA,UACA,SAAS;AAAA,UACT,cAAc,KAAK;AAAA,YACjB,IAAI;AAAA,YACJ,KAAK,IAAI,SAAS,4BAA4B,GAAM,IAClD,KAAK,KAAK,IAAI,GAAG,MAAM,oBAAoB,CAAC;AAAA,UAChD;AAAA,QACF,CAAC,EAAE,MAAM,MAAM,KAAK;AACpB,sBAAc,KAAK,6DAA6D;AAAA,UAC9E,WAAW,MAAM;AAAA,UACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,QAAI,OAAO,SAAS,uCAAwC;AAAA,EAC9D,SAAS,KAAK,IAAI,IAAI;AACxB;AAEA,eAAsB,uCACpB,OACA,YACA,kBACA,aAAa,oBAAI,KAAK,GACkC;AACxD,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,MAAM,MAAM;AAAA,IACZ;AAAA,EACF;AACA,QAAM,SACJ,MAAM,UAAU,aAAa,MAAM,mBAAmB,KAAK,IAAI,MAAM,kBAAkB,KAAK;AAC9F,SAAO,gCAAgC,QAAQ,UAAU;AAC3D;AAEO,SAAS,gCACd,QACA,YAC+C;AAC/C,UAAQ,OAAO,OAAO;AAAA,IACpB,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ;AACX,aAAO;AAAA,IACT,KAAK,QAAQ,mBAAmB;AAC9B,YAAM,OAAO,OAAO;AACpB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI,MAAM,oDAAoD;AAAA,MACtE;AACA,YAAM,cAAc,KAAK,YAAY,QAAQ,mBAAmB,GAAG,EAAE,MAAM,GAAG,GAAG;AACjF,YAAM,SAAS,cACX,cAAc,WAAW,KACzB,KAAK,YACH,iBACA,KAAK,WACH,iBACA;AACR,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU,KAAK;AAAA,QACf;AAAA,QACA,GAAI,cACA;AAAA,UACE,SAAS,EAAE,MAAM,aAAa,QAAQ,KAAK,eAAe,WAAW,MAAe;AAAA,QACtF,IACA,CAAC;AAAA,QACL;AAAA,MACF;AAAA,IACF;AAAA,IACA,KAAK,QAAQ;AACX,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,QACE,OAAO,eAAe,aAAa,yBAC/B,oBACA,OAAO,eAAe,aAAa,iCACjC,4BACA;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACE,YAAM,IAAI,MAAM,+CAA+C,OAAO,KAAK,EAAE;AAAA,EACjF;AACF;AAEA,eAAe,2BACb,IACA,UACA,eACA,OACA,SACe;AACf,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,oBAAoB,CAAC,CAAC;AACrE,QAAM,eAAe,KAAK;AAAA,IACxB,IAAI;AAAA,IACJ,KAAK,IAAI,SAAS,4BAA4B,GAAM,IAAI,KAAK;AAAA,EAC/D;AACA,MAAI;AACF,QACE,CAAE,MAAM,qDAAqD,IAAI;AAAA,MAC/D;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,GACD;AACA,YAAM,IAAI,MAAM,8CAA8C;AAAA,IAChE;AACA,yCAAqC,eAAe,OAAO;AAAA,EAC7D,SAAS,OAAO;AACd,yCAAqC,eAAe,cAAc;AAClE,kBAAc,KAAK,kDAAkD;AAAA,MACnE,WAAW,MAAM;AAAA,MACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AAMA,IAAM,iCAAiC;AACvC,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC,IAAI;AAChD,IAAM,qCAAqC,IAAI,KAAK;AAqBpD,eAAe,yBACb,IACA,UACA,eACA,+BACA,SACe;AAGf,MAAI,SAAS,mBAAmB,WAAW,CAAC,SAAS,gBAAgB,CAAC,SAAS,kBAAkB;AAC/F;AAAA,EACF;AACA,QAAM,cAAc,KAAK,IAAI;AAE7B,QAAM,WAAW,oBAAI,IAAoB;AACzC,QAAM,OAA+B,CAAC;AAEtC,iBAAe,QACb,YACA,SACwB;AACxB,UAAM,QAAQ,SAAS,IAAI,UAAU;AACrC,QAAI,UAAU,OAAW,QAAO;AAChC,UAAM,QAAQ,KAAK,IAAI;AACvB,SAAK,QAAQ,IAAI,UAAU,GAAG,iBAAiB,KAAK,MAAO,QAAO;AAClE,QAAI,QAAQ,eAAe,oCAAqC,QAAO;AACvE,QAAI;AACJ,QAAI;AACF,gBACE,MAAM;AAAA,QACJ,8BAA8B,UAAU,YAAY,IAAI;AAAA,MAC1D,GACA;AAAA,IACJ,SAAS,OAAO;AACd,eAAS,UAAU,iCAAiC,qBAAqB;AACzE,oBAAc,KAAK,gDAAgD;AAAA,QACjE,GAAG;AAAA,QACH,OACE,UAAU,iCACN,qBACA,iBAAiB,QACf,MAAM,UACN,OAAO,KAAK;AAAA,MACtB,CAAC;AAAA,IACH;AACA,aAAS,IAAI,YAAY,MAAM;AAC/B,QAAI,WAAW,cAAc;AAC3B,cAAQ,OAAO,UAAU;AAAA,IAC3B,OAAO;AACL,2BAAqB,SAAS,UAAU;AACxC,WAAK,MAAM,KAAK,KAAK,MAAM,KAAK,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,gCAAgC,IAAI,EAAE,OAAO,+BAA+B,CAAC;AAAA,EAC9F,SAAS,OAAO;AACd,kBAAc,KAAK,wDAAwD;AAAA,MACzE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,aAAS,CAAC;AAAA,EACZ;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,MAAM;AACzB,UAAM,UAAU;AAAA,MACd,aAAa,MAAM;AAAA,MACnB,gBAAgB,MAAM;AAAA,MACtB,WAAW,MAAM;AAAA,MACjB;AAAA,IACF;AACA,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,UAAI,WAAW,aAAc;AAC7B,YAAM,UAAU,MAAM,kCAAkC,IAAI,KAAK;AACjE,UAAI,SAAS;AACX,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,GAAG;AAAA,UACH,GAAG;AAAA,QACL,CAAC;AAAA,MACH;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,wCAAwC,IAAI;AAAA,MACzD,OAAO;AAAA,IACT,CAAC;AAAA,EACH,SAAS,OAAO;AACd,kBAAc,KAAK,sDAAsD;AAAA,MACvE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,aAAS,CAAC;AAAA,EACZ;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,MAAM,IAAI;AAG7B,UAAM,UAAU,SAAS,MAAM,MAAM,IAAI,MAAM,SAAS;AACxD,SAAK,QAAQ,IAAI,OAAO,GAAG,iBAAiB,KAAK,KAAK,IAAI,EAAG;AAC7D,UAAM,UAAU;AAAA,MACd,aAAa,MAAM;AAAA,MACnB,WAAW,MAAM;AAAA,MACjB,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM;AAAA,MACtB,WAAW,MAAM,IAAI;AAAA,MACrB;AAAA,IACF;AACA,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,UAAI,WAAW,aAAc;AAC7B,YAAM,EAAE,WAAW,IAAI,MAAM,+BAA+B,IAAI;AAAA,QAC9D,WAAW,MAAM;AAAA,QACjB,aAAa,MAAM;AAAA,QACnB,WAAW,MAAM;AAAA,QACjB,QAAQ,MAAM;AAAA,QACd,WAAW,MAAM;AAAA,QACjB,eAAe,MAAM;AAAA,MACvB,CAAC;AACD,UAAI,YAAY;AACd,gBAAQ,OAAO,OAAO;AACtB,sBAAc;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,OAAO;AACL,6BAAqB,SAAS,OAAO;AAAA,MACvC;AAAA,IACF,SAAS,OAAO;AACd,2BAAqB,SAAS,OAAO;AACrC,oBAAc,KAAK,2DAA2D;AAAA,QAC5E,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,OAAO,KAAK,IAAI,EAAE,QAAQ;AAC5B,kBAAc,KAAK,oEAAoE,IAAI;AAAA,EAC7F;AACF;AAEA,SAAS,qBAAqB,SAAiC,KAAmB;AAChF,QAAM,YAAY,QAAQ,IAAI,GAAG,GAAG,YAAY,KAAK;AACrD,UAAQ,IAAI,KAAK;AAAA,IACf;AAAA,IACA,eACE,KAAK,IAAI,IACT,KAAK;AAAA,MACH;AAAA,MACA,sCAAsC,KAAK,KAAK,IAAI,WAAW,GAAG,EAAE;AAAA,IACtE;AAAA,EACJ,CAAC;AACH;AAEA,eAAe,mCACb,IACA,UACA,eACA,OACA,+BACA,KACe;AACf,QAAM,UAAU,OAAO,WAAW;AAClC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,+BAA+B,IAAI;AAAA,MAChD;AAAA,MACA,OAAO;AAAA,MACP,YAAY;AAAA,IACd,CAAC;AAAA,EACH,SAAS,OAAO;AACd,wCAAoC,eAAe,cAAc;AACjE,kBAAc,KAAK,iDAAiD;AAAA,MAClE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD;AAAA,EACF;AAEA,QAAM,cAA0C;AAAA,IAC9C,OAAO,KAAK,IAAI,IAAI;AAAA,EACtB;AAGA,QAAM,YAAY,oBAAI,IAAY;AAClC,aAAW,SAAS,QAAQ;AAC1B,QAAI,UAAU,MAAM;AACpB,QAAI,UAAU,IAAI,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,EAAE;AACxF;AACF,UAAM,WAAW,kCAAkC,OAAO;AAC1D,QAAI,QAAQ,mCAAmC,OAAO;AAGtD,QAAI,eAAe;AACnB,UAAM,eAAe;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,IACrB;AAIA,QACE,QAAQ,mBAAmB,UAC3B,QAAQ,oBAAoB,WAC5B,CAAC,SACD,CAAC,MAAM,WAAW,WAAW,aAAa,GAC1C;AACA,UAAI;AACF,cAAM,UAAU,MAAM,uCAAuC,IAAI;AAAA,UAC/D,GAAG;AAAA,UACH;AAAA,UACA,uBAAuB,MAAM;AAAA,QAC/B,CAAC;AACD,YAAI,SAAS;AACX,gBAAM,aAAa,cAAc,QAAQ,KAAK;AAC9C,8CAAoC,eAAe,sBAAsB;AAAA,QAC3E;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,+DAA+D;AAAA,UAChF,WAAW,QAAQ;AAAA,UACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF;AACA,UAAM,2BAA2B;AAAA,MAC/B;AAAA,MACA;AAAA,MACA,MAAM,CAAC,WAAW,IAAI,QAAQ,QAAQ,aAAa,QAAQ,WAAW,MAAM,IAAI;AAAA,IAClF;AACA,UAAM,qBAAqB;AAAA,MACzB,GAAG;AAAA,MACH,qBAAqB,OAAO,WAAkD;AAC5E,YAAI,WAAW,qBAAqB;AAClC,gBAAM,2CAA2C,IAAI,YAAY;AACjE,cAAI,CAAE,MAAM,yBAAyB,sBAAsB;AACzD,kBAAM,IAAI;AAAA,cACR;AAAA,YACF;AAAA,QACJ,MAAO,OAAM,yBAAyB,oBAAoB,MAAM;AAAA,MAClE;AAAA,IACF;AACA,QAAI,SAAS,QAAQ,oBAAoB,SAAS;AAChD,UAAI;AACF,cAAM,UAAU,MAAM,mBAAmB,KAAK;AAC9C,YACE,SAAS,SAAS,qBAClB,QAAQ,gBACP,CAAE,MAAM,mBAAmB,uBAAuB,KACjD,CAAC,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,EAAE;AAAA,UAChD,CAAC,WAAW,OAAO,OAAO,OAAO,aAAa;AAAA,QAChD;AAEF,kBAAQ;AAAA,MACZ,QAAQ;AAGN,gBAAQ;AAAA,MACV;AAAA,IACF;AACA,QAAI,CAAC,OAAO;AACV,UAAI,cAAiD;AAMrD,UACE,QAAQ,oBAAoB,WAC5B,QAAQ,kBAAkB,QAC1B,CAAC,QAAQ,oBACT;AACA,YAAI;AACF,gBAAM,YAAY,MAAM;AAAA,YACtB,8BAA8B,UAAU,QAAQ,oBAAoB,IAAI;AAAA,UAC1E;AACA,cAAI,UAAU,WAAW,aAAa;AACpC,0BAAc;AAAA,cACZ,QAAQ;AAAA,cACR,QAAQ;AAAA,YACV;AAAA,UACF,OAAO;AACL,sBAAU,MAAM,oCAAoC,IAAI;AAAA,cACtD,WAAW,QAAQ;AAAA,cACnB,aAAa,QAAQ;AAAA,cACrB,WAAW,QAAQ;AAAA,cACnB,WAAW,QAAQ;AAAA,cACnB;AAAA,cACA,SAAS,MAAM;AAAA,cACf,oBAAoB,UAAU;AAAA,cAC9B,iBAAiB,UAAU;AAAA,YAC7B,CAAC;AACD,gDAAoC,eAAe,0BAA0B;AAC7E,gBAAI,UAAU,WAAW,cAAc;AACrC,4BAAc;AAAA,gBACZ,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,wBAAc;AAAA,YACZ,QAAQ;AAAA,YACR,QACE,UAAU,iCAAiC,qBAAqB;AAAA,UACpE;AAAA,QACF;AAAA,MACF;AAEA,UACE,aAAa,WAAW,cACxB,QAAQ,oBAAoB,WAC5B,CAAC,QAAQ,sBACT,CAAC,QAAQ,iCACT;AAIA,YAAI;AACF,gBAAM,2CAA2C,IAAI,YAAY;AAAA,QACnE,SAAS,OAAO;AACd,wBAAc,KAAK,8DAA8D;AAAA,YAC/E,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,gBAAgB,MAAM;AACxB,cAAM,QAAQ,MAAMF,WAAU,IAAI,QAAQ,aAAa,QAAQ,cAAc,EAAE;AAAA,UAC7E,MAAM;AAAA,QACR;AACA,cAAM,8BACJ,QAAQ,KAAK,KACb,MAAO,OAAO,QAAQ,WACtB,MAAO,eAAe,QAAQ,cAC9B,MAAO,YAAY,QAAQ,mBAC3B,MAAO,eAAe,QAAQ,sBAC9B,QAAQ,kBAAkB;AAC5B,YAAI,CAAC,6BAA6B;AAIhC,cAAI,QAAQ,oBAAoB,WAAW,QAAQ,kBAAkB,MAAM;AACzE,0BAAc,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,UAClE,OAAO;AACL,gBAAI;AACF,oBAAM,YAAY,MAAM;AAAA,gBACtB;AAAA,kBACE;AAAA,kBACA,QAAQ;AAAA,kBACR,QAAQ;AAAA,gBACV;AAAA,cACF;AACA,4BACE,UAAU,WAAW,cACjB;AAAA,gBACE,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF,IACA,UAAU,WAAW,eACnB;AAAA,gBACE,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF,IACA,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,YAC5D,SAAS,OAAO;AACd,4BAAc;AAAA,gBACZ,QAAQ;AAAA,gBACR,QACE,UAAU,iCAAiC,qBAAqB;AAAA,cACpE;AAAA,YACF;AAAA,UACF;AAAA,QACF,OAAO;AACL,cAAI;AACF,kBAAM,UAAU,MAAM,mBAAmB,KAAK;AAC9C,kBAAM,aAAa,SAAS,SAAS,qBAAqB,QAAQ,QAAQ,WAAW;AACrF,kBAAM,oBAAoB,CAAC,YACzB,cAAc,iBAAiB;AAAA,cAC7B,MAAM;AAAA,cACN,MAAM;AAAA,cACN,QAAQ,EAAE,QAAQ;AAAA,YACpB,CAAC;AACH,gBACE,eACC,MAAO,mBAAmB,uBACzB,MAAM,eAAe,wBACvB;AACA,oBAAM,mBAAmB;AAAA,gBACvB,MAAO,mBAAmB,sBACtB,sBACA;AAAA,cACN;AACA,gCAAkB,qBAAqB;AAAA,YACzC;AACA,kBAAM,qBACJ,MAAO,mBAAmB,uBAC1B,CAAC,cACD,CAAC,QAAQ;AACX,kBAAM,eAAe,MAAM;AAAA,cACzB;AAAA,cACA;AAAA,cACA;AAAA,cACC,MAAM,eAAe,yBACpB,CAAC,QAAQ,mCACT,qBACE,WACA;AAAA,cACJ,OAAO,QAAQ,SAAS,QAAQ,mBAAmB;AACjD,oBACE,OAAO,WAAW,YACjB,CAAC,UAAU,wBAAwB,QAAQ,QAAQ,iBAAiB;AAErE;AACF,sBAAM,SAAS,MAAM,2BAA2B,IAAI;AAAA,kBAClD,WAAW,QAAQ;AAAA,kBACnB,aAAa,QAAQ;AAAA,kBACrB,WAAW,QAAQ;AAAA,kBACnB,WAAW,QAAQ;AAAA,kBACnB;AAAA,kBACA,QAAQ,UAAU;AAAA,kBAClB,gBAAgB,mBAAmB,SAAS,aAAa;AAAA,kBACzD,OAAO,SAAS,SAAS,gBAAgB,MAAM;AAAA,gBACjD,CAAC;AACD,oBAAI,OAAO,UAAU;AACnB,wBAAM,IACH,QAAQ,QAAQ,aAAa,QAAQ,WAAW,MAAM,EACtD,MAAM,MAAM,MAAS;AAAA,cAC5B;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE;AAAA,cACA,CAAC,WAAW,EAAE,IAAI,MAAe,MAAM;AAAA,cACvC,CAAC,WAAoB,EAAE,IAAI,OAAgB,MAAM;AAAA,YACnD;AAIA,gBACE,sBACA,CAAE,MAAM,2CAA2C,IAAI,YAAY;AAEnE,oBAAM,IAAI,MAAM,yDAAyD;AAC3E,gBAAI,CAAC,aAAa,GAAI,OAAM,aAAa;AACzC,0BAAc,aAAa;AAC3B,2BACE,YAAY,WAAW,YACvB,YAAY,MAAM,YAAY,UAC9B,YAAY,MAAM,WAAW;AAC/B,gBAAI,YAAY;AACd;AAAA,gBACG,MAAM,mBAAmB,uBAAuB,IAC7C,mBACA;AAAA,cACN;AACA,kBAAI,YAAY,WAAW;AACzB;AAAA,kBACE,YAAY,WAAW,qBACnB,uBACA;AAAA,gBACN;AAAA,YACJ;AAAA,UACF,SAAS,OAAO;AACd,0BAAc,KAAK,0DAA0D;AAAA,cAC3E,WAAW,QAAQ;AAAA,cACnB,iBAAiB,QAAQ;AAAA,cACzB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AACD,0BAAc,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,UAC/D;AAAA,QACF;AAAA,MACF;AACA,UAAI,YAAY,WAAW,WAAW;AACpC,YAAI;AACF,oBAAU,MAAM,oCAAoC,IAAI;AAAA,YACtD,WAAW,QAAQ;AAAA,YACnB,aAAa,QAAQ;AAAA,YACrB,WAAW,QAAQ;AAAA,YACnB,WAAW,QAAQ;AAAA,YACnB;AAAA,YACA,SAAS,MAAM;AAAA,YACf,oBAAoB,YAAY;AAAA,YAChC,iBAAiB,YAAY;AAAA,UAC/B,CAAC;AACD,8CAAoC,eAAe,0BAA0B;AAC7E,wBAAc,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,QACjE,SAAS,OAAO;AACd,wBAAc,KAAK,4DAA4D;AAAA,YAC7E,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,wBAAc,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,QAC/D;AAAA,MACF;AACA,UAAI,YAAY,WAAW,YAAY;AACrC,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,YAAY,kBAAkB;AAAA,QAChC;AACA;AAAA,MACF;AACA,UAAI;AACF,kBAAU,MAAM,yCAAyC,IAAI;AAAA,UAC3D,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,WAAW,QAAQ;AAAA,UACnB,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,SAAS,MAAM;AAAA,UACf,OAAO,YAAY;AAAA,QACrB,CAAC;AACD,gBAAQ,YAAY;AACpB,4CAAoC,eAAe,SAAS,MAAM,OAAO,EAAE;AAAA,MAC7E,SAAS,OAAO;AACd,4CAAoC,eAAe,yBAAyB;AAC5E,wCAAgC,eAAe,OAAO,OAAO;AAC7D;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc;AAShB,YAAM,SAAS,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB;AACrF,iBAAW,SAAS,QAAQ;AAC1B,cAAM,UAAU,MAAM;AACtB,YACE,UAAU,SACV,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,OAAO,UAC7E,QAAQ,mBAAmB,UAC3B,QAAQ,oBAAoB,WAC5B,MAAM,WAAW,WAAW,aAAa,KACzC,mCAAmC,OAAO;AAE1C;AACF,YAAI;AACF,gBAAM,uCAAuC,IAAI;AAAA,YAC/C,WAAW,QAAQ;AAAA,YACnB,aAAa,QAAQ;AAAA,YACrB,WAAW,QAAQ;AAAA,YACnB,WAAW,QAAQ;AAAA,YACnB,UAAU,kCAAkC,OAAO;AAAA,YACnD,uBAAuB,MAAM;AAAA,UAC/B,CAAC;AAAA,QACH,SAAS,OAAO;AACd,wBAAc,KAAK,+DAA+D;AAAA,YAChF,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,UAAI;AACF,cAAM,SAAS,MAAM,0BAA0B,IAAI;AAAA,UACjD,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,gBAAgB,QAAQ;AAAA,UACxB,eAAe,QAAQ;AAAA,UACvB,oBAAoB,QAAQ;AAAA,UAC5B,iBAAiB,QAAQ;AAAA,QAC3B,CAAC;AACD,YAAI,OAAO,WAAW,UAAU;AAC9B,oBAAU,IAAI,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,EAAE;AACtF,8CAAoC,eAAe,yBAAyB;AAC5E,cAAI,OAAO,yBAAyB,QAAQ;AAC1C,kBAAM;AAAA,cACJ;AAAA,cACA,QAAQ;AAAA,cACR,OAAO;AAAA,cACP,CAAC,UAAU;AACT,8BAAc,KAAK,wDAAwD;AAAA,kBACzE,gBAAgB,QAAQ;AAAA,kBACxB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,gBAC9D,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF;AACA;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,WAAW,QAAQ;AAAA,UACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IAGF;AACA,QAAI;AACF,YAAM,aAAa,MAAM,sBAAsB,IAAI;AAAA,QACjD,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,QACrB,WAAW,QAAQ;AAAA,QACnB,WAAW,QAAQ;AAAA,QACnB;AAAA,QACA,uBAAuB,MAAM;AAAA,QAC7B,SAAS,MAAM;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,QAAQ,MAAM;AAAA,QACd,aAAa,SAAS;AAAA,MACxB,CAAC;AACD,UAAI,WAAW,QAAQ,UAAU,UAAU;AACzC,cAAM,IAAI,MAAM,oEAAoE;AAAA,MACtF;AACA,UAAI,WAAW,wBAAwB,SAAS,GAAG;AACjD,YAAI;AACF,gBAAM,IAAI;AAAA,YACR,QAAQ;AAAA,YACR,QAAQ;AAAA,YACR,WAAW;AAAA,UACb;AAAA,QACF,SAAS,OAAO;AACd,wBAAc,KAAK,wDAAwD;AAAA,YACzE,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,0CAAoC,eAAe,WAAW,MAAM,OAAO,EAAE;AAAA,IAC/E,SAAS,OAAO;AACd,0CAAoC,eAAe,mBAAmB;AACtE,oBAAc,KAAK,sDAAsD;AAAA,QACvE,WAAW,QAAQ;AAAA,QACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,0BACb,IACA,UACA,eACA,SACA,UACA,SACA,SACA,gBAAgB,OACD;AACf,QAAM,WAAW,sCAAsC,UAAU,SAAS,SAAS,aAAa;AAChG,MAAI;AAIF,UAAM,QACJ,QAAQ,oBAAoB,WAAW,QAAQ,kBAAkB,OAC7D,MAAMA,WAAU,IAAI,QAAQ,aAAa,QAAQ,cAAc,IAC/D;AACN,UAAM,mCAAmC,IAAI;AAAA,MAC3C,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA,SAAS,SAAS;AAAA,MAClB,cAAc,+BAA+B,SAAS,OAAO,UAAU,SAAS,YAAY;AAAA,IAC9F,CAAC;AACD,wCAAoC,eAAe,SAAS,aAAa;AAAA,EAC3E,SAAS,OAAO;AACd,wCAAoC,eAAe,cAAc;AACjE,kBAAc,KAAK,iDAAiD;AAAA,MAClE,WAAW,QAAQ;AAAA,MACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AAMO,SAAS,sCACd,UACA,SACA,SACA,gBAAgB,OAMhB;AACA,QAAM,oBACJ,QAAQ,qBAAqB,uDAC5B,YAAY,8BACX,YAAY,+BACZ,YAAY;AAChB,MAAI,mBAAmB;AACrB,WAAO;AAAA,MACL,gBAAgB,eAAe,OAAO;AAAA,MACtC,eACE,YAAY,6BACR,gCACA,YAAY,8BACV,iCACA;AAAA,MACR,cAAc;AAAA,MACd,aAAa;AAAA,IACf;AAAA,EACF;AACA,QAAM,cAAc,KAAK,IAAI,SAAS,4BAA4B,GAAM;AAGxE,QAAM,WACJ,YAAY,sBAAsB,gBAC9B,IACA,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,oBAAoB,CAAC,CAAC;AAC5D,QAAM,eAAe,KAAK,IAAI,IAAI,KAAQ,cAAc,KAAK,QAAQ;AACrE,SAAO;AAAA,IACL,gBAAgB;AAAA,IAChB,eAAe;AAAA,IACf;AAAA,IACA,aAAa;AAAA,EACf;AACF;AAkBA,IAAM,iCAAiC,OAAO,yCAAyC;AAOvF,eAAsB,4BACpB,SACA,mBACA,OACA,aACA,QACqC;AAErC,MAAI,UAAU,uBAAuB,SAAS,KAAK;AACnD,QAAM,YAAY,KAAK;AACvB,MAAI,cAAc,kCAAkC,OAAO,mBAAmB,OAAO;AACrF,MAAI,OAAO,QAAQ,eAAe,WAAY,QAAO;AACrD,QAAM,QAAQ,KAAK;AAAA,IACjB,KAAK,IAAI,IAAI;AAAA,IACb,QAAQ,SAAS,OAAO;AAAA,EAC1B;AACA,WACM,QAAQ,GACZ,WACA,YAAY,WAAW,cACvB,YAAY,WAAW,sBACvB,QAAQ,2CACR,KAAK,IAAI,IAAI,OACb,SACA;AACA,QAAI;AACJ,QAAI;AACF,aAAO,MAAM;AAAA,QACX,QAAQ,WAAW;AAAA,UACjB,WAAW;AAAA,UACX,OAAO;AAAA,UACP,aAAa;AAAA,UACb,iBAAiB;AAAA,QACnB,CAAC;AAAA,MACH;AAAA,IACF,QAAQ;AAGN;AAAA,IACF;AACA,cAAU,uBAAuB,SAAS,IAAI;AAC9C,UAAM,YAAY,IAAI;AACtB,kBAAc,kCAAkC,MAAM,mBAAmB,OAAO;AAAA,EAClF;AACA,SAAO,WAAW,YAAY,WAAW,cAAc,YAAY,WAAW,qBAC1E,EAAE,GAAG,aAAa,eAAe,KAAK,IACtC;AACN;AAMA,SAAS,uBAAuB,SAAsC,QAA0B;AAC9F,QAAM,OAAO,QAAQ,2BAA2B,MAAM;AACtD,MAAI,CAAC,MAAM,YAAY,KAAK,QAAQ,SAAS,kBAAmB,QAAO;AACvE,MAAI,KAAK,QAAQ,eAAe,KAAK,aAAa,KAAM,QAAO;AAC/D,MAAI,KAAK,eAAgB,QAAO;AAChC,QAAM,EAAE,SAAS,SAAS,IAAI;AAC9B,SAAQ,CAAC,UAAU,QAAQ,EAAY;AAAA,IACrC,CAAC,WACC,QAAQ,QAAQ,MAAM,EAAE,aAAa,SAAS,QAAQ,MAAM,EAAE,cAC9D;AAAA,EACJ;AACF;AAEA,eAAe,gCAAmC,SAAiC;AACjF,MAAI;AACJ,MAAI;AACF,WAAO,MAAM,QAAQ,KAAK;AAAA,MACxB;AAAA,MACA,IAAI,QAAe,CAAC,UAAU,WAAW;AACvC,gBAAQ;AAAA,UACN,MAAM,OAAO,8BAA8B;AAAA,UAC3C;AAAA,QACF;AACA,cAAM,QAAQ;AAAA,MAChB,CAAC;AAAA,IACH,CAAC;AAAA,EACH,UAAE;AACA,QAAI,MAAO,cAAa,KAAK;AAAA,EAC/B;AACF;AAIA,IAAM,6BAA6B,oBAAI,IAAkD;AAEzF,eAAsB,2BACpB,WACA,QACA,SACe;AACf,QAAM,UAAU,2BAA2B,IAAI,SAAS,KAAK;AAAA,IAC3D;AAAA,IACA,SAAS,OAAO,WAAW;AAAA,EAC7B;AACA,6BAA2B,IAAI,WAAW,OAAO;AACjD,QAAM,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAC7C,6BAA2B,OAAO,SAAS;AAC7C;AAEA,eAAsB,+BACpB,UACA,OACA,SACA,OAA6B,WAC7B,eAMA,qBACA,aACqC;AACrC,MACE,MAAM,OAAO,QAAQ,WACrB,MAAM,mBAAmB,QAAQ,kBACjC,MAAM,eAAe,QAAQ,cAC7B,MAAM,YAAY,QAAQ,mBAC1B,MAAM,eAAe,QAAQ,sBAC7B,CAAC,MAAM,mBACP,8BAA8B,MAAM,eAAe,MAAM,QAAQ,mBACjE,QAAQ,kBAAkB,MAC1B;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AACA,MAAI,CAAC,MAAM,aAAa;AACtB,WAAO,EAAE,QAAQ,YAAY,QAAQ,uBAAuB;AAAA,EAC9D;AACA,QAAM,UAAU,2BAA2B,IAAI,QAAQ,EAAE;AACzD,MAAI,SAAS;AACX,QAAI,CAAC,cAAe,OAAM,IAAI,MAAM,0DAA0D;AAC9F,UAAM,2BAA2B,QAAQ,IAAI,QAAQ,QAAQ,aAAa;AAC1E,UAAM,qBAAqB;AAAA,MACzB,QAAQ;AAAA,MACR,QAAQ;AAAA,IACV;AAGA,QAAI,SAAS,YAAY,mBAAmB,WAAW,SAAU,QAAO;AAAA,EAC1E;AACA,QAAM,kBAAmB,MAAM,YAAwC;AACvE,MACE,oBAAoB,WACnB,OAAO,oBAAoB,YAC1B,8BAA8B,eAAe,MAAM,QAAQ,kBAC7D;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AACA,MAAI,sCAAsC,MAAM,WAAW,MAAM,QAAQ,oBAAoB;AAC3F,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AAEA,MAAI,yBAEO;AACX,MAAI,QAAQ,oBAAoB,WAAW,QAAQ,oBAAoB;AACrE,QAAI;AACF,+BAAyB,MAAM;AAAA,QAC7B,sCAAsC,QAAQ;AAAA,MAChD;AAAA,IACF,SAAS,OAAO;AACd,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AACA,QAAI,uBAAuB,QAAQ,QAAQ,oBAAoB;AAC7D,aAAO,EAAE,QAAQ,YAAY,QAAQ,4BAA4B;AAAA,IACnE;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAASH,+BAA8B,QAAQ,iBAA0B,QAAQ;AAAA,EAGnF,QAAQ;AACN,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MACE,CAAC,UACD,8BAA8B,OAAO,SAAS,MAAM,QAAQ,mBAC5D,CAAC,OAAO,QACR;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,sBAAsB;AAAA,EAC7D;AACA,QAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,MAAI;AACJ,MAAI;AACF,mBAAe,MAAMI;AAAA,MACnB;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAAA,EACF,QAAQ;AACN,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MAAI,iBAAiB,QAAW;AAC9B,WAAO,EAAE,QAAQ,YAAY,QAAQ,uBAAuB;AAAA,EAC9D;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAM;AAAA,MACpBC;AAAA,QACE;AAAA,QACA,QAAQ;AAAA,QACR;AAAA,QACA,QAAQ;AAAA,MACV;AAAA,IACF;AACA,cAAU,QAAQ;AAAA,EACpB,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,aAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,IAC1D;AACA,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,UAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,eAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,MAClE;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,OAAO;AAAA,UACL,SAAS;AAAA,UACT,UAAU;AAAA,UACV,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MAAI,OAAO,QAAQ,eAAe,YAAY;AAC5C,WAAO,EAAE,QAAQ,YAAY,QAAQ,sBAAsB;AAAA,EAC7D;AACA,MAAI,YAAY;AAChB,MAAI,QAAQ,oBAAoB,SAAS;AACvC,QAAI;AACJ,QAAI;AACF,oBAAc,MAAM;AAAA,QAClB,gDAAgD,UAAU,OAAO;AAAA,MACnE;AAAA,IACF,SAAS,OAAO;AACd,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AACA,QAAI,CAAC,QAAQ,oBAAoB;AAC/B,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,oBAAoB,YAAY;AAAA,QAChC,iBAAiB,YAAY;AAAA,MAC/B;AAAA,IACF;AACA,QACE,YAAY,QAAQ,QAAQ,sBAC5B,wBAAwB,QAAQ,QAAQ,oBACxC;AACA,aAAO,EAAE,QAAQ,YAAY,QAAQ,4BAA4B;AAAA,IACnE;AAGA,UAAM,UAAU,MAAM,qBAAqB,KAAK;AAChD,QAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,QAAQ;AAC/C,aAAO,EAAE,QAAQ,YAAY,QAAQ,kCAAkC;AACzE,YAAQ,oBAAoB,QAAQ,mBAAmB,SAAS,mBAAmB;AACnF,gBAAY,QAAQ,QAAQ;AAAA,EAC9B;AACA,QAAM,cAAc,OAAO,UAAkC;AAC3D,QAAI,CAAC,eAAe;AAClB,UAAI,QAAQ,2BAA2B,KAAK;AAC1C,cAAM,IAAI,MAAM,oEAAoE;AACtF;AAAA,IACF;AACA,UAAM,OAAO,QAAQ,2BAA2B,KAAK;AACrD,QAAI,MAAM,QAAQ,SAAS,qBAAqB,OAAO,UAAU,UAAU;AACzE,UAAI,CAAC,QAAQ,wBAAwB,CAAE,MAAM,QAAQ,qBAAqB,KAAK;AAC7E,cAAM,IAAI,MAAM,8DAA8D;AAChF;AAAA,IACF;AACA,QAAI,MAAM;AACR,iBAAW,SAAS,KAAK;AACvB,YAAI,MAAM;AACR,gBAAM,cAAc,MAAM,MAAM,MAAM,SAAS,MAAM,QAAQ,KAAK,cAAc;AAAA,IACtF,OAAO;AACL,YAAM,2BAA2B,QAAQ,IAAI,OAAO,aAAa;AAAA,IACnE;AACA,QAAI,OAAO,UAAU,SAAU,OAAM,QAAQ,2BAA2B,KAAK;AAAA,EAC/E;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,sBACJ,SAAS,YAAY,MAAM,mBAAmB,sBACxC,MAAM,QAAQ;AAAA,MACd,QAAQ;AAAA,MACR,MAAM,mBAAmB,sBAAsB,sBAAsB;AAAA,IACvE,KAAM,QACN;AACN,aAAS,MAAM;AAAA,MACb,QAAQ,WAAW;AAAA,QACjB,WAAW,QAAQ;AAAA;AAAA;AAAA,QAGnB,OACE,SAAS,YACT,CAAC,wBACA,QAAQ,oBAAoB,WAAW,aACpC,MACA;AAAA,QACN,aAAa;AAAA,QACb,iBAAiB;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,EACF,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,aAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,IAC1D;AACA,QAAI,2CAA2C,OAAO,WAAW,KAAK,GAAG;AACvE,UAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,eAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,MAClE;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,OAAO;AAAA,UACL,SAAS;AAAA,UACT,UAAU;AAAA,UACV,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AAGA,QAAM,cAAc,MAAM;AAAA,IACxB;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,MACE,YAAY,WAAW,cACvB,YAAY,WAAW,sBACvB,MAAM,wBAAwB,QAC9B,SAAS,YACT,CAAC,QAAQ,2BACT,aACA,CAAE,MAAM,qBACJ,KAAK,EACN,KAAK,CAAC,YAAY,SAAS,SAAS,qBAAqB,QAAQ,WAAW,GAC/E;AAIA,QAAI;AACF,YAAM,cAAc,MAAM;AAAA,QACxB,QAAQ,WAAW;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,OAAO;AAAA,UACP,aAAa;AAAA,UACb,iBAAiB;AAAA,QACnB,CAAC;AAAA,MACH;AACA,YAAM,YAAY,WAAW;AAC7B,aAAO,kCAAkC,aAAa,QAAQ,mBAAmB,OAAO;AAAA,IAC1F,SAAS,OAAO;AACd,UAAI,2CAA2C,OAAO,WAAW,KAAK,GAAG;AACvE,YAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,iBAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,QAClE;AACA,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,OAAO;AAAA,YACL,SAAS;AAAA,YACT,UAAU;AAAA,YACV,QAAQ;AAAA,UACV;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAIO,SAAS,kCACd,QACA,mBACA,QAC4B;AAC5B,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,EAC1D;AACA,MAAI,wBAAwB,QAAQ,mBAAmB,MAAM,GAAG;AAC9D,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,OAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AACA,QAAM,WAAW,wBAAwB,MAAM;AAC/C,MAAI,aAAa,MAAM;AACrB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,OAAO,EAAE,SAAS,UAAU,UAAU,QAAQ,uBAAuB;AAAA,IACvE;AAAA,EACF;AACA,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,QAAQ,OAAO,SAAS,iCAAiC,IACrD,qBACA;AAAA,EACN;AACF;AAUA,eAAe,eACb,IACA,UACA,eACyD;AACzD,QAAM,eAAe,oBAAI,IAAY;AACrC,MAAI,UAAU;AACd,MAAI,SAA+B,CAAC;AACpC,MAAI;AACF,aAAS,MAAM,wBAAwB,EAAE;AAAA,EAC3C,SAAS,OAAO;AACd,kBAAc,KAAK,0CAA0C;AAAA,MAC3D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,WAAO,EAAE,SAAS,aAAa;AAAA,EACjC;AACA,aAAW,SAAS,QAAQ;AAC1B,iBAAa,IAAI,MAAM,WAAW;AAAA,EACpC;AACA,QAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,QAAI;AACF,YAAM,OACJ,SAAS,2BAA2B,eAChC,IACA,+BAA+B,UAAU,MAAM,OAAO;AAC5D,YAAM,SAAS,MAAM,kBAAkB,IAAI;AAAA,QACzC,WAAW,MAAM;AAAA,QACjB,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,eAAe,MAAM;AAAA,QACrB,yBAAyB;AAAA,QACzB,aAAa,SAAS;AAAA,QACtB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD,UAAI,OAAO,SAAS;AAClB,mBAAW;AAAA,MACb;AACA,UAAI,OAAO,aAAa,KAAK,SAAS,2BAA2B,WAAW;AAC1E,2BAAmB,eAAe,SAAS,OAAO,UAAU;AAAA,MAC9D;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,SAAO,EAAE,SAAS,aAAa;AACjC;AAOA,eAAsB,oCAAoC,OAKxC;AAChB,QAAM;AAAA,IACJ,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM,+BAA+B;AAAA,EACvC;AACF;AAQA,eAAsB,4BAA4B,OAI7B;AACnB,QAAM,EAAE,UAAU,cAAc,IAAI;AACpC,MAAI,CAAC,2BAA2B,QAAQ,EAAG,QAAO;AAClD,MAAI,OAAO,MAAM,uBAAuB,6BAA6B,QAAQ,EAAG,QAAO;AACvF,oCAAkC,eAAe;AAAA,IAC/C,SAAS;AAAA,IACT,UAAU;AAAA,IACV,2BAA2B;AAAA,EAC7B,CAAC;AACD,0CAAwC,eAAe,gBAAgB;AACvE,SAAO;AACT;AAEA,SAAS,2BAA2B,UAAiD;AACnF,SACE,SAAS,mBAAmB,WAAW,CAAC,CAAC,SAAS,gBAAgB,CAAC,CAAC,SAAS;AAEjF;AAEA,eAAe,iCACb,IACA,UACA,eACA,6BACe;AACf,QAAM,QAAQ,IAAI;AAAA,KACf,YAAY;AACX,UAAI;AACF,+BAAuB,eAAe,MAAM,iBAAiB,EAAE,CAAC;AAAA,MAClE,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,IACH,6BAA6B,eAAe,UAAU,iBAAiB,YAAY;AACjF,+BAAyB,eAAe,MAAM,6BAA6B,EAAE,CAAC;AAAA,IAChF,CAAC;AAAA,IACD;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,uCAAuC,EAAE;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,sCAAsC,EAAE;AAAA,QAChD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV,2CAAmC,eAAe,MAAM,2BAA2B,EAAE,CAAC;AAAA,MACxF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,+BAA+B,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,yCAAyC,EAAE;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,qCAAqC,IAAI,MAAM,wBAAwB,EAAE,CAAC;AAAA,QAClF;AAAA,MACF;AAAA,IACF;AAAA,IACA,6BAA6B,eAAe,kBAAkB,oBAAoB,YAAY;AAC5F;AAAA,QACE;AAAA,QACA,MAAM,kCAAkC,EAAE;AAAA,MAC5C;AAAA,IACF,CAAC;AAAA,IACD,GAAI,SAAS,mBAAmB,iBAChC,SAAS,0CACL;AAAA,MACE;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AACV;AAAA,YACE;AAAA,YACA,MAAM,4BAA4B;AAAA,cAChC,WAAW,SAAS;AAAA,YACtB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,IACA,CAAC;AAAA,KACJ,YAAY;AACX,UAAI;AACF,kCAA0B,eAAe,MAAM,4BAA4B,EAAE,CAAC;AAAA,MAChF,SAAS,OAAO;AACd,sBAAc,KAAK,uDAAuD;AAAA,UACxE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,+BAAuB,eAAe,MAAM,iBAAiB,EAAE,CAAC;AAAA,MAClE,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,gCAAwB,eAAe,MAAM,sBAAsB,EAAE,CAAC;AAAA,MACxE,SAAS,OAAO;AACd,sBAAc,KAAK,qDAAqD;AAAA,UACtE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX;AAAA,QACE;AAAA,QACA,SAAS;AAAA,MACX;AACA,UAAI;AACF,cAAM,cAAc,MAAM,8BAA8B,EAAE;AAC1D,6CAAqC,eAAe,aAAa,kBAAkB,IAAI;AAAA,MACzF,SAAS,OAAO;AACd,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,cAAM,gBAAgB,MAAM,gCAAgC,EAAE;AAC9D;AAAA,UACE;AAAA,UACA,eAAe,mBAAmB;AAAA,QACpC;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,2DAA2D;AAAA,UAC5E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,EACL,CAAC;AACH;AAEA,eAAe,6BACb,eACA,QACA,gBACA,SACe;AACf,MAAI;AACF,UAAM,QAAQ;AACd,4CAAwC,eAAe,MAAM;AAAA,EAC/D,SAAS,OAAO;AACd,4CAAwC,eAAe,MAAM;AAC7D,kBAAc,KAAK,mBAAmB,cAAc,yBAAyB;AAAA,MAC3E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AASA,eAAe,8BACb,IACA,UACA,cACA,eACiB;AACjB,QAAM,iBAAiB;AACvB,QAAM,MAAM,SAAS;AACrB,MAAI,eAAe;AACnB,QAAM;AAAA,IACJ,CAAC,GAAG,YAAY;AAAA,IAChB;AAAA,IACA,OAAO,gBAAgB;AACrB,UAAI;AACF,cAAM,SAAS,MAAM,mCAAmC,IAAI;AAAA,UAC1D;AAAA,UACA;AAAA,UACA,4BAA4B;AAAA,UAC5B,aAAa,SAAS;AAAA,QACxB,CAAC;AACD,YAAI,OAAO,aAAa,OAAO,QAAQ,SAAS,GAAG;AACjD,0BAAgB,OAAO,QAAQ;AAC/B,wBAAc,KAAK,gEAAgE;AAAA,YACjF;AAAA,YACA,QAAQ,OAAO;AAAA,YACf,SAAS,OAAO,QAAQ;AAAA,UAC1B,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,0DAA0D;AAAA,UAC3E;AAAA,UACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,8BACpB,IACA,UACA,eACA,WAAkD,gCACnC;AACf,MAAI,SAAS,mBAAmB,WAAW,CAAC,SAAS,gBAAgB,CAAC,SAAS;AAC7E;AACF,QAAM,aAAa,MAAM,gCAAgC,EAAE;AAC3D,QAAM,uBAAuB,YAAY,GAAG,OAAO,cAAc;AAC/D,QAAI;AACF,YAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,UAAU,WAAW;AAC5E,YAAM,WAAW,EAAE,GAAG,WAAW,UAAU;AAC3C,YAAM,UAAU,MAAM,iCAAiC,IAAI,QAAQ;AACnE,UAAI,CAAC,QAAS;AACd,YAAM,aAAa,MAAM,SAAS,UAAU,OAAO;AACnD,UAAI,CAAC,WAAY;AACjB,UAAI,MAAM,mCAAmC,IAAI,EAAE,GAAG,UAAU,SAAS,WAAW,CAAC,GAAG;AACtF,sBAAc,KAAK,kDAAkD;AAAA,UACnE,GAAG;AAAA,UACH,aAAa,QAAQ;AAAA,UACrB;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,iEAAiE;AAAA,QAClF,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;AAEO,SAAS,oCACd,QACA,WACS;AAKT,MAAI,OAAO,KAAK,CAAC,UAAU,MAAM,aAAa,aAAa,MAAM,eAAe,IAAI,GAAG;AACrF,WAAO;AAAA,EACT;AACA,SAAO,iCAAiC,QAAQ,SAAS;AAC3D;AAMO,SAAS,oCACd,QACA,oBACQ;AACR,SAAO,mBAAmB;AAAA,IAAO,CAAC,eAChC,OAAO,KAAK,CAAC,UAAU,MAAM,aAAa,UAAU,MAAM,eAAe,UAAU;AAAA,EACrF,EAAE;AACJ;AAKO,SAAS,iCACd,QACA,WACS;AACT,MAAI,OAAO,KAAK,CAAC,UAAU,MAAM,eAAe,UAAU,SAAS,GAAG;AACpE,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,UAAU,KAAK;AAC/B,QAAM,cAAc,UAAU,KAAK;AACnC,QAAM,iBAAiB,UAAU,KAAK;AACtC,MAAI,CAAC,WAAW,CAAC,eAAe,CAAC,gBAAgB;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,oBAAoB,OAAO;AAAA,IAC/B,CAAC,UACC,MAAM,YAAY,WAClB,MAAM,gBAAgB,eACtB,MAAM,mBAAmB;AAAA,EAC7B;AACA,SAAO;AAAA,IACL,CAAC,qBACA,kBAAkB,eAAe,QAAQ,kBAAkB,eAAe,UAAU;AAAA,EACvF;AACF;AAEA,eAAe,sCACb,UACA,IACA,eACiB;AAIjB,MAAI,CAAC,2BAA2B,QAAQ,GAAG;AACzC,WAAO;AAAA,EACT;AACA,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,sCAAsC,EAAE;AAAA,EAC7D,SAAS,OAAO;AACd,kBAAc,KAAK,4DAA4D;AAAA,MAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,MAAM,0BAA0B,UAAU,YAAY;AAAA,IACnE,uBAAuB,OAAO,cAAc;AAC1C,YAAM,SAAS,MAAM,sCAAsC,EAAE;AAC7D,aAAO,oCAAoC,QAAQ,SAAS;AAAA,IAC9D;AAAA,EACF,CAAC;AAKD,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI;AACF,YAAM,SAAS,MAAM,sCAAsC,EAAE;AAC7D,wCAAkC,eAAe;AAAA,QAC/C,SAAS,OAAO,UAAU;AAAA,QAC1B,UAAU,OAAO,UAAU,aAAa;AAAA,UAAO,CAAC,cAC9C,iCAAiC,QAAQ,SAAS;AAAA,QACpD,EAAE;AAAA,QACF,2BAA2B;AAAA,UACzB;AAAA,UACA,OAAO,UAAU;AAAA,QACnB;AAAA,MACF,CAAC;AACD,8CAAwC,eAAe,gBAAgB;AAAA,IACzE,SAAS,OAAO;AACd,8CAAwC,eAAe,gBAAgB;AACvE,oBAAc,KAAK,4DAA4D;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AACA,aAAW,cAAc,OAAO,YAAY;AAC1C,kBAAc,KAAK,mDAAmD;AAAA,MACpE,WAAW,WAAW;AAAA,MACtB,QAAQ,WAAW;AAAA,MACnB,MAAM,KAAK,UAAU,WAAW,IAAI;AAAA,IACtC,CAAC;AAAA,EACH;AACA,MAAI,OAAO,WAAW,GAAG;AACvB,kBAAc,KAAK,gDAAgD;AAAA,MACjE,UAAU,OAAO;AAAA,MACjB,YAAY,OAAO,WAAW;AAAA,MAC9B,SAAS,OAAO;AAAA,MAChB,SAAS,SAAS;AAAA,IACpB,CAAC;AAAA,EACH;AACA,SAAO,OAAO,WAAW;AAC3B;AAWA,eAAe,gCACb,UACA,OAC8B;AAC9B,MAAI,CAAC,MAAM,cAAc,CAAC,MAAM,aAAa;AAC3C,UAAM,IAAI,MAAM,8DAA8D;AAAA,EAChF;AACA,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,MAAI;AACJ,MAAI;AACF,kBAAc,MAAM,oCAAoC,UAAU,MAAM,aAAa;AAAA,MACnF,WAAW,4BAA4B,MAAM,cAAc;AAAA,MAC3D,UAAU;AAAA,MACV,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH,SAAS,OAAO;AACd,QAAI,+BAA+B,SAAS,KAAK,EAAG,QAAO;AAC3D,UAAM;AAAA,EACR;AACA,MAAI,YAAY,eAAe,MAAM,YAAY;AAC/C,UAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AACA,MAAI;AACF,QAAI,YAAY,cAAc,SAAS;AACrC,YAAM,2BAA2B,aAAa,0CAA0C;AACxF,aAAO;AAAA,IACT;AACA,UAAM,UAAU,YAAY;AAY5B,UAAM,MAAM,QAAQ,QAAQ,QAAQ;AACpC,QAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6DAA6D;AACvF,QAAI;AACJ,UAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,MAChC,IAAI,KAAK,SAAS;AAAA,QAChB,KAAK;AAAA,QACL,aAAa;AAAA,QACb,iBAAiB;AAAA,MACnB,CAAC;AAAA,MACD,IAAI,QAAe,CAAC,GAAG,WAAW;AAChC,gBAAQ;AAAA,UACN,MAAM,OAAO,IAAI,MAAM,qDAAqD,CAAC;AAAA,UAC7E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH,CAAC,EAAE,QAAQ,MAAM;AACf,UAAI,MAAO,cAAa,KAAK;AAAA,IAC/B,CAAC;AACD,QAAI,2BAA2B,MAAM,KAAK,uBAAuB,MAAM,MAAM,GAAG;AAC9E,YAAM,IAAI,MAAM,yDAAyD;AAAA,IAC3E;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,QAAI,+BAA+B,SAAS,KAAK,EAAG,QAAO;AAC3D,UAAM;AAAA,EACR;AACF;AAaO,SAAS,mBACd,OACoB;AACpB,MAAI,MAAM,wBAAwB,KAAM,QAAO;AAC/C,SAAO,MAAM,kBAAkB;AACjC;AAkBA,eAAe,sBACb,IACA,UACA,KACA,eACA,cACA,wBACA,SACA,eACA,MAAsC,MACpB;AAGlB,QAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,IAAI,WAAW;AAGtE,QAAM,QAAQ,MAAMF,WAAU,IAAI,IAAI,aAAa,IAAI,cAAc;AACrE,MAAI,CAAC,OAAO;AAEV,WAAO;AAAA,EACT;AACA,MACE,MAAM,aAAa,cAClB,MAAM,aAAa,KAAK,CAAC,MAAM,6BAA6B,UAC7D,MAAM,eAAe,IAAI,YACzB;AAIA,WAAO;AAAA,EACT;AAEA,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,QAAM,kBAAkB,YAAY,UAAU,YAAY;AAC1D,QAAM,gBAAgB,+BAA+B,SAAS,MAAM,WAAW;AAC/E,MAAI,mBAAmB,CAAC,eAAe;AACrC,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,eAAe,iBAAiB,YAAY;AAC9C,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAEA,QAAM,mBAAmB,QAAQ;AACjC,MACE,mBACA,CAAC,MAAM,gBAAgB,eACvB,qCAAqC,KAAK,GAC1C;AAMA,UAAM,IAAI;AAAA,MACR,WAAW,IAAI,cAAc,UAAU,IAAI,UAAU;AAAA,IACvD;AAAA,EACF;AACA,MAAI,eAIO;AACX,MAAI,+BAA+B;AACnC,MAAI,qBAA8C;AAClD,MAAI,MAAM,cAAc,iBAAiB;AACvC,QAAI,MAAM,gBAAgB;AACxB,YAAM,eAAe,MAAM;AAC3B,YAAM,aAAa,MAAM;AACzB,UAAI,aAAa,aAAa;AAK5B,uBAAe;AAAA,UACb,GAAG;AAAA,UACH,SAAS,MAAM;AAAA,UACf,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,mBAAmB,MAAM;AAAA,UACzB,iBAAiB,MAAM;AAAA,QACzB;AACA,6BAAqB;AAAA,MACvB,OAAO;AACL,YACE,aAAa,gBAAgB,QAAQ,eACrC,aAAa,WAAW,QAAQ,SAChC;AAKA,gBAAM,IAAI;AAAA,YACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE,wBAAwB,aAAa,OAAO;AAAA,UACtG;AAAA,QACF;AACA,cAAM,kBAAkB,MAAM,uCAAuC,IAAI;AAAA,UACvE;AAAA,UACA,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,WAAW,aAAa;AAAA,UACxB,eAAe,IAAI;AAAA,UACnB,oBAAoB;AAAA,QACtB,CAAC;AACD,cAAM,6BACJ,YAAY,WACZ,CAAC,sCAAsC,MAAM,aAAa,OAAO,KACjE,MAAM,SAAS,SAAS,WAAW,cACnC,MAAM,SAAS,SAAS,eAAe,cACvC,MAAM,SAAS,UAAU,WAAW;AAOtC,cAAM,UAAU,YACd,MAAM,yCAAyC,IAAI;AAAA,UACjD;AAAA,UACA,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,gBAAgB,aAAa;AAAA,UAC7B,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,SAAS,QAAQ;AAAA,UACjB,eAAe,IAAI;AAAA,UACnB,oBAAoB;AAAA,UACpB;AAAA,QACF,CAAC;AAMH,cAAM,sBACJ,kBAAkB,QAAQ,cAAc,aAAa;AACvD,YAAI,CAAC,MAAM,aAAa;AACtB,cAAI,8BAA8B,aAAa,gBAAgB,qBAAqB;AAClF,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2BAAe;AAAA,UACjB,OAAO;AACL,gBAAI,CAAC,iBAAiB;AACpB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,gBAAI,CAAC,4BAA4B;AAC/B,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AAIA,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2BAAe;AAAA,UACjB;AAAA,QACF,WAAW,aAAa,gBAAgB,qBAAqB;AAI3D,gBAAM,cAAc,MAAM,QAAQ;AAClC,cAAI,CAAC,aAAa;AAChB,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,0BAA0B,aAAa,EAAE;AAAA,YACxE;AAAA,UACF;AACA,yBAAe;AAAA,QACjB,OAAO;AACL,cAAI,CAAC,iBAAiB;AACpB,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,YAC1D;AAAA,UACF;AACA,gBAAM,gBAAgB,MAAM,uBAAuB,UAAU,KAAK;AAClE,cAAI,kBAAkB,WAAW;AAI/B,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2CAA+B;AAC/B,2BAAe;AAAA,UACjB,OAAO;AAML,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,sBAAsB,aAAa,EAAE;AAAA,YACpE;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,UAAU,MAAM,6BAA6B,IAAI;AAAA,QACrD;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,QACrB,SAAS,QAAQ;AAAA,QACjB,eAAe,IAAI;AAAA,QACnB,oBAAoB,MAAM;AAAA,QAC1B,UAAU;AAAA,QACV;AAAA,QACA,eAAe;AAAA,QACf,oBAAoB,sCAAsC,SAAS,MAAM,WAAW;AAAA,QACpF,gCAAgC,sCAAsC,QAAQ;AAAA,QAC9E,oBAAoB,eAAe,aAAa;AAAA,QAChD,cAAc,kBAAkB,QAAQ,cAAc,aAAa;AAAA,MACrE,CAAC;AACD,UAAI,QAAQ,WAAW,WAAW;AAChC,uBAAe,QAAQ;AAAA,MACzB,WAAW,QAAQ,WAAW,wBAAwB;AAOpD,cAAM,gBAAgB,MAAM,uBAAuB,UAAU,KAAK;AAClE,YAAI,kBAAkB,UAAW,QAAO;AACxC,uCAA+B;AAAA,MACjC,WAAW,QAAQ,WAAW,uBAAuB;AAInD,cAAM,IAAI;AAAA,UACR,WAAW,IAAI,cAAc;AAAA,QAC/B;AAAA,MACF,OAAO;AACL,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAUA,MAAI,YAAY,uBAAuB;AACvC,MAAI,kBACF,uBAAuB,sBAClB,MAAM,SAAS,QAAQ,SAAS,YAAY,OAC7C;AACN,QAAM,iBAAmC,OACvC,eACA,iBACA,kBACA,oBAGG;AAIH,QAAI,CAAC,MAAM,cAAc,CAAC,cAAc;AACtC,aAAO,EAAE,OAAO,OAAO,iBAAiB,KAAK;AAAA,IAC/C;AACA,UAAM,iBAAiB,oCAAoC,aAAa;AACxE,QAAI,uBAAsC;AAC1C,QACE,OAAO,kBAAkB,YACzB,iBACA,iBAAiB,YAAY,MAC5B,gBAAgB,aAAa,+BAC5B,gBAAgB,aAAa,6BAC/B;AACA,UAAI,CAAC,iBAAiB;AACpB,cAAM,IAAI,MAAM,8DAA8D;AAAA,MAChF;AACA,YAAM,YAAY,MAAM,kCAAkC,IAAI;AAAA,QAC5D;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,eAAe,MAAM;AAAA,QACrB,kBAAkB,IAAI;AAAA,QACtB,kBAAkB,MAAM;AAAA,QACxB,2BAA2B,aAAa;AAAA,QACxC,oBAAoB,gBAAgB;AAAA,QACpC,iBAAiB,gBAAgB;AAAA,QACjC,kBAAkB;AAAA,QAClB,sBAAsB;AAAA,MACxB,CAAC;AACD,6BAAuB,UAAU;AAAA,IACnC;AACA,UAAM,YAAY;AAAA,MAChB;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,iBAAiB,MAAM;AAAA,MACvB,eAAe,IAAI;AAAA,MACnB,oBAAoB,MAAM;AAAA,MAC1B,6BAA6B,aAAa;AAAA,MAC1C,WAAW,aAAa;AAAA,MACxB,mBAAmB,aAAa;AAAA,IAClC;AACA,QAAI;AACJ,QAAI,kBAAkB,MAAM;AAC1B,eAAS,MAAM,qBAAqB,IAAI;AAAA,QACtC,GAAG;AAAA,QACH,kBAAkB;AAAA,MACpB,CAAC;AAAA,IACH,OAAO;AACL,UAAI,CAAC,iBAAiB;AACpB,cAAM,IAAI,MAAM,qEAAqE;AAAA,MACvF;AACA,YAAM,YAAY,MAAM,8BAA8B;AAAA,QACpD,SAAS,MAAM;AAAA,QACf;AAAA,QACA;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,SACE,OAAO,kBAAkB,WACrB,gBACA;AAAA,UACE,OAAO,OAAO,KAAK,eAAe,QAAQ;AAAA,UAC1C,YAAY;AAAA,UACZ,QAAQ;AAAA,QACV;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AACD,eAAS,MAAM,iCAAiC;AAAA,QAC9C;AAAA,QACA,GAAI,UAAU,sBAAsB,EAAE,KAAK,UAAU,oBAAoB,IAAI,CAAC;AAAA,QAC9E,SAAS,MACP,qBAAqB,IAAI;AAAA,UACvB,GAAG;AAAA,UACH,sBAAsB;AAAA,UACtB,GAAG;AAAA,UACH,GAAI,uBAAuB,EAAE,qBAAqB,IAAI,CAAC;AAAA,QACzD,CAAC;AAAA,MACL,CAAC;AAAA,IACH;AACA,QAAI,CAAC,OAAO,SAAS,sBAAsB;AACzC,YAAM,2CAA2C,IAAI;AAAA,QACnD;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,YAAY;AAAA,QACZ,QAAQ;AAAA,MACV,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,IAC1B;AACA,QAAI,OAAO,SAAS,kBAAkB,MAAM;AAC1C,kBAAY;AACZ,wBAAkB,OAAO;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AAcA,MACE,CAAC,gCACD,YAAY,WACZ,MAAM,cACN,CAAE,MAAM,qCAAqC,IAAI;AAAA,IAC/C;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,gBAAgB,IAAI;AAAA,IACpB,eAAe,IAAI;AAAA,IACnB,oBAAoB,MAAM;AAAA,IAC1B,UAAU;AAAA,EACZ,CAAC,GACD;AACA,WAAO;AAAA,EACT;AAIA,MAAI,mBAAmB;AACvB,QAAM,qBAAqB,YAA2B;AACpD,UAAM,SAAS,MAAM,0BAA0B,IAAI;AAAA,MACjD;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,IACrB,CAAC;AACD,QAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,4DAA4D;AACzF,uBAAmB;AAAA,EACrB;AACA,QAAM,cAAoD,+BACtD,EAAE,YAAY,MAAM,8BAA8B,KAAK,IACvD,MAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,cAAc,qBAAqB,QAAQ;AAAA,IAC3C;AAAA,IACA;AAAA,IACA,QAAQ,aAAa;AAAA,IACrB,YAAY;AACV,UAAI,CAAC,gBAAgB,CAAC,MAAM,WAAY;AACxC,YAAM,+BAA+B,IAAI;AAAA,QACvC;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,WAAW,aAAa;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,oBAAoB,MAAM;AAAA,MAC5B,CAAC;AAAA,IACH;AAAA,IACA,IAAI;AAAA,IACJ;AAAA,IACA,YAAY,WACR,YAAY;AACV,UAAI,CAAC,gBAAgB,CAAC,MAAM;AAC1B,cAAM,IAAI,MAAM,6CAA6C;AAC/D,aAAO,MAAM,8BAA8B,IAAI;AAAA,QAC7C;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,SAAS,MAAM;AAAA,QACf,YAAY,IAAI;AAAA,QAChB,YAAY,MAAM;AAAA,QAClB,qBAAqB,aAAa;AAAA,QAClC,WAAW,aAAa;AAAA,QACxB,mBAAmB,aAAa;AAAA,MAClC,CAAC;AAAA,IACH,IACA;AAAA,EACN;AACJ,QAAM,aAAa,OAAO,gBAAgB,YAAY,cAAc,YAAY;AAChF,MAAI,CAAC,YAAY;AACf,WAAO;AAAA,EACT;AAKA,QAAM,4BACJ,gCACC,OAAO,gBAAgB,aAAa,YAAY;AACnD,MAAI,CAAC,oBAAoB,CAAC,2BAA2B;AACnD,QAAI,mBAAmB,MAAM,YAAY;AACvC,YAAM,IAAI,MAAM,qDAAqD;AAAA,IACvE;AACA,UAAM,mBAAmB;AAAA,EAC3B;AAMA,QAAM,YACJ,SAAS,2BAA2B,eAChC,IACA,+BAA+B,UAAU,OAAO;AACtD,MAAI,CAAC,2BAA2B;AAC9B,UAAM,YAAY,MAAM,kBAAkB,IAAI;AAAA,MAC5C;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,MACnB,yBAAyB;AAAA,MACzB,aAAa,SAAS;AAAA,MACtB,YAAY;AAAA,IACd,CAAC;AACD,QAAI,UAAU,aAAa,KAAK,SAAS,2BAA2B,WAAW;AAC7E,yBAAmB,eAAe,SAAS,UAAU,UAAU;AAAA,IACjE;AAAA,EACF;AAEA,QAAM,kBACJ,iCACC,OAAO,gBAAgB,YAAY,QAAQ,YAAY;AAK1D,QAAM,EAAE,UAAU,yBAAyB,wBAAwB,IAAI,MAAM;AAAA,IAC3E;AAAA,IACA;AAAA,MACE;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,MACnB,GAAI,eAAe,EAAE,mBAAmB,aAAa,GAAG,IAAI,CAAC;AAAA,MAC7D,8BAA8B;AAAA,MAC9B,0BAA0B,SAAS;AAAA;AAAA,MAEnC,iBACE,mBAAmB,MAAM,YAAY,gBAAgB,MAAM,oBAAoB;AAAA,IACnF;AAAA,EACF;AAGA,QAAM,4BAA4B,KAAK,IAAI,aAAa,yBAAyB,CAAC,UAAU;AAC1F,kBAAc,KAAK,yDAAyD;AAAA,MAC1E,gBAAgB,IAAI;AAAA,MACpB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH,CAAC;AACD,MAAI,UAAU;AAMZ,QAAI,mBAAmB,CAAC,yBAAyB;AAC/C,gDAA0C,eAAe,OAAO;AAAA,IAClE;AACA,QAAI,MAAM,6BAA6B,QAAQ;AAC7C;AAAA,QACE;AAAA,QACA,kBAAkB,qBAAqB;AAAA,MACzC;AAAA,IACF;AAKA,QAAI;AACF,YAAM,iCAAiC,IAAI,IAAI,aAAa,IAAI,gBAAgB;AAAA,QAC9E,MAAM;AAAA,QACN,SAAS;AAAA,UACP,OAAO;AAAA,UACP;AAAA,UACA;AAAA,UACA,YAAY,MAAM;AAAA,UAClB,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA,UAK9B,aAAa,mBAAmB,KAAK;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH,SAAS,YAAY;AACnB,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,gBAAgB,IAAI;AAAA,UACpB,YAAY,IAAI;AAAA,UAChB,SAAS,MAAM;AAAA,UACf,YAAY,MAAM;AAAA,UAClB,OAAO,sBAAsB,QAAQ,WAAW,UAAU,OAAO,UAAU;AAAA,QAC7E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAiBA,SAAS,yBACP,OACA,SAOA;AACA,SAAO;AAAA,IACL,SAAS,MAAM;AAAA,IACf,gBAAgB,MAAM;AAAA,IACtB,YAAY,MAAM;AAAA,IAClB;AAAA,IACA,YAAY,MAAM;AAAA,EACpB;AACF;AAmCA,eAAsB,8BACpB,IACA,OAWA,UAAU,EAAE,WAAAA,YAAW,qCAAqC,GACpB;AACxC,QAAM,UAAU,MAAM,QAAQ,UAAU,IAAI,MAAM,aAAa,MAAM,cAAc;AACnF,MACE,CAAC,WACD,QAAQ,OAAO,MAAM,WACrB,QAAQ,aAAa,cACrB,QAAQ,eAAe,MAAM,cAC7B,QAAQ,eAAe,MAAM,cAC7B,QAAQ,YAAY,YACpB,QAAQ,wBAAwB,MAAM,uBACtC,QAAQ,gBAAgB,OAAO,MAAM,aACrC,QAAQ,eAAe,sBAAsB,MAAM,qBACnD,QAAQ,eAAe,wBAAwB,MAAM;AAErD,UAAM,IAAI,MAAM,gDAAgD;AAClE,QAAM,YAAY,MAAM,QAAQ,qCAAqC,IAAI;AAAA,IACvE,WAAW,MAAM;AAAA,IACjB,aAAa,MAAM;AAAA,IACnB,gBAAgB,MAAM;AAAA,IACtB,eAAe,MAAM;AAAA,IACrB,oBAAoB,MAAM;AAAA,IAC1B,UAAU;AAAA,EACZ,CAAC;AACD,MAAI,CAAC,aAAa,UAAU,wBAAwB,MAAM;AACxD,UAAM,IAAI,MAAM,iEAAiE;AACnF,SAAO;AAAA,IACL,kBACE,QAAQ,eAAe,uBAAuB,QAC9C,OAAO,SAAS,QAAQ,eAAe,YAAY,QAAQ,CAAC,KAC5D,QAAQ,oBAAoB;AAAA,EAChC;AACF;AAEA,SAAS,8BACP,eACA,SACA,aACM;AACN,QAAM,UAAW,QACd;AACH,MAAI,CAAC,QAAS;AACd,sCAAoC,eAAe,QAAQ,MAAM;AACjE,MAAI,QAAQ,WAAW,YAAY;AACjC,kBAAc;AAAA,MACZ;AAAA,MACA,EAAE,GAAG,aAAa,QAAQ,QAAQ,OAAO;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,UACA,OACA,eACA,gBACA,yBAA0DH,gCAC1D,qBAAuD,2BACvD,0BACA,qBAA8C,oBAC9C,sBACA,qBAAqB,OACrB,sBACA,aACA,oBACA,wBACqC;AACrC,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,QAAM,cAAc,yBAAyB,OAAO,OAAO;AAC3D,QAAM,qBAAqB,CAAC,SAAiB,UAC3C,yBAAyB,eAAe,SAAS,OAAO;AAAA,IACtD,gBAAgB,MAAM;AAAA,IACtB,YAAY,MAAM;AAAA,IAClB,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,EACvC,CAAC;AACH,QAAM,6BAA6B,YAA2B;AAC5D,QAAI;AACF,YAAM,uBAAuB;AAAA,IAC/B,SAAS,OAAO;AACd,yBAAmB,uDAAuD,KAAK;AAAA,IACjF;AAAA,EACF;AAEA,MAAI,CAAC,WAAW,YAAY,QAAQ;AAClC,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAWA,MACE,YAAY,gBACZ,MAAM,YAAY,gBAClB,MAAM,oBAAoB,cAC1B;AACA,kBAAc;AAAA,MACZ;AAAA,MACA;AAAA,QACE,GAAG;AAAA,MACL;AAAA,IACF;AACA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAEA,QAAM,uBACJ,wBAAwB,+BAA+B,SAAS,MAAM,WAAW;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,qBAAqB,iBAAiB,aAAa;AACrD,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAEA,QAAM,sBAAsB,wCAAwC,SAAS,MAAM,WAAW;AAO9F,MAAI,CAAC,MAAM,YAAY;AACrB,QAAI,MAAM,uBAAuB,eAAe,MAAM;AACpD,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,QAAI,qBAAqB;AACvB,YAAM,IAAI;AAAA,QACR,mBAAmB,OAAO,oCAAoC,mBAAmB;AAAA,MACnF;AAAA,IACF;AACA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AACA,MAAI,uBAAuB,wBAAwB,MAAM,YAAY;AACnE,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO,mBAAmB,MAAM,UAAU,+CAA+C,mBAAmB;AAAA,IACjI;AAAA,EACF;AAMA,QAAM,+BACJ,YAAY,WACZ,CAAC,uBACD,MAAM,SAAS,SAAS,WAAW,cACnC,MAAM,SAAS,SAAS,eAAe,MAAM,cAC7C,MAAM,SAAS,UAAU,WAAW;AACtC,MAAI,8BAA8B;AAChC,UAAM,EAAE,MAAM,IAAI,MAAM,eAAe,IAAI;AAC3C,QAAI,CAAC,OAAO;AACV,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,GAAG;AAAA,QACL;AAAA,MACF;AACA,aAAO,EAAE,YAAY,OAAO,8BAA8B,MAAM;AAAA,IAClE;AACA,QAAI;AACF,YAAM,qBAAqB;AAC3B,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,QACN;AAAA,QACA,MAAM,uBAAuB;AAAA,MAC/B;AAAA,IACF,SAAS,OAAO;AACd,UAAI,CAAC,+BAA+B,SAAS,KAAK,GAAG;AACnD,cAAM;AAAA,MACR;AACA,oBAAc,KAAK,sEAAsE;AAAA,QACvF,GAAG;AAAA,MACL,CAAC;AAAA,IACH;AAIA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAMA,MAAI,CAAC,uBAAuB,CAAC,MAAM,aAAa;AAC9C,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO,aAAa,MAAM,UAAU;AAAA,IACzD;AAAA,EACF;AAEA,QAAM,SAAS,uBAAuB,SAAkB,QAAQ;AAGhE,MAAI,CAAC,QAAQ;AAEX,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAIA,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI,CAAC,OAAO,2BAA4B,YAAY,YAAY,CAAC,OAAO,QAAS;AAI/E,YAAM,IAAI,MAAM,mBAAmB,OAAO,yCAAyC;AAAA,IACrF;AAYA,UAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,UAAM,eAAe,MAAMI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA,QAAI,iBAAiB,QAAW;AAC9B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,QAAI,YAAY,UAAU;AACxB,UAAI,CAAC,OAAO,2BAA2B,CAAC,wBAAwB;AAC9D,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,uBAAuB;AAC7B,gBAAU,MAAM,OAAO,wBAAwB,cAAc,sBAAsB;AACnF,qBAAe;AAAA,IACjB,OAAO;AACL,YAAM,UAAU,MAAMC;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,MACR;AACA,gBAAU,QAAQ;AAClB,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF,SAAS,OAAO;AACd,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,uEAAuE;AAAA,QACxF,GAAG;AAAA,MACL,CAAC;AACD,aAAO;AAAA,QACL,YAAY;AAAA,QACZ,8BAA8B,uBAAuB;AAAA,MACvD;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AAEA,QAAM,oBAAoB,+BAA+B,SAAS,OAAO;AACzE,MACE,CAAC,qBACD,kBAAkB,aAAa,qBAAqB,YACpD,kBAAkB,aAAa,qBAAqB,YACpD,kBAAkB,iBAAiB,qBAAqB,cACxD;AACA,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAMA,MAAI;AACJ,QAAM,8BACJ,SACC,OAAO;AACV,QAAM,oBACJ,uBAAuB,sBACvB,YAAY,YACX,gCAAgC,yBAC/B,gCAAgC,wBAC9B,MAAM;AAAA,IACJ,gDAAgD,UAAU,OAAO;AAAA,EACnE,IACA;AAGN,MAAI,yBAAqC,MAAM;AAC/C,MAAI;AACF,QAAI,uBAAuB,uBAAuB,SAAS,kBAAkB;AAC3E,+BAAyB,0BAA0B,KAAK;AAIxD,YAAM,UAAU;AAAA,QACd;AAAA,QACA,KAAK,IAAI;AAAA,QACT;AAAA,UACE,WAAW,4BAA4B,WAAW;AAAA,UAClD,UAAU,kBAAkB;AAAA,QAC9B;AAAA,QACA;AAAA,MACF;AACA,wBAAkB,MAAM,wCAAwC;AAAA,QAC9D;AAAA,QACA,WAAW,SAAS;AAAA,QACpB,cAAc,IAAI;AAAA,UAChB,MAAM;AAAA,UACN;AAAA,UACA,SAAS;AAAA,UACT,MAAM;AAAA,UACN,MAAM;AAAA,QACR;AAAA,QACA,aAAa,OAAO,YAAY;AAC9B,cAAI;AACF,kBAAM,SAAS,MAAM;AAAA,cACnB,QAAQ,SAAS,eAAe,UAAU,QAAQ;AAAA,cAClD,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AACA,0BAAc;AAAA,cACZ,OAAO,QACH,yEACA;AAAA,cACJ;AAAA,gBACE,GAAG;AAAA,cACL;AAAA,YACF;AAAA,UACF,SAAS,OAAO;AACd,+BAAmB,6DAA6D,KAAK;AAAA,UACvF,UAAE;AACA,kBAAM,wBAAwB,OAAO;AAAA,UACvC;AAAA,QACF;AAAA,QACA,+BAA+B,CAAC,UAAU;AACxC,6BAAmB,2DAA2D,KAAK;AAAA,QACrF;AAAA,QACA,oBAAoB,CAAC,UAAU;AAI7B,eAAK,2BAA2B;AAChC,6BAAmB,2DAA2D,KAAK;AAAA,QACrF;AAAA,QACA,eAAe,MAAM,uBAAuB;AAAA,MAC9C,CAAC;AAAA,IACH;AAAA,EACF,SAAS,OAAO;AAGd,QAAI,EAAE,iBAAiB,oCAAqC,wBAAuB;AACnF,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,qDAAqD;AAAA,QACtE,GAAG;AAAA,MACL,CAAC;AACD,aAAO,EAAE,YAAY,MAAM,8BAA8B,KAAK;AAAA,IAChE;AACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAIA,QAAI,EAAE,iBAAiB,qCAAqC;AAC1D,YAAM,2BAA2B;AAAA,IACnC;AAEA,UAAM;AAAA,EACR;AASA,MAAI,uBAAuB,qBAAqB;AAC9C,kBAAc,KAAK,sEAAsE;AAAA,MACvF,GAAG;AAAA,IACL,CAAC;AAAA,EACH,WAAW,iBAAiB;AAC1B,QAAI;AACJ,QAAI;AACF,OAAC,EAAE,MAAM,IAAI,MAAM;AAAA,QACjB,gBAAgB,SAAS,eAAe,kBAAkB,gBAAgB;AAAA,QAC1E,gBAAgB;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF,UAAE;AACA,UAAI;AACF,cAAM,wBAAwB,eAAe;AAAA,MAC/C,UAAE;AACA,+BAAuB;AAAA,MACzB;AAAA,IACF;AACA,QAAI,CAAC,OAAO;AACV,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,GAAG;AAAA,QACL;AAAA,MACF;AACA,aAAO,EAAE,YAAY,OAAO,8BAA8B,MAAM;AAAA,IAClE;AAAA,EACF,OAAO;AAIL,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAIA,MAAI;AACF,QAAI,CAAC,WAAW,CAAC,cAAc;AAC7B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,2CAAuC,SAAS,OAAO;AACvD,UAAM,qBAAqB;AAC3B,UAAM,+BAA+B,QAAQ,cAAc,OAAO;AAClE,QAAI,YAAY,SAAU,+BAA8B,eAAe,SAAS,WAAW;AAC3F,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE,SAAS,OAAO;AACd,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,0DAA0D;AAAA,QAC3E,GAAG;AAAA,MACL,CAAC;AAGD,aAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,IACjE;AACA,UAAM;AAAA,EACR;AACF;","names":["Context","readLease","createSandboxClientForBackend","deserializeSandboxSessionStateEnvelope","resumeExactSandboxSession","sandboxLeaseTelemetryKey","Context","heartbeat","createSandboxClientForBackend","drainableInventory","drainable","readLease","deserializeSandboxSessionStateEnvelope","resumeExactSandboxSession"]}
1
+ {"version":3,"sources":["../src/retained-process-diagnostics.ts","../src/retained-process-retry.ts","../src/activities/sandbox-lease.ts","../src/activities/browser-deadline-checkpoint.ts"],"sourcesContent":["import {\n databaseFailureCode,\n nestedPostgresSqlState,\n safeDatabaseErrorFacts,\n SandboxWorkspaceMutationFencedError,\n type SandboxRetainedProcess,\n} from \"@opengeni/db\";\nimport type { Observability } from \"@opengeni/observability\";\n\n/** Protected cause locations and schema facts; public logs receive only the diagnostic key. */\nexport function warnRetainedProcessProofFailure(\n observability: Pick<Observability, \"recordFailureDiagnostic\" | \"warn\">,\n error: unknown,\n process: Pick<\n SandboxRetainedProcess,\n \"id\" | \"sessionId\" | \"ownerAttemptId\" | \"ownerTurnId\" | \"reconcileAttempts\"\n >,\n): void {\n let correlationId: string | undefined;\n try {\n const sqlState = nestedPostgresSqlState(error);\n const constraint = safeDatabaseErrorFacts(error).constraint;\n correlationId = observability.recordFailureDiagnostic({\n code:\n error instanceof SandboxWorkspaceMutationFencedError\n ? \"retained_process_fenced\"\n : sqlState\n ? databaseFailureCode(sqlState)\n : \"retained_process_proof_failed\",\n stage: \"sandbox_retained_processes.proof\",\n error,\n processId: process.id,\n sessionId: process.sessionId,\n ...(process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {}),\n ...(process.ownerTurnId ? { turnId: process.ownerTurnId } : {}),\n attempts: process.reconcileAttempts,\n sqlState,\n ...(constraint ? { constraint } : {}),\n });\n } catch {\n // A diagnostic/export failure must not change reconciliation or release its fence.\n }\n try {\n observability.warn(\"sandbox reaper: retained-process proof checkpoint failed\", {\n ...(correlationId ? { correlationId } : {}),\n });\n } catch {\n // Keep the original failure behavior even when the public sink fails.\n }\n}\n","import type { LeaseSnapshot, SandboxRetainedProcess } from \"@opengeni/db\";\n\n/** Observation backoff must leave the exact provider's rotation window usable.\n * This changes only the next probe time, never cancellation or settlement proof. */\nexport function retainedProcessDeadlineRetryMs(\n process: Pick<\n SandboxRetainedProcess,\n \"leaseId\" | \"leaseEpoch\" | \"providerBackend\" | \"providerInstanceId\" | \"routeTargetId\"\n >,\n lease: Pick<\n LeaseSnapshot,\n \"id\" | \"leaseEpoch\" | \"backend\" | \"instanceId\" | \"providerDeadlineAt\"\n > | null,\n settings: { sandboxRotationLeadMs: number; sandboxLeaseReaperPeriodMs: number },\n retryAfterMs: number,\n now = Date.now(),\n): number {\n if (\n process.providerBackend !== \"modal\" ||\n process.routeTargetId !== null ||\n !lease ||\n lease.id !== process.leaseId ||\n lease.leaseEpoch !== process.leaseEpoch ||\n lease.backend !== process.providerBackend ||\n lease.instanceId !== process.providerInstanceId ||\n !lease.providerDeadlineAt\n )\n return retryAfterMs;\n\n const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;\n if (!Number.isFinite(untilRotation)) return retryAfterMs;\n return Math.min(\n retryAfterMs,\n Math.max(1_000, settings.sandboxLeaseReaperPeriodMs, untilRotation),\n );\n}\n","// apps/worker/src/activities/sandbox-lease.ts — the sole global lease inventory,\n// GC and cost-stop activity set (P1.3 / OD-3).\n//\n// One scheduled workflow inventories at most the DB reaper's fair 500-row batch,\n// then starts one durable child per exact (workspace, group, epoch). Each child\n// claims, snapshots and terminates only its own provider box. Temporal's control\n// worker bounds real provider concurrency; a slow or dead provider cannot hold\n// the global scan or any sibling drain. Activity heartbeats make worker loss\n// visible promptly, while the DB capture id remains the authoritative teardown\n// fence through provider termination and the final draining->cold commit.\n//\n// The legacy `reapSandboxLeases` composite remains only as a direct/embedded test\n// harness and executes admitted drains concurrently. There is no per-session\n// timer, viewer activity, owner task queue, or provider-specific lifecycle path\n// in the normal drain state machine.\nimport { warnDrainSnapshotFailure } from \"../sandbox-snapshot-diagnostics\";\nimport { maybePersistWarmWorkspaceSnapshot } from \"../sandbox-resume\";\nimport { publishDurableSessionEvents } from \"../session-event-fanout\";\nimport { warnRetainedProcessProofFailure } from \"../retained-process-diagnostics\";\nimport { retainedProcessDeadlineRetryMs } from \"../retained-process-retry\";\n\nimport { createHash, randomUUID } from \"node:crypto\";\nimport { requestRetainedProcessDeadlineCancellation } from \"@opengeni/db/retained-provider-commands\";\nimport type { ProviderCommandPersistence, ProviderCommandSession } from \"@opengeni/runtime\";\nimport { Context } from \"@temporalio/activity\";\nimport { OpLostReason, OpState, type OpStatus } from \"@opengeni/agent-proto\";\nimport {\n accrueWarmSeconds,\n retainedProviderCommandPersistence,\n adoptLegacyModalCheckpointArtifact,\n confirmDrainCold,\n markWarmLeaseInstanceLost,\n appendSessionEventToSandboxGroup,\n bindRetainedProcessProviderIdentity,\n claimWorkspaceArchiveCapture,\n leaseMayHaveUntrackedWriters,\n releaseWorkspaceArchiveCapture,\n listIdleCheckpointCandidates,\n recordIdleCheckpointAttempt,\n claimSandboxCheckpointArtifactsForGc,\n claimTerminalRetainedProcesses,\n getEnrollment,\n countActiveRetainedProcessesByOwnerState,\n countSandboxCheckpointArtifactsByState,\n countExpiredDrainingSandboxLeases,\n countQueuedTurns,\n countSandboxLeasesByLiveness,\n deferRetainedProcessReconciliation,\n forceDrainOverLimitViewerOnlyBoxes,\n listCreditBalancesByAccount,\n countActiveUsers,\n readCreditGrantTotals,\n readManagedAuthNewSignupsSwitch,\n readVerifiedSignupTrialSwitch,\n listLegacyModalCheckpointSlots,\n listLiveModalSandboxLeaseAttributions,\n listPendingModalProviderCreates,\n claimModalProviderCreateRecovery,\n recordRecoveredModalProviderCreate,\n markWarmBillingStopCutoff,\n listMeterableWarmLeases,\n countInteractionOnlyWarmLeasesByIdle,\n listSandboxViewerForceDrainWorkspaceIds,\n markSandboxCheckpointArtifactDeletePending,\n persistDrainSnapshot,\n pruneDeletedSandboxCheckpointArtifacts,\n registerSandboxCheckpointArtifact,\n recordRetainedProcessReconciliationProof,\n replaceWorkspaceArchiveCaptureAfterProof,\n listEndedEpochWorkspaceBlockers,\n listEndedEpochSandboxSetupUncertainties,\n reconcileCompletedSandboxSetup,\n readLease,\n settleEndedEpochWorkspaceBlockers,\n readWorkspaceArchiveCapturePreflight,\n reapExpiredSessionListSnapshots,\n reapStaleLeaseHoldersGlobal,\n requestDueSandboxRotationsGlobal,\n readSandboxRotationBacklog,\n readSandboxCheckpointStaleness,\n readRecentSandboxRecoveryObservations,\n workspaceArchiveCaptureDeadlineElapsed,\n retainedProcessReconciliationProof,\n retainedProcessSettlementIdentity,\n recoverManagedSessionBackgroundCommand,\n settleClaimedConnectedMachineBackgroundCommand,\n settleSandboxCheckpointArtifactGc,\n rlsContextForWorkspace,\n settleRetainedProcess,\n type MeterableWarmLease,\n type ReapDrainable,\n type RetainedProcessProviderProof,\n type SandboxRetainedProcess,\n type LeaseSnapshot,\n} from \"@opengeni/db\";\nimport {\n claimConnectedCommandOutputReleases,\n claimConnectedMachineSessionBackgroundCommands,\n deferConnectedMachineBackgroundCommandReconciliation,\n recordConnectedMachineBackgroundCommandProof,\n recordConnectedCommandOutputConsumption,\n settleConnectedCommandOutputReleaseClaim,\n type ConnectedMachineBackgroundCommandClaim,\n type ConnectedMachineBackgroundCommandProof,\n} from \"@opengeni/db/session-background-commands\";\nimport {\n SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,\n sandboxArchiveCaptureTimeoutMs,\n sandboxDeadlineMandatoryCaptureLeadMs,\n sandboxWarmRateMicrosPerSecond,\n} from \"@opengeni/config\";\nimport { appendSessionCommandOutput } from \"@opengeni/db/session-command-output\";\nimport { captureConnectedCommandOutput } from \"../sandbox-routing\";\nimport { sandboxLeaseTelemetryKey } from \"@opengeni/observability\";\nimport {\n // Normal drain teardown builds the client and resumes the envelope directly:\n // a live box gets its /workspace persisted before termination, while a gone\n // box is typed as provider loss. Stale capture reconciliation separately uses\n // establishSandboxSessionFromEnvelope in strict resume-only mode so it can\n // reuse the runtime's provider-ready proof without ever cold-restoring.\n captureWorkspaceArchiveForStorage,\n disposeWorkspaceArchive,\n type VerifiedWorkspaceArchivePayload,\n type VerifiedHostWorkspaceArchive,\n cancelSelfhostedOp,\n assertConsistentSandboxProviderIdentity,\n createSandboxClientForBackend,\n deleteModalCheckpointSnapshot,\n deserializeSandboxSessionStateEnvelope,\n establishSandboxSessionFromEnvelope,\n inspectModalSandboxLifecycle,\n isExecSessionLostBanner,\n isProviderSandboxNotFoundError,\n isProviderSandboxGoneDuringRoutedOperation,\n NatsControlRpc,\n NatsOpStreamTransport,\n OpStreamExecClient,\n defaultSelfhostedRetryClock,\n stripExecBanner,\n parseExecBannerExitCode,\n prepareProviderForTeardownAfterCapture,\n providerWorkspaceCapturePolicy,\n providerWorkspaceCaptureIsPointInTime,\n resolveModalCheckpointProviderBindingForLiveSandbox,\n resolveModalCheckpointProviderBinding,\n findModalProviderCreateReceipt,\n resolveModalCheckpointProviderBindingForSession,\n querySelfhostedOp,\n resumeExactSandboxSession,\n sandboxBackendForSdkBackendId,\n sandboxProviderInstanceIdFromEnvelope,\n sandboxCommandExitCode,\n sandboxCommandStillRunning,\n SelfhostedControlError,\n subjectFor,\n modalAppHasRunningSandboxes,\n sweepModalOrphanSandboxes,\n terminateManagedSandboxSession,\n terminateModalSandboxById,\n verifySandboxExecReadiness,\n type DockerHostWorkspaceRelease,\n type ModalOrphanSweepTermination,\n type ModalCheckpointProviderBinding,\n type ProviderWorkspaceCapturePolicy,\n type WorkspaceArchiveDescriptor,\n type ControlRpc,\n type NatsRequestConnection,\n} from \"@opengeni/runtime/sandbox\";\nimport {\n SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,\n SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,\n type SandboxDrainActivityInput,\n type SandboxLeaseSweepMaintenanceInput,\n sandboxDrainTimeoutClass,\n sandboxIdleCheckpointWorkflowId,\n type SandboxIdleCheckpointActivityInput,\n type SandboxIdleCheckpointPlan,\n} from \"../sandbox-reaper-contract\";\nimport { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from \"../concurrency\";\nimport { browserCheckpointPrerequisites } from \"./browser-deadline-checkpoint\";\nimport { assertSandboxDrainInputTiming, sandboxDrainTiming } from \"../sandbox-reaper-timeout\";\nimport type { ControlActivityServices as ActivityServices } from \"./types\";\nimport { reconcilePendingParentSystemUpdates } from \"./parent-wake\";\nimport {\n recordActiveUserGauges,\n recordCreditBalanceGauges,\n recordCreditGrantGauges,\n recordCreditMicros,\n recordExpiredDrainingSandboxLeaseGauges,\n recordOpenSandboxKubernetesInventoryGauges,\n recordRetainedProcessInventoryGauges,\n recordRetainedProcessReconciliation,\n recordSandboxCheckpointArtifactGauges,\n recordSandboxCheckpointArtifactOutcome,\n recordSandboxDeadlineRotationsRequested,\n recordSandboxInventoryProjectionFailure,\n recordSandboxInventoryProjectionSuccess,\n type RetainedProcessReconciliationOutcome,\n type SandboxInventoryProjectionDomain,\n recordSandboxLeaseGauges,\n recordModalSandboxInventoryGauges,\n recordInteractionOnlyLeaseGauges,\n recordSandboxOrphansTerminated,\n recordSandboxCommandContainment,\n recordSandboxIdleInteractionRelease,\n recordSandboxProviderMissingBeforeCapture,\n recordSandboxDockerWorkspaceRelease,\n recordSandboxRecoveryObservationGauges,\n recordSandboxRotationBacklogGauges,\n recordSandboxCheckpointStalenessGauges,\n recordTurnsQueuedGauge,\n recordVerifiedSignupTrialDeploymentFlagGauge,\n recordManagedAuthNewSignupsSwitchGauge,\n recordVerifiedSignupTrialSwitchGauge,\n runtimeMetricsHooksForObservability,\n} from \"../observability-metrics\";\nimport {\n inspectOpenSandboxKubernetesInventory,\n type OpenSandboxKubernetesInventory,\n} from \"../opensandbox-kubernetes-inventory\";\nimport type { ObjectStorage } from \"@opengeni/storage\";\nimport {\n persistWorkspaceArchiveCandidate,\n putVersion1TarArchiveOrInline,\n} from \"../sandbox-archive-storage\";\n\nexport { sandboxLeaseTelemetryKey } from \"@opengeni/observability\";\n\nexport type ReapSandboxLeasesResult = {\n /** Stale viewer holders + warming-death rows the sweep touched is folded into\n * the DB pass; here we report the terminate outcomes. */\n examined: number;\n /** Boxes whose provider terminate fired AND whose lease went cold. */\n terminated: number;\n /** Drainable rows skipped because the CAS no longer held (re-armed / newer\n * epoch / already drained by a concurrent sweep) — provider stop() did NOT\n * fire. */\n skipped: number;\n /** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */\n metered: number;\n /** Holderless boxes accelerated to draining by the optional warm cap. */\n forceDrained: number;\n /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */\n modalOrphansTerminated: number;\n};\n\ntype CreateSandboxClientForBackendFn = typeof createSandboxClientForBackend;\n\ntype DrainSandboxClient = {\n backendId: string;\n resume?: (state: unknown) => Promise<unknown>;\n deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;\n delete?: (state: unknown) => Promise<void>;\n attachWorkspaceForDrain?: (\n state: unknown,\n assertCurrentCapture: () => Promise<{ archivePublished: boolean }>,\n ) => Promise<PersistableSession>;\n};\n\n/**\n * The CAS-write the terminate seam calls to PERSIST the captured /workspace\n * archive onto the lease under the epoch fence, BEFORE the box is terminated\n * (sandbox-file-persistence). The caller (`terminateDrainableBox`) closes over\n * db/accountId/row and delegates to persistDrainSnapshot. Returns:\n * `wrote:false` means the CAS missed (re-armed / newer epoch / vanished). The\n * box is wanted again, so the seam MUST NOT terminate it. Provider-native\n * deletion is owned independently by artifact-ledger GC.\n *\n * Pass null for archiveBase64 to CAS-check WITHOUT writing (re-arm guard for\n * backends with no persistWorkspace — ensures a re-arm during the snapshot window\n * aborts the terminate before client.delete()).\n */\nexport type PersistArchiveFn = (\n archiveBase64: string | VerifiedHostWorkspaceArchive | null,\n archiveMetadata?: WorkspaceArchiveDescriptor,\n providerSession?: unknown,\n providerBinding?: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBindingForSession>\n > | null,\n) => Promise<{\n wrote: boolean;\n archiveRevision?: string | null;\n}>;\n\nexport type ProviderTerminationOutcome = {\n /** false means an epoch/refcount CAS proved the box was re-armed, so it was\n * deliberately left running and the cold commit must not execute. */\n terminated: boolean;\n /** True only when a definitive provider NotFound proved the cloud box had\n * already vanished before this drain could capture its workspace. */\n providerMissingBeforeCapture: boolean;\n};\n\nexport type DrainCaptureDisposition = \"capture_required\" | \"archive_published\";\n\n/** The provider-terminate seam. Production wires the real resume-by-id ->\n * persistWorkspace -> persist-onto-lease (epoch-fenced) -> snapshot-GC ->\n * provider stop() (`terminateProviderBox`); a unit test injects a spy so the\n * drain/CAS logic is exercised against a real DB without a live provider box.\n * Test seams may return the legacy boolean shorthand. Production returns the\n * typed outcome so the cold commit can distinguish a clean stop from definitive\n * provider disappearance before capture. */\nexport type TerminateBoxFn = (\n settings: ActivityServices[\"settings\"],\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,\n observability: ActivityServices[\"observability\"],\n persistArchive: PersistArchiveFn,\n providerCaptureRequestId?: string,\n captureDisposition?: DrainCaptureDisposition,\n capturePolicy?: ProviderWorkspaceCapturePolicy | null,\n diskBackedArchives?: boolean,\n releaseFailedCapture?: () => Promise<void>,\n workspaceId?: string,\n beforeProviderStop?: () => Promise<void>,\n beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,\n) => Promise<boolean | ProviderTerminationOutcome>;\n\nexport type SweepModalOrphansFn = (\n settings: ActivityServices[\"settings\"],\n db: ActivityServices[\"db\"],\n observability: ActivityServices[\"observability\"],\n) => Promise<number>;\n\nexport type InspectOpenSandboxKubernetesInventoryFn = (input: {\n namespace: string;\n}) => Promise<OpenSandboxKubernetesInventory>;\n\nexport type SandboxLeaseActivityOptions = {\n /** Override the provider terminate (tests spy this; defaults to the real\n * resume-by-id + provider stop()). */\n terminateBox?: TerminateBoxFn;\n /** Override the provider-side Modal orphan sweep (tests spy this; defaults to\n * Modal list+tag comparison when Modal is configured). */\n sweepModalOrphans?: SweepModalOrphansFn;\n /** Only provider discovery is injected; durable recovery remains real. */\n findModalProviderCreateReceipt?: typeof findModalProviderCreateReceipt;\n /** Override the read-only bounded OpenSandbox Kubernetes projection. */\n inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;\n /** Override only the read-only provider process probe. The canonical DB\n * settlement remains real in tests and production. */\n probeRetainedProcess?: RetainedProcessProbeFn;\n /** Observe a historical Modal box after its lease row has advanced to a\n * successor identity. Lifecycle-only; a live box remains ambiguous. */\n inspectHistoricalModalSandbox?: HistoricalModalSandboxLifecycleProbeFn;\n /** Override the read-only exact-instance readiness probe used before the\n * reaper settles blockers for a provider that has definitively vanished. */\n probeDrainableProvider?: DrainableProviderProbeFn;\n /** Override only the resume-by-id of the box an idle checkpoint captures.\n * Returns the live session, or null when the provider no longer has it. The\n * capture claim, publication and release remain real. */\n resumeIdleCheckpointSession?: ResumeIdleCheckpointSessionFn;\n /** Override only the provider client factory of the default drain. The\n * claim, capture, publication and teardown ordering remain real. */\n createClientForBackend?: CreateSandboxClientForBackendFn;\n};\n\nexport type ResumeIdleCheckpointSessionFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n) => Promise<unknown | null>;\n\nexport type RetainedProcessProbeResult =\n | { status: \"proved\"; proof: RetainedProcessProviderProof }\n | {\n status: \"binding\";\n providerBindingKey: string;\n providerBinding: ModalCheckpointProviderBinding;\n }\n | {\n status: \"deferred\";\n reason:\n | \"identity_mismatch\"\n | \"resume_state_missing\"\n | \"backend_unsupported\"\n | \"provider_running\"\n | \"provider_unknown\"\n | \"provider_timeout\"\n | \"provider_error\"\n | \"provider_binding_missing\"\n | \"provider_binding_mismatch\"\n | \"process_observation_unavailable\";\n /** A running command whose last read still returned a large backlog. */\n outputBacklog?: boolean;\n };\n\nexport type RetainedProcessProbeFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n process: SandboxRetainedProcess,\n mode?: \"observe\" | \"cancel\",\n captureOutput?: (\n result: unknown,\n chunkId: string,\n stream?: \"stdout\" | \"stderr\",\n streamFidelity?: \"separate\" | \"merged\",\n ) => Promise<void>,\n providerPersistence?: ProviderCommandPersistence,\n drainBudget?: RetainedProcessDrainBudget,\n) => Promise<RetainedProcessProbeResult>;\n\n/** Shared wall-clock bound on output draining within one reconciliation sweep. */\nexport type RetainedProcessDrainBudget = { until: number };\n\nexport type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;\n\nexport type DrainableProviderProbeFn = (\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n) => Promise<\"ready\" | \"missing\">;\n\nexport const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;\nexport const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;\nexport const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;\n/** Bounds on reading a command's output backlog: per claim, and across one\n * reconciliation sweep so the reaper activity stays well inside its timeout.\n * Each read returns up to one provider page per stream. */\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 10_000;\nexport const RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 60_000;\n/** A live process whose read returned at least this much output left a backlog\n * behind it. A server or watcher that only trickles output stays on ordinary\n * backoff; a process the provider reports exited is always read to EOF. */\nexport const RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;\nexport const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;\nexport const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;\nexport const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;\nexport const CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;\n\nexport class SandboxProviderCaptureTimeoutError extends Error {\n readonly name = \"SandboxProviderCaptureTimeoutError\";\n\n constructor(\n public readonly sandboxGroupId: string,\n public readonly backend: string,\n public readonly timeoutMs: number,\n public readonly leaseEpoch: number,\n public readonly instanceId: string,\n ) {\n super(\n `sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} ` +\n `workspace capture exceeded ${timeoutMs}ms; ` +\n \"the exact durable claim remains fenced while its provider response settles\",\n );\n }\n}\n\nexport async function awaitProviderCaptureWithLatePublication<T>(input: {\n capture: Promise<T>;\n timeoutMs: number;\n timeoutError: SandboxProviderCaptureTimeoutError;\n publishLate: (value: T) => Promise<void>;\n observeLateFailure: (error: unknown) => void;\n observeLatePublicationFailure: (error: unknown) => void;\n /** Called once the detached late path has fully settled (after publication\n * or failure handling), never on the in-budget path. */\n onLateSettled?: () => void;\n}): Promise<T> {\n let timeout: ReturnType<typeof setTimeout> | undefined;\n const timeoutError = input.timeoutError;\n try {\n return await Promise.race([\n input.capture,\n new Promise<never>((_, reject) => {\n timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);\n }),\n ]);\n } catch (error) {\n if (error !== timeoutError) throw error;\n // Promise cancellation is not a generic SandboxSession contract. Keep the\n // exact failed activity's result path alive in-process, but permit only the\n // durable archive publication: never provider teardown or a cold commit from\n // an activity Temporal has already rejected. A replaced claim fences this\n // callback; Modal candidates then flow to artifact-ledger GC.\n const settled = () => {\n try {\n input.onLateSettled?.();\n } catch {\n // Settlement bookkeeping cannot surface as an unhandled rejection.\n }\n };\n void input.capture\n .then(input.publishLate, input.observeLateFailure)\n .catch(input.observeLatePublicationFailure)\n .then(settled, settled);\n throw error;\n } finally {\n if (timeout) clearTimeout(timeout);\n }\n}\n\nfunction startSandboxReaperHeartbeat(details: Record<string, string | number | null>): () => void {\n let context: Context;\n try {\n context = Context.current();\n } catch {\n // Direct unit/embedded harnesses intentionally run without Temporal.\n return () => undefined;\n }\n const heartbeat = (): void => {\n try {\n context.heartbeat({ ...details, heartbeatAt: new Date().toISOString() });\n } catch {\n // Heartbeat can synchronously surface Temporal cancellation. This timer\n // must never turn that into an uncaught process-level exception while an\n // already-issued provider capture/teardown is settling under its durable\n // DB claim. The activity result/cancellation boundary remains Temporal's;\n // claim recovery remains the database's.\n }\n };\n heartbeat();\n const timer = setInterval(heartbeat, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);\n if (\"unref\" in timer && typeof timer.unref === \"function\") timer.unref();\n return () => clearInterval(timer);\n}\n\nexport type SandboxLeaseSweepPlan = {\n drainable: ReapDrainable[];\n timeoutClass: SandboxDrainActivityInput[\"timeoutClass\"];\n snapshotTimeoutMs: number;\n captureTimeoutMs: number;\n /** Deadline rotations admitted immediately before inventory, so zero-holder\n * boxes enter the same sweep instead of paying another schedule period. */\n rotationsRequested: number;\n};\n\nexport type SandboxIdleCheckpointActivityResult = {\n status: \"checkpointed\" | \"skipped\";\n reason?: string;\n};\n\n/** Children per sweep; the next sweep continues with the least recently tried\n * boxes (each box is tried at most once per snapshot interval). Half the\n * lifecycle worker's activity slots, so a burst of captures never queues the\n * drains that stop idle boxes behind it. */\nconst SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(\n 1,\n Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2),\n);\n\nexport type SandboxLeaseSweepMaintenanceResult = {\n metered: number;\n forceDrained: number;\n rotationsRequested: number;\n modalOrphansTerminated: number;\n};\n\nexport type SandboxDrainActivityResult = { status: \"terminated\" | \"skipped\" };\n\ntype SandboxDrainCaptureAttempt = {\n operationId: string;\n captureId: string;\n attempt: number;\n /** Durable child input, frozen when the workflow was created. Never\n * re-derive this from a later deployment's config/formula. */\n captureTimeoutMs: number;\n};\n\nclass SandboxDrainRecoveryDeferredError extends Error {\n readonly name = \"SandboxDrainRecoveryDeferredError\";\n}\n\n/**\n * Drain captures still physically running in this worker, keyed by exact lease\n * epoch and provider instance. A capture that outlives its attempt's budget\n * keeps running and may publish late through its claim's provider-request\n * lineage. A parallel-read provider (Docker, for one) also lets the next\n * attempt replace that unpublished claim at once and start its own full read,\n * so a capture slower than the retry interval used to gain one more concurrent\n * read and host spool per retry. This keeps one physical capture per lease\n * epoch and instance in each worker; the durable claim still decides which\n * archive publishes, and a capture started by another worker (or one that died\n * with its process) is handled by the ordinary claim replacement. An entry\n * older than the longest durable capture window no longer defers, so a\n * provider call that never settles cannot pin the lease to this worker forever.\n */\nconst settlingDrainCaptures = new Map<string, { startedAtMs: number }>();\n\ntype DrainCaptureIdentity = {\n sandboxGroupId: string;\n leaseEpoch: number;\n instanceId: string | null;\n};\n\nfunction drainCaptureKey(lease: DrainCaptureIdentity & { instanceId: string }): string {\n return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);\n}\n\n/** True while this worker still runs (or publishes, or removes the spool of)\n * a drain capture of this exact lease epoch and instance. */\nfunction sandboxDrainCaptureSettlingInProcess(\n lease: DrainCaptureIdentity,\n nowMs = Date.now(),\n): boolean {\n if (!lease.instanceId) return false;\n const entry = settlingDrainCaptures.get(\n drainCaptureKey({ ...lease, instanceId: lease.instanceId }),\n );\n return entry !== undefined && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;\n}\n\nfunction trackSettlingDrainCapture(lease: DrainCaptureIdentity): () => void {\n if (!lease.instanceId) return () => undefined;\n const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });\n const entry = { startedAtMs: Date.now() };\n settlingDrainCaptures.set(key, entry);\n return () => {\n if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);\n };\n}\n\n/** Stable UUID per logical operation + Temporal attempt. Attempt one uses the\n * operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.\n * This makes the DB receipt idempotent even if an activity delivery is replayed. */\nexport function sandboxDrainCaptureId(operationId: string, attempt: number): string {\n if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {\n throw new Error(\"Sandbox drain operation id is invalid\");\n }\n if (!Number.isSafeInteger(attempt) || attempt <= 0) {\n throw new Error(\"Sandbox drain activity attempt is invalid\");\n }\n if (attempt === 1) return operationId.toLowerCase();\n const bytes = createHash(\"sha256\")\n .update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`)\n .digest()\n .subarray(0, 16);\n bytes[6] = (bytes[6]! & 0x0f) | 0x50;\n bytes[8] = (bytes[8]! & 0x3f) | 0x80;\n const hex = bytes.toString(\"hex\");\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;\n}\n\nfunction sandboxDrainActivityAttempt(): number {\n try {\n const attempt = Context.current().info.attempt;\n return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;\n } catch {\n return 1;\n }\n}\n\nexport function createSandboxLeaseActivities(\n services: () => Promise<ActivityServices>,\n options: SandboxLeaseActivityOptions = {},\n) {\n const terminateBox: TerminateBoxFn =\n (options.terminateBox\n ? async (...args: Parameters<TerminateBoxFn>) => {\n // The injected test seam represents the stop itself; the real\n // implementation invokes this only after capture has succeeded.\n await args[10]?.();\n return options.terminateBox!(...args);\n }\n : null) ??\n (async (\n settings,\n lease,\n observability,\n persistArchive,\n providerCaptureRequestId,\n captureDisposition,\n capturePolicy,\n diskBackedArchives,\n releaseFailedCapture,\n workspaceId,\n beforeProviderStop,\n beforeWorkspaceCapture,\n ) =>\n await terminateProviderBox(\n settings,\n lease,\n observability,\n persistArchive,\n options.createClientForBackend ?? createSandboxClientForBackend,\n terminateModalSandboxById,\n providerCaptureRequestId,\n captureDisposition,\n capturePolicy,\n diskBackedArchives,\n releaseFailedCapture,\n workspaceId,\n beforeProviderStop,\n beforeWorkspaceCapture,\n ));\n const sweepModalOrphans: SweepModalOrphansFn =\n options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;\n const inspectOpenSandboxInventory =\n options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;\n const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;\n const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;\n const resumeIdleCheckpointSession =\n options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;\n /** Embedded idle checkpoints still running, by exact lease epoch. */\n const embeddedIdleCheckpoints = new Set<string>();\n const endedEpochProbeBackoff: EndedEpochProbeBackoff = new Map();\n async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {\n const stopHeartbeat = startSandboxReaperHeartbeat({ phase: \"prepare\" });\n try {\n const { db, settings, observability, objectStorage } = await services();\n const timing = sandboxDrainTiming(settings);\n const commandContainment = {\n // Browsers and desktops follow the general idle grace: unused for it,\n // they stop holding the box (saved browsers resume on demand).\n // Never under a minute, so a visible live view's 30 s heartbeat\n // always lands inside the window.\n idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 60_000),\n idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),\n onIdleInteractionRelease: (\n outcome: Parameters<typeof recordSandboxIdleInteractionRelease>[1],\n ) => recordSandboxIdleInteractionRelease(observability, outcome),\n idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,\n deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),\n onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>\n recordSandboxCommandContainment(observability, outcome),\n onCommandContainmentError: (error: unknown) => {\n observability.warn(\"sandbox reaper: retained command containment inspection failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n },\n };\n if (!settings.sandboxOwnershipEnabled) {\n // Turns skip leases when the flag is off, but Computer/Browser attach\n // still acquire them. Leaving drainable rows untouched strands Desktop\n // behind rotation_in_progress forever. Inventory and drain those rows;\n // do not request NEW deadline rotations or meter warm time.\n const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {\n ...commandContainment,\n viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,\n turnHolderTtlMs: settings.sandboxLeaseTtlMs,\n interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);\n return {\n drainable,\n timeoutClass: timing.timeoutClass,\n snapshotTimeoutMs: timing.snapshotTimeoutMs,\n captureTimeoutMs: timing.captureTimeoutMs,\n rotationsRequested: 0,\n };\n }\n\n // Admit finite-lifetime rotations immediately before inventory. The DB\n // function is bounded and provider-I/O-free; a due zero-holder box becomes\n // draining there and is therefore dispatched in THIS sweep, eliminating\n // the old extra schedule-period availability gap. Failure is isolated: the\n // inventory must still dispatch every row that was already drainable.\n const rotationsRequested = await requestDueSandboxRotationsGlobal(\n db,\n settings.sandboxRotationLeadMs,\n settings.sandboxRotationBatchSize,\n ).catch((error) => {\n observability.warn(\"sandbox reaper: provider-deadline rotation request failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n });\n recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);\n\n // Keep the remaining dispatch-critical path to one bounded DB inventory.\n // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges\n // run only after every drainable box has its own durable child.\n const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {\n ...commandContainment,\n viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,\n // Dead-worker turn holders: a live holder is touched every 10s from the\n // moment it is registered (resumeBoxForTurn's holder-liveness loop covers\n // the whole warmup — waitForWarm/establish/display-stack — and the turn\n // heartbeat covers the run), so NO live path is ever silent for more than\n // one tick. The ordinary warm lease TTL is therefore already a generous\n // dead-worker horizon. Canonical holders whose turn is durably closed are\n // removed immediately by the DB function; no path-duration guess belongs\n // in this lifecycle contract.\n turnHolderTtlMs: settings.sandboxLeaseTtlMs,\n interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);\n if (drainableInventory.length > drainable.length) {\n observability.info(\"sandbox reaper: drain inventory continues next sweep\", {\n admitted: drainable.length,\n deferred: drainableInventory.length - drainable.length,\n });\n }\n return {\n drainable,\n timeoutClass: timing.timeoutClass,\n snapshotTimeoutMs: timing.snapshotTimeoutMs,\n captureTimeoutMs: timing.captureTimeoutMs,\n rotationsRequested,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Inventory for the idle checkpoint sweep. DB only: warm boxes no turn\n * holds, kept warm by viewers, interactions or running background commands,\n * with writes no capture may cover yet and neither a capture attempt nor an\n * idle checkpoint attempt within the snapshot interval.\n * Least recently tried first; the next sweep continues past the batch. */\n async function listIdleSandboxCheckpoints(): Promise<SandboxIdleCheckpointPlan> {\n const stopHeartbeat = startSandboxReaperHeartbeat({ phase: \"idle_checkpoint_inventory\" });\n try {\n const { db, settings } = await services();\n // The idle capture claims the warm checkpoint budget, not the drain's;\n // freeze the matching activity class with each child.\n const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));\n if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {\n return { targets: [], timeoutClass };\n }\n const rows = await listIdleCheckpointCandidates(db, {\n limit: SANDBOX_IDLE_CHECKPOINT_BATCH,\n intervalMs: settings.sandboxSnapshotIntervalMs,\n });\n return {\n targets: rows.map(({ accountId: _accountId, ...target }) => target),\n timeoutClass,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Checkpoint one warm box between turns. Reuses the turn's warm checkpoint\n * path with an idle owner: the exact capture claim requires that only\n * viewers, interactions and active background commands hold the\n * box and that no other request is open, then the native Modal snapshot runs\n * around them and is published one generation behind the workspace. */\n async function checkpointIdleSandboxLease(\n input: SandboxIdleCheckpointActivityInput,\n ): Promise<SandboxIdleCheckpointActivityResult> {\n const target = input.target;\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"idle_checkpoint\",\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n leaseEpoch: target.leaseEpoch,\n instanceId: target.instanceId,\n });\n try {\n const { db, settings, observability, objectStorage } = await services();\n const fields = {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),\n leaseEpoch: target.leaseEpoch,\n };\n const skipped = (reason: string): SandboxIdleCheckpointActivityResult => {\n observability.info(\"sandbox reaper: idle checkpoint skipped\", { ...fields, reason });\n return { status: \"skipped\", reason };\n };\n if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {\n return skipped(\"disabled\");\n }\n const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);\n if (\n !lease ||\n lease.liveness !== \"warm\" ||\n lease.leaseEpoch !== target.leaseEpoch ||\n lease.instanceId !== target.instanceId\n ) {\n return skipped(\"lease_fenced\");\n }\n // A turn's own checkpoints cover a box it holds; a box nothing holds\n // belongs to the idle drain.\n if (lease.turnHolders > 0) return skipped(\"turn_held\");\n if (lease.refcount === 0) return skipped(\"drain_owned\");\n const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);\n // Counted as tried whatever happens next, so a refused or failing box\n // waits a full interval instead of taking a batch slot every sweep.\n const stamped = await recordIdleCheckpointAttempt(db, {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n expectedEpoch: target.leaseEpoch,\n expectedInstanceId: target.instanceId,\n });\n if (!stamped) return skipped(\"lease_fenced\");\n if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {\n return skipped(\"capture_policy\");\n }\n if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped(\"clean\");\n if (lease.archiveCapture) {\n const tookOver = await releaseWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n captureId: lease.archiveCapture.id,\n expectedEpoch: target.leaseEpoch,\n expectedInstanceId: target.instanceId,\n onlyExpiredWarmTakeover: true,\n });\n if (!tookOver) return skipped(\"capture_in_progress\");\n observability.warn(\"sandbox reaper: released an expired warm capture claim\", {\n ...fields,\n captureId: lease.archiveCapture.id,\n });\n }\n let session: unknown;\n try {\n session = await resumeIdleCheckpointSession(settings, lease);\n } catch (error) {\n observability.warn(\"sandbox reaper: idle checkpoint resume failed\", {\n ...fields,\n error: error instanceof Error ? error.message : String(error),\n });\n return skipped(\"provider_error\");\n }\n if (session === null) return skipped(\"provider_missing\");\n const capture = maybePersistWarmWorkspaceSnapshot(\n { db, settings, objectStorage, observability },\n {\n accountId,\n workspaceId: target.workspaceId,\n sandboxGroupId: target.sandboxGroupId,\n idleCheckpoint: true,\n },\n session,\n target.leaseEpoch,\n );\n const persisted = await capture;\n // Return only after the provider capture settled and its exact claim\n // was released, so the activity never outlives its own gate unobserved.\n await capture.settled;\n if (!persisted) return skipped(\"not_captured\");\n observability.info(\"sandbox reaper: idle checkpoint published\", fields);\n return { status: \"checkpointed\" };\n } finally {\n stopHeartbeat();\n }\n }\n\n async function drainSandboxLease(\n input: SandboxDrainActivityInput,\n ): Promise<SandboxDrainActivityResult> {\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"drain\",\n workspaceId: input.target.workspaceId,\n sandboxGroupId: input.target.sandboxGroupId,\n leaseEpoch: input.target.leaseEpoch,\n instanceId: input.target.instanceId,\n });\n try {\n const { db, settings, observability, objectStorage, bus } = await services();\n assertSandboxDrainInputTiming(input);\n const drainSettings =\n settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs\n ? settings\n : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };\n const row = input.target;\n const activityAttempt = sandboxDrainActivityAttempt();\n const captureAttempt: SandboxDrainCaptureAttempt = {\n operationId: input.operationId,\n captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),\n attempt: activityAttempt,\n captureTimeoutMs: input.captureTimeoutMs,\n };\n try {\n const drainedCold = await terminateDrainableBox(\n db,\n drainSettings,\n row,\n observability,\n terminateBox,\n probeDrainableProvider,\n captureAttempt,\n objectStorage,\n bus,\n );\n return { status: drainedCold ? \"terminated\" : \"skipped\" };\n } catch (error) {\n const lease = await readLease(db, row.workspaceId, row.sandboxGroupId).catch(() => null);\n const details = {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n leaseEpoch: row.leaseEpoch,\n backend: lease?.backend ?? \"unknown\",\n instanceId: lease?.instanceId ?? row.instanceId,\n captureId: lease?.archiveCapture?.id ?? null,\n captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,\n captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,\n timeoutClass: input.timeoutClass,\n error: error instanceof Error ? error.message : String(error),\n };\n if (error instanceof SandboxDrainRecoveryDeferredError) {\n observability.info(\"sandbox reaper: per-sandbox recovery deferred safely\", details);\n } else {\n observability.warn(\"sandbox reaper: per-sandbox drain failed\", details);\n }\n throw error;\n }\n } finally {\n stopHeartbeat();\n }\n }\n\n async function maintainSandboxLeaseSweep(\n input: SandboxLeaseSweepMaintenanceInput,\n ): Promise<SandboxLeaseSweepMaintenanceResult> {\n const stopHeartbeat = startSandboxReaperHeartbeat({\n phase: \"maintain\",\n examined: input.examined,\n started: input.started,\n });\n try {\n const service = await services();\n const { db, settings, observability } = service;\n const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(\n (error) => {\n observability.warn(\"system-update outbox reconciliation failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { claimed: 0, delivered: 0, failed: 1 };\n },\n );\n const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(\n (error) => {\n observability.warn(\"session-list snapshot reconciliation failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n },\n );\n if (expiredSessionListSnapshots > 0) {\n observability.info(\"expired session-list snapshots reaped\", {\n deleted: expiredSessionListSnapshots,\n });\n }\n if (parentUpdates.claimed > 0) {\n observability.info(\"system-update outbox reconciled\", parentUpdates);\n }\n\n let metered = 0;\n let forceDrained = 0;\n const rotationsRequested = input.rotationsRequested;\n let modalOrphansTerminated = 0;\n\n // Connected Machine commands are not managed-sandbox lease state. Their\n // immutable launch locators must reconcile even when managed ownership is\n // disabled, otherwise an exact runner exit/loss proof can remain stranded.\n await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);\n await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);\n\n // Disabling new lease ownership cannot strand a previously dispatched\n // operation. This only attributes a positive receipt; normal draining\n // already runs in both ownership modes.\n try {\n await reconcileModalProviderCreates(\n db,\n settings,\n observability,\n options.findModalProviderCreateReceipt,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal create recovery inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n\n if (!settings.sandboxOwnershipEnabled) {\n // Inventory remains useful while provider ownership is intentionally\n // disabled, and this scheduled activity is its single projection owner.\n await refreshQueueLeaseAndCreditGauges(\n db,\n settings,\n observability,\n inspectOpenSandboxInventory,\n );\n return {\n metered,\n forceDrained,\n rotationsRequested,\n modalOrphansTerminated,\n };\n }\n\n // Warm metering and cost-stop run after current drain children launch.\n // Freshly force-drained billing rows are admission-fenced immediately and\n // receive their own provider child on the next bounded schedule tick.\n const meterResult = await accrueWarmTick(db, settings, observability);\n metered = meterResult.accrued;\n const forceDrainWorkspaceIds = new Set<string>();\n if (\n settings.sandboxWarmBillingMode === \"credits\" ||\n settings.sandboxMaxWarmSecondsPerWorkspace > 0\n ) {\n for (const workspaceId of meterResult.workspaceIds) {\n forceDrainWorkspaceIds.add(workspaceId);\n }\n }\n try {\n for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {\n forceDrainWorkspaceIds.add(workspaceId);\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: viewer force-drain workspace read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n forceDrained = await forceDrainOverLimitWorkspaces(\n db,\n settings,\n forceDrainWorkspaceIds,\n observability,\n );\n\n // Terminal-owner process reconciliation is ancillary to lease dispatch.\n // Exact provider proof is durably checkpointed before settlement; every\n // ambiguous outcome remains claimed/deferred for another sweep.\n await reconcileTerminalRetainedProcesses(\n db,\n settings,\n observability,\n probeRetainedProcess,\n options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,\n service.bus,\n );\n\n // A request or PTY left on a Modal box whose lease epoch has ended can\n // never progress once that box is gone; settle it on exact proof.\n await settleEndedEpochBlockers(\n db,\n settings,\n observability,\n options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,\n endedEpochProbeBackoff,\n );\n\n try {\n modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);\n recordSandboxOrphansTerminated(observability, modalOrphansTerminated);\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal orphan sweep failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n\n const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(\n (error) => {\n observability.warn(\"sandbox reaper: checkpoint artifact GC failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { claimed: 0, deleted: 0, failed: 0 };\n },\n );\n\n await refreshQueueLeaseAndCreditGauges(\n db,\n settings,\n observability,\n inspectOpenSandboxInventory,\n );\n\n if (\n input.examined > 0 ||\n metered > 0 ||\n forceDrained > 0 ||\n modalOrphansTerminated > 0 ||\n rotationsRequested > 0 ||\n checkpointGc.claimed > 0\n ) {\n observability.info(\"sandbox reaper dispatched\", {\n drainable: input.examined,\n childrenStarted: input.started,\n childrenAlreadyRunning: input.alreadyRunning,\n childrenStartFailed: input.startFailed,\n metered,\n forceDrained,\n modalOrphansTerminated,\n rotationsRequested,\n checkpointArtifactsClaimed: checkpointGc.claimed,\n checkpointArtifactsDeleted: checkpointGc.deleted,\n checkpointArtifactsFailed: checkpointGc.failed,\n });\n }\n\n return {\n metered,\n forceDrained,\n rotationsRequested,\n modalOrphansTerminated,\n };\n } finally {\n stopHeartbeat();\n }\n }\n\n /** Direct/embedded compatibility harness. Production uses the split Temporal\n * workflow above. The production activity is also the rolling-deploy bridge:\n * its workflow command stays byte-for-byte compatible while this activity\n * starts V2 through the injected client edge. Old workers may still execute\n * the bounded composite during rollout; exact DB claims keep both paths safe. */\n async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {\n const service = await services();\n if (service.startSandboxReaperWorkflow) {\n const dispatch = await service.startSandboxReaperWorkflow();\n service.observability.info(\"sandbox reaper routed to versioned workflow\", {\n dispatch,\n });\n return {\n examined: 0,\n terminated: 0,\n skipped: 0,\n metered: 0,\n forceDrained: 0,\n modalOrphansTerminated: 0,\n };\n }\n\n const plan = await prepareSandboxLeaseSweep();\n const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);\n const outcomes = await Promise.allSettled(\n admitted.map((target) =>\n drainSandboxLease({\n target,\n timeoutClass: plan.timeoutClass,\n snapshotTimeoutMs: plan.snapshotTimeoutMs,\n captureTimeoutMs: plan.captureTimeoutMs,\n operationId: randomUUID(),\n }),\n ),\n );\n // The embedded path has no idle checkpoint sweep workflow; run the same\n // inventory and per-box activity. Captures run detached so a provider\n // snapshot never holds up this sweep's drains; a box whose capture is\n // still running is not started twice. Failure defers to the next sweep.\n try {\n const { targets, timeoutClass } = await listIdleSandboxCheckpoints();\n for (const target of targets) {\n const key = sandboxIdleCheckpointWorkflowId(target);\n if (embeddedIdleCheckpoints.has(key)) continue;\n embeddedIdleCheckpoints.add(key);\n void checkpointIdleSandboxLease({ target, timeoutClass })\n .catch((error: unknown) => {\n service.observability.warn(\"sandbox reaper: idle checkpoint failed\", {\n sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),\n error: error instanceof Error ? error.message : String(error),\n });\n })\n .finally(() => embeddedIdleCheckpoints.delete(key));\n }\n } catch (error) {\n service.observability.warn(\"sandbox reaper: idle checkpoint inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n const terminated = outcomes.filter(\n (outcome) => outcome.status === \"fulfilled\" && outcome.value.status === \"terminated\",\n ).length;\n const skipped = outcomes.length - terminated;\n const maintenance = await maintainSandboxLeaseSweep({\n examined: admitted.length,\n started: admitted.length,\n alreadyRunning: 0,\n startFailed: 0,\n rotationsRequested: plan.rotationsRequested,\n });\n return {\n examined: admitted.length,\n terminated,\n skipped,\n metered: maintenance.metered,\n forceDrained: maintenance.forceDrained,\n modalOrphansTerminated: maintenance.modalOrphansTerminated,\n };\n }\n\n return {\n prepareSandboxLeaseSweep,\n drainSandboxLease,\n listIdleSandboxCheckpoints,\n checkpointIdleSandboxLease,\n maintainSandboxLeaseSweep,\n reapSandboxLeases,\n };\n}\n\n/** Resume the exact warm box by id for an idle checkpoint, with no cold\n * restore and no replacement. Returns null when the provider no longer has\n * it; the ordinary loss paths own that box. The session is not closed: the\n * box keeps running its commands. */\nexport async function resumeIdleCheckpointSessionAtProvider(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n): Promise<unknown | null> {\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n if (backend !== \"modal\" || !lease.instanceId || !lease.resumeState) {\n throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);\n }\n if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {\n throw new Error(\"idle checkpoint lease envelope does not name its provider instance\");\n }\n const client = createSandboxClientForBackend(backend as never, settings) as\n | RetainedProcessProbeClient\n | undefined;\n if (!client?.resume) {\n throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);\n }\n try {\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n const resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n lease.instanceId,\n );\n if (resumedState === undefined) {\n throw new Error(`sandbox backend ${backend} returned no resumable provider state`);\n }\n const resumed = await withRetainedProcessProbeTimeout(\n resumeExactSandboxSession(client, backend, resumedState, lease.instanceId),\n );\n return resumed.session;\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n throw new Error(\"idle checkpoint provider resume timed out\", { cause: error });\n }\n if (isProviderSandboxNotFoundError(client.backendId, error)) return null;\n throw error;\n }\n}\n\nconst CHECKPOINT_GC_LIMIT = 50;\nconst CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;\n/** The checkpoint artifact GC claim TTL, which the reaper's work check previews. */\nexport const SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;\nconst CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;\nconst CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;\nconst SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;\n\ntype ConnectedCommandReconciliationOutcome =\n | \"claim_failed\"\n | \"provider_running\"\n | \"provider_offline\"\n | \"provider_error\"\n | \"proof_exited\"\n | \"proof_lost\"\n | \"proof_checkpoint_failed\"\n | \"settled_exited\"\n | \"settled_lost\"\n | \"output_unavailable\"\n | \"output_ack_published\"\n | \"output_released\"\n | \"settlement_failed\"\n | \"defer_failed\";\n\nfunction recordConnectedCommandReconciliation(\n observability: ActivityServices[\"observability\"],\n outcome: ConnectedCommandReconciliationOutcome,\n): void {\n observability.incrementCounter({\n name: \"opengeni_connected_background_command_reconciliation_total\",\n help: \"Exact-locator Connected Machine background-command reconciliation outcomes.\",\n labels: { outcome },\n });\n}\n\n/** Why a connected command's runner can never answer for it again. */\ntype ConnectedRunnerRetirement =\n | \"enrollment_revoked\"\n | \"connection_replaced\"\n | \"connection_stopped\";\ntype ConnectedRunnerLookups = Map<string, Promise<ConnectedRunnerRetirement | null>>;\n\n/** Offline reasons after which the runner process has exited for good. */\nconst CONNECTED_RUNNER_STOPPED_REASONS = new Set([\n \"GOING_OFFLINE_REASON_USER_STOP\",\n \"GOING_OFFLINE_REASON_UPDATE\",\n \"GOING_OFFLINE_REASON_HOST_SHUTDOWN\",\n]);\n\n/**\n * A connected command's operation lives in the memory of the runner process\n * that launched it and is addressed through that process's connection\n * instance. No responder on that exact address is not proof by itself: the\n * machine may be offline or reconnecting. It is proof once the enrollment\n * also retires that process, by the same rules the running-command claim\n * applies: the enrollment was revoked, a different runner process now holds\n * the connection, or the runner went offline by stopping, updating or shutting\n * down. Anything else, including a failed read, keeps the existing retry. One\n * read per runner address per sweep.\n */\nasync function connectedCommandRunnerRetirement(\n db: ActivityServices[\"db\"],\n claim: { controlWorkspaceId: string; enrollmentId: string; connectionInstanceId: string },\n error: unknown,\n lookups: ConnectedRunnerLookups,\n): Promise<ConnectedRunnerRetirement | null> {\n if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {\n return null;\n }\n const key = JSON.stringify([\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ]);\n let lookup = lookups.get(key);\n if (!lookup) {\n lookup = (async (): Promise<ConnectedRunnerRetirement | null> => {\n const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);\n if (!enrollment) return null;\n if (enrollment.status === \"revoked\") return \"enrollment_revoked\";\n if (enrollment.connectionInstanceId) {\n return enrollment.connectionInstanceId === claim.connectionInstanceId\n ? null\n : \"connection_replaced\";\n }\n return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? \"\")\n ? \"connection_stopped\"\n : null;\n })().catch(() => null);\n lookups.set(key, lookup);\n }\n return await lookup;\n}\n\nasync function forEachWithConcurrency<T>(\n values: readonly T[],\n concurrency: number,\n visit: (value: T, index: number) => Promise<void>,\n): Promise<void> {\n if (values.length === 0) return;\n const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));\n let nextIndex = 0;\n await Promise.all(\n Array.from({ length: workerCount }, async () => {\n while (nextIndex < values.length) {\n const index = nextIndex;\n nextIndex += 1;\n await visit(values[index]!, index);\n }\n }),\n );\n}\n\nasync function gcSandboxCheckpointArtifacts(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<{ claimed: number; deleted: number; failed: number }> {\n // Explicit OPENGENI_MODAL_TOKEN_* settings are optional: the SDK also supports\n // an operator-provisioned Modal profile. Durable artifacts must not become\n // immortal merely because credentials arrive through that supported path.\n const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(\n (error) => {\n observability.warn(\"sandbox reaper: legacy Modal checkpoint discovery failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n },\n );\n if (adopted > 0) {\n recordSandboxCheckpointArtifactOutcome(observability, \"legacy_adopted\", adopted);\n observability.info(\"sandbox reaper: adopted legacy Modal checkpoint receipts\", { adopted });\n }\n const claimId = crypto.randomUUID();\n const claims = await claimSandboxCheckpointArtifactsForGc(db, {\n claimId,\n limit: CHECKPOINT_GC_LIMIT,\n claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS,\n });\n recordSandboxCheckpointArtifactOutcome(observability, \"claimed\", claims.length);\n let deleted = 0;\n let failed = 0;\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n try {\n if (claim.providerBackend !== \"modal\") {\n throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);\n }\n await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);\n const settled = await settleSandboxCheckpointArtifactGc(db, {\n artifactId: claim.id,\n claimId,\n deleted: true,\n retryAfterMs: 1_000,\n });\n if (settled) {\n deleted += 1;\n recordSandboxCheckpointArtifactOutcome(observability, \"deleted\");\n }\n } catch (error) {\n failed += 1;\n recordSandboxCheckpointArtifactOutcome(observability, \"delete_failed\");\n const retryAfterMs = Math.min(\n 6 * 60 * 60_000,\n 5_000 * 2 ** Math.min(claim.deleteAttempts, 17),\n );\n await settleSandboxCheckpointArtifactGc(db, {\n artifactId: claim.id,\n claimId,\n deleted: false,\n error: error instanceof Error ? error.message : String(error),\n retryAfterMs,\n }).catch(() => false);\n observability.warn(\"sandbox reaper: checkpoint artifact delete failed\", {\n artifactId: claim.id,\n providerBackend: claim.providerBackend,\n objectKind: claim.objectKind,\n attempt: claim.deleteAttempts,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n const pruned = await pruneDeletedSandboxCheckpointArtifacts(\n db,\n CHECKPOINT_TOMBSTONE_RETENTION_MS,\n CHECKPOINT_TOMBSTONE_PRUNE_LIMIT,\n );\n recordSandboxCheckpointArtifactOutcome(observability, \"tombstone_pruned\", pruned);\n return { claimed: claims.length, deleted, failed };\n}\n\nasync function adoptLegacyModalCheckpointReceipts(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n const slots = await listLegacyModalCheckpointSlots(db, 25);\n if (slots.length === 0) return 0;\n const bindings = new Map<\n string,\n Promise<Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForLiveSandbox>>>\n >();\n let adopted = 0;\n await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {\n try {\n let binding = bindings.get(slot.instanceId);\n if (!binding) {\n binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);\n bindings.set(slot.instanceId, binding);\n }\n const resolvedBinding = await binding;\n if (\n await adoptLegacyModalCheckpointArtifact(db, {\n accountId: slot.accountId,\n workspaceId: slot.workspaceId,\n sandboxGroupId: slot.sandboxGroupId,\n leaseId: slot.leaseId,\n leaseEpoch: slot.leaseEpoch,\n workspaceGeneration: slot.workspaceGeneration,\n slot: slot.slot,\n archiveBase64: slot.archiveBase64,\n descriptor: slot.descriptor,\n providerBindingKey: resolvedBinding.key,\n providerBinding: resolvedBinding.binding,\n })\n ) {\n adopted += 1;\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: legacy Modal checkpoint adoption failed\", {\n workspaceId: slot.workspaceId,\n sandboxGroupId: slot.sandboxGroupId,\n slot: slot.slot,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n return adopted;\n}\n\n/** Reconcile commands only against the immutable physical locator copied at\n * adoption. A successor enrollment connection is intentionally irrelevant: no\n * query or cancellation is ever rebound to it. */\nexport async function reconcileConnectedMachineBackgroundCommands(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n bus: ActivityServices[\"bus\"],\n controlRpcOverride?: ControlRpc,\n): Promise<void> {\n const dueBefore = new Date();\n const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;\n const offlineConnections = new Set<string>();\n const firstConnectionProbes = new Map<string, Promise<void>>();\n // Drain the fixed due frontier with the existing concurrency. Yield to the\n // remaining maintenance work each period; retries belong to a later sweep.\n do {\n const claimId = crypto.randomUUID();\n let claims: ConnectedMachineBackgroundCommandClaim[];\n try {\n claims = await claimConnectedMachineSessionBackgroundCommands(db, {\n claimId,\n dueBefore,\n limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,\n claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: connected-command claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n\n const controlRpc =\n controlRpcOverride ??\n new NatsControlRpc(\n async (): Promise<NatsRequestConnection | null> => bus.getRequestConnection(),\n );\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n let proof = claim.proof;\n let outputReplay: Awaited<ReturnType<OpStreamExecClient[\"readExisting\"]>> | null = null;\n const connectionKey = JSON.stringify([\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ]);\n if (!proof) {\n const firstProbe = firstConnectionProbes.get(connectionKey);\n if (firstProbe) await firstProbe;\n if (offlineConnections.has(connectionKey)) {\n await deferConnectedCommandClaim(db, settings, observability, claim, \"provider_offline\");\n return;\n }\n // Share only the initial connection observation, never another command's result.\n let finishFirstProbe: (() => void) | undefined;\n if (!firstProbe) {\n firstConnectionProbes.set(\n connectionKey,\n new Promise<void>((resolve) => {\n finishFirstProbe = resolve;\n }),\n );\n }\n try {\n proof = await probeConnectedMachineBackgroundCommand(\n claim,\n controlRpc,\n Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n ),\n );\n // Replay currently retained frames for running as well as completed\n // commands. This is attach-only and leaves final ACK/observation alone.\n if (proof?.outcome !== \"lost\") {\n if (!bus.getOpStreamConnection)\n throw new Error(\"Connected command output transport unavailable\");\n const client = new OpStreamExecClient({\n workspaceId: claim.controlWorkspaceId,\n agentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n epoch: 0,\n controlRpc,\n rpcSubject: subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ),\n transport: new NatsOpStreamTransport(\n async () => bus.getOpStreamConnection?.() ?? null,\n ),\n controlTimeoutMs: Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n ),\n retryClock: defaultSelfhostedRetryClock,\n journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },\n });\n outputReplay = await replayConnectedCommandOutput(\n client,\n claim.opId,\n Boolean(proof),\n async (frames) => {\n await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);\n },\n );\n }\n if (!proof) {\n await deferConnectedCommandClaim(\n db,\n settings,\n observability,\n claim,\n \"provider_running\",\n );\n return;\n }\n } catch (error) {\n const offline =\n error instanceof SelfhostedControlError &&\n (error.agentOffline || error.reason === \"agent_reconnecting\" || error.draining);\n if (offline) offlineConnections.add(connectionKey);\n if (!offline) {\n observability.warn(\"sandbox reaper: connected-command provider probe failed\", {\n commandId: claim.commandId,\n enrollmentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n await deferConnectedCommandClaim(\n db,\n settings,\n observability,\n claim,\n offline ? \"provider_offline\" : \"provider_error\",\n );\n return;\n } finally {\n finishFirstProbe?.();\n }\n\n try {\n await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });\n recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"proof_checkpoint_failed\");\n observability.warn(\"sandbox reaper: connected-command proof checkpoint failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n }\n\n try {\n const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });\n if (!settlement.settled) {\n throw new Error(\"Connected command settlement lost its exact claim or proof\");\n }\n const receipt =\n outputReplay?.outputReceipt ??\n (outputReplay?.status === \"running\" ? outputReplay.terminal?.outputReceipt : undefined);\n const terminal =\n outputReplay?.status === \"completed\" ? outputReplay : outputReplay?.terminal;\n if (proof.outcome === \"exited\" && receipt && terminal) {\n await recordConnectedCommandOutputConsumption(db, {\n ...claim,\n receipt,\n exitCode: terminal.outcome.response.exitCode,\n });\n }\n if (settlement.events.length > 0) {\n try {\n await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);\n } catch (error) {\n observability.warn(\"sandbox reaper: connected-command event fanout failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"settlement_failed\");\n observability.warn(\"sandbox reaper: connected-command settlement failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n await deferConnectedCommandClaim(db, settings, observability, claim, \"settlement_failed\");\n }\n });\n if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;\n } while (Date.now() < deadline);\n}\n\nexport async function replayConnectedCommandOutput(\n client: Pick<OpStreamExecClient, \"readExisting\">,\n opId: string,\n terminalKnown: boolean,\n capture: Parameters<OpStreamExecClient[\"readExisting\"]>[2],\n): Promise<Awaited<ReturnType<OpStreamExecClient[\"readExisting\"]>>> {\n // A completed operation can need several bounded reads to replay its retained\n // frames. Keep the reader's integrity checkpoint until the exit is verified;\n // replacing it after each partial read would restart forever at frame one.\n let capturedThrough = 0n;\n while (true) {\n const before = capturedThrough;\n const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, capture);\n if (!terminalKnown || replay.status === \"completed\" || replay.terminal) return replay;\n // Quiet jobs can retain many heartbeat frames (or an incomplete UTF-8\n // chunk). These are real replay progress even when capture receives no\n // stdout/stderr. Use the reader's verified, contiguous protocol frontier.\n capturedThrough = BigInt(replay.replaySequence);\n // This is a finite terminal drain, not a poller for a running command.\n // Failed persistence, transport/integrity errors, and no-progress reads\n // return control to normal reconciliation without licensing settlement.\n if (capturedThrough === before) {\n throw new Error(\"Connected command terminal output replay has not reached its exit frontier\");\n }\n }\n}\n\n/** Drain the durable terminal-output obligation independently of the launching\n * attempt or model completion notice. Existing terminal rows receive no inferred\n * receipt: they must replay and capture all retained bytes before any final ACK. */\nexport async function reconcileConnectedCommandOutputReleases(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n bus: ActivityServices[\"bus\"],\n controlRpcOverride?: ControlRpc,\n): Promise<void> {\n const dueBefore = new Date();\n const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;\n const controlRpc =\n controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());\n const runnerLookups: ConnectedRunnerLookups = new Map();\n do {\n let claims: Awaited<ReturnType<typeof claimConnectedCommandOutputReleases>>;\n try {\n claims = await claimConnectedCommandOutputReleases(db, {\n claimId: crypto.randomUUID(),\n dueBefore,\n limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,\n claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: connected-command output claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {\n let receipt = claim.receipt;\n try {\n if (!bus.getOpStreamConnection)\n throw new Error(\"Connected command output transport unavailable\");\n const rpcSubject = subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n );\n const controlTimeoutMs = Math.min(\n settings.sandboxSelfhostedControlTimeoutMs,\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n const client = new OpStreamExecClient({\n workspaceId: claim.controlWorkspaceId,\n agentId: claim.enrollmentId,\n connectionInstanceId: claim.connectionInstanceId,\n epoch: 0,\n controlRpc,\n rpcSubject,\n transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),\n controlTimeoutMs,\n retryClock: defaultSelfhostedRetryClock,\n journal: {\n attachGeneration: () => String(Date.now()),\n persistSettled: () => {\n throw new Error(\"Background output release requires its database receipt\");\n },\n },\n });\n if (!receipt) {\n const status = await querySelfhostedOp({\n controlRpc,\n rpcSubject,\n opId: claim.opId,\n controlTimeoutMs,\n });\n if (status.state === OpState.OP_STATE_LOST) {\n if (\n !(await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: \"unavailable\",\n retryAfterMs: 0,\n }))\n )\n throw new Error(\"Connected command output loss no longer owns its exact claim\");\n recordConnectedCommandReconciliation(observability, \"output_unavailable\");\n return;\n }\n if (status.state !== OpState.OP_STATE_COMPLETE)\n throw new Error(\"Terminal command output returned a live operation\");\n const replay = await replayConnectedCommandOutput(\n client,\n claim.opId,\n true,\n async (frames) =>\n captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames),\n );\n const terminal = replay.status === \"completed\" ? replay : replay.terminal;\n receipt =\n replay.outputReceipt ??\n (replay.status === \"running\" ? replay.terminal?.outputReceipt : undefined) ??\n null;\n if (!terminal || !receipt)\n throw new Error(\"Connected command output replay has no complete custody receipt\");\n await recordConnectedCommandOutputConsumption(db, {\n ...claim,\n receipt,\n exitCode: terminal.outcome.response.exitCode,\n });\n }\n const outcome = await client.releaseCapturedOutput(claim.opId, receipt);\n if (\n !(await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome,\n retryAfterMs: outcome === \"published\" ? settings.sandboxLeaseReaperPeriodMs : 0,\n }))\n )\n throw new Error(\"Connected command output release no longer owns its exact claim\");\n recordConnectedCommandReconciliation(\n observability,\n outcome === \"published\" ? \"output_ack_published\" : \"output_released\",\n );\n } catch (error) {\n const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);\n // A retired runner retains nothing: captured output is released, and\n // output it never handed over is recorded as missing.\n if (\n retired &&\n (await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: receipt ? \"not_retained\" : \"unavailable\",\n retryAfterMs: 0,\n }).catch(() => false))\n ) {\n recordConnectedCommandReconciliation(\n observability,\n receipt ? \"output_released\" : \"output_unavailable\",\n );\n return;\n }\n await settleConnectedCommandOutputReleaseClaim(db, {\n claim,\n outcome: \"retry\",\n retryAfterMs: Math.min(\n 5 * 60_000,\n Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) *\n 2 ** Math.min(4, claim.reconcileAttempts - 1),\n ),\n }).catch(() => false);\n observability.warn(\"sandbox reaper: connected-command output release deferred\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;\n } while (Date.now() < deadline);\n}\n\nexport async function probeConnectedMachineBackgroundCommand(\n claim: ConnectedMachineBackgroundCommandClaim,\n controlRpc: ControlRpc,\n controlTimeoutMs: number,\n observedAt = new Date(),\n): Promise<ConnectedMachineBackgroundCommandProof | null> {\n const input = {\n controlRpc,\n rpcSubject: subjectFor(\n claim.controlWorkspaceId,\n claim.enrollmentId,\n claim.connectionInstanceId,\n ),\n opId: claim.opId,\n controlTimeoutMs,\n };\n const status =\n claim.state === \"stopping\" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);\n return connectedCommandProofFromStatus(status, observedAt);\n}\n\nexport function connectedCommandProofFromStatus(\n status: OpStatus,\n observedAt: Date,\n): ConnectedMachineBackgroundCommandProof | null {\n switch (status.state) {\n case OpState.OP_STATE_ACCEPTED:\n case OpState.OP_STATE_RUNNING:\n return null;\n case OpState.OP_STATE_COMPLETE: {\n const exit = status.exit;\n if (!exit) {\n throw new Error(\"Connected command completed without an exit record\");\n }\n const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, \"_\").slice(0, 128);\n const reason = failureCode\n ? `op_failure_${failureCode}`\n : exit.cancelled\n ? \"op_cancelled\"\n : exit.timedOut\n ? \"op_timed_out\"\n : \"op_exit\";\n return {\n outcome: \"exited\",\n exitCode: exit.exitCode,\n reason,\n ...(failureCode\n ? {\n failure: { code: failureCode, detail: exit.failureDetail, retryable: false as const },\n }\n : {}),\n observedAt,\n };\n }\n case OpState.OP_STATE_LOST:\n return {\n outcome: \"lost\",\n exitCode: null,\n reason:\n status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED\n ? \"op_lost_evicted\"\n : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED\n ? \"op_lost_agent_restarted\"\n : \"op_lost_unspecified\",\n observedAt,\n };\n default:\n throw new Error(`Connected command returned invalid op state ${status.state}`);\n }\n}\n\nasync function deferConnectedCommandClaim(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n claim: ConnectedMachineBackgroundCommandClaim,\n outcome: \"provider_running\" | \"provider_offline\" | \"provider_error\" | \"settlement_failed\",\n): Promise<void> {\n const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));\n const retryAfterMs = Math.min(\n 5 * 60_000,\n Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,\n );\n try {\n if (\n !(await deferConnectedMachineBackgroundCommandReconciliation(db, {\n claim,\n outcome,\n retryAfterMs,\n }))\n ) {\n throw new Error(\"Connected command defer lost its exact claim\");\n }\n recordConnectedCommandReconciliation(observability, outcome);\n } catch (error) {\n recordConnectedCommandReconciliation(observability, \"defer_failed\");\n observability.warn(\"sandbox reaper: connected-command defer failed\", {\n commandId: claim.commandId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\n/** Tuples listed per sweep (random order, the next sweep lists again) and the\n * share of the reaper tick the probes may use, so the pass never crowds out\n * the orphan sweep and GC after it. A box the provider cannot yet prove gone is\n * re-probed on an exponential backoff instead of on every tick. */\nconst ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;\nconst ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 60_000;\nconst ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 60_000;\nconst ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 60_000;\n\n/** Per worker process: when each historical box may be probed again. */\nexport type EndedEpochProbeBackoff = Map<string, { failures: number; nextProbeAtMs: number }>;\n\n/**\n * Settle open requests still recorded on a Modal box whose lease\n * epoch has ended (a legacy loss, or a cold commit that left them). Lease\n * succession alone proves nothing about the old box, so each exact tuple is\n * settled only after its historical sandbox is observed terminated. NotFound\n * is ambiguous here (deletion or a rotated Modal credential workspace) and,\n * like a running or unreachable box, keeps the rows for a later probe (with\n * backoff) or an operator.\n *\n * The same observation also ends a turn's wait on sandbox setup whose start\n * outcome is unknown: once the exact box all its setup ran on is terminated,\n * nothing it may have started can still run, so the turn's setup reruns its\n * idempotent preparation (`reconcileCompletedSandboxSetup`). A start that\n * failed before any process was recorded otherwise had nothing to exit and\n * stayed recovering forever.\n */\nasync function settleEndedEpochBlockers(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,\n backoff: EndedEpochProbeBackoff,\n): Promise<void> {\n // Same scope as the Modal orphan sweep: no Modal API calls from workers of\n // a deployment that never configured Modal.\n if (settings.sandboxBackend !== \"modal\" && !settings.modalTokenId && !settings.modalTokenSecret) {\n return;\n }\n const startedAtMs = Date.now();\n // One observation per box: several tuples and turns can name the same box.\n const observed = new Map<string, string>();\n const kept: Record<string, number> = {};\n /** The box's status, or null when it is backing off or the budget is spent. */\n async function observe(\n instanceId: string,\n details: Record<string, unknown>,\n ): Promise<string | null> {\n const known = observed.get(instanceId);\n if (known !== undefined) return known;\n const nowMs = Date.now();\n if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;\n if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;\n let status: string;\n try {\n status = (\n await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(settings, instanceId, null),\n )\n ).status;\n } catch (error) {\n status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"error\";\n observability.warn(\"sandbox reaper: ended-epoch box probe failed\", {\n ...details,\n error:\n error === RETAINED_PROCESS_PROBE_TIMEOUT\n ? \"provider_timeout\"\n : error instanceof Error\n ? error.message\n : String(error),\n });\n }\n observed.set(instanceId, status);\n if (status === \"terminated\") {\n backoff.delete(instanceId);\n } else {\n deferEndedEpochProbe(backoff, instanceId);\n kept[status] = (kept[status] ?? 0) + 1;\n }\n return status;\n }\n\n let tuples: Awaited<ReturnType<typeof listEndedEpochWorkspaceBlockers>>;\n try {\n tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch blocker inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n tuples = [];\n }\n for (const tuple of tuples) {\n const instanceId = tuple.lostInstanceId;\n const details = {\n workspaceId: tuple.workspaceId,\n sandboxGroupId: tuple.sandboxGroupId,\n lostEpoch: tuple.lostEpoch,\n instanceId,\n };\n try {\n const status = await observe(instanceId, details);\n if (status !== \"terminated\") continue;\n const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);\n if (settled) {\n observability.info(\"sandbox reaper: settled blockers of an ended lease epoch\", {\n ...details,\n ...settled,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch blocker settlement failed\", {\n ...details,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n let setups: Awaited<ReturnType<typeof listEndedEpochSandboxSetupUncertainties>>;\n try {\n setups = await listEndedEpochSandboxSetupUncertainties(db, {\n limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT,\n });\n } catch (error) {\n observability.warn(\"sandbox reaper: ended-epoch setup inventory failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n setups = [];\n }\n for (const setup of setups) {\n const instanceId = setup.box.providerInstanceId;\n // A turn whose reconciliation is refused on a terminated box (for example\n // an ancestor is paused) backs off by turn, so it never re-probes per tick.\n const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;\n if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;\n const details = {\n workspaceId: setup.workspaceId,\n sessionId: setup.sessionId,\n turnId: setup.turnId,\n sandboxGroupId: setup.sandboxGroupId,\n lostEpoch: setup.box.leaseEpoch,\n instanceId,\n };\n try {\n const status = await observe(instanceId, details);\n if (status !== \"terminated\") continue;\n const { reconciled } = await reconcileCompletedSandboxSetup(db, {\n accountId: setup.accountId,\n workspaceId: setup.workspaceId,\n sessionId: setup.sessionId,\n turnId: setup.turnId,\n attemptId: setup.attemptId,\n terminatedBox: setup.box,\n });\n if (reconciled) {\n backoff.delete(turnKey);\n observability.info(\n \"sandbox reaper: resumed a turn whose unknown setup ran on a terminated box\",\n details,\n );\n } else {\n deferEndedEpochProbe(backoff, turnKey);\n }\n } catch (error) {\n deferEndedEpochProbe(backoff, turnKey);\n observability.warn(\"sandbox reaper: ended-epoch setup reconciliation failed\", {\n ...details,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n if (Object.keys(kept).length) {\n observability.info(\"sandbox reaper: ended-epoch blockers kept without terminal proof\", kept);\n }\n}\n\nfunction deferEndedEpochProbe(backoff: EndedEpochProbeBackoff, key: string): void {\n const failures = (backoff.get(key)?.failures ?? 0) + 1;\n backoff.set(key, {\n failures,\n nextProbeAtMs:\n Date.now() +\n Math.min(\n ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,\n ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16),\n ),\n });\n}\n\nasync function reconcileTerminalRetainedProcesses(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n probe: RetainedProcessProbeFn,\n inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,\n bus: ActivityServices[\"bus\"],\n): Promise<void> {\n const claimId = crypto.randomUUID();\n let claims: Awaited<ReturnType<typeof claimTerminalRetainedProcesses>>;\n try {\n claims = await claimTerminalRetainedProcesses(db, {\n claimId,\n limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,\n claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,\n });\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"claim_failed\");\n observability.warn(\"sandbox reaper: retained-process claim failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return;\n }\n\n const drainBudget: RetainedProcessDrainBudget = {\n until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,\n };\n // Boxes already retired as lost in this batch: their other commands were\n // settled by that same transaction and need no probe of their own.\n const lostBoxes = new Set<string>();\n for (const claim of claims) {\n let process = claim.process;\n if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))\n continue;\n const expected = retainedProcessSettlementIdentity(process);\n let proof = retainedProcessReconciliationProof(process);\n // The exact current box itself answered NotFound to this probe: that is\n // loss proof for the whole box, not just this command.\n let wholeBoxGone = false;\n const processScope = {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n };\n // Recover a yield whose observation failed before normal receipt adoption.\n // Do this before provider I/O: an unavailable observer must not fence every\n // subsequent turn. Physical retention and unknown outcome stay unchanged.\n if (\n process.ownerActorKind === \"turn\" &&\n process.providerBackend === \"modal\" &&\n !proof &&\n !claim.ownerState.startsWith(\"background_\")\n ) {\n try {\n const command = await recoverManagedSessionBackgroundCommand(db, {\n ...processScope,\n expected,\n reconciliationClaimId: claim.claimId,\n });\n if (command) {\n claim.ownerState = `background_${command.state}`;\n recordRetainedProcessReconciliation(observability, \"background_recovered\");\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command background recovery failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n const storedCommandPersistence = retainedProviderCommandPersistence(\n db,\n processScope,\n bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : undefined,\n );\n const commandPersistence = {\n ...storedCommandPersistence,\n requestCancellation: async (reason: \"provider_deadline\" | \"explicit_stop\") => {\n if (reason === \"provider_deadline\") {\n await requestRetainedProcessDeadlineCancellation(db, processScope);\n if (!(await storedCommandPersistence.cancellationRequested()))\n throw new Error(\n \"Supervised deadline cancellation no longer owns its original rotating lease\",\n );\n } else await storedCommandPersistence.requestCancellation(reason);\n },\n };\n if (proof && process.providerBackend === \"modal\") {\n try {\n const command = await commandPersistence.load();\n if (\n command?.kind === \"modal-router-v1\" &&\n command.supervision &&\n (!(await commandPersistence.loadSupervisionReceipt()) ||\n ![command.streams.stdout, command.streams.stderr].every(\n (stream) => stream.eof && stream.exitCode === 0,\n ))\n )\n proof = null;\n } catch {\n // An earlier terminal observation cannot make missing control/output\n // proof unreachable. Reprobe the original binding; DB remains the gate.\n proof = null;\n }\n }\n if (!proof) {\n let observation: RetainedProcessProbeResult | null = null;\n\n // Legacy rows created before provider bindings existed are adoptable only\n // when the exact historical sandbox resolves in the configured Modal\n // namespace. NotFound is ambiguous here: it can mean either deletion or\n // credential rotation, so it never creates ownership or loss proof.\n if (\n process.providerBackend === \"modal\" &&\n process.routeTargetId === null &&\n !process.providerBindingKey\n ) {\n try {\n const lifecycle = await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(settings, process.providerInstanceId, null),\n );\n if (lifecycle.status === \"not_found\") {\n observation = {\n status: \"deferred\",\n reason: \"provider_binding_missing\",\n };\n } else {\n process = await bindRetainedProcessProviderIdentity(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n providerBindingKey: lifecycle.providerBindingKey,\n providerBinding: lifecycle.providerBinding,\n });\n recordRetainedProcessReconciliation(observability, \"provider_binding_adopted\");\n if (lifecycle.status === \"terminated\") {\n observation = {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_terminated\",\n },\n };\n }\n }\n } catch (error) {\n observation = {\n status: \"deferred\",\n reason:\n error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n\n if (\n observation?.status === \"deferred\" &&\n process.providerBackend === \"modal\" &&\n !process.providerBindingKey &&\n !process.deadlineCancellationRequestedAt\n ) {\n // An unbound historical command cannot be signalled safely. Still\n // record deadline intent when its lease matches: a failed observer\n // must not block capture until the provider expires.\n try {\n await requestRetainedProcessDeadlineCancellation(db, processScope);\n } catch (error) {\n observability.warn(\"sandbox reaper: legacy deadline cancellation intent failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n\n if (observation === null) {\n const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(\n () => null,\n );\n const currentLeaseIdentityMatches =\n Boolean(lease) &&\n lease!.id === process.leaseId &&\n lease!.leaseEpoch === process.leaseEpoch &&\n lease!.backend === process.providerBackend &&\n lease!.instanceId === process.providerInstanceId &&\n process.routeTargetId === null;\n if (!currentLeaseIdentityMatches) {\n // The lease may already be cold or belong to a successor. Its current\n // resume envelope cannot interrogate the historical process, but a\n // bound process can safely inspect the exact old Modal sandbox id.\n if (process.providerBackend !== \"modal\" || process.routeTargetId !== null) {\n observation = { status: \"deferred\", reason: \"identity_mismatch\" };\n } else {\n try {\n const lifecycle = await withRetainedProcessProbeTimeout(\n inspectHistoricalModalSandbox(\n settings,\n process.providerInstanceId,\n process.providerBindingKey,\n ),\n );\n observation =\n lifecycle.status === \"not_found\"\n ? {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n }\n : lifecycle.status === \"terminated\"\n ? {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_terminated\",\n },\n }\n : { status: \"deferred\", reason: \"identity_mismatch\" };\n } catch (error) {\n observation = {\n status: \"deferred\",\n reason:\n error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n } else {\n try {\n const command = await commandPersistence.load();\n const supervised = command?.kind === \"modal-router-v1\" && Boolean(command.supervision);\n const supervisionMetric = (outcome: string) =>\n observability.incrementCounter({\n name: \"opengeni_command_supervision_total\",\n help: \"Native retained command supervision reconciliation outcomes.\",\n labels: { outcome },\n });\n if (\n supervised &&\n (lease!.rotationReason === \"provider_deadline\" ||\n claim.ownerState === \"background_stopping\")\n ) {\n await commandPersistence.requestCancellation(\n lease!.rotationReason === \"provider_deadline\"\n ? \"provider_deadline\"\n : \"explicit_stop\",\n );\n supervisionMetric(\"cancellation_intent\");\n }\n const legacyDeadlineStop =\n lease!.rotationReason === \"provider_deadline\" &&\n !supervised &&\n !process.deadlineCancellationRequestedAt;\n const probeOutcome = await probe(\n settings,\n lease!,\n process,\n (claim.ownerState === \"background_stopping\" &&\n !process.deadlineCancellationRequestedAt) ||\n legacyDeadlineStop\n ? \"cancel\"\n : \"observe\",\n async (result, chunkId, stream, streamFidelity) => {\n if (\n typeof result !== \"string\" ||\n (!stream && isExecSessionLostBanner(result, process.providerSessionId))\n )\n return;\n const events = await appendSessionCommandOutput(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n commandId: process.id,\n chunkId,\n stream: stream ?? \"stdout\",\n streamFidelity: streamFidelity ?? (stream ? \"separate\" : \"merged\"),\n chunk: stream ? result : stripExecBanner(result),\n });\n if (events.length && bus)\n await bus\n .publish(process.workspaceId, process.sessionId, events)\n .catch(() => undefined);\n },\n commandPersistence,\n drainBudget,\n ).then(\n (value) => ({ ok: true as const, value }),\n (error: unknown) => ({ ok: false as const, error }),\n );\n // Legacy PTY input reserves Ctrl-C before the cancellation fence.\n // Non-PTY has no command signal in the Modal SDK: probe it without\n // injecting Ctrl-C as data, then fence and save after the grace.\n if (\n legacyDeadlineStop &&\n !(await requestRetainedProcessDeadlineCancellation(db, processScope))\n )\n throw new Error(\"Deadline cancellation no longer owns its rotating lease\");\n if (!probeOutcome.ok) throw probeOutcome.error;\n observation = probeOutcome.value;\n wholeBoxGone =\n observation.status === \"proved\" &&\n observation.proof.outcome === \"lost\" &&\n observation.proof.reason === \"provider_instance_not_found\";\n if (supervised) {\n supervisionMetric(\n (await commandPersistence.loadSupervisionReceipt())\n ? \"proof_retained\"\n : \"proof_missing\",\n );\n if (observation.status === \"deferred\")\n supervisionMetric(\n observation.reason === \"provider_running\"\n ? \"checkpoint_blocked\"\n : \"provider_failure\",\n );\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-process provider probe failed\", {\n processId: process.id,\n providerBackend: process.providerBackend,\n error: error instanceof Error ? error.message : String(error),\n });\n observation = { status: \"deferred\", reason: \"provider_error\" };\n }\n }\n }\n if (observation.status === \"binding\") {\n try {\n process = await bindRetainedProcessProviderIdentity(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n providerBindingKey: observation.providerBindingKey,\n providerBinding: observation.providerBinding,\n });\n recordRetainedProcessReconciliation(observability, \"provider_binding_adopted\");\n observation = { status: \"deferred\", reason: \"provider_running\" };\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-process binding adoption failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n observation = { status: \"deferred\", reason: \"provider_error\" };\n }\n }\n if (observation.status === \"deferred\") {\n await deferRetainedProcessClaim(\n db,\n settings,\n observability,\n process,\n expected,\n claim.claimId,\n observation.reason,\n observation.outputBacklog === true,\n );\n continue;\n }\n try {\n process = await recordRetainedProcessReconciliationProof(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId: claim.claimId,\n proof: observation.proof,\n });\n proof = observation.proof;\n recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"proof_checkpoint_failed\");\n warnRetainedProcessProofFailure(observability, error, process);\n continue;\n }\n }\n\n if (wholeBoxGone) {\n // Settle every blocker of the exact lost box in one transaction: all its\n // commands, open requests, PTYs and process holders, then cold the lease\n // and wake waiters. Settling one command per probe took a 20-per-sweep\n // queue 11 minutes for 35 commands in staging session 5040c525 and kept\n // the session waiting on a box that was already gone.\n // Commands of this box that a closed turn never adopted get their\n // session background record first, exactly as their own claim would\n // have, so the agent hears about them when the box settles below.\n const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;\n for (const other of claims) {\n const sibling = other.process;\n if (\n other === claim ||\n `${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey ||\n sibling.ownerActorKind !== \"turn\" ||\n sibling.providerBackend !== \"modal\" ||\n other.ownerState.startsWith(\"background_\") ||\n retainedProcessReconciliationProof(sibling)\n )\n continue;\n try {\n await recoverManagedSessionBackgroundCommand(db, {\n accountId: sibling.accountId,\n workspaceId: sibling.workspaceId,\n sessionId: sibling.sessionId,\n processId: sibling.id,\n expected: retainedProcessSettlementIdentity(sibling),\n reconciliationClaimId: other.claimId,\n });\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command background recovery failed\", {\n processId: sibling.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n try {\n const marked = await markWarmLeaseInstanceLost(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sandboxGroupId: process.sandboxGroupId,\n expectedEpoch: process.leaseEpoch,\n expectedInstanceId: process.providerInstanceId,\n expectedBackend: process.providerBackend,\n });\n if (marked.status === \"marked\") {\n lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);\n recordRetainedProcessReconciliation(observability, \"provider_lost_whole_box\");\n if (marked.backgroundCommandEvents?.length) {\n await publishDurableSessionEvents(\n bus,\n process.workspaceId,\n marked.backgroundCommandEvents,\n (error) => {\n observability.warn(\"sandbox reaper: lost-box command event fanout failed\", {\n sandboxGroupId: process.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n },\n );\n }\n continue;\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: whole-box loss settlement failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n // Stale (a draining drain or another observer owns the box): fall back\n // to this command's own exact settlement below.\n }\n try {\n const settlement = await settleRetainedProcess(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n reconciliationClaimId: claim.claimId,\n outcome: proof.outcome,\n exitCode: proof.exitCode,\n reason: proof.reason,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n if (settlement.process.state === \"active\") {\n throw new Error(\"Retained-process reconciliation returned an active durable process\");\n }\n if (settlement.backgroundCommandEvents.length > 0) {\n try {\n await bus.publish(\n process.workspaceId,\n process.sessionId,\n settlement.backgroundCommandEvents,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: retained-command event fanout failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"settlement_failed\");\n observability.warn(\"sandbox reaper: retained-process settlement failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n await deferRetainedProcessClaim(\n db,\n settings,\n observability,\n process,\n expected,\n claim.claimId,\n \"settlement_failed\",\n );\n }\n }\n}\n\nasync function deferRetainedProcessClaim(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n process: SandboxRetainedProcess,\n expected: ReturnType<typeof retainedProcessSettlementIdentity>,\n claimId: string,\n outcome: RetainedProcessDeferralOutcome,\n outputBacklog = false,\n): Promise<void> {\n const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);\n try {\n // A healthy long-lived command can already be on a five-minute backoff\n // when rotation becomes due. Wake it at the lead boundary, then at the\n // reaper cadence, so observation does not consume the capture window.\n const lease =\n process.providerBackend === \"modal\" && process.routeTargetId === null\n ? await readLease(db, process.workspaceId, process.sandboxGroupId)\n : null;\n await deferRetainedProcessReconciliation(db, {\n accountId: process.accountId,\n workspaceId: process.workspaceId,\n sessionId: process.sessionId,\n processId: process.id,\n expected,\n claimId,\n outcome: deferral.durableOutcome,\n retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs),\n });\n recordRetainedProcessReconciliation(observability, deferral.metricOutcome);\n } catch (error) {\n recordRetainedProcessReconciliation(observability, \"defer_failed\");\n observability.warn(\"sandbox reaper: retained-process defer failed\", {\n processId: process.id,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\ntype RetainedProcessDeferralOutcome =\n | Extract<RetainedProcessProbeResult, { status: \"deferred\" }>[\"reason\"]\n | \"settlement_failed\";\n\nexport function retainedProcessReconciliationDeferral(\n settings: Pick<ActivityServices[\"settings\"], \"sandboxLeaseReaperPeriodMs\">,\n process: Pick<SandboxRetainedProcess, \"reconcileAttempts\">,\n outcome: RetainedProcessDeferralOutcome,\n outputBacklog = false,\n): {\n durableOutcome: string;\n metricOutcome: RetainedProcessReconciliationOutcome;\n retryAfterMs: number;\n quarantined: boolean;\n} {\n const bindingQuarantine =\n process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS &&\n (outcome === \"provider_binding_missing\" ||\n outcome === \"provider_binding_mismatch\" ||\n outcome === \"process_observation_unavailable\");\n if (bindingQuarantine) {\n return {\n durableOutcome: `quarantined_${outcome}`,\n metricOutcome:\n outcome === \"provider_binding_missing\"\n ? \"quarantined_binding_missing\"\n : outcome === \"provider_binding_mismatch\"\n ? \"quarantined_binding_mismatch\"\n : \"quarantined_process_observation_unavailable\",\n retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,\n quarantined: true,\n };\n }\n const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000);\n // A backlog still being drained is progress, not an unhealthy provider:\n // come back at the reaper cadence, without backoff.\n const exponent =\n outcome === \"provider_running\" && outputBacklog\n ? 0\n : Math.min(4, Math.max(0, process.reconcileAttempts - 1));\n const retryAfterMs = Math.min(5 * 60_000, baseRetryMs * 2 ** exponent);\n return {\n durableOutcome: outcome,\n metricOutcome: outcome,\n retryAfterMs,\n quarantined: false,\n };\n}\n\ntype RetainedProcessProbeClient = {\n backendId: string;\n resume?: (state: unknown) => Promise<unknown>;\n deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;\n};\n\ntype RetainedProcessProbeSession = ProviderCommandSession & {\n acknowledgeCommandOutput?: (result: string) => Promise<void>;\n writeStdin?: (args: {\n sessionId: number;\n chars: string;\n yieldTimeMs: number;\n maxOutputTokens: number;\n }) => Promise<unknown>;\n};\n\nconst RETAINED_PROCESS_PROBE_TIMEOUT = Symbol(\"retained-process-provider-probe-timeout\");\n\n/** A command's exit is reported only after both output streams reach EOF, so a\n * finished command with a large backlog looks running until it is read. While a\n * backlog remains, keep reading within the per-claim and per-sweep budgets, so\n * it settles in this claim instead of one page per backoff step. Every page,\n * including `first`, is captured here. */\nexport async function drainRetainedCommandBacklog(\n session: RetainedProcessProbeSession,\n providerSessionId: number,\n first: unknown,\n capturePage: (value: unknown) => Promise<void>,\n budget?: RetainedProcessDrainBudget,\n): Promise<RetainedProcessProbeResult> {\n // Measure before capture: capturing a byte-offset page consumes its receipt.\n let backlog = retainedBacklogRemains(session, first);\n await capturePage(first);\n let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);\n if (typeof session.writeStdin !== \"function\") return observation;\n const until = Math.min(\n Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,\n budget?.until ?? Number.POSITIVE_INFINITY,\n );\n for (\n let reads = 0;\n backlog &&\n observation.status === \"deferred\" &&\n observation.reason === \"provider_running\" &&\n reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS &&\n Date.now() < until;\n reads++\n ) {\n let next: unknown;\n try {\n next = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: providerSessionId,\n chars: \"\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n } catch {\n // The observation already made stays authoritative; the next sweep\n // classifies any provider failure through the ordinary path.\n break;\n }\n backlog = retainedBacklogRemains(session, next);\n await capturePage(next);\n observation = classifyRetainedProcessPollResult(next, providerSessionId, session);\n }\n return backlog && observation.status === \"deferred\" && observation.reason === \"provider_running\"\n ? { ...observation, outputBacklog: true }\n : observation;\n}\n\n/** Whether a byte-offset page leaves a backlog worth reading now: the provider\n * reports the process exited but its output is not yet read to EOF, or a live\n * process produced a large page. Must run before the page is captured, which\n * consumes its receipt. Supervised commands are left to their proof path. */\nfunction retainedBacklogRemains(session: RetainedProcessProbeSession, result: unknown): boolean {\n const page = session.getProviderCommandOutput?.(result);\n if (!page?.expected || page.command.kind !== \"modal-router-v1\") return false;\n if (page.command.supervision || page.exitCode !== null) return false;\n if (page.providerExited) return true;\n const { command, expected } = page;\n return ([\"stdout\", \"stderr\"] as const).some(\n (stream) =>\n command.streams[stream].byteOffset - expected.streams[stream].byteOffset >=\n RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,\n );\n}\n\nasync function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {\n let timer: ReturnType<typeof setTimeout> | undefined;\n try {\n return await Promise.race([\n promise,\n new Promise<never>((_resolve, reject) => {\n timer = setTimeout(\n () => reject(RETAINED_PROCESS_PROBE_TIMEOUT),\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n timer.unref?.();\n }),\n ]);\n } finally {\n if (timer) clearTimeout(timer);\n }\n}\n\n/** Consumed SDK output cannot be polled again. Retain the receipt across\n * persistence retries in this worker; a worker crash remains unreplayable. */\nconst pendingRetainedProbeOutput = new Map<string, { result: unknown; chunkId: string }>();\n\nexport async function captureRetainedProbeOutput(\n processId: string,\n result: unknown,\n capture: (result: unknown, chunkId: string) => Promise<void>,\n): Promise<void> {\n const pending = pendingRetainedProbeOutput.get(processId) ?? {\n result,\n chunkId: crypto.randomUUID(),\n };\n pendingRetainedProbeOutput.set(processId, pending);\n await capture(pending.result, pending.chunkId);\n pendingRetainedProbeOutput.delete(processId);\n}\n\nexport async function probeRetainedProcessAtProvider(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n process: SandboxRetainedProcess,\n mode: \"observe\" | \"cancel\" = \"observe\",\n captureOutput?: (\n result: unknown,\n chunkId: string,\n stream?: \"stdout\" | \"stderr\",\n streamFidelity?: \"separate\" | \"merged\",\n ) => Promise<void>,\n providerPersistence?: ProviderCommandPersistence,\n drainBudget?: RetainedProcessDrainBudget,\n): Promise<RetainedProcessProbeResult> {\n if (\n lease.id !== process.leaseId ||\n lease.sandboxGroupId !== process.sandboxGroupId ||\n lease.leaseEpoch !== process.leaseEpoch ||\n lease.backend !== process.providerBackend ||\n lease.instanceId !== process.providerInstanceId ||\n !lease.resumeBackendId ||\n sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend ||\n process.routeTargetId !== null\n ) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n if (!lease.resumeState) {\n return { status: \"deferred\", reason: \"resume_state_missing\" };\n }\n const pending = pendingRetainedProbeOutput.get(process.id);\n if (pending) {\n if (!captureOutput) throw new Error(\"Pending command output requires its persistence callback\");\n await captureRetainedProbeOutput(process.id, pending.result, captureOutput);\n const pendingObservation = classifyRetainedProcessPollResult(\n pending.result,\n process.providerSessionId,\n );\n // Capture of a previous page must not consume the only deadline stop\n // attempt. Terminal proof needs no further provider call.\n if (mode !== \"cancel\" || pendingObservation.status === \"proved\") return pendingObservation;\n }\n const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;\n if (\n envelopeBackend !== undefined &&\n (typeof envelopeBackend !== \"string\" ||\n sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)\n ) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {\n return { status: \"deferred\", reason: \"identity_mismatch\" };\n }\n\n let configuredModalBinding: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBinding>\n > | null = null;\n if (process.providerBackend === \"modal\" && process.providerBindingKey) {\n try {\n configuredModalBinding = await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBinding(settings),\n );\n } catch (error) {\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n if (configuredModalBinding.key !== process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_mismatch\" };\n }\n }\n\n let client: RetainedProcessProbeClient | undefined;\n try {\n client = createSandboxClientForBackend(process.providerBackend as never, settings) as\n | RetainedProcessProbeClient\n | undefined;\n } catch {\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (\n !client ||\n sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend ||\n !client.resume\n ) {\n return { status: \"deferred\", reason: \"backend_unsupported\" };\n }\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n let resumedState: unknown;\n try {\n resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n process.providerInstanceId,\n );\n } catch {\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (resumedState === undefined) {\n return { status: \"deferred\", reason: \"resume_state_missing\" };\n }\n\n let session: RetainedProcessProbeSession;\n try {\n const resumed = await withRetainedProcessProbeTimeout(\n resumeExactSandboxSession(\n client,\n process.providerBackend,\n resumedState,\n process.providerInstanceId,\n ),\n );\n session = resumed.session as RetainedProcessProbeSession;\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n return { status: \"deferred\", reason: \"provider_timeout\" };\n }\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n if (typeof session.writeStdin !== \"function\") {\n return { status: \"deferred\", reason: \"backend_unsupported\" };\n }\n let legacyPty = false;\n if (process.providerBackend === \"modal\") {\n let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;\n try {\n liveBinding = await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBindingForSession(settings, session),\n );\n } catch (error) {\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n if (!process.providerBindingKey) {\n return {\n status: \"binding\",\n providerBindingKey: liveBinding.key,\n providerBinding: liveBinding.binding,\n };\n }\n if (\n liveBinding.key !== process.providerBindingKey ||\n configuredModalBinding?.key !== process.providerBindingKey\n ) {\n return { status: \"deferred\", reason: \"provider_binding_mismatch\" };\n }\n // Legacy SDK-local handles cannot be reconstructed. New commands carry\n // an opaque provider locator retained outside the sandbox's authority.\n const command = await providerPersistence?.load();\n if (!command || !providerPersistence || !session.bindProviderCommand)\n return { status: \"deferred\", reason: \"process_observation_unavailable\" };\n session.bindProviderCommand(process.providerSessionId, command, providerPersistence);\n legacyPty = command.pty === true;\n }\n const capturePage = async (value: unknown): Promise<void> => {\n if (!captureOutput) {\n if (session.getProviderCommandOutput?.(value))\n throw new Error(\"Provider command output requires durable capture before settlement\");\n return;\n }\n const page = session.getProviderCommandOutput?.(value);\n if (page?.command.kind === \"modal-router-v1\" && typeof value === \"string\") {\n if (!session.captureCommandOutput || !(await session.captureCommandOutput(value)))\n throw new Error(\"Byte-offset output requires atomic capture before settlement\");\n return;\n }\n if (page) {\n for (const chunk of page.chunks)\n if (chunk.text)\n await captureOutput(chunk.text, chunk.chunkId, chunk.stream, page.streamFidelity);\n } else {\n await captureRetainedProbeOutput(process.id, value, captureOutput);\n }\n if (typeof value === \"string\") await session.acknowledgeCommandOutput?.(value);\n };\n\n let result: unknown;\n try {\n const supervisedCancelled =\n mode === \"cancel\" || lease.rotationReason === \"provider_deadline\"\n ? ((await session.cancelSupervisedCommand?.(\n process.providerSessionId,\n lease.rotationReason === \"provider_deadline\" ? \"provider_deadline\" : \"explicit_stop\",\n )) ?? false)\n : false;\n result = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: process.providerSessionId,\n // Ctrl-C is a signal only through a PTY. For a non-PTY command it is\n // ordinary stdin data and can corrupt the command's input.\n chars:\n mode === \"cancel\" &&\n !supervisedCancelled &&\n (process.providerBackend !== \"modal\" || legacyPty)\n ? \"\\u0003\"\n : \"\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n } catch (error) {\n if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {\n return { status: \"deferred\", reason: \"provider_timeout\" };\n }\n if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return { status: \"deferred\", reason: \"provider_error\" };\n }\n // Cancellation keeps draining with empty reads too: a stopped command's exit\n // is also reported only after its unread output reaches EOF.\n const observation = await drainRetainedCommandBacklog(\n session,\n process.providerSessionId,\n result,\n capturePage,\n drainBudget,\n );\n if (\n observation.status === \"deferred\" &&\n observation.reason === \"provider_running\" &&\n lease.rotationRequestedAt !== null &&\n mode !== \"cancel\" &&\n !process.cancellationRequestedAt &&\n legacyPty &&\n !(await providerPersistence\n ?.load()\n .then((command) => command?.kind === \"modal-router-v1\" && command.supervision))\n ) {\n // A background PTY cannot outlive the finite provider box. Interrupt only\n // this exact durable provider session after rotation admission is fenced;\n // settlement still requires an exit/loss banner on this or a later sweep.\n try {\n const interrupted = await withRetainedProcessProbeTimeout(\n session.writeStdin({\n sessionId: process.providerSessionId,\n chars: \"\\u0003\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 2_000,\n }),\n );\n await capturePage(interrupted);\n return classifyRetainedProcessPollResult(interrupted, process.providerSessionId, session);\n } catch (error) {\n if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {\n if (process.providerBackend === \"modal\" && !process.providerBindingKey) {\n return { status: \"deferred\", reason: \"provider_binding_missing\" };\n }\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_instance_not_found\",\n },\n };\n }\n return {\n status: \"deferred\",\n reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? \"provider_timeout\" : \"provider_error\",\n };\n }\n }\n return observation;\n}\n\n/** Classify only exact SDK control banners. Arbitrary output, malformed text,\n * and a running marker are observations to retry; none is physical exit proof. */\nexport function classifyRetainedProcessPollResult(\n result: unknown,\n providerSessionId: number,\n source?: object,\n): RetainedProcessProbeResult {\n if (typeof result !== \"string\") {\n return { status: \"deferred\", reason: \"provider_unknown\" };\n }\n if (isExecSessionLostBanner(result, providerSessionId, source)) {\n return {\n status: \"proved\",\n proof: {\n outcome: \"lost\",\n exitCode: null,\n reason: \"provider_session_lost_banner\",\n },\n };\n }\n const exitCode = parseExecBannerExitCode(result);\n if (exitCode !== null) {\n return {\n status: \"proved\",\n proof: { outcome: \"exited\", exitCode, reason: \"provider_exit_banner\" },\n };\n }\n return {\n status: \"deferred\",\n reason: result.includes(\"Process running with session ID\")\n ? \"provider_running\"\n : \"provider_unknown\",\n };\n}\n\n/**\n * The reaper-tick warm-meter pass (P2.1). Accrues warm-seconds for every WARM\n * viewer-only lease cross-workspace (the list fn excludes turn-held boxes — those\n * meter on the turn heartbeat). Returns the count accrued + the distinct\n * workspaces touched (so the force-drain pass only checks workspaces that have a\n * live warm box, not the whole fleet). Per-row best-effort: one row's metering\n * error must not abort the sweep.\n */\nasync function accrueWarmTick(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n): Promise<{ accrued: number; workspaceIds: Set<string> }> {\n const workspaceIds = new Set<string>();\n let accrued = 0;\n let leases: MeterableWarmLease[] = [];\n try {\n leases = await listMeterableWarmLeases(db);\n } catch (error) {\n observability.warn(\"sandbox reaper: warm-lease read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return { accrued, workspaceIds };\n }\n for (const lease of leases) {\n workspaceIds.add(lease.workspaceId);\n }\n await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {\n try {\n const rate =\n settings.sandboxWarmBillingMode === \"usage_only\"\n ? 0\n : sandboxWarmRateMicrosPerSecond(settings, lease.backend);\n const result = await accrueWarmSeconds(db, {\n accountId: lease.accountId,\n workspaceId: lease.workspaceId,\n sandboxGroupId: lease.sandboxGroupId,\n expectedEpoch: lease.leaseEpoch,\n warmRateMicrosPerSecond: rate,\n billingMode: settings.sandboxWarmBillingMode,\n subjectId: lease.sandboxGroupId,\n });\n if (result.accrued) {\n accrued += 1;\n }\n if (result.costMicros > 0 && settings.sandboxWarmBillingMode === \"credits\") {\n recordCreditMicros(observability, \"usage\", result.costMicros);\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: warm-seconds accrual failed for lease\", {\n workspaceId: lease.workspaceId,\n sandboxGroupId: lease.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n return { accrued, workspaceIds };\n}\n\n/**\n * The reaper maintenance pass's read-only projections, for a reaper tick that\n * started no workflow because there was no work. Same gauges, same process:\n * the control worker both runs the cadence and polls the lifecycle queue.\n */\nexport async function refreshIdleSandboxReaperProjections(input: {\n db: ActivityServices[\"db\"];\n settings: ActivityServices[\"settings\"];\n observability: ActivityServices[\"observability\"];\n inspectOpenSandboxInventory?: InspectOpenSandboxKubernetesInventoryFn;\n}): Promise<void> {\n await refreshQueueLeaseAndCreditGauges(\n input.db,\n input.settings,\n input.observability,\n input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory,\n );\n}\n\n/**\n * Whether the Modal orphan sweep could act: the deployment uses Modal and its\n * app has an unfinished sandbox. Called only once the database checks found no\n * warm lease, so an empty listing is the sweep's complete, empty inventory and\n * is projected exactly as the sweep would project it.\n */\nexport async function modalOrphanSweepMayHaveWork(input: {\n settings: ActivityServices[\"settings\"];\n observability: ActivityServices[\"observability\"];\n hasRunningSandboxes?: (settings: ActivityServices[\"settings\"]) => Promise<boolean>;\n}): Promise<boolean> {\n const { settings, observability } = input;\n if (!modalOrphanSweepConfigured(settings)) return false;\n if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;\n recordModalSandboxInventoryGauges(observability, {\n running: 0,\n unleased: 0,\n liveLeaseInstancesMissing: 0,\n });\n recordSandboxInventoryProjectionSuccess(observability, \"modal_provider\");\n return false;\n}\n\nfunction modalOrphanSweepConfigured(settings: ActivityServices[\"settings\"]): boolean {\n return (\n settings.sandboxBackend === \"modal\" || !!settings.modalTokenId || !!settings.modalTokenSecret\n );\n}\n\nasync function refreshQueueLeaseAndCreditGauges(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n inspectOpenSandboxInventory: InspectOpenSandboxKubernetesInventoryFn,\n): Promise<void> {\n await Promise.all([\n (async () => {\n try {\n recordTurnsQueuedGauge(observability, await countQueuedTurns(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: queued-turn gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n refreshSandboxInventoryGauge(observability, \"leases\", \"sandbox-lease\", async () => {\n recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));\n }),\n refreshSandboxInventoryGauge(\n observability,\n \"checkpoint_artifacts\",\n \"checkpoint-artifact\",\n async () => {\n recordSandboxCheckpointArtifactGauges(\n observability,\n await countSandboxCheckpointArtifactsByState(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"recovery_observations\",\n \"recovery-observations\",\n async () => {\n recordSandboxRecoveryObservationGauges(\n observability,\n await readRecentSandboxRecoveryObservations(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"rotation_backlog\",\n \"rotation-backlog\",\n async () => {\n recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"checkpoint_staleness\",\n \"checkpoint-staleness\",\n async () => {\n recordSandboxCheckpointStalenessGauges(\n observability,\n await readSandboxCheckpointStaleness(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"retained_processes\",\n \"retained-process\",\n async () => {\n recordRetainedProcessInventoryGauges(\n observability,\n await countActiveRetainedProcessesByOwnerState(db),\n );\n },\n ),\n refreshSandboxInventoryGauge(\n observability,\n \"interaction_idle\",\n \"interaction-only-lease\",\n async () => {\n recordInteractionOnlyLeaseGauges(\n observability,\n await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db)),\n );\n },\n ),\n refreshSandboxInventoryGauge(observability, \"expired_drains\", \"expired-draining\", async () => {\n recordExpiredDrainingSandboxLeaseGauges(\n observability,\n await countExpiredDrainingSandboxLeases(db),\n );\n }),\n ...(settings.sandboxBackend === \"opensandbox\" &&\n settings.openSandboxKubernetesInventoryNamespace\n ? [\n refreshSandboxInventoryGauge(\n observability,\n \"opensandbox_kubernetes\",\n \"OpenSandbox Kubernetes\",\n async () => {\n recordOpenSandboxKubernetesInventoryGauges(\n observability,\n await inspectOpenSandboxInventory({\n namespace: settings.openSandboxKubernetesInventoryNamespace!,\n }),\n );\n },\n ),\n ]\n : []),\n (async () => {\n try {\n recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: credit-balance gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n recordActiveUserGauges(observability, await countActiveUsers(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: active-user gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n recordCreditGrantGauges(observability, await readCreditGrantTotals(db));\n } catch (error) {\n observability.warn(\"sandbox reaper: credit-grant gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n recordVerifiedSignupTrialDeploymentFlagGauge(\n observability,\n settings.verifiedSignupTrialCreditsEnabled,\n );\n try {\n const trialSwitch = await readVerifiedSignupTrialSwitch(db);\n recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);\n } catch (error) {\n observability.warn(\"sandbox reaper: trial-credit switch gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n (async () => {\n try {\n const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);\n recordManagedAuthNewSignupsSwitchGauge(\n observability,\n signupsSwitch?.signupsEnabled !== false,\n );\n } catch (error) {\n observability.warn(\"sandbox reaper: new-signups switch gauge refresh failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n })(),\n ]);\n}\n\nasync function refreshSandboxInventoryGauge(\n observability: ActivityServices[\"observability\"],\n domain: SandboxInventoryProjectionDomain,\n diagnosticName: string,\n refresh: () => Promise<void>,\n): Promise<void> {\n try {\n await refresh();\n recordSandboxInventoryProjectionSuccess(observability, domain);\n } catch (error) {\n recordSandboxInventoryProjectionFailure(observability, domain);\n observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\n/**\n * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a\n * live warm box, under the usage lock: if it is over its optional warm cap,\n * accelerate only holderless boxes to draining\n * (guarded turn_holders=0 — a paying turn is never killed). Returns the count of\n * viewer-only boxes force-drained. Per-workspace best-effort.\n */\nasync function forceDrainOverLimitWorkspaces(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n workspaceIds: Set<string>,\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n const enforceBalance = false;\n const cap = settings.sandboxMaxWarmSecondsPerWorkspace;\n let forceDrained = 0;\n await forEachWithConcurrency(\n [...workspaceIds],\n SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,\n async (workspaceId) => {\n try {\n const result = await forceDrainOverLimitViewerOnlyBoxes(db, {\n workspaceId,\n enforceBalance,\n maxWarmSecondsPerWorkspace: cap,\n idleGraceMs: settings.sandboxIdleGraceMs,\n });\n if (result.overLimit && result.drained.length > 0) {\n forceDrained += result.drained.length;\n observability.info(\"sandbox reaper: force-drained viewer-only boxes (over limit)\", {\n workspaceId,\n reason: result.reason,\n drained: result.drained.length,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: force-drain check failed for workspace\", {\n workspaceId,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n },\n );\n return forceDrained;\n}\n\nexport async function reconcileModalProviderCreates(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n observability: ActivityServices[\"observability\"],\n discover: typeof findModalProviderCreateReceipt = findModalProviderCreateReceipt,\n): Promise<void> {\n if (settings.sandboxBackend !== \"modal\" && !settings.modalTokenId && !settings.modalTokenSecret)\n return;\n const candidates = await listPendingModalProviderCreates(db);\n await forEachWithConcurrency(candidates, 2, async (candidate) => {\n try {\n const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);\n const identity = { ...candidate, accountId };\n const attempt = await claimModalProviderCreateRecovery(db, identity);\n if (!attempt) return;\n const instanceId = await discover(settings, attempt);\n if (!instanceId) return; // Absence never settles an unknown dispatch.\n if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {\n observability.info(\"sandbox reaper: recovered Modal create receipt\", {\n ...candidate,\n operationId: attempt.operationId,\n instanceId,\n });\n }\n } catch (error) {\n observability.warn(\"sandbox reaper: Modal create remains fenced pending discovery\", {\n ...candidate,\n error: error instanceof Error ? error.message : String(error),\n });\n }\n });\n}\n\nexport function modalOrphanTerminationStillEligible(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n candidate: ModalOrphanSweepTermination,\n): boolean {\n // The provider may have accepted a create whose reply has not arrived.\n // Before attribution, neither its ID nor its separately-applied tags are\n // available here. Postpone orphan deletion while any such owner exists;\n // age/absence cannot distinguish its provider from an abandoned one.\n if (latest.some((lease) => lease.liveness === \"warming\" && lease.instanceId === null)) {\n return false;\n }\n return modalOrphanCandidateStillUnowned(latest, candidate);\n}\n\n/** Live leases still WARM on an instance the complete listing did not show.\n * A draining lease legitimately loses its box mid-drain (capture, terminate,\n * then cold), and warming has not published a box yet, so only a warm lease\n * whose exact box is gone is a zombie. */\nexport function countWarmModalLeasesMissingInstance(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n missingInstanceIds: string[],\n): number {\n return missingInstanceIds.filter((instanceId) =>\n latest.some((lease) => lease.liveness === \"warm\" && lease.instanceId === instanceId),\n ).length;\n}\n\n/** Whether no live lease owns the candidate box by exact instance or by its\n * attribution tags. Unlike termination eligibility this ignores the pending\n * create postponement: a postponed orphan is still running without a lease. */\nexport function modalOrphanCandidateStillUnowned(\n latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,\n candidate: ModalOrphanSweepTermination,\n): boolean {\n if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {\n return false;\n }\n\n const leaseId = candidate.tags.opengeni_lease_id;\n const workspaceId = candidate.tags.opengeni_workspace_id;\n const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;\n if (!leaseId || !workspaceId || !sandboxGroupId) {\n return true;\n }\n const activeAttribution = latest.find(\n (lease) =>\n lease.leaseId === leaseId &&\n lease.workspaceId === workspaceId &&\n lease.sandboxGroupId === sandboxGroupId,\n );\n return Boolean(\n !activeAttribution ||\n (activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId),\n );\n}\n\nasync function sweepModalOrphansForConfiguredBackend(\n settings: ActivityServices[\"settings\"],\n db: ActivityServices[\"db\"],\n observability: ActivityServices[\"observability\"],\n): Promise<number> {\n // Keep the provider-side sweep tightly scoped to deployments that have a Modal\n // app path configured. Local/docker-only workers should not attempt Modal API\n // calls just because modalAppName has a default.\n if (!modalOrphanSweepConfigured(settings)) {\n return 0;\n }\n let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;\n try {\n liveLeases = await listLiveModalSandboxLeaseAttributions(db);\n } catch (error) {\n observability.warn(\"sandbox reaper: live Modal lease attribution read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n return 0;\n }\n\n const result = await sweepModalOrphanSandboxes(settings, liveLeases, {\n revalidateTermination: async (candidate) => {\n const latest = await listLiveModalSandboxLeaseAttributions(db);\n return modalOrphanTerminationStillEligible(latest, candidate);\n },\n });\n // Only a complete listing is an inventory; a pass cut short by the\n // termination budget leaves the previous projection to age out as stale.\n // Re-read durable ownership after the listing so a lease that started or\n // finished while the app was being listed is not reported as a leak.\n if (result.inventory.complete) {\n try {\n const latest = await listLiveModalSandboxLeaseAttributions(db);\n recordModalSandboxInventoryGauges(observability, {\n running: result.inventory.running,\n unleased: result.inventory.unterminated.filter((candidate) =>\n modalOrphanCandidateStillUnowned(latest, candidate),\n ).length,\n liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(\n latest,\n result.inventory.missingLiveLeaseInstanceIds,\n ),\n });\n recordSandboxInventoryProjectionSuccess(observability, \"modal_provider\");\n } catch (error) {\n recordSandboxInventoryProjectionFailure(observability, \"modal_provider\");\n observability.warn(\"sandbox reaper: Modal inventory ownership re-read failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n }\n }\n for (const terminated of result.terminated) {\n observability.warn(\"sandbox reaper: terminated Modal orphan sandbox\", {\n sandboxId: terminated.sandboxId,\n reason: terminated.reason,\n tags: JSON.stringify(terminated.tags),\n });\n }\n if (result.examined > 0) {\n observability.info(\"sandbox reaper: Modal orphan sweep completed\", {\n examined: result.examined,\n terminated: result.terminated.length,\n skipped: result.skipped,\n appName: settings.modalAppName,\n });\n }\n return result.terminated.length;\n}\n\n/**\n * A capture claim whose owning worker disappeared must not fence a workspace\n * forever. The drain reaper reaches this seam only after every holder is gone.\n * It resumes the exact attributed provider without replacement and proves that\n * a no-op command can execute. This distinguishes typed provider loss from a\n * still-addressable instance; it does not prove that an old capture RPC has\n * stopped. The caller applies the provider's durable takeover policy and the\n * claim deadline. Ambiguity preserves the old claim and therefore the files.\n */\nasync function probeDrainableProviderReadiness(\n settings: ActivityServices[\"settings\"],\n lease: LeaseSnapshot,\n): Promise<\"ready\" | \"missing\"> {\n if (!lease.instanceId || !lease.resumeState) {\n throw new Error(\"Expired workspace capture has no resumable provider identity\");\n }\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n let established: Awaited<ReturnType<typeof establishSandboxSessionFromEnvelope>>;\n try {\n established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {\n sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,\n recovery: \"resume-only\",\n backendOverride: backend as never,\n });\n } catch (error) {\n if (isProviderSandboxNotFoundError(backend, error)) return \"missing\";\n throw error;\n }\n if (established.instanceId !== lease.instanceId) {\n throw new Error(\"Expired workspace capture resumed a different provider instance\");\n }\n try {\n if (established.backendId === \"modal\") {\n await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);\n return \"ready\";\n }\n const session = established.session as {\n exec?: (args: {\n cmd: string;\n yieldTimeMs?: number;\n maxOutputTokens?: number;\n }) => Promise<unknown>;\n execCommand?: (args: {\n cmd: string;\n yieldTimeMs?: number;\n maxOutputTokens?: number;\n }) => Promise<unknown>;\n };\n const run = session.exec ?? session.execCommand;\n if (!run) throw new Error(\"Expired workspace capture provider has no readiness command\");\n let timer: ReturnType<typeof setTimeout> | undefined;\n const result = await Promise.race([\n run.call(session, {\n cmd: \"true\",\n yieldTimeMs: 1_000,\n maxOutputTokens: 1_000,\n }),\n new Promise<never>((_, reject) => {\n timer = setTimeout(\n () => reject(new Error(\"Expired workspace capture readiness probe timed out\")),\n RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,\n );\n }),\n ]).finally(() => {\n if (timer) clearTimeout(timer);\n });\n if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {\n throw new Error(\"Expired workspace capture provider is not command-ready\");\n }\n return \"ready\";\n } catch (error) {\n if (isProviderSandboxNotFoundError(backend, error)) return \"missing\";\n throw error;\n }\n}\n\nexport type SandboxDrainReason =\n | \"idle\"\n | \"provider_deadline\"\n | \"operator\"\n | \"teardown_claim\"\n | \"rotation_requested\";\n\n/** The bounded drain reason carried on `sandbox.box.terminated`: the lease's\n * rotation reason when a rotation was requested, otherwise the ordinary idle\n * drain. A requested rotation without a recorded reason (legacy rows) is\n * reported as `rotation_requested` rather than guessed. */\nexport function sandboxDrainReason(\n lease: Pick<LeaseSnapshot, \"rotationRequestedAt\" | \"rotationReason\">,\n): SandboxDrainReason {\n if (lease.rotationRequestedAt === null) return \"idle\";\n return lease.rotationReason ?? \"rotation_requested\";\n}\n\n/**\n * Terminate one drainable box by id, then CAS its lease draining->cold under the\n * epoch fence. Returns true when the lease went cold (the box is ours to stop\n * and was stopped), false when a concurrent sweep / re-arm / newer epoch means\n * we must NOT stop it (provider terminate is skipped).\n *\n * The ordering is deliberately receipt-gated on BOTH ends:\n * - BEFORE provider I/O: re-read the exact draining epoch and acquire/retain\n * its durable capture/teardown claim. Before that claim an arrival may\n * re-arm for availability; afterward every arrival waits and cannot race\n * provider termination.\n * - AFTER provider terminate: confirmDrainCold commits only the exact current\n * capture id together with draining->cold. A miss means a recovery attempt\n * or epoch already replaced this callback; it says nothing about provider\n * liveness, so only that durable successor may reconcile and commit cold.\n */\nasync function terminateDrainableBox(\n db: ActivityServices[\"db\"],\n settings: ActivityServices[\"settings\"],\n row: ReapDrainable,\n observability: ActivityServices[\"observability\"],\n terminateBox: TerminateBoxFn,\n probeDrainableProvider: DrainableProviderProbeFn,\n attempt: SandboxDrainCaptureAttempt,\n objectStorage: ObjectStorage | null,\n bus: ActivityServices[\"bus\"] | null = null,\n): Promise<boolean> {\n // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep\n // returns no account_id; the workspace->account map is the bootstrap read).\n const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);\n\n // Re-read the lease for the resume envelope AND the pre-terminate CAS guard.\n const lease = await readLease(db, row.workspaceId, row.sandboxGroupId);\n if (!lease) {\n // Lease vanished (cold-reaped by a concurrent sweep). Nothing to stop.\n return false;\n }\n if (\n lease.liveness !== \"draining\" ||\n (lease.refcount !== 0 && !lease.unobservableCommandDrainIds?.length) ||\n lease.leaseEpoch !== row.leaseEpoch\n ) {\n // Re-armed (warm again) / a newer epoch / already drained by a concurrent\n // sweep. Skip — provider stop() must fire ONLY past the drain grace at\n // refcount=0 on the box we observed, never while a turn or viewer holds it.\n return false;\n }\n\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n const managedProvider = backend !== \"none\" && backend !== \"selfhosted\";\n const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);\n if (managedProvider && !capturePolicy) {\n throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);\n }\n if (capturePolicy?.liveInstance === \"replaced\") {\n throw new Error(\n `sandbox backend ${backend} workspace capture would replace the fenced live instance`,\n );\n }\n\n const captureTimeoutMs = attempt.captureTimeoutMs;\n if (\n managedProvider &&\n !lease.archiveCapture?.publishedAt &&\n sandboxDrainCaptureSettlingInProcess(lease)\n ) {\n // An earlier attempt's capture is still reading, publishing late, or\n // removing its spool in this worker. Leave its claim untouched and do not\n // start a second physical capture of the same workspace; a later retry\n // tears down from its published archive or, if it failed, captures anew.\n // An already published archive still resumes teardown immediately.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`,\n );\n }\n let captureClaim:\n | NonNullable<\n Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: \"claimed\" }>\n >[\"claim\"]\n | null = null;\n let providerMissingBeforeCapture = false;\n let captureDisposition: DrainCaptureDisposition = \"capture_required\";\n if (lease.instanceId && managedProvider) {\n if (lease.archiveCapture) {\n const priorCapture = lease.archiveCapture;\n const instanceId = lease.instanceId;\n if (priorCapture.publishedAt) {\n // Publication is an irreversible durable phase transition. The archive\n // already represents this exact fenced generation, so a crash/retry\n // resumes teardown immediately: no deadline wait, provider probe, claim\n // replacement, or duplicate snapshot request.\n captureClaim = {\n ...priorCapture,\n leaseId: lease.id,\n leaseEpoch: lease.leaseEpoch,\n instanceId,\n archiveGeneration: lease.archiveGeneration,\n archiveComplete: lease.archiveComplete,\n };\n captureDisposition = \"archive_published\";\n } else {\n if (\n priorCapture.operationId === attempt.operationId &&\n priorCapture.attempt >= attempt.attempt\n ) {\n // Never let an accepted duplicate/stale delivery complete the durable\n // child while its exact claim still fences the lease. Throwing asks\n // Temporal for a strictly newer attempt; an actually stale task token is\n // rejected by Temporal and cannot affect the workflow either way.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`,\n );\n }\n const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: priorCapture.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: instanceId,\n });\n const unpublishedWarmingProvider =\n backend === \"modal\" &&\n !sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) &&\n lease.recovery.provider.status === \"creating\" &&\n lease.recovery.provider.instanceId === instanceId &&\n lease.recovery.workspace.status === \"not_ready\";\n // Temporal knows that the prior activity attempt is no longer accepted,\n // but it cannot generically cancel provider I/O. Immediate replacement is\n // legal only when BOTH the durable claim and the current adapter declare\n // takeover safety: either the same idempotent request, or an independently\n // repeatable read-only capture. A readiness probe alone never proves an\n // unrelated exclusive snapshot RPC has settled.\n const replace = async () =>\n await replaceWorkspaceArchiveCaptureAfterProof(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n priorCaptureId: priorCapture.id,\n captureId: attempt.captureId,\n operationId: attempt.operationId,\n attempt: attempt.attempt,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: instanceId,\n captureTimeoutMs,\n });\n\n // If no durable complete archive exists, replacement is safe only when\n // this exact provider can be recaptured. A typed missing provider before\n // the old deadline may mean the dead worker still holds the sole archive\n // bytes in memory, so preserve its callback until that bounded window.\n const currentTakeoverSafe =\n capturePolicy !== null && capturePolicy.takeover !== \"exclusive\";\n if (!lease.resumeState) {\n if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`,\n );\n }\n captureClaim = replacement;\n } else {\n if (!deadlineExpired) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,\n );\n }\n if (!unpublishedWarmingProvider) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`,\n );\n }\n // Legacy/unrelated owner on a never-published Modal warming box. Its\n // deadline elapsed and no user-visible workspace ever existed. Fence\n // the old callback before the by-id termination below.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`,\n );\n }\n captureClaim = replacement;\n }\n } else if (priorCapture.takeoverSafe && currentTakeoverSafe) {\n // The stable lineage reuses the exact external idempotency key where\n // supported. Parallel-read adapters may instead recapture independently;\n // both cases preserve the addressed instance and cannot overlap writes.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`,\n );\n }\n captureClaim = replacement;\n } else {\n if (!deadlineExpired) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`,\n );\n }\n const providerState = await probeDrainableProvider(settings, lease);\n if (providerState === \"missing\") {\n // Take over on the DB clock before committing loss. This fences an\n // old teardown callback. Its provider-request lineage remains\n // durable so a verified late archive can still repair the cold row.\n const replacement = await replace();\n if (!replacement) {\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`,\n );\n }\n providerMissingBeforeCapture = true;\n captureClaim = replacement;\n } else {\n // An exclusive provider operation can outlive both its Temporal\n // activity and its diagnostic deadline. A live command channel says\n // nothing about whether that operation is still reading/pausing the\n // workspace. Keep the durable fence; an operator/provider-specific\n // receipt is required to resolve this deliberately fail-closed case.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`,\n );\n }\n }\n }\n } else {\n const claimed = await claimWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: attempt.captureId,\n operationId: attempt.operationId,\n attempt: attempt.attempt,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n liveness: \"draining\",\n captureTimeoutMs,\n minIntervalMs: 0,\n pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),\n deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),\n // An unknown-outcome setup request of a closed attempt never settles\n // on a live box; after the idle grace it no longer pins the drain.\n abandonedSetupCaptureAfterMs: Math.max(settings.sandboxIdleGraceMs, 60_000),\n providerReplaySafe: capturePolicy?.takeover === \"same_request\",\n takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== \"exclusive\",\n });\n if (claimed.status === \"claimed\") {\n captureClaim = claimed.claim;\n } else if (claimed.status === \"mutation_in_progress\") {\n // An admission is normally authoritative proof that a provider\n // operation may still be running. It cannot, however, pin a draining\n // lease forever after the exact provider has disappeared. Probe the\n // attributed instance without replacement. Only typed NotFound permits\n // the cold commit to reject the exact stale admissions; a live or\n // ambiguous provider remains fenced for a later sweep.\n const providerState = await probeDrainableProvider(settings, lease);\n if (providerState !== \"missing\") return false;\n providerMissingBeforeCapture = true;\n } else if (claimed.status === \"capture_in_progress\") {\n // A legacy activity or a racing delivery installed the durable claim.\n // Its owner may still complete; this child remains the repair owner if\n // it does not. Completing here would recreate a schedule-sized gap.\n throw new SandboxDrainRecoveryDeferredError(\n `sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`,\n );\n } else {\n return false;\n }\n }\n }\n\n // The epoch-fenced PERSIST-onto-lease CAS the terminate seam calls AFTER it has\n // snapshotted the live box and BEFORE it terminates (sandbox-file-persistence).\n // Same exact lease/capture guard as confirmDrainCold. A stale capture owner or\n // newer epoch writes ZERO rows → wrote:false → the seam leaves the box RUNNING\n // and skips the cold-commit below.\n // `persisted` tracks whether a real archive landed on the lease this drain —\n // the durable sandbox.box.terminated event below carries it, so a \"terminated\n // with NOTHING persisted\" (box already dead at drain) is visible in the DB.\n let persisted = captureDisposition === \"archive_published\";\n let archiveRevision: string | null =\n captureDisposition === \"archive_published\"\n ? (lease.recovery.archive.current?.revision ?? null)\n : null;\n const persistArchive: PersistArchiveFn = async (\n archiveBase64: string | VerifiedHostWorkspaceArchive | null,\n archiveMetadata?: WorkspaceArchiveDescriptor,\n _providerSession?: unknown,\n providerBinding?: Awaited<\n ReturnType<typeof resolveModalCheckpointProviderBindingForSession>\n > | null,\n ) => {\n // A live persistable provider must own the exact durable capture claim.\n // Without it we cannot prove that these bytes came from a provider that was\n // admission-fenced for the full pause/capture interval.\n if (!lease.instanceId || !captureClaim) {\n return { wrote: false, archiveRevision: null };\n }\n const archiveMetrics = runtimeMetricsHooksForObservability(observability);\n let checkpointArtifactId: string | null = null;\n if (\n typeof archiveBase64 === \"string\" &&\n archiveBase64 &&\n archiveMetadata?.version === 2 &&\n (archiveMetadata.provider === \"modal_snapshot_filesystem\" ||\n archiveMetadata.provider === \"modal_snapshot_directory\")\n ) {\n if (!providerBinding) {\n throw new Error(\"Modal native snapshot has no exact session provider identity\");\n }\n const candidate = await registerSandboxCheckpointArtifact(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n sourceLeaseId: lease.id,\n sourceLeaseEpoch: row.leaseEpoch,\n sourceInstanceId: lease.instanceId,\n sourceWorkspaceGeneration: captureClaim.workspaceGeneration,\n providerBindingKey: providerBinding.key,\n providerBinding: providerBinding.binding,\n workspaceArchive: archiveBase64,\n workspaceArchiveMeta: archiveMetadata,\n });\n checkpointArtifactId = candidate.id;\n }\n const baseInput = {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedLeaseId: lease.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n expectedWorkspaceGeneration: captureClaim.workspaceGeneration,\n captureId: captureClaim.id,\n providerRequestId: captureClaim.providerRequestId,\n };\n let result: Awaited<ReturnType<typeof persistDrainSnapshot>>;\n if (archiveBase64 === null) {\n result = await persistDrainSnapshot(db, {\n ...baseInput,\n workspaceArchive: null,\n });\n } else {\n if (!archiveMetadata) {\n throw new Error(\"Sandbox snapshot publication requires a verified archive descriptor\");\n }\n const published = await putVersion1TarArchiveOrInline({\n backend: lease.backend,\n objectStorage,\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n archive:\n typeof archiveBase64 === \"object\"\n ? archiveBase64\n : {\n bytes: Buffer.from(archiveBase64, \"base64\"),\n descriptor: archiveMetadata,\n base64: archiveBase64,\n },\n metrics: archiveMetrics,\n });\n result = await persistWorkspaceArchiveCandidate({\n objectStorage,\n ...(published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {}),\n persist: () =>\n persistDrainSnapshot(db, {\n ...baseInput,\n workspaceArchiveMeta: archiveMetadata,\n ...published,\n ...(checkpointArtifactId ? { checkpointArtifactId } : {}),\n }),\n });\n }\n if (!result.wrote && checkpointArtifactId) {\n await markSandboxCheckpointArtifactDeletePending(db, {\n accountId,\n workspaceId: row.workspaceId,\n artifactId: checkpointArtifactId,\n reason: \"drain_snapshot_publication_fenced\",\n }).catch(() => undefined);\n }\n if (result.wrote && archiveBase64 !== null) {\n persisted = true;\n archiveRevision = result.archiveRevision;\n }\n return result;\n };\n\n // Resume-by-id -> verified capture -> persist-onto-lease -> provider\n // terminate. Persisting atomically hands any displaced provider snapshot to\n // the provider-bound artifact ledger; its global GC worker owns deletion.\n // Definitive NotFound before capture is a typed loss, not ordinary teardown\n // success: confirmDrainCold preserves any prior archive or marks recovery\n // unrecoverable when none exists. A genuine capture/resume failure leaves the\n // lease draining for a later sweep (NEVER terminate a box whose files we\n // could not capture). A persist CAS miss means the box was re-armed and left\n // running, so the cold commit is skipped.\n // A published capture retry skips persistence. Revalidate legacy containment\n // even on that path: stale enrollment is not native-supervisor exit proof.\n // Typed provider absence is independently fenced by the canonical loss commit.\n if (\n !providerMissingBeforeCapture &&\n backend === \"modal\" &&\n lease.instanceId &&\n !(await readWorkspaceArchiveCapturePreflight(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n liveness: \"draining\",\n }))\n ) {\n return false;\n }\n // Stamp the charge horizon after verified capture and immediately before\n // provider stop. Capture can fail and release its claim, allowing re-arm.\n // A retry after stop success cannot charge post-termination wall time.\n let stopCutoffMarked = false;\n const beforeProviderStop = async (): Promise<void> => {\n const cutoff = await markWarmBillingStopCutoff(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n });\n if (!cutoff) throw new Error(\"sandbox stop cutoff was fenced before provider termination\");\n stopCutoffMarked = true;\n };\n const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture\n ? { terminated: true, providerMissingBeforeCapture: true }\n : await terminateBox(\n settings,\n lease,\n observability,\n persistArchive,\n captureClaim?.providerRequestId ?? attempt.operationId,\n captureDisposition,\n capturePolicy,\n Boolean(objectStorage),\n async () => {\n if (!captureClaim || !lease.instanceId) return;\n await releaseWorkspaceArchiveCapture(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n captureId: captureClaim.id,\n expectedEpoch: row.leaseEpoch,\n expectedInstanceId: lease.instanceId,\n });\n },\n row.workspaceId,\n beforeProviderStop,\n backend === \"docker\"\n ? async () => {\n if (!captureClaim || !lease.instanceId)\n throw new Error(\"Docker drain has no current claimed capture\");\n return await verifyDockerDrainCaptureFence(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n leaseId: lease.id,\n leaseEpoch: row.leaseEpoch,\n instanceId: lease.instanceId,\n workspaceGeneration: captureClaim.workspaceGeneration,\n captureId: captureClaim.id,\n providerRequestId: captureClaim.providerRequestId,\n });\n }\n : undefined,\n );\n const terminated = typeof termination === \"boolean\" ? termination : termination.terminated;\n if (!terminated) {\n return false;\n }\n\n // Self-hosted/none and an unassigned lease have no provider stop callback;\n // finish their usage at logical detach without touching customer compute.\n // A managed instance that was stopped without its callback must fail closed.\n const providerMissingDuringStop =\n providerMissingBeforeCapture ||\n (typeof termination !== \"boolean\" && termination.providerMissingBeforeCapture);\n if (!stopCutoffMarked && !providerMissingDuringStop) {\n if (managedProvider && lease.instanceId) {\n throw new Error(\"managed sandbox stopped without a warm usage cutoff\");\n }\n await beforeProviderStop();\n }\n\n // Stop was confirmed but the lease is not cold yet. This final epoch-fenced\n // tick settles the interval since the last heartbeat/reaper sweep, including\n // a box drained before its first periodic tick. If settlement fails, keep\n // the draining receipt for retry instead of silently losing the charge.\n const finalRate =\n settings.sandboxWarmBillingMode === \"usage_only\"\n ? 0\n : sandboxWarmRateMicrosPerSecond(settings, backend);\n if (!providerMissingDuringStop) {\n const finalTick = await accrueWarmSeconds(db, {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n warmRateMicrosPerSecond: finalRate,\n billingMode: settings.sandboxWarmBillingMode,\n finalDrain: true,\n });\n if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === \"credits\") {\n recordCreditMicros(observability, \"usage\", finalTick.costMicros);\n }\n }\n\n const providerMissing =\n providerMissingBeforeCapture ||\n (typeof termination === \"boolean\" ? false : termination.providerMissingBeforeCapture);\n // The authoritative commit clears the durable capture/teardown claim together\n // with draining->cold. Until this succeeds, arrivals remain fenced by that\n // exact claim; a timestamp or a failed provider call can never reopen a box\n // while termination may still be in flight.\n const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(\n db,\n {\n accountId,\n workspaceId: row.workspaceId,\n sandboxGroupId: row.sandboxGroupId,\n expectedEpoch: row.leaseEpoch,\n ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),\n providerMissingBeforeCapture: providerMissing,\n idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,\n // Mirrors terminateProviderBox: a selfhosted machine is never stopped.\n providerStopped:\n managedProvider && lease.backend !== \"selfhosted\" && lease.resumeBackendId !== \"selfhosted\",\n },\n );\n // The command terminal events and agent inputs are already durable in the\n // cold commit; this is only best-effort live fanout.\n await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {\n observability.warn(\"sandbox reaper: contained command event fanout failed\", {\n sandboxGroupId: row.sandboxGroupId,\n error: error instanceof Error ? error.message : String(error),\n });\n });\n if (wentCold) {\n // Only the exact successful cold commit counts provider loss. A missing\n // probe, a stale capture, a failed commit, or a retried child is not another\n // observed loss. Keep this outside the best-effort session event writer.\n // An unpublished warming replacement (its creator died before warm\n // publication) held no workspace; its absence is not a capture loss.\n if (providerMissing && !unpublishedProviderLost) {\n recordSandboxProviderMissingBeforeCapture(observability, backend);\n }\n if (lease.unobservableCommandDrainIds?.length) {\n recordSandboxCommandContainment(\n observability,\n providerMissing ? \"provider_missing\" : \"contained\",\n );\n }\n // Durable termination record (sandbox-file-persistence observability): who\n // ended this box and whether its /workspace was captured first, appended to\n // every session sharing the group's box. Best-effort: attribution must\n // never affect the drain outcome.\n try {\n await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {\n type: \"sandbox.box.terminated\",\n payload: {\n actor: \"reaper\",\n persisted,\n archiveRevision,\n instanceId: lease.instanceId,\n providerMissingBeforeCapture: providerMissing,\n // Why the box drained: the lease's bounded rotation reason when one\n // was requested (provider deadline, operator/image conflict, teardown\n // claim), otherwise the ordinary idle drain. Additive diagnostic\n // fact; no identifiers beyond the ones already carried above.\n drainReason: sandboxDrainReason(lease),\n },\n });\n } catch (eventError) {\n observability.warn(\n \"sandbox reaper: box-terminated event write failed (drain outcome unaffected)\",\n {\n sandboxGroupId: row.sandboxGroupId,\n leaseEpoch: row.leaseEpoch,\n backend: lease.backend,\n instanceId: lease.instanceId,\n error: eventError instanceof Error ? eventError.message : String(eventError),\n },\n );\n }\n }\n return wentCold;\n}\n\n// The persist-capable slice of a live provider session. persistWorkspace()\n// snapshots /workspace (snapshot_filesystem → a Modal snapshot-ref; tar → a tar\n// archive) and returns the archive bytes. The session also carries the Modal SDK\n// client used to bind the snapshot receipt to the exact provider namespace.\ntype PersistableSession = {\n state?: { workspacePersistence?: unknown };\n persistWorkspace?: (options?: { requestId: string }) => Promise<Uint8Array | undefined>;\n runPreStopHooks?: () => Promise<void>;\n preStop?: (options?: { reason?: string }) => Promise<void>;\n stop?: (options?: { reason?: string }) => Promise<void>;\n shutdown?: (options?: { reason?: string }) => Promise<void>;\n delete?: (options?: { reason?: string }) => Promise<void>;\n close?: () => Promise<void>;\n};\n\nfunction providerDrainLogIdentity(\n lease: LeaseSnapshot,\n backend: string,\n): {\n leaseId: string;\n sandboxGroupId: string;\n leaseEpoch: number;\n backend: string;\n instanceId: string | null;\n} {\n return {\n leaseId: lease.id,\n sandboxGroupId: lease.sandboxGroupId,\n leaseEpoch: lease.leaseEpoch,\n backend,\n instanceId: lease.instanceId,\n };\n}\n\n/**\n * Resume/attach the box by id, PERSIST its /workspace, fold the snapshot onto the\n * lease under the epoch fence, hand the superseded snapshot to durable GC, THEN\n * terminate. Resumes DIRECTLY (no cold-restore fallback) so a gone box is a clean\n * no-op, never a wasteful create-then-kill:\n * - build the client for the lease's backend;\n * - deserialize + resume the envelope (warm reattach by id, R4-safe);\n * - session.persistWorkspace() -> capture the /workspace archive;\n * - persistArchive(base64) -> CAS-fold it onto the lease and atomically mark\n * the displaced artifact delete-pending;\n * - the provider-bound global GC deletes that object asynchronously;\n * - terminate the live handle through the standard Agents SDK lifecycle.\n *\n * Returns true when the box was terminated (or was already gone), false when the\n * persist CAS found the lease re-armed and the box was deliberately LEFT RUNNING.\n *\n * Failure discipline (NEVER lose files, NEVER leak a box):\n * - resume NotFound -> the box is already down; SUCCESS (return true), nothing\n * to persist, the caller colds the lease.\n * - a transient/auth/network resume failure on a box that may be ALIVE -> throw\n * (the caller skips; the lease stays draining for the next sweep).\n * - a persistWorkspace failure (snapshot timeout / provider error) -> throw\n * BEFORE any terminate, so the box is NOT torn down with un-captured files;\n * the next sweep retries, and the provider idle-timeout is the ultimate\n * backstop. We never terminate a box whose snapshot we could not capture.\n * A provider-backed row with instance_id but no envelope is never guessed away:\n * Modal has an explicit by-id rescue adapter above; every other backend stays\n * draining until its adapter can prove termination. A truly instance-less or\n * `none`-backed row is a no-op.\n */\n/** Internal server pre-read fence; not a grant to arbitrary callers. The\n * owning drain already derived account/workspace/group authority and claimed\n * capture through the canonical DB lifecycle. No claim is fabricated here. */\nexport async function verifyDockerDrainCaptureFence(\n db: ActivityServices[\"db\"],\n input: {\n accountId: string;\n workspaceId: string;\n sandboxGroupId: string;\n leaseId: string;\n leaseEpoch: number;\n instanceId: string;\n workspaceGeneration: number;\n captureId: string;\n providerRequestId: string;\n },\n readers = { readLease, readWorkspaceArchiveCapturePreflight },\n): Promise<{ archivePublished: boolean }> {\n const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);\n if (\n !current ||\n current.id !== input.leaseId ||\n current.liveness !== \"draining\" ||\n current.leaseEpoch !== input.leaseEpoch ||\n current.instanceId !== input.instanceId ||\n current.backend !== \"docker\" ||\n current.workspaceGeneration !== input.workspaceGeneration ||\n current.archiveCapture?.id !== input.captureId ||\n current.archiveCapture.providerRequestId !== input.providerRequestId ||\n current.archiveCapture.workspaceGeneration !== input.workspaceGeneration\n )\n throw new Error(\"Docker drain capture lease/epoch/claim changed\");\n const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {\n accountId: input.accountId,\n workspaceId: input.workspaceId,\n sandboxGroupId: input.sandboxGroupId,\n expectedEpoch: input.leaseEpoch,\n expectedInstanceId: input.instanceId,\n liveness: \"draining\",\n });\n if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)\n throw new Error(\"Docker drain capture has an active writer or changed generation\");\n return {\n archivePublished:\n current.archiveCapture.publishedAt instanceof Date &&\n Number.isFinite(current.archiveCapture.publishedAt.getTime()) &&\n current.archiveComplete === true,\n };\n}\n\nfunction observeDockerWorkspaceRelease(\n observability: ActivityServices[\"observability\"],\n session: unknown,\n logIdentity: ReturnType<typeof providerDrainLogIdentity>,\n): void {\n const release = (session as { hostWorkspaceRelease?: DockerHostWorkspaceRelease | null })\n .hostWorkspaceRelease;\n if (!release) return;\n recordSandboxDockerWorkspaceRelease(observability, release.status);\n if (release.status === \"retained\") {\n observability.warn(\n \"sandbox reaper: Docker drain kept its host workspace directory after teardown\",\n { ...logIdentity, reason: release.reason },\n );\n }\n}\n\nexport async function terminateProviderBox(\n settings: ActivityServices[\"settings\"],\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,\n observability: ActivityServices[\"observability\"],\n persistArchive: PersistArchiveFn,\n createClientForBackend: CreateSandboxClientForBackendFn = createSandboxClientForBackend,\n terminateModalById: typeof terminateModalSandboxById = terminateModalSandboxById,\n providerCaptureRequestId?: string,\n captureDisposition: DrainCaptureDisposition = \"capture_required\",\n claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,\n diskBackedArchives = false,\n releaseFailedCapture?: () => Promise<void>,\n workspaceId?: string,\n beforeProviderStop?: () => Promise<void>,\n beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,\n): Promise<ProviderTerminationOutcome> {\n const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;\n const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;\n const logIdentity = providerDrainLogIdentity(lease, backend);\n const warnCaptureFailure = (message: string, error: unknown): void =>\n warnDrainSnapshotFailure(observability, message, error, {\n sandboxGroupId: lease.sandboxGroupId,\n leaseEpoch: lease.leaseEpoch,\n ...(workspaceId ? { workspaceId } : {}),\n });\n const releaseAfterCaptureFailure = async (): Promise<void> => {\n try {\n await releaseFailedCapture?.();\n } catch (error) {\n warnCaptureFailure(\"sandbox reaper: failed capture claim release failed\", error);\n }\n };\n // 'none' / no backend -> nothing to terminate.\n if (!backend || backend === \"none\") {\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // CRITICAL SAFETY (bring-your-own-compute): NEVER provider-stop\n // a selfhosted box. The \"box\" is a user's PHYSICAL machine — you cannot kill it,\n // and a delete()/kill() reaching the agent would be catastrophic. The reaper\n // DRAINS the lease to cold only (refcount→0 → draining → this returns true so\n // the caller's confirmDrainCold flips draining→cold) but issues NO provider stop:\n // no client is built, no resume, no persistWorkspace (the machine IS the\n // persistence — nothing to snapshot), no delete/kill. The session simply detaches;\n // the machine stays up under the agent's own process lifetime (§23.0). Checked on\n // BOTH the lease backend and the resume envelope's backend so neither path leaks.\n if (\n backend === \"selfhosted\" ||\n lease.backend === \"selfhosted\" ||\n lease.resumeBackendId === \"selfhosted\"\n ) {\n observability.info(\n \"sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped — it is the user's machine)\",\n {\n ...logIdentity,\n },\n );\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n const durableCapturePolicy =\n claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);\n if (!durableCapturePolicy) {\n throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);\n }\n if (durableCapturePolicy.liveInstance !== \"preserved\") {\n throw new Error(\n `sandbox backend ${backend} workspace capture would replace the fenced live instance`,\n );\n }\n\n const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);\n // `instance_id` is the lease's authoritative provider address. A recovery\n // envelope without it is archive/config state, never permission to target a\n // provider inferred only from stale JSON. Conversely, a provider identity in\n // the envelope with no authoritative lease identity is inconsistent state:\n // fail closed instead of silently leaving a live provider behind while the\n // caller commits the lease cold.\n if (!lease.instanceId) {\n if (lease.providerCreateAttempt?.instanceId === null) {\n throw new Error(\n \"provider_create_outcome_unknown: cannot infer termination from missing instance ID\",\n );\n }\n if (persistedInstanceId) {\n throw new Error(\n `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,\n );\n }\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {\n throw new Error(\n `sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`,\n );\n }\n\n // A warming-death row can have a provider instance id recorded immediately\n // after create() but no resumable envelope yet. For Modal, instance_id is\n // enough to terminate directly; CAS-check first so a re-arm during the sweep\n // leaves the box running.\n const isUnpublishedWarmingProvider =\n backend === \"modal\" &&\n !persistedInstanceId &&\n lease.recovery.provider.status === \"creating\" &&\n lease.recovery.provider.instanceId === lease.instanceId &&\n lease.recovery.workspace.status === \"not_ready\";\n if (isUnpublishedWarmingProvider) {\n const { wrote } = await persistArchive(null);\n if (!wrote) {\n observability.info(\n \"sandbox reaper: Modal lease re-armed before direct terminate — leaving sandbox RUNNING\",\n {\n ...logIdentity,\n },\n );\n return { terminated: false, providerMissingBeforeCapture: false };\n }\n try {\n await beforeProviderStop?.();\n await terminateModalById(\n settings,\n lease.instanceId,\n undefined,\n lease.providerCreateAttempt?.providerBindingKey,\n );\n } catch (error) {\n if (!isProviderSandboxNotFoundError(\"modal\", error)) {\n throw error;\n }\n observability.info(\"sandbox reaper: Modal sandbox already gone during direct terminate\", {\n ...logIdentity,\n });\n }\n // This branch is a warming-death box recorded before any resumable envelope\n // was published. It was never a ready workspace, so a missing instance does\n // not imply loss of a previously exposed workspace revision.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // resume_state is the folded group box-envelope (the provider sessionState the\n // box was last persisted as). An attributed provider with no envelope may\n // still be live. Only Modal exposes a verified by-id rescue adapter above;\n // never cold another backend merely because generic resume is impossible.\n if (!persistedInstanceId || !lease.resumeState) {\n throw new Error(\n `sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`,\n );\n }\n\n const client = createClientForBackend(backend as never, settings) as\n | DrainSandboxClient\n | undefined;\n if (!client) {\n // 'none' backend resolved to no client.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n\n // Resume by id (warm reattach) — NO cold-restore. A NotFound here = the box is\n // already gone; success.\n let session: PersistableSession | undefined;\n let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;\n try {\n if (!client.deserializeSessionState || (backend !== \"docker\" && !client.resume)) {\n // A cloud backend that cannot prove provider state is not safely\n // terminable: treating this as success would cold the lease while the\n // provider may still be live. Leave it draining for a later retry.\n throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);\n }\n // resume_state is the lease ENVELOPE: `{ backendId, sessionState: {\n // providerState: { sandboxId, ... }, manifest, ... } }` (the shape\n // serializeEstablishedSandboxEnvelope folds onto the lease). The provider\n // payload deserializeSandboxSessionStateEnvelope re-hydrates is the INNER\n // `sessionState` — pass the WHOLE envelope and it reads `state.providerState`\n // at the top level, finds nothing (providerState is nested under\n // `sessionState`), drops `sandboxId`, and `client.resume(state)` throws\n // \"requires a persisted sandboxId\". This is exactly what the working\n // resume-by-id paths (establishSandboxSessionFromEnvelope) avoid: they unwrap\n // `envelope.sessionState` first. Mirror that. (`?? lease.resumeState` keeps a\n // legacy flat envelope — providerState at the top — resumable too.)\n const envelopeSessionState =\n (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;\n const resumedState = await deserializeSandboxSessionStateEnvelope(\n client as never,\n envelopeSessionState,\n lease.instanceId,\n );\n if (resumedState === undefined) {\n throw new Error(`sandbox backend ${backend} returned no resumable provider state`);\n }\n if (backend === \"docker\") {\n if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {\n throw new Error(\n \"Docker drain requires a creation-free attachment and current capture fence\",\n );\n }\n await beforeWorkspaceCapture();\n session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);\n sessionState = resumedState;\n } else {\n const resumed = await resumeExactSandboxSession(\n client,\n backend,\n resumedState,\n lease.instanceId,\n );\n session = resumed.session as PersistableSession;\n sessionState = resumed.sessionState;\n }\n } catch (error) {\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: drainable box already gone before workspace capture\", {\n ...logIdentity,\n });\n return {\n terminated: true,\n providerMissingBeforeCapture: captureDisposition !== \"archive_published\",\n };\n }\n // Re-throw a non-NotFound resume failure so the caller SKIPS (the lease stays\n // draining for the next sweep) — never cold a box we could not prove is gone.\n throw error;\n }\n\n const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);\n if (\n !liveCapturePolicy ||\n liveCapturePolicy.takeover !== durableCapturePolicy.takeover ||\n liveCapturePolicy.strategy !== durableCapturePolicy.strategy ||\n liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance\n ) {\n throw new Error(\n `sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`,\n );\n }\n\n // PERSIST /workspace BEFORE terminating (sandbox-file-persistence). A failure to\n // snapshot must re-throw BEFORE any terminate so files are never lost — the next\n // sweep retries, the provider idle-timeout is the backstop. A NotFound here (the\n // box raced gone between resume and persist) is success: nothing to persist.\n let verifiedArchive: VerifiedWorkspaceArchivePayload | undefined;\n const sessionWorkspacePersistence = (\n session as { state?: { workspacePersistence?: unknown } } | undefined\n )?.state?.workspacePersistence;\n const checkpointBinding =\n captureDisposition === \"capture_required\" &&\n backend === \"modal\" &&\n (sessionWorkspacePersistence === \"snapshot_filesystem\" ||\n sessionWorkspacePersistence === \"snapshot_directory\")\n ? await withRetainedProcessProbeTimeout(\n resolveModalCheckpointProviderBindingForSession(settings, session),\n )\n : null;\n // Physical capture bookkeeping for this worker: released once the capture\n // and its spool are settled, or by the late path that inherits them.\n let releaseSettlingCapture: () => void = () => undefined;\n try {\n if (captureDisposition !== \"archive_published\" && session?.persistWorkspace) {\n releaseSettlingCapture = trackSettlingDrainCapture(lease);\n // The budget bounds only this attempt's wait, never the capture: a slow\n // but finite capture keeps reading and publishes late, and the retry\n // that finds it still settling here defers instead of starting another.\n const capture = captureWorkspaceArchiveForStorage(\n session,\n Date.now(),\n {\n requestId: providerCaptureRequestId ?? randomUUID(),\n strategy: liveCapturePolicy.strategy,\n },\n diskBackedArchives,\n );\n verifiedArchive = await awaitProviderCaptureWithLatePublication({\n capture,\n timeoutMs: settings.sandboxSnapshotTimeoutMs,\n timeoutError: new SandboxProviderCaptureTimeoutError(\n lease.sandboxGroupId,\n backend,\n settings.sandboxSnapshotTimeoutMs,\n lease.leaseEpoch,\n lease.instanceId,\n ),\n publishLate: async (archive) => {\n try {\n const result = await persistArchive(\n archive.kind === \"host_spool\" ? archive : archive.base64,\n archive.descriptor,\n session,\n checkpointBinding,\n );\n observability.info(\n result.wrote\n ? \"sandbox reaper: late workspace capture published under durable claim\"\n : \"sandbox reaper: late workspace capture fenced by successor\",\n {\n ...logIdentity,\n },\n );\n } catch (error) {\n warnCaptureFailure(\"sandbox reaper: late workspace capture publication failed\", error);\n } finally {\n await disposeWorkspaceArchive(archive);\n }\n },\n observeLatePublicationFailure: (error) => {\n warnCaptureFailure(\"sandbox reaper: late capture publication cleanup failed\", error);\n },\n observeLateFailure: (error) => {\n // Capture has now rejected and this timed-out path can never reach\n // provider teardown. Release only our unpublished claim so a waiting\n // turn can re-arm the intact live box; stale callbacks are DB-fenced.\n void releaseAfterCaptureFailure();\n warnCaptureFailure(\"sandbox reaper: timed-out provider capture later failed\", error);\n },\n onLateSettled: () => releaseSettlingCapture(),\n });\n }\n } catch (error) {\n // Every rejection except the caller's own timeout means the capture has\n // settled; a timed-out capture is released by its late path instead.\n if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: box gone during workspace capture\", {\n ...logIdentity,\n });\n return { terminated: true, providerMissingBeforeCapture: true };\n }\n warnCaptureFailure(\n \"sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)\",\n error,\n );\n // An unresolved timeout is not proof of capture failure: retain the fence\n // until its provider promise settles. A settled failure cannot enter the\n // teardown path below, so its unpublished claim may safely be released.\n if (!(error instanceof SandboxProviderCaptureTimeoutError)) {\n await releaseAfterCaptureFailure();\n }\n // NEVER terminate a box whose snapshot we could not capture.\n throw error;\n }\n\n // Fold the captured archive onto the lease under the exact capture fence, then\n // GC the superseded snapshot. A CAS miss means this callback no longer owns\n // teardown — DO NOT terminate (the caller skips confirmDrainCold).\n //\n // Ownership guard for the no-archive path: even when persistWorkspace returns\n // no bytes, require the exact capture receipt before delete(). The null-archive\n // path performs that CAS without changing archive bytes; wrote:false aborts.\n if (captureDisposition === \"archive_published\") {\n observability.info(\"sandbox reaper: resuming teardown from durable archive publication\", {\n ...logIdentity,\n });\n } else if (verifiedArchive) {\n let wrote: boolean;\n try {\n ({ wrote } = await persistArchive(\n verifiedArchive.kind === \"host_spool\" ? verifiedArchive : verifiedArchive.base64,\n verifiedArchive.descriptor,\n session,\n checkpointBinding,\n ));\n } finally {\n try {\n await disposeWorkspaceArchive(verifiedArchive);\n } finally {\n releaseSettlingCapture();\n }\n }\n if (!wrote) {\n observability.info(\n \"sandbox reaper: lease re-armed during persist — leaving box RUNNING (no terminate)\",\n {\n ...logIdentity,\n },\n );\n return { terminated: false, providerMissingBeforeCapture: false };\n }\n } else {\n // A resumable cloud box that produces no verified archive cannot be safely\n // deleted: it may contain work newer than the lease's last durable revision.\n // Keep the lease draining and retry instead of knowingly discarding bytes.\n throw new Error(\n `sandbox backend ${backend} produced no verified workspace archive during drain`,\n );\n }\n\n // Provider terminate through the declared Agents SDK lifecycle. A terminate\n // that fails because the box is already gone is success.\n try {\n if (!session || !sessionState) {\n throw new Error(`sandbox backend ${backend} returned no terminable session state`);\n }\n prepareProviderForTeardownAfterCapture(backend, session);\n await beforeProviderStop?.();\n await terminateManagedSandboxSession(client, sessionState, session);\n if (backend === \"docker\") observeDockerWorkspaceRelease(observability, session, logIdentity);\n return { terminated: true, providerMissingBeforeCapture: false };\n } catch (error) {\n if (isProviderSandboxNotFoundError(client.backendId, error)) {\n observability.info(\"sandbox reaper: provider already gone during terminate\", {\n ...logIdentity,\n });\n // Capture was already durably folded above, so this is not a\n // missing-before-capture outcome.\n return { terminated: true, providerMissingBeforeCapture: false };\n }\n throw error;\n }\n}\n","import { Context } from \"@temporalio/activity\";\nimport {\n environmentsEncryptionKeyBytes,\n resolveFirstPartyDelegationSecret,\n type Settings,\n} from \"@opengeni/config\";\nimport { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from \"@opengeni/contracts\";\nimport {\n browserDeadlineCheckpoint,\n clearSuspendedBrowserSessionController,\n commitBrowserSessionSuspension,\n dispatchBrowserSessionOperation,\n failBrowserSessionSuspension,\n listBrowserDeadlineCheckpoints,\n readLease,\n releaseLeaseHolder,\n withRlsContext,\n type BrowserDeadlineCheckpointTarget,\n} from \"@opengeni/db\";\nimport { deriveBrowserControllerAdminToken } from \"@opengeni/interaction/browser-controller-authority\";\nimport {\n browserStateArtifactAad,\n browserStateManifestDigest,\n browserStateObjectKey,\n deriveBrowserStateDataKey,\n wrapBrowserStateDataKey,\n} from \"@opengeni/interaction/browser-state-authority\";\nimport {\n BrowserControlClient,\n BrowserControlRequestError,\n createSandboxClientForBackend,\n deserializeSandboxSessionStateEnvelope,\n resumeExactSandboxSession,\n type BrowserControlPlacementSession,\n} from \"@opengeni/runtime\";\nimport type { ControlActivityServices } from \"./types\";\n\nconst UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 60_000;\n\nfunction heartbeat(): void {\n try {\n Context.current().heartbeat();\n } catch {\n // Direct embedded calls have no Temporal context. In-flight controller\n // operations still settle under their exact durable database claim.\n }\n}\n\nasync function withActivityHeartbeat<T>(run: () => Promise<T>): Promise<T> {\n heartbeat();\n const timer = setInterval(heartbeat, 5_000);\n try {\n return await run();\n } finally {\n clearInterval(timer);\n }\n}\n\ntype CheckpointController = Pick<BrowserControlClient, \"captureState\" | \"endSession\">;\ntype CheckpointPlacement = {\n target: BrowserDeadlineCheckpointTarget;\n lease: NonNullable<Awaited<ReturnType<typeof readLease>>>;\n settings: Settings;\n controllerRoot: string;\n};\n\nasync function resumeCheckpointController({\n target,\n lease,\n settings,\n controllerRoot,\n}: CheckpointPlacement): Promise<CheckpointController> {\n // This edge cannot create a box or route through a user's current machine.\n const provider = createSandboxClientForBackend(\"modal\", settings) as Parameters<\n typeof deserializeSandboxSessionStateEnvelope\n >[0];\n const envelope = lease.resumeState as { sessionState?: unknown };\n const state = await deserializeSandboxSessionStateEnvelope(\n provider,\n envelope.sessionState ?? lease.resumeState,\n target.instanceId,\n );\n const resumed = await resumeExactSandboxSession(provider, \"modal\", state, target.instanceId);\n return new BrowserControlClient(resumed.session as BrowserControlPlacementSession, {\n adminToken: deriveBrowserControllerAdminToken({\n rootSecret: controllerRoot,\n accountId: target.accountId,\n workspaceId: target.workspaceId,\n placement: { kind: \"sandbox_group\", sandboxGroupId: target.sandboxGroupId },\n placementInstanceId: target.instanceId,\n }),\n });\n}\n\n/** Whether this worker can save browser profiles: ownership, object storage,\n * the controller delegation secret and the environments encryption key. */\nexport function browserCheckpointPrerequisites(\n settings: Settings,\n objectStorage: ControlActivityServices[\"objectStorage\"],\n): boolean {\n if (\n !settings.sandboxOwnershipEnabled ||\n !objectStorage ||\n !resolveFirstPartyDelegationSecret(settings)\n )\n return false;\n const root = environmentsEncryptionKeyBytes(settings);\n if (!root) return false;\n root.fill(0);\n return true;\n}\n\nexport function createBrowserDeadlineCheckpointActivities(\n services: () => Promise<ControlActivityServices>,\n connectController: (\n placement: CheckpointPlacement,\n ) => Promise<CheckpointController> = resumeCheckpointController,\n) {\n return {\n async listDueBrowserCheckpoints() {\n return await withActivityHeartbeat(async () => {\n const { db, settings, objectStorage } = await services();\n if (!browserCheckpointPrerequisites(settings, objectStorage)) return [];\n return await listBrowserDeadlineCheckpoints(db);\n });\n },\n async checkpointBrowserBeforeDeadline(target: BrowserDeadlineCheckpointTarget) {\n return await withActivityHeartbeat(async () => {\n const { db, settings, objectStorage } = await services();\n const rootKey = environmentsEncryptionKeyBytes(settings);\n const controllerRoot = resolveFirstPartyDelegationSecret(settings);\n if (!rootKey || !objectStorage || !controllerRoot) {\n rootKey?.fill(0);\n throw new Error(\"browser deadline checkpoint storage or encryption is unavailable\");\n }\n let dataKey: Buffer | null = null;\n let aad: Buffer | null = null;\n let timer: ReturnType<typeof setInterval> | null = null;\n try {\n // Only the reaper's locked idle decision prepares an idle save; this\n // activity continues it. A deadline save prepares here.\n const claim = await browserDeadlineCheckpoint(db, target, {\n prepare: target.reason !== \"idle\",\n touch: true,\n });\n if (!claim) return { status: \"skipped\" as const };\n let pulseRunning = false;\n let authorityLost = false;\n const pulse = async () => {\n if (pulseRunning) return;\n pulseRunning = true;\n try {\n if (!(await browserDeadlineCheckpoint(db, target, { touch: true })))\n authorityLost = true;\n } finally {\n pulseRunning = false;\n }\n };\n timer = setInterval(() => {\n void pulse().catch(() => undefined);\n }, 5_000);\n const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);\n if (\n !lease ||\n lease.instanceId !== target.instanceId ||\n lease.leaseEpoch !== target.leaseEpoch ||\n !lease.resumeState\n ) {\n throw new Error(\"browser deadline checkpoint placement changed\");\n }\n const placement = {\n kind: \"sandbox_group\" as const,\n sandboxGroupId: target.sandboxGroupId,\n };\n const client = await connectController({ target, lease, settings, controllerRoot });\n if (authorityLost || !(await browserDeadlineCheckpoint(db, target, { touch: true }))) {\n throw new Error(\"browser deadline checkpoint authority changed\");\n }\n if (claim.state !== \"completed\") {\n const scope = { ...target, operationId: claim.operationId };\n const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);\n dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });\n aad = browserStateArtifactAad({ ...scope, objectKey });\n await dispatchBrowserSessionOperation(db, {\n ...scope,\n deadlineTarget: target,\n stateUpload: {\n objectKey,\n cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS),\n },\n });\n const signed = await objectStorage.createPutUrl({\n key: objectKey,\n contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,\n audience: \"sandbox\",\n });\n const receipt = await client\n .captureState({\n ...scope,\n objectKey,\n afterCapture: \"stop\",\n dataKey,\n aad,\n upload: {\n url: signed.url,\n requiredHeaders: signed.requiredHeaders,\n expiresAt: signed.expiresAt.toISOString(),\n },\n })\n .catch(async (error: unknown) => {\n if (\n error instanceof BrowserControlRequestError &&\n (error.error.code === \"outcome_unknown\" || !error.error.retryable)\n ) {\n await failBrowserSessionSuspension(db, { ...scope, error: error.error });\n }\n throw error;\n });\n if (authorityLost) throw new Error(\"browser deadline checkpoint authority changed\");\n const portable = receipt.manifest.profileCrypto !== \"platform_bound\";\n await commitBrowserSessionSuspension(db, {\n ...scope,\n deadlineTarget: target,\n artifact: {\n kind: \"chromium_profile\",\n format: receipt.format,\n objectKey,\n artifactDigest: receipt.artifactDigest,\n contentDigest: receipt.contentDigest,\n manifestDigest: browserStateManifestDigest(receipt.manifest),\n sizeBytes: receipt.sizeBytes,\n encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {\n ...scope,\n objectKey,\n artifactDigest: receipt.artifactDigest,\n contentDigest: receipt.contentDigest,\n }),\n materialization: {\n portability: portable ? \"portable\" : \"placement_bound\",\n reason: portable\n ? null\n : \"Profile encryption depends on the source operating-system credential store.\",\n platform: receipt.manifest.platform,\n architecture: receipt.manifest.architecture,\n engine: receipt.manifest.engine,\n engineVersion: receipt.manifest.engineVersion,\n driverId: receipt.manifest.driverId,\n driverSchemaVersion: receipt.manifest.driverSchemaVersion,\n profileCrypto: receipt.manifest.profileCrypto,\n providerId: null,\n placement: portable ? null : placement,\n },\n },\n });\n }\n // Only remove the local profile after the encrypted artifact and its\n // authority committed. A crash resumes this cleanup, never capture.\n await client.endSession(target, { removeState: true }).catch((error: unknown) => {\n if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;\n });\n await withRlsContext(\n db,\n target,\n async (tx) => {\n if ((await browserDeadlineCheckpoint(tx, target))?.state !== \"completed\") {\n throw new Error(\"browser deadline checkpoint cleanup authority changed\");\n }\n await clearSuspendedBrowserSessionController(tx, {\n ...target,\n expectedControllerGeneration: target.controllerGeneration,\n });\n // One transaction prevents a human resume from acquiring a new\n // same-browser holder between clearing and releasing the old one.\n await releaseLeaseHolder(tx, {\n ...target,\n kind: \"interaction\",\n holderId: `browser-session:${target.browserSessionId}`,\n // The idle window already elapsed before this save began, so\n // a box the saved browser was last to hold drains now.\n idleGraceMs: target.reason === \"idle\" ? 0 : settings.sandboxIdleGraceMs,\n preserveHoldersChangedAt: target.reason === \"idle\",\n });\n },\n undefined,\n \"none\",\n );\n return { status: \"suspended\" as const };\n } finally {\n if (timer) clearInterval(timer);\n dataKey?.fill(0);\n aad?.fill(0);\n rootKey.fill(0);\n }\n });\n },\n };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAIA,SAAS,gCACd,eACA,OACA,SAIM;AACN,MAAI;AACJ,MAAI;AACF,UAAM,WAAW,uBAAuB,KAAK;AAC7C,UAAM,aAAa,uBAAuB,KAAK,EAAE;AACjD,oBAAgB,cAAc,wBAAwB;AAAA,MACpD,MACE,iBAAiB,sCACb,4BACA,WACE,oBAAoB,QAAQ,IAC5B;AAAA,MACR,OAAO;AAAA,MACP;AAAA,MACA,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,GAAI,QAAQ,iBAAiB,EAAE,WAAW,QAAQ,eAAe,IAAI,CAAC;AAAA,MACtE,GAAI,QAAQ,cAAc,EAAE,QAAQ,QAAQ,YAAY,IAAI,CAAC;AAAA,MAC7D,UAAU,QAAQ;AAAA,MAClB;AAAA,MACA,GAAI,aAAa,EAAE,WAAW,IAAI,CAAC;AAAA,IACrC,CAAC;AAAA,EACH,QAAQ;AAAA,EAER;AACA,MAAI;AACF,kBAAc,KAAK,4DAA4D;AAAA,MAC7E,GAAI,gBAAgB,EAAE,cAAc,IAAI,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH,QAAQ;AAAA,EAER;AACF;;;AC7CO,SAAS,+BACd,SAIA,OAIA,UACA,cACA,MAAM,KAAK,IAAI,GACP;AACR,MACE,QAAQ,oBAAoB,WAC5B,QAAQ,kBAAkB,QAC1B,CAAC,SACD,MAAM,OAAO,QAAQ,WACrB,MAAM,eAAe,QAAQ,cAC7B,MAAM,YAAY,QAAQ,mBAC1B,MAAM,eAAe,QAAQ,sBAC7B,CAAC,MAAM;AAEP,WAAO;AAET,QAAM,gBAAgB,MAAM,mBAAmB,QAAQ,IAAI,SAAS,wBAAwB;AAC5F,MAAI,CAAC,OAAO,SAAS,aAAa,EAAG,QAAO;AAC5C,SAAO,KAAK;AAAA,IACV;AAAA,IACA,KAAK,IAAI,KAAO,SAAS,4BAA4B,aAAa;AAAA,EACpE;AACF;;;ACdA,SAAS,YAAY,kBAAkB;AACvC,SAAS,kDAAkD;AAE3D,SAAS,WAAAA,gBAAe;AACxB,SAAS,cAAc,eAA8B;AACrD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,aAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAMK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,kCAAkC;AAE3C,SAAS,gCAAgC;AACzC;AAAA,EAME;AAAA,EACA;AAAA,EAGA;AAAA,EACA;AAAA,EACA,iCAAAC;AAAA,EACA;AAAA,EACA,0CAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA,6BAAAC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAQK;;;ACvKP,SAAS,eAAe;AACxB;AAAA,EACE;AAAA,EACA;AAAA,OAEK;AACP,SAAS,2CAA2C;AACpD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,yCAAyC;AAClD;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AAGP,IAAM,0BAA0B,KAAK,KAAK;AAE1C,SAAS,YAAkB;AACzB,MAAI;AACF,YAAQ,QAAQ,EAAE,UAAU;AAAA,EAC9B,QAAQ;AAAA,EAGR;AACF;AAEA,eAAe,sBAAyB,KAAmC;AACzE,YAAU;AACV,QAAM,QAAQ,YAAY,WAAW,GAAK;AAC1C,MAAI;AACF,WAAO,MAAM,IAAI;AAAA,EACnB,UAAE;AACA,kBAAc,KAAK;AAAA,EACrB;AACF;AAUA,eAAe,2BAA2B;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,GAAuD;AAErD,QAAM,WAAW,8BAA8B,SAAS,QAAQ;AAGhE,QAAM,WAAW,MAAM;AACvB,QAAM,QAAQ,MAAM;AAAA,IAClB;AAAA,IACA,SAAS,gBAAgB,MAAM;AAAA,IAC/B,OAAO;AAAA,EACT;AACA,QAAM,UAAU,MAAM,0BAA0B,UAAU,SAAS,OAAO,OAAO,UAAU;AAC3F,SAAO,IAAI,qBAAqB,QAAQ,SAA2C;AAAA,IACjF,YAAY,kCAAkC;AAAA,MAC5C,YAAY;AAAA,MACZ,WAAW,OAAO;AAAA,MAClB,aAAa,OAAO;AAAA,MACpB,WAAW,EAAE,MAAM,iBAAiB,gBAAgB,OAAO,eAAe;AAAA,MAC1E,qBAAqB,OAAO;AAAA,IAC9B,CAAC;AAAA,EACH,CAAC;AACH;AAIO,SAAS,+BACd,UACA,eACS;AACT,MACE,CAAC,SAAS,2BACV,CAAC,iBACD,CAAC,kCAAkC,QAAQ;AAE3C,WAAO;AACT,QAAM,OAAO,+BAA+B,QAAQ;AACpD,MAAI,CAAC,KAAM,QAAO;AAClB,OAAK,KAAK,CAAC;AACX,SAAO;AACT;AAEO,SAAS,0CACd,UACA,oBAEqC,4BACrC;AACA,SAAO;AAAA,IACL,MAAM,4BAA4B;AAChC,aAAO,MAAM,sBAAsB,YAAY;AAC7C,cAAM,EAAE,IAAI,UAAU,cAAc,IAAI,MAAM,SAAS;AACvD,YAAI,CAAC,+BAA+B,UAAU,aAAa,EAAG,QAAO,CAAC;AACtE,eAAO,MAAM,+BAA+B,EAAE;AAAA,MAChD,CAAC;AAAA,IACH;AAAA,IACA,MAAM,gCAAgC,QAAyC;AAC7E,aAAO,MAAM,sBAAsB,YAAY;AAC7C,cAAM,EAAE,IAAI,UAAU,cAAc,IAAI,MAAM,SAAS;AACvD,cAAM,UAAU,+BAA+B,QAAQ;AACvD,cAAM,iBAAiB,kCAAkC,QAAQ;AACjE,YAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,gBAAgB;AACjD,mBAAS,KAAK,CAAC;AACf,gBAAM,IAAI,MAAM,kEAAkE;AAAA,QACpF;AACA,YAAI,UAAyB;AAC7B,YAAI,MAAqB;AACzB,YAAI,QAA+C;AACnD,YAAI;AAGF,gBAAM,QAAQ,MAAM,0BAA0B,IAAI,QAAQ;AAAA,YACxD,SAAS,OAAO,WAAW;AAAA,YAC3B,OAAO;AAAA,UACT,CAAC;AACD,cAAI,CAAC,MAAO,QAAO,EAAE,QAAQ,UAAmB;AAChD,cAAI,eAAe;AACnB,cAAI,gBAAgB;AACpB,gBAAM,QAAQ,YAAY;AACxB,gBAAI,aAAc;AAClB,2BAAe;AACf,gBAAI;AACF,kBAAI,CAAE,MAAM,0BAA0B,IAAI,QAAQ,EAAE,OAAO,KAAK,CAAC;AAC/D,gCAAgB;AAAA,YACpB,UAAE;AACA,6BAAe;AAAA,YACjB;AAAA,UACF;AACA,kBAAQ,YAAY,MAAM;AACxB,iBAAK,MAAM,EAAE,MAAM,MAAM,MAAS;AAAA,UACpC,GAAG,GAAK;AACR,gBAAM,QAAQ,MAAM,UAAU,IAAI,OAAO,aAAa,OAAO,cAAc;AAC3E,cACE,CAAC,SACD,MAAM,eAAe,OAAO,cAC5B,MAAM,eAAe,OAAO,cAC5B,CAAC,MAAM,aACP;AACA,kBAAM,IAAI,MAAM,+CAA+C;AAAA,UACjE;AACA,gBAAM,YAAY;AAAA,YAChB,MAAM;AAAA,YACN,gBAAgB,OAAO;AAAA,UACzB;AACA,gBAAM,SAAS,MAAM,kBAAkB,EAAE,QAAQ,OAAO,UAAU,eAAe,CAAC;AAClF,cAAI,iBAAiB,CAAE,MAAM,0BAA0B,IAAI,QAAQ,EAAE,OAAO,KAAK,CAAC,GAAI;AACpF,kBAAM,IAAI,MAAM,+CAA+C;AAAA,UACjE;AACA,cAAI,MAAM,UAAU,aAAa;AAC/B,kBAAM,QAAQ,EAAE,GAAG,QAAQ,aAAa,MAAM,YAAY;AAC1D,kBAAM,YAAY,sBAAsB,OAAO,aAAa,MAAM,WAAW;AAC7E,sBAAU,0BAA0B,SAAS,EAAE,GAAG,OAAO,UAAU,CAAC;AACpE,kBAAM,wBAAwB,EAAE,GAAG,OAAO,UAAU,CAAC;AACrD,kBAAM,gCAAgC,IAAI;AAAA,cACxC,GAAG;AAAA,cACH,gBAAgB;AAAA,cAChB,aAAa;AAAA,gBACX;AAAA,gBACA,cAAc,IAAI,KAAK,KAAK,IAAI,IAAI,uBAAuB;AAAA,cAC7D;AAAA,YACF,CAAC;AACD,kBAAM,SAAS,MAAM,cAAc,aAAa;AAAA,cAC9C,KAAK;AAAA,cACL,aAAa;AAAA,cACb,UAAU;AAAA,YACZ,CAAC;AACD,kBAAM,UAAU,MAAM,OACnB,aAAa;AAAA,cACZ,GAAG;AAAA,cACH;AAAA,cACA,cAAc;AAAA,cACd;AAAA,cACA;AAAA,cACA,QAAQ;AAAA,gBACN,KAAK,OAAO;AAAA,gBACZ,iBAAiB,OAAO;AAAA,gBACxB,WAAW,OAAO,UAAU,YAAY;AAAA,cAC1C;AAAA,YACF,CAAC,EACA,MAAM,OAAO,UAAmB;AAC/B,kBACE,iBAAiB,+BAChB,MAAM,MAAM,SAAS,qBAAqB,CAAC,MAAM,MAAM,YACxD;AACA,sBAAM,6BAA6B,IAAI,EAAE,GAAG,OAAO,OAAO,MAAM,MAAM,CAAC;AAAA,cACzE;AACA,oBAAM;AAAA,YACR,CAAC;AACH,gBAAI,cAAe,OAAM,IAAI,MAAM,+CAA+C;AAClF,kBAAM,WAAW,QAAQ,SAAS,kBAAkB;AACpD,kBAAM,+BAA+B,IAAI;AAAA,cACvC,GAAG;AAAA,cACH,gBAAgB;AAAA,cAChB,UAAU;AAAA,gBACR,MAAM;AAAA,gBACN,QAAQ,QAAQ;AAAA,gBAChB;AAAA,gBACA,gBAAgB,QAAQ;AAAA,gBACxB,eAAe,QAAQ;AAAA,gBACvB,gBAAgB,2BAA2B,QAAQ,QAAQ;AAAA,gBAC3D,WAAW,QAAQ;AAAA,gBACnB,kBAAkB,wBAAwB,SAAS,SAAS;AAAA,kBAC1D,GAAG;AAAA,kBACH;AAAA,kBACA,gBAAgB,QAAQ;AAAA,kBACxB,eAAe,QAAQ;AAAA,gBACzB,CAAC;AAAA,gBACD,iBAAiB;AAAA,kBACf,aAAa,WAAW,aAAa;AAAA,kBACrC,QAAQ,WACJ,OACA;AAAA,kBACJ,UAAU,QAAQ,SAAS;AAAA,kBAC3B,cAAc,QAAQ,SAAS;AAAA,kBAC/B,QAAQ,QAAQ,SAAS;AAAA,kBACzB,eAAe,QAAQ,SAAS;AAAA,kBAChC,UAAU,QAAQ,SAAS;AAAA,kBAC3B,qBAAqB,QAAQ,SAAS;AAAA,kBACtC,eAAe,QAAQ,SAAS;AAAA,kBAChC,YAAY;AAAA,kBACZ,WAAW,WAAW,OAAO;AAAA,gBAC/B;AAAA,cACF;AAAA,YACF,CAAC;AAAA,UACH;AAGA,gBAAM,OAAO,WAAW,QAAQ,EAAE,aAAa,KAAK,CAAC,EAAE,MAAM,CAAC,UAAmB;AAC/E,gBAAI,EAAE,iBAAiB,8BAA8B,MAAM,WAAW,KAAM,OAAM;AAAA,UACpF,CAAC;AACD,gBAAM;AAAA,YACJ;AAAA,YACA;AAAA,YACA,OAAO,OAAO;AACZ,mBAAK,MAAM,0BAA0B,IAAI,MAAM,IAAI,UAAU,aAAa;AACxE,sBAAM,IAAI,MAAM,uDAAuD;AAAA,cACzE;AACA,oBAAM,uCAAuC,IAAI;AAAA,gBAC/C,GAAG;AAAA,gBACH,8BAA8B,OAAO;AAAA,cACvC,CAAC;AAGD,oBAAM,mBAAmB,IAAI;AAAA,gBAC3B,GAAG;AAAA,gBACH,MAAM;AAAA,gBACN,UAAU,mBAAmB,OAAO,gBAAgB;AAAA;AAAA;AAAA,gBAGpD,aAAa,OAAO,WAAW,SAAS,IAAI,SAAS;AAAA,gBACrD,0BAA0B,OAAO,WAAW;AAAA,cAC9C,CAAC;AAAA,YACH;AAAA,YACA;AAAA,YACA;AAAA,UACF;AACA,iBAAO,EAAE,QAAQ,YAAqB;AAAA,QACxC,UAAE;AACA,cAAI,MAAO,eAAc,KAAK;AAC9B,mBAAS,KAAK,CAAC;AACf,eAAK,KAAK,CAAC;AACX,kBAAQ,KAAK,CAAC;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ADtEA,SAAS,4BAAAC,iCAAgC;AAwLlC,IAAM,wCAAwC;AAC9C,IAAM,+CAA+C,IAAI;AACzD,IAAM,6CAA6C;AAInD,IAAM,0CAA0C;AAChD,IAAM,0CAA0C;AAChD,IAAM,gDAAgD;AAItD,IAAM,wCAAwC,MAAM;AACpD,IAAM,qDAAqD;AAC3D,IAAM,+CAA+C,KAAK,KAAK;AAC/D,IAAM,yCAAyC;AAC/C,IAAM,gDAAgD,IAAI;AAE1D,IAAM,qCAAN,cAAiD,MAAM;AAAA,EAG5D,YACkB,gBACA,SACA,WACA,YACA,YAChB;AACA;AAAA,MACE,WAAW,cAAc,UAAU,UAAU,IAAI,OAAO,aAAa,UAAU,+BAC/C,SAAS;AAAA,IAE3C;AAVgB;AACA;AACA;AACA;AACA;AAAA,EAOlB;AAAA,EAdS,OAAO;AAelB;AAEA,eAAsB,wCAA2C,OAUlD;AACb,MAAI;AACJ,QAAM,eAAe,MAAM;AAC3B,MAAI;AACF,WAAO,MAAM,QAAQ,KAAK;AAAA,MACxB,MAAM;AAAA,MACN,IAAI,QAAe,CAAC,GAAG,WAAW;AAChC,kBAAU,WAAW,MAAM,OAAO,YAAY,GAAG,MAAM,SAAS;AAAA,MAClE,CAAC;AAAA,IACH,CAAC;AAAA,EACH,SAAS,OAAO;AACd,QAAI,UAAU,aAAc,OAAM;AAMlC,UAAM,UAAU,MAAM;AACpB,UAAI;AACF,cAAM,gBAAgB;AAAA,MACxB,QAAQ;AAAA,MAER;AAAA,IACF;AACA,SAAK,MAAM,QACR,KAAK,MAAM,aAAa,MAAM,kBAAkB,EAChD,MAAM,MAAM,6BAA6B,EACzC,KAAK,SAAS,OAAO;AACxB,UAAM;AAAA,EACR,UAAE;AACA,QAAI,QAAS,cAAa,OAAO;AAAA,EACnC;AACF;AAEA,SAAS,4BAA4B,SAA6D;AAChG,MAAI;AACJ,MAAI;AACF,cAAUC,SAAQ,QAAQ;AAAA,EAC5B,QAAQ;AAEN,WAAO,MAAM;AAAA,EACf;AACA,QAAMC,aAAY,MAAY;AAC5B,QAAI;AACF,cAAQ,UAAU,EAAE,GAAG,SAAS,cAAa,oBAAI,KAAK,GAAE,YAAY,EAAE,CAAC;AAAA,IACzE,QAAQ;AAAA,IAMR;AAAA,EACF;AACA,EAAAA,WAAU;AACV,QAAM,QAAQ,YAAYA,YAAW,6CAA6C;AAClF,MAAI,WAAW,SAAS,OAAO,MAAM,UAAU,WAAY,OAAM,MAAM;AACvE,SAAO,MAAM,cAAc,KAAK;AAClC;AAqBA,IAAM,gCAAgC,KAAK;AAAA,EACzC;AAAA,EACA,KAAK,MAAM,2CAA2C,CAAC;AACzD;AAoBA,IAAM,oCAAN,cAAgD,MAAM;AAAA,EAC3C,OAAO;AAClB;AAgBA,IAAM,wBAAwB,oBAAI,IAAqC;AAQvE,SAAS,gBAAgB,OAA8D;AACrF,SAAO,KAAK,UAAU,CAAC,MAAM,gBAAgB,MAAM,YAAY,MAAM,UAAU,CAAC;AAClF;AAIA,SAAS,qCACP,OACA,QAAQ,KAAK,IAAI,GACR;AACT,MAAI,CAAC,MAAM,WAAY,QAAO;AAC9B,QAAM,QAAQ,sBAAsB;AAAA,IAClC,gBAAgB,EAAE,GAAG,OAAO,YAAY,MAAM,WAAW,CAAC;AAAA,EAC5D;AACA,SAAO,UAAU,UAAa,QAAQ,MAAM,cAAc;AAC5D;AAEA,SAAS,0BAA0B,OAAyC;AAC1E,MAAI,CAAC,MAAM,WAAY,QAAO,MAAM;AACpC,QAAM,MAAM,gBAAgB,EAAE,GAAG,OAAO,YAAY,MAAM,WAAW,CAAC;AACtE,QAAM,QAAQ,EAAE,aAAa,KAAK,IAAI,EAAE;AACxC,wBAAsB,IAAI,KAAK,KAAK;AACpC,SAAO,MAAM;AACX,QAAI,sBAAsB,IAAI,GAAG,MAAM,MAAO,uBAAsB,OAAO,GAAG;AAAA,EAChF;AACF;AAKO,SAAS,sBAAsB,aAAqB,SAAyB;AAClF,MAAI,CAAC,kEAAkE,KAAK,WAAW,GAAG;AACxF,UAAM,IAAI,MAAM,uCAAuC;AAAA,EACzD;AACA,MAAI,CAAC,OAAO,cAAc,OAAO,KAAK,WAAW,GAAG;AAClD,UAAM,IAAI,MAAM,2CAA2C;AAAA,EAC7D;AACA,MAAI,YAAY,EAAG,QAAO,YAAY,YAAY;AAClD,QAAM,QAAQ,WAAW,QAAQ,EAC9B,OAAO,0BAA0B,YAAY,YAAY,CAAC,IAAI,OAAO,EAAE,EACvE,OAAO,EACP,SAAS,GAAG,EAAE;AACjB,QAAM,CAAC,IAAK,MAAM,CAAC,IAAK,KAAQ;AAChC,QAAM,CAAC,IAAK,MAAM,CAAC,IAAK,KAAQ;AAChC,QAAM,MAAM,MAAM,SAAS,KAAK;AAChC,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AAC1G;AAEA,SAAS,8BAAsC;AAC7C,MAAI;AACF,UAAM,UAAUD,SAAQ,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,cAAc,OAAO,KAAK,UAAU,IAAI,UAAU;AAAA,EAClE,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,SAAS,6BACd,UACA,UAAuC,CAAC,GACxC;AACA,QAAM,gBACH,QAAQ,eACL,UAAU,SAAqC;AAG7C,UAAM,KAAK,EAAE,IAAI;AACjB,WAAO,QAAQ,aAAc,GAAG,IAAI;AAAA,EACtC,IACA,UACH,OACC,UACA,OACA,eACA,gBACA,0BACA,oBACA,eACA,oBACA,sBACA,aACA,oBACA,2BAEA,MAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,0BAA0BE;AAAA,IAClC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACJ,QAAM,oBACJ,QAAQ,qBAAqB;AAC/B,QAAM,8BACJ,QAAQ,yCAAyC;AACnD,QAAM,uBAAuB,QAAQ,wBAAwB;AAC7D,QAAM,yBAAyB,QAAQ,0BAA0B;AACjE,QAAM,8BACJ,QAAQ,+BAA+B;AAEzC,QAAM,0BAA0B,oBAAI,IAAY;AAChD,QAAM,yBAAiD,oBAAI,IAAI;AAC/D,iBAAe,2BAA2D;AACxE,UAAM,gBAAgB,4BAA4B,EAAE,OAAO,UAAU,CAAC;AACtE,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,cAAc,IAAI,MAAM,SAAS;AACtE,YAAM,SAAS,mBAAmB,QAAQ;AAC1C,YAAM,qBAAqB;AAAA;AAAA;AAAA;AAAA;AAAA,QAKzB,0BAA0B,KAAK,IAAI,SAAS,oBAAoB,GAAM;AAAA,QACtE,wBAAwB,+BAA+B,UAAU,aAAa;AAAA,QAC9E,0BAA0B,CACxB,YACG,oCAAoC,eAAe,OAAO;AAAA,QAC/D,0BAA0B,SAAS;AAAA,QACnC,gCAAgC,sCAAsC,QAAQ;AAAA,QAC9E,sBAAsB,CAAC,YACrB,gCAAgC,eAAe,OAAO;AAAA,QACxD,2BAA2B,CAAC,UAAmB;AAC7C,wBAAc,KAAK,kEAAkE;AAAA,YACnF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,UAAI,CAAC,SAAS,yBAAyB;AAKrC,cAAMC,sBAAqB,MAAM,4BAA4B,IAAI;AAAA,UAC/D,GAAG;AAAA,UACH,mBAAmB,SAAS;AAAA,UAC5B,iBAAiB,SAAS;AAAA,UAC1B,wBAAwB,SAAS;AAAA,UACjC,aAAa,SAAS;AAAA,QACxB,CAAC;AACD,cAAMC,aAAYD,oBAAmB,MAAM,GAAG,mCAAmC;AACjF,eAAO;AAAA,UACL,WAAAC;AAAA,UACA,cAAc,OAAO;AAAA,UACrB,mBAAmB,OAAO;AAAA,UAC1B,kBAAkB,OAAO;AAAA,UACzB,oBAAoB;AAAA,QACtB;AAAA,MACF;AAOA,YAAM,qBAAqB,MAAM;AAAA,QAC/B;AAAA,QACA,SAAS;AAAA,QACT,SAAS;AAAA,MACX,EAAE,MAAM,CAAC,UAAU;AACjB,sBAAc,KAAK,6DAA6D;AAAA,UAC9E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,eAAO;AAAA,MACT,CAAC;AACD,8CAAwC,eAAe,kBAAkB;AAKzE,YAAM,qBAAqB,MAAM,4BAA4B,IAAI;AAAA,QAC/D,GAAG;AAAA,QACH,mBAAmB,SAAS;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,QAS5B,iBAAiB,SAAS;AAAA,QAC1B,wBAAwB,SAAS;AAAA,QACjC,aAAa,SAAS;AAAA,MACxB,CAAC;AACD,YAAM,YAAY,mBAAmB,MAAM,GAAG,mCAAmC;AACjF,UAAI,mBAAmB,SAAS,UAAU,QAAQ;AAChD,sBAAc,KAAK,wDAAwD;AAAA,UACzE,UAAU,UAAU;AAAA,UACpB,UAAU,mBAAmB,SAAS,UAAU;AAAA,QAClD,CAAC;AAAA,MACH;AACA,aAAO;AAAA,QACL;AAAA,QACA,cAAc,OAAO;AAAA,QACrB,mBAAmB,OAAO;AAAA,QAC1B,kBAAkB,OAAO;AAAA,QACzB;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,6BAAiE;AAC9E,UAAM,gBAAgB,4BAA4B,EAAE,OAAO,4BAA4B,CAAC;AACxF,QAAI;AACF,YAAM,EAAE,IAAI,SAAS,IAAI,MAAM,SAAS;AAGxC,YAAM,eAAe,yBAAyB,+BAA+B,QAAQ,CAAC;AACtF,UAAI,CAAC,SAAS,2BAA2B,SAAS,6BAA6B,GAAG;AAChF,eAAO,EAAE,SAAS,CAAC,GAAG,aAAa;AAAA,MACrC;AACA,YAAM,OAAO,MAAM,6BAA6B,IAAI;AAAA,QAClD,OAAO;AAAA,QACP,YAAY,SAAS;AAAA,MACvB,CAAC;AACD,aAAO;AAAA,QACL,SAAS,KAAK,IAAI,CAAC,EAAE,WAAW,YAAY,GAAG,OAAO,MAAM,MAAM;AAAA,QAClE;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,2BACb,OAC8C;AAC9C,UAAM,SAAS,MAAM;AACrB,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,aAAa,OAAO;AAAA,MACpB,gBAAgB,OAAO;AAAA,MACvB,YAAY,OAAO;AAAA,MACnB,YAAY,OAAO;AAAA,IACrB,CAAC;AACD,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,cAAc,IAAI,MAAM,SAAS;AACtE,YAAM,SAAS;AAAA,QACb,iBAAiB,yBAAyB,OAAO,aAAa,OAAO,cAAc;AAAA,QACnF,YAAY,OAAO;AAAA,MACrB;AACA,YAAM,UAAU,CAAC,WAAwD;AACvE,sBAAc,KAAK,2CAA2C,EAAE,GAAG,QAAQ,OAAO,CAAC;AACnF,eAAO,EAAE,QAAQ,WAAW,OAAO;AAAA,MACrC;AACA,UAAI,CAAC,SAAS,2BAA2B,SAAS,6BAA6B,GAAG;AAChF,eAAO,QAAQ,UAAU;AAAA,MAC3B;AACA,YAAM,QAAQ,MAAMC,WAAU,IAAI,OAAO,aAAa,OAAO,cAAc;AAC3E,UACE,CAAC,SACD,MAAM,aAAa,UACnB,MAAM,eAAe,OAAO,cAC5B,MAAM,eAAe,OAAO,YAC5B;AACA,eAAO,QAAQ,cAAc;AAAA,MAC/B;AAGA,UAAI,MAAM,cAAc,EAAG,QAAO,QAAQ,WAAW;AACrD,UAAI,MAAM,aAAa,EAAG,QAAO,QAAQ,aAAa;AACtD,YAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,OAAO,WAAW;AAGzE,YAAM,UAAU,MAAM,4BAA4B,IAAI;AAAA,QACpD;AAAA,QACA,aAAa,OAAO;AAAA,QACpB,gBAAgB,OAAO;AAAA,QACvB,eAAe,OAAO;AAAA,QACtB,oBAAoB,OAAO;AAAA,MAC7B,CAAC;AACD,UAAI,CAAC,QAAS,QAAO,QAAQ,cAAc;AAC3C,UAAI,CAAC,sCAAsC,MAAM,SAAS,MAAM,WAAW,GAAG;AAC5E,eAAO,QAAQ,gBAAgB;AAAA,MACjC;AACA,UAAI,MAAM,mBAAmB,CAAC,6BAA6B,KAAK,EAAG,QAAO,QAAQ,OAAO;AACzF,UAAI,MAAM,gBAAgB;AACxB,cAAM,WAAW,MAAM,+BAA+B,IAAI;AAAA,UACxD;AAAA,UACA,aAAa,OAAO;AAAA,UACpB,gBAAgB,OAAO;AAAA,UACvB,WAAW,MAAM,eAAe;AAAA,UAChC,eAAe,OAAO;AAAA,UACtB,oBAAoB,OAAO;AAAA,UAC3B,yBAAyB;AAAA,QAC3B,CAAC;AACD,YAAI,CAAC,SAAU,QAAO,QAAQ,qBAAqB;AACnD,sBAAc,KAAK,0DAA0D;AAAA,UAC3E,GAAG;AAAA,UACH,WAAW,MAAM,eAAe;AAAA,QAClC,CAAC;AAAA,MACH;AACA,UAAI;AACJ,UAAI;AACF,kBAAU,MAAM,4BAA4B,UAAU,KAAK;AAAA,MAC7D,SAAS,OAAO;AACd,sBAAc,KAAK,iDAAiD;AAAA,UAClE,GAAG;AAAA,UACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,eAAO,QAAQ,gBAAgB;AAAA,MACjC;AACA,UAAI,YAAY,KAAM,QAAO,QAAQ,kBAAkB;AACvD,YAAM,UAAU;AAAA,QACd,EAAE,IAAI,UAAU,eAAe,cAAc;AAAA,QAC7C;AAAA,UACE;AAAA,UACA,aAAa,OAAO;AAAA,UACpB,gBAAgB,OAAO;AAAA,UACvB,gBAAgB;AAAA,QAClB;AAAA,QACA;AAAA,QACA,OAAO;AAAA,MACT;AACA,YAAM,YAAY,MAAM;AAGxB,YAAM,QAAQ;AACd,UAAI,CAAC,UAAW,QAAO,QAAQ,cAAc;AAC7C,oBAAc,KAAK,6CAA6C,MAAM;AACtE,aAAO,EAAE,QAAQ,eAAe;AAAA,IAClC,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAEA,iBAAe,kBACb,OACqC;AACrC,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,aAAa,MAAM,OAAO;AAAA,MAC1B,gBAAgB,MAAM,OAAO;AAAA,MAC7B,YAAY,MAAM,OAAO;AAAA,MACzB,YAAY,MAAM,OAAO;AAAA,IAC3B,CAAC;AACD,QAAI;AACF,YAAM,EAAE,IAAI,UAAU,eAAe,eAAe,IAAI,IAAI,MAAM,SAAS;AAC3E,oCAA8B,KAAK;AACnC,YAAM,gBACJ,SAAS,6BAA6B,MAAM,oBACxC,WACA,EAAE,GAAG,UAAU,0BAA0B,MAAM,kBAAkB;AACvE,YAAM,MAAM,MAAM;AAClB,YAAM,kBAAkB,4BAA4B;AACpD,YAAM,iBAA6C;AAAA,QACjD,aAAa,MAAM;AAAA,QACnB,WAAW,sBAAsB,MAAM,aAAa,eAAe;AAAA,QACnE,SAAS;AAAA,QACT,kBAAkB,MAAM;AAAA,MAC1B;AACA,UAAI;AACF,cAAM,cAAc,MAAM;AAAA,UACxB;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,eAAO,EAAE,QAAQ,cAAc,eAAe,UAAU;AAAA,MAC1D,SAAS,OAAO;AACd,cAAM,QAAQ,MAAMA,WAAU,IAAI,IAAI,aAAa,IAAI,cAAc,EAAE,MAAM,MAAM,IAAI;AACvF,cAAM,UAAU;AAAA,UACd,iBAAiB,yBAAyB,IAAI,aAAa,IAAI,cAAc;AAAA,UAC7E,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,YAAY,IAAI;AAAA,UAChB,SAAS,OAAO,WAAW;AAAA,UAC3B,YAAY,OAAO,cAAc,IAAI;AAAA,UACrC,WAAW,OAAO,gBAAgB,MAAM;AAAA,UACxC,oBAAoB,OAAO,gBAAgB,eAAe,MAAM;AAAA,UAChE,gBAAgB,OAAO,gBAAgB,WAAW;AAAA,UAClD,cAAc,MAAM;AAAA,UACpB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D;AACA,YAAI,iBAAiB,mCAAmC;AACtD,wBAAc,KAAK,wDAAwD,OAAO;AAAA,QACpF,OAAO;AACL,wBAAc,KAAK,4CAA4C,OAAO;AAAA,QACxE;AACA,cAAM;AAAA,MACR;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAEA,iBAAe,0BACb,OAC6C;AAC7C,UAAM,gBAAgB,4BAA4B;AAAA,MAChD,OAAO;AAAA,MACP,UAAU,MAAM;AAAA,MAChB,SAAS,MAAM;AAAA,IACjB,CAAC;AACD,QAAI;AACF,YAAM,UAAU,MAAM,SAAS;AAC/B,YAAM,EAAE,IAAI,UAAU,cAAc,IAAI;AACxC,YAAM,gBAAgB,MAAM,oCAAoC,SAAS,GAAG,EAAE;AAAA,QAC5E,CAAC,UAAU;AACT,wBAAc,KAAK,8CAA8C;AAAA,YAC/D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO,EAAE,SAAS,GAAG,WAAW,GAAG,QAAQ,EAAE;AAAA,QAC/C;AAAA,MACF;AACA,YAAM,8BAA8B,MAAM,gCAAgC,IAAI,GAAG,EAAE;AAAA,QACjF,CAAC,UAAU;AACT,wBAAc,KAAK,+CAA+C;AAAA,YAChE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,8BAA8B,GAAG;AACnC,sBAAc,KAAK,yCAAyC;AAAA,UAC1D,SAAS;AAAA,QACX,CAAC;AAAA,MACH;AACA,UAAI,cAAc,UAAU,GAAG;AAC7B,sBAAc,KAAK,mCAAmC,aAAa;AAAA,MACrE;AAEA,UAAI,UAAU;AACd,UAAI,eAAe;AACnB,YAAM,qBAAqB,MAAM;AACjC,UAAI,yBAAyB;AAK7B,YAAM,4CAA4C,IAAI,UAAU,eAAe,QAAQ,GAAG;AAC1F,YAAM,wCAAwC,IAAI,UAAU,eAAe,QAAQ,GAAG;AAKtF,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ;AAAA,QACV;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,0DAA0D;AAAA,UAC3E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAEA,UAAI,CAAC,SAAS,yBAAyB;AAGrC,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AACA,eAAO;AAAA,UACL;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,QACF;AAAA,MACF;AAKA,YAAM,cAAc,MAAM,eAAe,IAAI,UAAU,aAAa;AACpE,gBAAU,YAAY;AACtB,YAAM,yBAAyB,oBAAI,IAAY;AAC/C,UACE,SAAS,2BAA2B,aACpC,SAAS,oCAAoC,GAC7C;AACA,mBAAW,eAAe,YAAY,cAAc;AAClD,iCAAuB,IAAI,WAAW;AAAA,QACxC;AAAA,MACF;AACA,UAAI;AACF,mBAAW,eAAe,MAAM,wCAAwC,EAAE,GAAG;AAC3E,iCAAuB,IAAI,WAAW;AAAA,QACxC;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AACA,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAKA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,iCAAiC;AAAA,QACzC,QAAQ;AAAA,MACV;AAIA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA,QAAQ,iCAAiC;AAAA,QACzC;AAAA,MACF;AAEA,UAAI;AACF,iCAAyB,MAAM,kBAAkB,UAAU,IAAI,aAAa;AAC5E,uCAA+B,eAAe,sBAAsB;AAAA,MACtE,SAAS,OAAO;AACd,sBAAc,KAAK,6CAA6C;AAAA,UAC9D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAEA,YAAM,eAAe,MAAM,6BAA6B,IAAI,UAAU,aAAa,EAAE;AAAA,QACnF,CAAC,UAAU;AACT,wBAAc,KAAK,iDAAiD;AAAA,YAClE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,iBAAO,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ,EAAE;AAAA,QAC7C;AAAA,MACF;AAEA,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAEA,UACE,MAAM,WAAW,KACjB,UAAU,KACV,eAAe,KACf,yBAAyB,KACzB,qBAAqB,KACrB,aAAa,UAAU,GACvB;AACA,sBAAc,KAAK,6BAA6B;AAAA,UAC9C,WAAW,MAAM;AAAA,UACjB,iBAAiB,MAAM;AAAA,UACvB,wBAAwB,MAAM;AAAA,UAC9B,qBAAqB,MAAM;AAAA,UAC3B;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,4BAA4B,aAAa;AAAA,UACzC,4BAA4B,aAAa;AAAA,UACzC,2BAA2B,aAAa;AAAA,QAC1C,CAAC;AAAA,MACH;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF,UAAE;AACA,oBAAc;AAAA,IAChB;AAAA,EACF;AAOA,iBAAe,oBAAsD;AACnE,UAAM,UAAU,MAAM,SAAS;AAC/B,QAAI,QAAQ,4BAA4B;AACtC,YAAM,WAAW,MAAM,QAAQ,2BAA2B;AAC1D,cAAQ,cAAc,KAAK,+CAA+C;AAAA,QACxE;AAAA,MACF,CAAC;AACD,aAAO;AAAA,QACL,UAAU;AAAA,QACV,YAAY;AAAA,QACZ,SAAS;AAAA,QACT,SAAS;AAAA,QACT,cAAc;AAAA,QACd,wBAAwB;AAAA,MAC1B;AAAA,IACF;AAEA,UAAM,OAAO,MAAM,yBAAyB;AAC5C,UAAM,WAAW,KAAK,UAAU,MAAM,GAAG,wCAAwC;AACjF,UAAM,WAAW,MAAM,QAAQ;AAAA,MAC7B,SAAS;AAAA,QAAI,CAAC,WACZ,kBAAkB;AAAA,UAChB;AAAA,UACA,cAAc,KAAK;AAAA,UACnB,mBAAmB,KAAK;AAAA,UACxB,kBAAkB,KAAK;AAAA,UACvB,aAAa,WAAW;AAAA,QAC1B,CAAC;AAAA,MACH;AAAA,IACF;AAKA,QAAI;AACF,YAAM,EAAE,SAAS,aAAa,IAAI,MAAM,2BAA2B;AACnE,iBAAW,UAAU,SAAS;AAC5B,cAAM,MAAM,gCAAgC,MAAM;AAClD,YAAI,wBAAwB,IAAI,GAAG,EAAG;AACtC,gCAAwB,IAAI,GAAG;AAC/B,aAAK,2BAA2B,EAAE,QAAQ,aAAa,CAAC,EACrD,MAAM,CAAC,UAAmB;AACzB,kBAAQ,cAAc,KAAK,0CAA0C;AAAA,YACnE,iBAAiB,yBAAyB,OAAO,aAAa,OAAO,cAAc;AAAA,YACnF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH,CAAC,EACA,QAAQ,MAAM,wBAAwB,OAAO,GAAG,CAAC;AAAA,MACtD;AAAA,IACF,SAAS,OAAO;AACd,cAAQ,cAAc,KAAK,oDAAoD;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AACA,UAAM,aAAa,SAAS;AAAA,MAC1B,CAAC,YAAY,QAAQ,WAAW,eAAe,QAAQ,MAAM,WAAW;AAAA,IAC1E,EAAE;AACF,UAAM,UAAU,SAAS,SAAS;AAClC,UAAM,cAAc,MAAM,0BAA0B;AAAA,MAClD,UAAU,SAAS;AAAA,MACnB,SAAS,SAAS;AAAA,MAClB,gBAAgB;AAAA,MAChB,aAAa;AAAA,MACb,oBAAoB,KAAK;AAAA,IAC3B,CAAC;AACD,WAAO;AAAA,MACL,UAAU,SAAS;AAAA,MACnB;AAAA,MACA;AAAA,MACA,SAAS,YAAY;AAAA,MACrB,cAAc,YAAY;AAAA,MAC1B,wBAAwB,YAAY;AAAA,IACtC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAMA,eAAsB,sCACpB,UACA,OACyB;AACzB,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,MAAI,YAAY,WAAW,CAAC,MAAM,cAAc,CAAC,MAAM,aAAa;AAClE,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,sCAAsC,MAAM,WAAW,MAAM,MAAM,YAAY;AACjF,UAAM,IAAI,MAAM,oEAAoE;AAAA,EACtF;AACA,QAAM,SAASH,+BAA8B,SAAkB,QAAQ;AAGvE,MAAI,CAAC,QAAQ,QAAQ;AACnB,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI;AACF,UAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,UAAM,eAAe,MAAMI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA,QAAI,iBAAiB,QAAW;AAC9B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,UAAM,UAAU,MAAM;AAAA,MACpBC,2BAA0B,QAAQ,SAAS,cAAc,MAAM,UAAU;AAAA,IAC3E;AACA,WAAO,QAAQ;AAAA,EACjB,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,YAAM,IAAI,MAAM,6CAA6C,EAAE,OAAO,MAAM,CAAC;AAAA,IAC/E;AACA,QAAI,+BAA+B,OAAO,WAAW,KAAK,EAAG,QAAO;AACpE,UAAM;AAAA,EACR;AACF;AAEA,IAAM,sBAAsB;AAC5B,IAAM,6BAA6B,KAAK;AAEjC,IAAM,qCAAqC;AAClD,IAAM,oCAAoC,KAAK,KAAK,KAAK;AACzD,IAAM,mCAAmC;AACzC,IAAM,uCAAuC;AAkB7C,SAAS,qCACP,eACA,SACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAUA,IAAM,mCAAmC,oBAAI,IAAI;AAAA,EAC/C;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAaD,eAAe,iCACb,IACA,OACA,OACA,SAC2C;AAC3C,MAAI,EAAE,iBAAiB,2BAA2B,CAAC,MAAM,gBAAgB,CAAC,MAAM,WAAW;AACzF,WAAO;AAAA,EACT;AACA,QAAM,MAAM,KAAK,UAAU;AAAA,IACzB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,EACR,CAAC;AACD,MAAI,SAAS,QAAQ,IAAI,GAAG;AAC5B,MAAI,CAAC,QAAQ;AACX,cAAU,YAAuD;AAC/D,YAAM,aAAa,MAAM,cAAc,IAAI,MAAM,oBAAoB,MAAM,YAAY;AACvF,UAAI,CAAC,WAAY,QAAO;AACxB,UAAI,WAAW,WAAW,UAAW,QAAO;AAC5C,UAAI,WAAW,sBAAsB;AACnC,eAAO,WAAW,yBAAyB,MAAM,uBAC7C,OACA;AAAA,MACN;AACA,aAAO,iCAAiC,IAAI,WAAW,qBAAqB,EAAE,IAC1E,uBACA;AAAA,IACN,GAAG,EAAE,MAAM,MAAM,IAAI;AACrB,YAAQ,IAAI,KAAK,MAAM;AAAA,EACzB;AACA,SAAO,MAAM;AACf;AAEA,eAAe,uBACb,QACA,aACA,OACe;AACf,MAAI,OAAO,WAAW,EAAG;AACzB,QAAM,cAAc,KAAK,IAAI,OAAO,QAAQ,KAAK,IAAI,GAAG,KAAK,MAAM,WAAW,CAAC,CAAC;AAChF,MAAI,YAAY;AAChB,QAAM,QAAQ;AAAA,IACZ,MAAM,KAAK,EAAE,QAAQ,YAAY,GAAG,YAAY;AAC9C,aAAO,YAAY,OAAO,QAAQ;AAChC,cAAM,QAAQ;AACd,qBAAa;AACb,cAAM,MAAM,OAAO,KAAK,GAAI,KAAK;AAAA,MACnC;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,eAAe,6BACb,IACA,UACA,eAC+D;AAI/D,QAAM,UAAU,MAAM,mCAAmC,IAAI,UAAU,aAAa,EAAE;AAAA,IACpF,CAAC,UAAU;AACT,oBAAc,KAAK,4DAA4D;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD,aAAO;AAAA,IACT;AAAA,EACF;AACA,MAAI,UAAU,GAAG;AACf,2CAAuC,eAAe,kBAAkB,OAAO;AAC/E,kBAAc,KAAK,4DAA4D,EAAE,QAAQ,CAAC;AAAA,EAC5F;AACA,QAAM,UAAU,OAAO,WAAW;AAClC,QAAM,SAAS,MAAM,qCAAqC,IAAI;AAAA,IAC5D;AAAA,IACA,OAAO;AAAA,IACP,YAAY;AAAA,EACd,CAAC;AACD,yCAAuC,eAAe,WAAW,OAAO,MAAM;AAC9E,MAAI,UAAU;AACd,MAAI,SAAS;AACb,QAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,QAAI;AACF,UAAI,MAAM,oBAAoB,SAAS;AACrC,cAAM,IAAI,MAAM,mCAAmC,MAAM,eAAe,EAAE;AAAA,MAC5E;AACA,YAAM,8BAA8B,UAAU,MAAM,oBAAoB,MAAM,QAAQ;AACtF,YAAM,UAAU,MAAM,kCAAkC,IAAI;AAAA,QAC1D,YAAY,MAAM;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,cAAc;AAAA,MAChB,CAAC;AACD,UAAI,SAAS;AACX,mBAAW;AACX,+CAAuC,eAAe,SAAS;AAAA,MACjE;AAAA,IACF,SAAS,OAAO;AACd,gBAAU;AACV,6CAAuC,eAAe,eAAe;AACrE,YAAM,eAAe,KAAK;AAAA,QACxB,IAAI,KAAK;AAAA,QACT,MAAQ,KAAK,KAAK,IAAI,MAAM,gBAAgB,EAAE;AAAA,MAChD;AACA,YAAM,kCAAkC,IAAI;AAAA,QAC1C,YAAY,MAAM;AAAA,QAClB;AAAA,QACA,SAAS;AAAA,QACT,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC5D;AAAA,MACF,CAAC,EAAE,MAAM,MAAM,KAAK;AACpB,oBAAc,KAAK,qDAAqD;AAAA,QACtE,YAAY,MAAM;AAAA,QAClB,iBAAiB,MAAM;AAAA,QACvB,YAAY,MAAM;AAAA,QAClB,SAAS,MAAM;AAAA,QACf,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,yCAAuC,eAAe,oBAAoB,MAAM;AAChF,SAAO,EAAE,SAAS,OAAO,QAAQ,SAAS,OAAO;AACnD;AAEA,eAAe,mCACb,IACA,UACA,eACiB;AACjB,QAAM,QAAQ,MAAM,+BAA+B,IAAI,EAAE;AACzD,MAAI,MAAM,WAAW,EAAG,QAAO;AAC/B,QAAM,WAAW,oBAAI,IAGnB;AACF,MAAI,UAAU;AACd,QAAM,uBAAuB,OAAO,sCAAsC,OAAO,SAAS;AACxF,QAAI;AACF,UAAI,UAAU,SAAS,IAAI,KAAK,UAAU;AAC1C,UAAI,CAAC,SAAS;AACZ,kBAAU,oDAAoD,UAAU,KAAK,UAAU;AACvF,iBAAS,IAAI,KAAK,YAAY,OAAO;AAAA,MACvC;AACA,YAAM,kBAAkB,MAAM;AAC9B,UACE,MAAM,mCAAmC,IAAI;AAAA,QAC3C,WAAW,KAAK;AAAA,QAChB,aAAa,KAAK;AAAA,QAClB,gBAAgB,KAAK;AAAA,QACrB,SAAS,KAAK;AAAA,QACd,YAAY,KAAK;AAAA,QACjB,qBAAqB,KAAK;AAAA,QAC1B,MAAM,KAAK;AAAA,QACX,eAAe,KAAK;AAAA,QACpB,YAAY,KAAK;AAAA,QACjB,oBAAoB,gBAAgB;AAAA,QACpC,iBAAiB,gBAAgB;AAAA,MACnC,CAAC,GACD;AACA,mBAAW;AAAA,MACb;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,2DAA2D;AAAA,QAC5E,aAAa,KAAK;AAAA,QAClB,gBAAgB,KAAK;AAAA,QACrB,MAAM,KAAK;AAAA,QACX,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,SAAO;AACT;AAKA,eAAsB,4CACpB,IACA,UACA,eACA,KACA,oBACe;AACf,QAAM,YAAY,oBAAI,KAAK;AAC3B,QAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AACvC,QAAM,qBAAqB,oBAAI,IAAY;AAC3C,QAAM,wBAAwB,oBAAI,IAA2B;AAG7D,KAAG;AACD,UAAM,UAAU,OAAO,WAAW;AAClC,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,+CAA+C,IAAI;AAAA,QAChE;AAAA,QACA;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH,SAAS,OAAO;AACd,2CAAqC,eAAe,cAAc;AAClE,oBAAc,KAAK,kDAAkD;AAAA,QACnE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD;AAAA,IACF;AAEA,UAAM,aACJ,sBACA,IAAI;AAAA,MACF,YAAmD,IAAI,qBAAqB;AAAA,IAC9E;AACF,UAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,UAAI,QAAQ,MAAM;AAClB,UAAI,eAA+E;AACnF,YAAM,gBAAgB,KAAK,UAAU;AAAA,QACnC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,MAAM;AAAA,MACR,CAAC;AACD,UAAI,CAAC,OAAO;AACV,cAAM,aAAa,sBAAsB,IAAI,aAAa;AAC1D,YAAI,WAAY,OAAM;AACtB,YAAI,mBAAmB,IAAI,aAAa,GAAG;AACzC,gBAAM,2BAA2B,IAAI,UAAU,eAAe,OAAO,kBAAkB;AACvF;AAAA,QACF;AAEA,YAAI;AACJ,YAAI,CAAC,YAAY;AACf,gCAAsB;AAAA,YACpB;AAAA,YACA,IAAI,QAAc,CAAC,YAAY;AAC7B,iCAAmB;AAAA,YACrB,CAAC;AAAA,UACH;AAAA,QACF;AACA,YAAI;AACF,kBAAQ,MAAM;AAAA,YACZ;AAAA,YACA;AAAA,YACA,KAAK;AAAA,cACH,SAAS;AAAA,cACT;AAAA,YACF;AAAA,UACF;AAGA,cAAI,OAAO,YAAY,QAAQ;AAC7B,gBAAI,CAAC,IAAI;AACP,oBAAM,IAAI,MAAM,gDAAgD;AAClE,kBAAM,SAAS,IAAI,mBAAmB;AAAA,cACpC,aAAa,MAAM;AAAA,cACnB,SAAS,MAAM;AAAA,cACf,sBAAsB,MAAM;AAAA,cAC5B,OAAO;AAAA,cACP;AAAA,cACA,YAAY;AAAA,gBACV,MAAM;AAAA,gBACN,MAAM;AAAA,gBACN,MAAM;AAAA,cACR;AAAA,cACA,WAAW,IAAI;AAAA,gBACb,YAAY,IAAI,wBAAwB,KAAK;AAAA,cAC/C;AAAA,cACA,kBAAkB,KAAK;AAAA,gBACrB,SAAS;AAAA,gBACT;AAAA,cACF;AAAA,cACA,YAAY;AAAA,cACZ,SAAS,EAAE,kBAAkB,MAAM,OAAO,KAAK,IAAI,CAAC,GAAG,gBAAgB,MAAM;AAAA,cAAC,EAAE;AAAA,YAClF,CAAC;AACD,2BAAe,MAAM;AAAA,cACnB;AAAA,cACA,MAAM;AAAA,cACN,QAAQ,KAAK;AAAA,cACb,OAAO,WAAW;AAChB,sBAAM,8BAA8B,IAAI,OAAO,GAAG,EAAE,MAAM,WAAW,MAAM;AAAA,cAC7E;AAAA,YACF;AAAA,UACF;AACA,cAAI,CAAC,OAAO;AACV,kBAAM;AAAA,cACJ;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,cACA;AAAA,YACF;AACA;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,gBAAM,UACJ,iBAAiB,2BAChB,MAAM,gBAAgB,MAAM,WAAW,wBAAwB,MAAM;AACxE,cAAI,QAAS,oBAAmB,IAAI,aAAa;AACjD,cAAI,CAAC,SAAS;AACZ,0BAAc,KAAK,2DAA2D;AAAA,cAC5E,WAAW,MAAM;AAAA,cACjB,cAAc,MAAM;AAAA,cACpB,sBAAsB,MAAM;AAAA,cAC5B,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AAAA,UACH;AACA,gBAAM;AAAA,YACJ;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,UAAU,qBAAqB;AAAA,UACjC;AACA;AAAA,QACF,UAAE;AACA,6BAAmB;AAAA,QACrB;AAEA,YAAI;AACF,gBAAM,6CAA6C,IAAI,EAAE,OAAO,MAAM,CAAC;AACvE,+CAAqC,eAAe,SAAS,MAAM,OAAO,EAAE;AAAA,QAC9E,SAAS,OAAO;AACd,+CAAqC,eAAe,yBAAyB;AAC7E,wBAAc,KAAK,6DAA6D;AAAA,YAC9E,WAAW,MAAM;AAAA,YACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD;AAAA,QACF;AAAA,MACF;AAEA,UAAI;AACF,cAAM,aAAa,MAAM,+CAA+C,IAAI,EAAE,MAAM,CAAC;AACrF,YAAI,CAAC,WAAW,SAAS;AACvB,gBAAM,IAAI,MAAM,4DAA4D;AAAA,QAC9E;AACA,cAAM,UACJ,cAAc,kBACb,cAAc,WAAW,YAAY,aAAa,UAAU,gBAAgB;AAC/E,cAAM,WACJ,cAAc,WAAW,cAAc,eAAe,cAAc;AACtE,YAAI,MAAM,YAAY,YAAY,WAAW,UAAU;AACrD,gBAAM,wCAAwC,IAAI;AAAA,YAChD,GAAG;AAAA,YACH;AAAA,YACA,UAAU,SAAS,QAAQ,SAAS;AAAA,UACtC,CAAC;AAAA,QACH;AACA,YAAI,WAAW,OAAO,SAAS,GAAG;AAChC,cAAI;AACF,kBAAM,IAAI,QAAQ,MAAM,aAAa,MAAM,WAAW,WAAW,MAAM;AAAA,UACzE,SAAS,OAAO;AACd,0BAAc,KAAK,yDAAyD;AAAA,cAC1E,WAAW,MAAM;AAAA,cACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AAAA,UACH;AAAA,QACF;AACA,6CAAqC,eAAe,WAAW,MAAM,OAAO,EAAE;AAAA,MAChF,SAAS,OAAO;AACd,6CAAqC,eAAe,mBAAmB;AACvE,sBAAc,KAAK,uDAAuD;AAAA,UACxE,WAAW,MAAM;AAAA,UACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AACD,cAAM,2BAA2B,IAAI,UAAU,eAAe,OAAO,mBAAmB;AAAA,MAC1F;AAAA,IACF,CAAC;AACD,QAAI,OAAO,SAAS,uCAAwC;AAAA,EAC9D,SAAS,KAAK,IAAI,IAAI;AACxB;AAEA,eAAsB,6BACpB,QACA,MACA,eACA,SACkE;AAIlE,MAAI,kBAAkB;AACtB,SAAO,MAAM;AACX,UAAM,SAAS;AACf,UAAM,SAAS,MAAM,OAAO,aAAa,MAAM,gBAAgB,MAAQ,KAAK,OAAO;AACnF,QAAI,CAAC,iBAAiB,OAAO,WAAW,eAAe,OAAO,SAAU,QAAO;AAI/E,sBAAkB,OAAO,OAAO,cAAc;AAI9C,QAAI,oBAAoB,QAAQ;AAC9B,YAAM,IAAI,MAAM,4EAA4E;AAAA,IAC9F;AAAA,EACF;AACF;AAKA,eAAsB,wCACpB,IACA,UACA,eACA,KACA,oBACe;AACf,QAAM,YAAY,oBAAI,KAAK;AAC3B,QAAM,WAAW,KAAK,IAAI,IAAI,SAAS;AACvC,QAAM,aACJ,sBAAsB,IAAI,eAAe,YAAY,IAAI,qBAAqB,CAAC;AACjF,QAAM,gBAAwC,oBAAI,IAAI;AACtD,KAAG;AACD,QAAI;AACJ,QAAI;AACF,eAAS,MAAM,oCAAoC,IAAI;AAAA,QACrD,SAAS,OAAO,WAAW;AAAA,QAC3B;AAAA,QACA,OAAO;AAAA,QACP,YAAY;AAAA,MACd,CAAC;AAAA,IACH,SAAS,OAAO;AACd,2CAAqC,eAAe,cAAc;AAClE,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD;AAAA,IACF;AACA,UAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,UAAI,UAAU,MAAM;AACpB,UAAI;AACF,YAAI,CAAC,IAAI;AACP,gBAAM,IAAI,MAAM,gDAAgD;AAClE,cAAM,aAAa;AAAA,UACjB,MAAM;AAAA,UACN,MAAM;AAAA,UACN,MAAM;AAAA,QACR;AACA,cAAM,mBAAmB,KAAK;AAAA,UAC5B,SAAS;AAAA,UACT;AAAA,QACF;AACA,cAAM,SAAS,IAAI,mBAAmB;AAAA,UACpC,aAAa,MAAM;AAAA,UACnB,SAAS,MAAM;AAAA,UACf,sBAAsB,MAAM;AAAA,UAC5B,OAAO;AAAA,UACP;AAAA,UACA;AAAA,UACA,WAAW,IAAI,sBAAsB,YAAY,IAAI,wBAAwB,KAAK,IAAI;AAAA,UACtF;AAAA,UACA,YAAY;AAAA,UACZ,SAAS;AAAA,YACP,kBAAkB,MAAM,OAAO,KAAK,IAAI,CAAC;AAAA,YACzC,gBAAgB,MAAM;AACpB,oBAAM,IAAI,MAAM,yDAAyD;AAAA,YAC3E;AAAA,UACF;AAAA,QACF,CAAC;AACD,YAAI,CAAC,SAAS;AACZ,gBAAM,SAAS,MAAM,kBAAkB;AAAA,YACrC;AAAA,YACA;AAAA,YACA,MAAM,MAAM;AAAA,YACZ;AAAA,UACF,CAAC;AACD,cAAI,OAAO,UAAU,QAAQ,eAAe;AAC1C,gBACE,CAAE,MAAM,yCAAyC,IAAI;AAAA,cACnD;AAAA,cACA,SAAS;AAAA,cACT,cAAc;AAAA,YAChB,CAAC;AAED,oBAAM,IAAI,MAAM,8DAA8D;AAChF,iDAAqC,eAAe,oBAAoB;AACxE;AAAA,UACF;AACA,cAAI,OAAO,UAAU,QAAQ;AAC3B,kBAAM,IAAI,MAAM,mDAAmD;AACrE,gBAAM,SAAS,MAAM;AAAA,YACnB;AAAA,YACA,MAAM;AAAA,YACN;AAAA,YACA,OAAO,WACL,8BAA8B,IAAI,OAAO,GAAG,EAAE,MAAM,WAAW,MAAM;AAAA,UACzE;AACA,gBAAM,WAAW,OAAO,WAAW,cAAc,SAAS,OAAO;AACjE,oBACE,OAAO,kBACN,OAAO,WAAW,YAAY,OAAO,UAAU,gBAAgB,WAChE;AACF,cAAI,CAAC,YAAY,CAAC;AAChB,kBAAM,IAAI,MAAM,iEAAiE;AACnF,gBAAM,wCAAwC,IAAI;AAAA,YAChD,GAAG;AAAA,YACH;AAAA,YACA,UAAU,SAAS,QAAQ,SAAS;AAAA,UACtC,CAAC;AAAA,QACH;AACA,cAAM,UAAU,MAAM,OAAO,sBAAsB,MAAM,MAAM,OAAO;AACtE,YACE,CAAE,MAAM,yCAAyC,IAAI;AAAA,UACnD;AAAA,UACA;AAAA,UACA,cAAc,YAAY,cAAc,SAAS,6BAA6B;AAAA,QAChF,CAAC;AAED,gBAAM,IAAI,MAAM,iEAAiE;AACnF;AAAA,UACE;AAAA,UACA,YAAY,cAAc,yBAAyB;AAAA,QACrD;AAAA,MACF,SAAS,OAAO;AACd,cAAM,UAAU,MAAM,iCAAiC,IAAI,OAAO,OAAO,aAAa;AAGtF,YACE,WACC,MAAM,yCAAyC,IAAI;AAAA,UAClD;AAAA,UACA,SAAS,UAAU,iBAAiB;AAAA,UACpC,cAAc;AAAA,QAChB,CAAC,EAAE,MAAM,MAAM,KAAK,GACpB;AACA;AAAA,YACE;AAAA,YACA,UAAU,oBAAoB;AAAA,UAChC;AACA;AAAA,QACF;AACA,cAAM,yCAAyC,IAAI;AAAA,UACjD;AAAA,UACA,SAAS;AAAA,UACT,cAAc,KAAK;AAAA,YACjB,IAAI;AAAA,YACJ,KAAK,IAAI,SAAS,4BAA4B,GAAM,IAClD,KAAK,KAAK,IAAI,GAAG,MAAM,oBAAoB,CAAC;AAAA,UAChD;AAAA,QACF,CAAC,EAAE,MAAM,MAAM,KAAK;AACpB,sBAAc,KAAK,6DAA6D;AAAA,UAC9E,WAAW,MAAM;AAAA,UACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AACD,QAAI,OAAO,SAAS,uCAAwC;AAAA,EAC9D,SAAS,KAAK,IAAI,IAAI;AACxB;AAEA,eAAsB,uCACpB,OACA,YACA,kBACA,aAAa,oBAAI,KAAK,GACkC;AACxD,QAAM,QAAQ;AAAA,IACZ;AAAA,IACA,YAAY;AAAA,MACV,MAAM;AAAA,MACN,MAAM;AAAA,MACN,MAAM;AAAA,IACR;AAAA,IACA,MAAM,MAAM;AAAA,IACZ;AAAA,EACF;AACA,QAAM,SACJ,MAAM,UAAU,aAAa,MAAM,mBAAmB,KAAK,IAAI,MAAM,kBAAkB,KAAK;AAC9F,SAAO,gCAAgC,QAAQ,UAAU;AAC3D;AAEO,SAAS,gCACd,QACA,YAC+C;AAC/C,UAAQ,OAAO,OAAO;AAAA,IACpB,KAAK,QAAQ;AAAA,IACb,KAAK,QAAQ;AACX,aAAO;AAAA,IACT,KAAK,QAAQ,mBAAmB;AAC9B,YAAM,OAAO,OAAO;AACpB,UAAI,CAAC,MAAM;AACT,cAAM,IAAI,MAAM,oDAAoD;AAAA,MACtE;AACA,YAAM,cAAc,KAAK,YAAY,QAAQ,mBAAmB,GAAG,EAAE,MAAM,GAAG,GAAG;AACjF,YAAM,SAAS,cACX,cAAc,WAAW,KACzB,KAAK,YACH,iBACA,KAAK,WACH,iBACA;AACR,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU,KAAK;AAAA,QACf;AAAA,QACA,GAAI,cACA;AAAA,UACE,SAAS,EAAE,MAAM,aAAa,QAAQ,KAAK,eAAe,WAAW,MAAe;AAAA,QACtF,IACA,CAAC;AAAA,QACL;AAAA,MACF;AAAA,IACF;AAAA,IACA,KAAK,QAAQ;AACX,aAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,QACE,OAAO,eAAe,aAAa,yBAC/B,oBACA,OAAO,eAAe,aAAa,iCACjC,4BACA;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACE,YAAM,IAAI,MAAM,+CAA+C,OAAO,KAAK,EAAE;AAAA,EACjF;AACF;AAEA,eAAe,2BACb,IACA,UACA,eACA,OACA,SACe;AACf,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,MAAM,oBAAoB,CAAC,CAAC;AACrE,QAAM,eAAe,KAAK;AAAA,IACxB,IAAI;AAAA,IACJ,KAAK,IAAI,SAAS,4BAA4B,GAAM,IAAI,KAAK;AAAA,EAC/D;AACA,MAAI;AACF,QACE,CAAE,MAAM,qDAAqD,IAAI;AAAA,MAC/D;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC,GACD;AACA,YAAM,IAAI,MAAM,8CAA8C;AAAA,IAChE;AACA,yCAAqC,eAAe,OAAO;AAAA,EAC7D,SAAS,OAAO;AACd,yCAAqC,eAAe,cAAc;AAClE,kBAAc,KAAK,kDAAkD;AAAA,MACnE,WAAW,MAAM;AAAA,MACjB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AAMA,IAAM,iCAAiC;AACvC,IAAM,sCAAsC;AAC5C,IAAM,sCAAsC,IAAI;AAChD,IAAM,qCAAqC,IAAI,KAAK;AAqBpD,eAAe,yBACb,IACA,UACA,eACA,+BACA,SACe;AAGf,MAAI,SAAS,mBAAmB,WAAW,CAAC,SAAS,gBAAgB,CAAC,SAAS,kBAAkB;AAC/F;AAAA,EACF;AACA,QAAM,cAAc,KAAK,IAAI;AAE7B,QAAM,WAAW,oBAAI,IAAoB;AACzC,QAAM,OAA+B,CAAC;AAEtC,iBAAe,QACb,YACA,SACwB;AACxB,UAAM,QAAQ,SAAS,IAAI,UAAU;AACrC,QAAI,UAAU,OAAW,QAAO;AAChC,UAAM,QAAQ,KAAK,IAAI;AACvB,SAAK,QAAQ,IAAI,UAAU,GAAG,iBAAiB,KAAK,MAAO,QAAO;AAClE,QAAI,QAAQ,eAAe,oCAAqC,QAAO;AACvE,QAAI;AACJ,QAAI;AACF,gBACE,MAAM;AAAA,QACJ,8BAA8B,UAAU,YAAY,IAAI;AAAA,MAC1D,GACA;AAAA,IACJ,SAAS,OAAO;AACd,eAAS,UAAU,iCAAiC,qBAAqB;AACzE,oBAAc,KAAK,gDAAgD;AAAA,QACjE,GAAG;AAAA,QACH,OACE,UAAU,iCACN,qBACA,iBAAiB,QACf,MAAM,UACN,OAAO,KAAK;AAAA,MACtB,CAAC;AAAA,IACH;AACA,aAAS,IAAI,YAAY,MAAM;AAC/B,QAAI,WAAW,cAAc;AAC3B,cAAQ,OAAO,UAAU;AAAA,IAC3B,OAAO;AACL,2BAAqB,SAAS,UAAU;AACxC,WAAK,MAAM,KAAK,KAAK,MAAM,KAAK,KAAK;AAAA,IACvC;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,gCAAgC,IAAI,EAAE,OAAO,+BAA+B,CAAC;AAAA,EAC9F,SAAS,OAAO;AACd,kBAAc,KAAK,wDAAwD;AAAA,MACzE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,aAAS,CAAC;AAAA,EACZ;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,MAAM;AACzB,UAAM,UAAU;AAAA,MACd,aAAa,MAAM;AAAA,MACnB,gBAAgB,MAAM;AAAA,MACtB,WAAW,MAAM;AAAA,MACjB;AAAA,IACF;AACA,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,UAAI,WAAW,aAAc;AAC7B,YAAM,UAAU,MAAM,kCAAkC,IAAI,KAAK;AACjE,UAAI,SAAS;AACX,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,GAAG;AAAA,UACH,GAAG;AAAA,QACL,CAAC;AAAA,MACH;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,wCAAwC,IAAI;AAAA,MACzD,OAAO;AAAA,IACT,CAAC;AAAA,EACH,SAAS,OAAO;AACd,kBAAc,KAAK,sDAAsD;AAAA,MACvE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,aAAS,CAAC;AAAA,EACZ;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,MAAM,IAAI;AAG7B,UAAM,UAAU,SAAS,MAAM,MAAM,IAAI,MAAM,SAAS;AACxD,SAAK,QAAQ,IAAI,OAAO,GAAG,iBAAiB,KAAK,KAAK,IAAI,EAAG;AAC7D,UAAM,UAAU;AAAA,MACd,aAAa,MAAM;AAAA,MACnB,WAAW,MAAM;AAAA,MACjB,QAAQ,MAAM;AAAA,MACd,gBAAgB,MAAM;AAAA,MACtB,WAAW,MAAM,IAAI;AAAA,MACrB;AAAA,IACF;AACA,QAAI;AACF,YAAM,SAAS,MAAM,QAAQ,YAAY,OAAO;AAChD,UAAI,WAAW,aAAc;AAC7B,YAAM,EAAE,WAAW,IAAI,MAAM,+BAA+B,IAAI;AAAA,QAC9D,WAAW,MAAM;AAAA,QACjB,aAAa,MAAM;AAAA,QACnB,WAAW,MAAM;AAAA,QACjB,QAAQ,MAAM;AAAA,QACd,WAAW,MAAM;AAAA,QACjB,eAAe,MAAM;AAAA,MACvB,CAAC;AACD,UAAI,YAAY;AACd,gBAAQ,OAAO,OAAO;AACtB,sBAAc;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,OAAO;AACL,6BAAqB,SAAS,OAAO;AAAA,MACvC;AAAA,IACF,SAAS,OAAO;AACd,2BAAqB,SAAS,OAAO;AACrC,oBAAc,KAAK,2DAA2D;AAAA,QAC5E,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AACA,MAAI,OAAO,KAAK,IAAI,EAAE,QAAQ;AAC5B,kBAAc,KAAK,oEAAoE,IAAI;AAAA,EAC7F;AACF;AAEA,SAAS,qBAAqB,SAAiC,KAAmB;AAChF,QAAM,YAAY,QAAQ,IAAI,GAAG,GAAG,YAAY,KAAK;AACrD,UAAQ,IAAI,KAAK;AAAA,IACf;AAAA,IACA,eACE,KAAK,IAAI,IACT,KAAK;AAAA,MACH;AAAA,MACA,sCAAsC,KAAK,KAAK,IAAI,WAAW,GAAG,EAAE;AAAA,IACtE;AAAA,EACJ,CAAC;AACH;AAEA,eAAe,mCACb,IACA,UACA,eACA,OACA,+BACA,KACe;AACf,QAAM,UAAU,OAAO,WAAW;AAClC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,+BAA+B,IAAI;AAAA,MAChD;AAAA,MACA,OAAO;AAAA,MACP,YAAY;AAAA,IACd,CAAC;AAAA,EACH,SAAS,OAAO;AACd,wCAAoC,eAAe,cAAc;AACjE,kBAAc,KAAK,iDAAiD;AAAA,MAClE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD;AAAA,EACF;AAEA,QAAM,cAA0C;AAAA,IAC9C,OAAO,KAAK,IAAI,IAAI;AAAA,EACtB;AAGA,QAAM,YAAY,oBAAI,IAAY;AAClC,aAAW,SAAS,QAAQ;AAC1B,QAAI,UAAU,MAAM;AACpB,QAAI,UAAU,IAAI,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,EAAE;AACxF;AACF,UAAM,WAAW,kCAAkC,OAAO;AAC1D,QAAI,QAAQ,mCAAmC,OAAO;AAGtD,QAAI,eAAe;AACnB,UAAM,eAAe;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,IACrB;AAIA,QACE,QAAQ,mBAAmB,UAC3B,QAAQ,oBAAoB,WAC5B,CAAC,SACD,CAAC,MAAM,WAAW,WAAW,aAAa,GAC1C;AACA,UAAI;AACF,cAAM,UAAU,MAAM,uCAAuC,IAAI;AAAA,UAC/D,GAAG;AAAA,UACH;AAAA,UACA,uBAAuB,MAAM;AAAA,QAC/B,CAAC;AACD,YAAI,SAAS;AACX,gBAAM,aAAa,cAAc,QAAQ,KAAK;AAC9C,8CAAoC,eAAe,sBAAsB;AAAA,QAC3E;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,+DAA+D;AAAA,UAChF,WAAW,QAAQ;AAAA,UACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF;AACA,UAAM,2BAA2B;AAAA,MAC/B;AAAA,MACA;AAAA,MACA,MAAM,CAAC,WAAW,IAAI,QAAQ,QAAQ,aAAa,QAAQ,WAAW,MAAM,IAAI;AAAA,IAClF;AACA,UAAM,qBAAqB;AAAA,MACzB,GAAG;AAAA,MACH,qBAAqB,OAAO,WAAkD;AAC5E,YAAI,WAAW,qBAAqB;AAClC,gBAAM,2CAA2C,IAAI,YAAY;AACjE,cAAI,CAAE,MAAM,yBAAyB,sBAAsB;AACzD,kBAAM,IAAI;AAAA,cACR;AAAA,YACF;AAAA,QACJ,MAAO,OAAM,yBAAyB,oBAAoB,MAAM;AAAA,MAClE;AAAA,IACF;AACA,QAAI,SAAS,QAAQ,oBAAoB,SAAS;AAChD,UAAI;AACF,cAAM,UAAU,MAAM,mBAAmB,KAAK;AAC9C,YACE,SAAS,SAAS,qBAClB,QAAQ,gBACP,CAAE,MAAM,mBAAmB,uBAAuB,KACjD,CAAC,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,EAAE;AAAA,UAChD,CAAC,WAAW,OAAO,OAAO,OAAO,aAAa;AAAA,QAChD;AAEF,kBAAQ;AAAA,MACZ,QAAQ;AAGN,gBAAQ;AAAA,MACV;AAAA,IACF;AACA,QAAI,CAAC,OAAO;AACV,UAAI,cAAiD;AAMrD,UACE,QAAQ,oBAAoB,WAC5B,QAAQ,kBAAkB,QAC1B,CAAC,QAAQ,oBACT;AACA,YAAI;AACF,gBAAM,YAAY,MAAM;AAAA,YACtB,8BAA8B,UAAU,QAAQ,oBAAoB,IAAI;AAAA,UAC1E;AACA,cAAI,UAAU,WAAW,aAAa;AACpC,0BAAc;AAAA,cACZ,QAAQ;AAAA,cACR,QAAQ;AAAA,YACV;AAAA,UACF,OAAO;AACL,sBAAU,MAAM,oCAAoC,IAAI;AAAA,cACtD,WAAW,QAAQ;AAAA,cACnB,aAAa,QAAQ;AAAA,cACrB,WAAW,QAAQ;AAAA,cACnB,WAAW,QAAQ;AAAA,cACnB;AAAA,cACA,SAAS,MAAM;AAAA,cACf,oBAAoB,UAAU;AAAA,cAC9B,iBAAiB,UAAU;AAAA,YAC7B,CAAC;AACD,gDAAoC,eAAe,0BAA0B;AAC7E,gBAAI,UAAU,WAAW,cAAc;AACrC,4BAAc;AAAA,gBACZ,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF;AAAA,YACF;AAAA,UACF;AAAA,QACF,SAAS,OAAO;AACd,wBAAc;AAAA,YACZ,QAAQ;AAAA,YACR,QACE,UAAU,iCAAiC,qBAAqB;AAAA,UACpE;AAAA,QACF;AAAA,MACF;AAEA,UACE,aAAa,WAAW,cACxB,QAAQ,oBAAoB,WAC5B,CAAC,QAAQ,sBACT,CAAC,QAAQ,iCACT;AAIA,YAAI;AACF,gBAAM,2CAA2C,IAAI,YAAY;AAAA,QACnE,SAAS,OAAO;AACd,wBAAc,KAAK,8DAA8D;AAAA,YAC/E,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AAEA,UAAI,gBAAgB,MAAM;AACxB,cAAM,QAAQ,MAAMF,WAAU,IAAI,QAAQ,aAAa,QAAQ,cAAc,EAAE;AAAA,UAC7E,MAAM;AAAA,QACR;AACA,cAAM,8BACJ,QAAQ,KAAK,KACb,MAAO,OAAO,QAAQ,WACtB,MAAO,eAAe,QAAQ,cAC9B,MAAO,YAAY,QAAQ,mBAC3B,MAAO,eAAe,QAAQ,sBAC9B,QAAQ,kBAAkB;AAC5B,YAAI,CAAC,6BAA6B;AAIhC,cAAI,QAAQ,oBAAoB,WAAW,QAAQ,kBAAkB,MAAM;AACzE,0BAAc,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,UAClE,OAAO;AACL,gBAAI;AACF,oBAAM,YAAY,MAAM;AAAA,gBACtB;AAAA,kBACE;AAAA,kBACA,QAAQ;AAAA,kBACR,QAAQ;AAAA,gBACV;AAAA,cACF;AACA,4BACE,UAAU,WAAW,cACjB;AAAA,gBACE,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF,IACA,UAAU,WAAW,eACnB;AAAA,gBACE,QAAQ;AAAA,gBACR,OAAO;AAAA,kBACL,SAAS;AAAA,kBACT,UAAU;AAAA,kBACV,QAAQ;AAAA,gBACV;AAAA,cACF,IACA,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,YAC5D,SAAS,OAAO;AACd,4BAAc;AAAA,gBACZ,QAAQ;AAAA,gBACR,QACE,UAAU,iCAAiC,qBAAqB;AAAA,cACpE;AAAA,YACF;AAAA,UACF;AAAA,QACF,OAAO;AACL,cAAI;AACF,kBAAM,UAAU,MAAM,mBAAmB,KAAK;AAC9C,kBAAM,aAAa,SAAS,SAAS,qBAAqB,QAAQ,QAAQ,WAAW;AACrF,kBAAM,oBAAoB,CAAC,YACzB,cAAc,iBAAiB;AAAA,cAC7B,MAAM;AAAA,cACN,MAAM;AAAA,cACN,QAAQ,EAAE,QAAQ;AAAA,YACpB,CAAC;AACH,gBACE,eACC,MAAO,mBAAmB,uBACzB,MAAM,eAAe,wBACvB;AACA,oBAAM,mBAAmB;AAAA,gBACvB,MAAO,mBAAmB,sBACtB,sBACA;AAAA,cACN;AACA,gCAAkB,qBAAqB;AAAA,YACzC;AACA,kBAAM,qBACJ,MAAO,mBAAmB,uBAC1B,CAAC,cACD,CAAC,QAAQ;AACX,kBAAM,eAAe,MAAM;AAAA,cACzB;AAAA,cACA;AAAA,cACA;AAAA,cACC,MAAM,eAAe,yBACpB,CAAC,QAAQ,mCACT,qBACE,WACA;AAAA,cACJ,OAAO,QAAQ,SAAS,QAAQ,mBAAmB;AACjD,oBACE,OAAO,WAAW,YACjB,CAAC,UAAU,wBAAwB,QAAQ,QAAQ,iBAAiB;AAErE;AACF,sBAAM,SAAS,MAAM,2BAA2B,IAAI;AAAA,kBAClD,WAAW,QAAQ;AAAA,kBACnB,aAAa,QAAQ;AAAA,kBACrB,WAAW,QAAQ;AAAA,kBACnB,WAAW,QAAQ;AAAA,kBACnB;AAAA,kBACA,QAAQ,UAAU;AAAA,kBAClB,gBAAgB,mBAAmB,SAAS,aAAa;AAAA,kBACzD,OAAO,SAAS,SAAS,gBAAgB,MAAM;AAAA,gBACjD,CAAC;AACD,oBAAI,OAAO,UAAU;AACnB,wBAAM,IACH,QAAQ,QAAQ,aAAa,QAAQ,WAAW,MAAM,EACtD,MAAM,MAAM,MAAS;AAAA,cAC5B;AAAA,cACA;AAAA,cACA;AAAA,YACF,EAAE;AAAA,cACA,CAAC,WAAW,EAAE,IAAI,MAAe,MAAM;AAAA,cACvC,CAAC,WAAoB,EAAE,IAAI,OAAgB,MAAM;AAAA,YACnD;AAIA,gBACE,sBACA,CAAE,MAAM,2CAA2C,IAAI,YAAY;AAEnE,oBAAM,IAAI,MAAM,yDAAyD;AAC3E,gBAAI,CAAC,aAAa,GAAI,OAAM,aAAa;AACzC,0BAAc,aAAa;AAC3B,2BACE,YAAY,WAAW,YACvB,YAAY,MAAM,YAAY,UAC9B,YAAY,MAAM,WAAW;AAC/B,gBAAI,YAAY;AACd;AAAA,gBACG,MAAM,mBAAmB,uBAAuB,IAC7C,mBACA;AAAA,cACN;AACA,kBAAI,YAAY,WAAW;AACzB;AAAA,kBACE,YAAY,WAAW,qBACnB,uBACA;AAAA,gBACN;AAAA,YACJ;AAAA,UACF,SAAS,OAAO;AACd,0BAAc,KAAK,0DAA0D;AAAA,cAC3E,WAAW,QAAQ;AAAA,cACnB,iBAAiB,QAAQ;AAAA,cACzB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,YAC9D,CAAC;AACD,0BAAc,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,UAC/D;AAAA,QACF;AAAA,MACF;AACA,UAAI,YAAY,WAAW,WAAW;AACpC,YAAI;AACF,oBAAU,MAAM,oCAAoC,IAAI;AAAA,YACtD,WAAW,QAAQ;AAAA,YACnB,aAAa,QAAQ;AAAA,YACrB,WAAW,QAAQ;AAAA,YACnB,WAAW,QAAQ;AAAA,YACnB;AAAA,YACA,SAAS,MAAM;AAAA,YACf,oBAAoB,YAAY;AAAA,YAChC,iBAAiB,YAAY;AAAA,UAC/B,CAAC;AACD,8CAAoC,eAAe,0BAA0B;AAC7E,wBAAc,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,QACjE,SAAS,OAAO;AACd,wBAAc,KAAK,4DAA4D;AAAA,YAC7E,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AACD,wBAAc,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,QAC/D;AAAA,MACF;AACA,UAAI,YAAY,WAAW,YAAY;AACrC,cAAM;AAAA,UACJ;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,UACN,YAAY;AAAA,UACZ,YAAY,kBAAkB;AAAA,QAChC;AACA;AAAA,MACF;AACA,UAAI;AACF,kBAAU,MAAM,yCAAyC,IAAI;AAAA,UAC3D,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,WAAW,QAAQ;AAAA,UACnB,WAAW,QAAQ;AAAA,UACnB;AAAA,UACA,SAAS,MAAM;AAAA,UACf,OAAO,YAAY;AAAA,QACrB,CAAC;AACD,gBAAQ,YAAY;AACpB,4CAAoC,eAAe,SAAS,MAAM,OAAO,EAAE;AAAA,MAC7E,SAAS,OAAO;AACd,4CAAoC,eAAe,yBAAyB;AAC5E,wCAAgC,eAAe,OAAO,OAAO;AAC7D;AAAA,MACF;AAAA,IACF;AAEA,QAAI,cAAc;AAShB,YAAM,SAAS,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB;AACrF,iBAAW,SAAS,QAAQ;AAC1B,cAAM,UAAU,MAAM;AACtB,YACE,UAAU,SACV,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,OAAO,UAC7E,QAAQ,mBAAmB,UAC3B,QAAQ,oBAAoB,WAC5B,MAAM,WAAW,WAAW,aAAa,KACzC,mCAAmC,OAAO;AAE1C;AACF,YAAI;AACF,gBAAM,uCAAuC,IAAI;AAAA,YAC/C,WAAW,QAAQ;AAAA,YACnB,aAAa,QAAQ;AAAA,YACrB,WAAW,QAAQ;AAAA,YACnB,WAAW,QAAQ;AAAA,YACnB,UAAU,kCAAkC,OAAO;AAAA,YACnD,uBAAuB,MAAM;AAAA,UAC/B,CAAC;AAAA,QACH,SAAS,OAAO;AACd,wBAAc,KAAK,+DAA+D;AAAA,YAChF,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,UAAI;AACF,cAAM,SAAS,MAAM,0BAA0B,IAAI;AAAA,UACjD,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,gBAAgB,QAAQ;AAAA,UACxB,eAAe,QAAQ;AAAA,UACvB,oBAAoB,QAAQ;AAAA,UAC5B,iBAAiB,QAAQ;AAAA,QAC3B,CAAC;AACD,YAAI,OAAO,WAAW,UAAU;AAC9B,oBAAU,IAAI,GAAG,QAAQ,OAAO,IAAI,QAAQ,UAAU,IAAI,QAAQ,kBAAkB,EAAE;AACtF,8CAAoC,eAAe,yBAAyB;AAC5E,cAAI,OAAO,yBAAyB,QAAQ;AAC1C,kBAAM;AAAA,cACJ;AAAA,cACA,QAAQ;AAAA,cACR,OAAO;AAAA,cACP,CAAC,UAAU;AACT,8BAAc,KAAK,wDAAwD;AAAA,kBACzE,gBAAgB,QAAQ;AAAA,kBACxB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,gBAC9D,CAAC;AAAA,cACH;AAAA,YACF;AAAA,UACF;AACA;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,WAAW,QAAQ;AAAA,UACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IAGF;AACA,QAAI;AACF,YAAM,aAAa,MAAM,sBAAsB,IAAI;AAAA,QACjD,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,QACrB,WAAW,QAAQ;AAAA,QACnB,WAAW,QAAQ;AAAA,QACnB;AAAA,QACA,uBAAuB,MAAM;AAAA,QAC7B,SAAS,MAAM;AAAA,QACf,UAAU,MAAM;AAAA,QAChB,QAAQ,MAAM;AAAA,QACd,aAAa,SAAS;AAAA,MACxB,CAAC;AACD,UAAI,WAAW,QAAQ,UAAU,UAAU;AACzC,cAAM,IAAI,MAAM,oEAAoE;AAAA,MACtF;AACA,UAAI,WAAW,wBAAwB,SAAS,GAAG;AACjD,YAAI;AACF,gBAAM,IAAI;AAAA,YACR,QAAQ;AAAA,YACR,QAAQ;AAAA,YACR,WAAW;AAAA,UACb;AAAA,QACF,SAAS,OAAO;AACd,wBAAc,KAAK,wDAAwD;AAAA,YACzE,WAAW,QAAQ;AAAA,YACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,UAC9D,CAAC;AAAA,QACH;AAAA,MACF;AACA,0CAAoC,eAAe,WAAW,MAAM,OAAO,EAAE;AAAA,IAC/E,SAAS,OAAO;AACd,0CAAoC,eAAe,mBAAmB;AACtE,oBAAc,KAAK,sDAAsD;AAAA,QACvE,WAAW,QAAQ;AAAA,QACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AACD,YAAM;AAAA,QACJ;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,eAAe,0BACb,IACA,UACA,eACA,SACA,UACA,SACA,SACA,gBAAgB,OACD;AACf,QAAM,WAAW,sCAAsC,UAAU,SAAS,SAAS,aAAa;AAChG,MAAI;AAIF,UAAM,QACJ,QAAQ,oBAAoB,WAAW,QAAQ,kBAAkB,OAC7D,MAAMA,WAAU,IAAI,QAAQ,aAAa,QAAQ,cAAc,IAC/D;AACN,UAAM,mCAAmC,IAAI;AAAA,MAC3C,WAAW,QAAQ;AAAA,MACnB,aAAa,QAAQ;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA;AAAA,MACA,SAAS,SAAS;AAAA,MAClB,cAAc,+BAA+B,SAAS,OAAO,UAAU,SAAS,YAAY;AAAA,IAC9F,CAAC;AACD,wCAAoC,eAAe,SAAS,aAAa;AAAA,EAC3E,SAAS,OAAO;AACd,wCAAoC,eAAe,cAAc;AACjE,kBAAc,KAAK,iDAAiD;AAAA,MAClE,WAAW,QAAQ;AAAA,MACnB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AAMO,SAAS,sCACd,UACA,SACA,SACA,gBAAgB,OAMhB;AACA,QAAM,oBACJ,QAAQ,qBAAqB,uDAC5B,YAAY,8BACX,YAAY,+BACZ,YAAY;AAChB,MAAI,mBAAmB;AACrB,WAAO;AAAA,MACL,gBAAgB,eAAe,OAAO;AAAA,MACtC,eACE,YAAY,6BACR,gCACA,YAAY,8BACV,iCACA;AAAA,MACR,cAAc;AAAA,MACd,aAAa;AAAA,IACf;AAAA,EACF;AACA,QAAM,cAAc,KAAK,IAAI,SAAS,4BAA4B,GAAM;AAGxE,QAAM,WACJ,YAAY,sBAAsB,gBAC9B,IACA,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,QAAQ,oBAAoB,CAAC,CAAC;AAC5D,QAAM,eAAe,KAAK,IAAI,IAAI,KAAQ,cAAc,KAAK,QAAQ;AACrE,SAAO;AAAA,IACL,gBAAgB;AAAA,IAChB,eAAe;AAAA,IACf;AAAA,IACA,aAAa;AAAA,EACf;AACF;AAkBA,IAAM,iCAAiC,OAAO,yCAAyC;AAOvF,eAAsB,4BACpB,SACA,mBACA,OACA,aACA,QACqC;AAErC,MAAI,UAAU,uBAAuB,SAAS,KAAK;AACnD,QAAM,YAAY,KAAK;AACvB,MAAI,cAAc,kCAAkC,OAAO,mBAAmB,OAAO;AACrF,MAAI,OAAO,QAAQ,eAAe,WAAY,QAAO;AACrD,QAAM,QAAQ,KAAK;AAAA,IACjB,KAAK,IAAI,IAAI;AAAA,IACb,QAAQ,SAAS,OAAO;AAAA,EAC1B;AACA,WACM,QAAQ,GACZ,WACA,YAAY,WAAW,cACvB,YAAY,WAAW,sBACvB,QAAQ,2CACR,KAAK,IAAI,IAAI,OACb,SACA;AACA,QAAI;AACJ,QAAI;AACF,aAAO,MAAM;AAAA,QACX,QAAQ,WAAW;AAAA,UACjB,WAAW;AAAA,UACX,OAAO;AAAA,UACP,aAAa;AAAA,UACb,iBAAiB;AAAA,QACnB,CAAC;AAAA,MACH;AAAA,IACF,QAAQ;AAGN;AAAA,IACF;AACA,cAAU,uBAAuB,SAAS,IAAI;AAC9C,UAAM,YAAY,IAAI;AACtB,kBAAc,kCAAkC,MAAM,mBAAmB,OAAO;AAAA,EAClF;AACA,SAAO,WAAW,YAAY,WAAW,cAAc,YAAY,WAAW,qBAC1E,EAAE,GAAG,aAAa,eAAe,KAAK,IACtC;AACN;AAMA,SAAS,uBAAuB,SAAsC,QAA0B;AAC9F,QAAM,OAAO,QAAQ,2BAA2B,MAAM;AACtD,MAAI,CAAC,MAAM,YAAY,KAAK,QAAQ,SAAS,kBAAmB,QAAO;AACvE,MAAI,KAAK,QAAQ,eAAe,KAAK,aAAa,KAAM,QAAO;AAC/D,MAAI,KAAK,eAAgB,QAAO;AAChC,QAAM,EAAE,SAAS,SAAS,IAAI;AAC9B,SAAQ,CAAC,UAAU,QAAQ,EAAY;AAAA,IACrC,CAAC,WACC,QAAQ,QAAQ,MAAM,EAAE,aAAa,SAAS,QAAQ,MAAM,EAAE,cAC9D;AAAA,EACJ;AACF;AAEA,eAAe,gCAAmC,SAAiC;AACjF,MAAI;AACJ,MAAI;AACF,WAAO,MAAM,QAAQ,KAAK;AAAA,MACxB;AAAA,MACA,IAAI,QAAe,CAAC,UAAU,WAAW;AACvC,gBAAQ;AAAA,UACN,MAAM,OAAO,8BAA8B;AAAA,UAC3C;AAAA,QACF;AACA,cAAM,QAAQ;AAAA,MAChB,CAAC;AAAA,IACH,CAAC;AAAA,EACH,UAAE;AACA,QAAI,MAAO,cAAa,KAAK;AAAA,EAC/B;AACF;AAIA,IAAM,6BAA6B,oBAAI,IAAkD;AAEzF,eAAsB,2BACpB,WACA,QACA,SACe;AACf,QAAM,UAAU,2BAA2B,IAAI,SAAS,KAAK;AAAA,IAC3D;AAAA,IACA,SAAS,OAAO,WAAW;AAAA,EAC7B;AACA,6BAA2B,IAAI,WAAW,OAAO;AACjD,QAAM,QAAQ,QAAQ,QAAQ,QAAQ,OAAO;AAC7C,6BAA2B,OAAO,SAAS;AAC7C;AAEA,eAAsB,+BACpB,UACA,OACA,SACA,OAA6B,WAC7B,eAMA,qBACA,aACqC;AACrC,MACE,MAAM,OAAO,QAAQ,WACrB,MAAM,mBAAmB,QAAQ,kBACjC,MAAM,eAAe,QAAQ,cAC7B,MAAM,YAAY,QAAQ,mBAC1B,MAAM,eAAe,QAAQ,sBAC7B,CAAC,MAAM,mBACP,8BAA8B,MAAM,eAAe,MAAM,QAAQ,mBACjE,QAAQ,kBAAkB,MAC1B;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AACA,MAAI,CAAC,MAAM,aAAa;AACtB,WAAO,EAAE,QAAQ,YAAY,QAAQ,uBAAuB;AAAA,EAC9D;AACA,QAAM,UAAU,2BAA2B,IAAI,QAAQ,EAAE;AACzD,MAAI,SAAS;AACX,QAAI,CAAC,cAAe,OAAM,IAAI,MAAM,0DAA0D;AAC9F,UAAM,2BAA2B,QAAQ,IAAI,QAAQ,QAAQ,aAAa;AAC1E,UAAM,qBAAqB;AAAA,MACzB,QAAQ;AAAA,MACR,QAAQ;AAAA,IACV;AAGA,QAAI,SAAS,YAAY,mBAAmB,WAAW,SAAU,QAAO;AAAA,EAC1E;AACA,QAAM,kBAAmB,MAAM,YAAwC;AACvE,MACE,oBAAoB,WACnB,OAAO,oBAAoB,YAC1B,8BAA8B,eAAe,MAAM,QAAQ,kBAC7D;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AACA,MAAI,sCAAsC,MAAM,WAAW,MAAM,QAAQ,oBAAoB;AAC3F,WAAO,EAAE,QAAQ,YAAY,QAAQ,oBAAoB;AAAA,EAC3D;AAEA,MAAI,yBAEO;AACX,MAAI,QAAQ,oBAAoB,WAAW,QAAQ,oBAAoB;AACrE,QAAI;AACF,+BAAyB,MAAM;AAAA,QAC7B,sCAAsC,QAAQ;AAAA,MAChD;AAAA,IACF,SAAS,OAAO;AACd,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AACA,QAAI,uBAAuB,QAAQ,QAAQ,oBAAoB;AAC7D,aAAO,EAAE,QAAQ,YAAY,QAAQ,4BAA4B;AAAA,IACnE;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAASH,+BAA8B,QAAQ,iBAA0B,QAAQ;AAAA,EAGnF,QAAQ;AACN,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MACE,CAAC,UACD,8BAA8B,OAAO,SAAS,MAAM,QAAQ,mBAC5D,CAAC,OAAO,QACR;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,sBAAsB;AAAA,EAC7D;AACA,QAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,MAAI;AACJ,MAAI;AACF,mBAAe,MAAMI;AAAA,MACnB;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,IACV;AAAA,EACF,QAAQ;AACN,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MAAI,iBAAiB,QAAW;AAC9B,WAAO,EAAE,QAAQ,YAAY,QAAQ,uBAAuB;AAAA,EAC9D;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,MAAM;AAAA,MACpBC;AAAA,QACE;AAAA,QACA,QAAQ;AAAA,QACR;AAAA,QACA,QAAQ;AAAA,MACV;AAAA,IACF;AACA,cAAU,QAAQ;AAAA,EACpB,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,aAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,IAC1D;AACA,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,UAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,eAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,MAClE;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,OAAO;AAAA,UACL,SAAS;AAAA,UACT,UAAU;AAAA,UACV,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AACA,MAAI,OAAO,QAAQ,eAAe,YAAY;AAC5C,WAAO,EAAE,QAAQ,YAAY,QAAQ,sBAAsB;AAAA,EAC7D;AACA,MAAI,YAAY;AAChB,MAAI,QAAQ,oBAAoB,SAAS;AACvC,QAAI;AACJ,QAAI;AACF,oBAAc,MAAM;AAAA,QAClB,gDAAgD,UAAU,OAAO;AAAA,MACnE;AAAA,IACF,SAAS,OAAO;AACd,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AACA,QAAI,CAAC,QAAQ,oBAAoB;AAC/B,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,oBAAoB,YAAY;AAAA,QAChC,iBAAiB,YAAY;AAAA,MAC/B;AAAA,IACF;AACA,QACE,YAAY,QAAQ,QAAQ,sBAC5B,wBAAwB,QAAQ,QAAQ,oBACxC;AACA,aAAO,EAAE,QAAQ,YAAY,QAAQ,4BAA4B;AAAA,IACnE;AAGA,UAAM,UAAU,MAAM,qBAAqB,KAAK;AAChD,QAAI,CAAC,WAAW,CAAC,uBAAuB,CAAC,QAAQ;AAC/C,aAAO,EAAE,QAAQ,YAAY,QAAQ,kCAAkC;AACzE,YAAQ,oBAAoB,QAAQ,mBAAmB,SAAS,mBAAmB;AACnF,gBAAY,QAAQ,QAAQ;AAAA,EAC9B;AACA,QAAM,cAAc,OAAO,UAAkC;AAC3D,QAAI,CAAC,eAAe;AAClB,UAAI,QAAQ,2BAA2B,KAAK;AAC1C,cAAM,IAAI,MAAM,oEAAoE;AACtF;AAAA,IACF;AACA,UAAM,OAAO,QAAQ,2BAA2B,KAAK;AACrD,QAAI,MAAM,QAAQ,SAAS,qBAAqB,OAAO,UAAU,UAAU;AACzE,UAAI,CAAC,QAAQ,wBAAwB,CAAE,MAAM,QAAQ,qBAAqB,KAAK;AAC7E,cAAM,IAAI,MAAM,8DAA8D;AAChF;AAAA,IACF;AACA,QAAI,MAAM;AACR,iBAAW,SAAS,KAAK;AACvB,YAAI,MAAM;AACR,gBAAM,cAAc,MAAM,MAAM,MAAM,SAAS,MAAM,QAAQ,KAAK,cAAc;AAAA,IACtF,OAAO;AACL,YAAM,2BAA2B,QAAQ,IAAI,OAAO,aAAa;AAAA,IACnE;AACA,QAAI,OAAO,UAAU,SAAU,OAAM,QAAQ,2BAA2B,KAAK;AAAA,EAC/E;AAEA,MAAI;AACJ,MAAI;AACF,UAAM,sBACJ,SAAS,YAAY,MAAM,mBAAmB,sBACxC,MAAM,QAAQ;AAAA,MACd,QAAQ;AAAA,MACR,MAAM,mBAAmB,sBAAsB,sBAAsB;AAAA,IACvE,KAAM,QACN;AACN,aAAS,MAAM;AAAA,MACb,QAAQ,WAAW;AAAA,QACjB,WAAW,QAAQ;AAAA;AAAA;AAAA,QAGnB,OACE,SAAS,YACT,CAAC,wBACA,QAAQ,oBAAoB,WAAW,aACpC,MACA;AAAA,QACN,aAAa;AAAA,QACb,iBAAiB;AAAA,MACnB,CAAC;AAAA,IACH;AAAA,EACF,SAAS,OAAO;AACd,QAAI,UAAU,gCAAgC;AAC5C,aAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,IAC1D;AACA,QAAI,2CAA2C,OAAO,WAAW,KAAK,GAAG;AACvE,UAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,eAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,MAClE;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,OAAO;AAAA,UACL,SAAS;AAAA,UACT,UAAU;AAAA,UACV,QAAQ;AAAA,QACV;AAAA,MACF;AAAA,IACF;AACA,WAAO,EAAE,QAAQ,YAAY,QAAQ,iBAAiB;AAAA,EACxD;AAGA,QAAM,cAAc,MAAM;AAAA,IACxB;AAAA,IACA,QAAQ;AAAA,IACR;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,MACE,YAAY,WAAW,cACvB,YAAY,WAAW,sBACvB,MAAM,wBAAwB,QAC9B,SAAS,YACT,CAAC,QAAQ,2BACT,aACA,CAAE,MAAM,qBACJ,KAAK,EACN,KAAK,CAAC,YAAY,SAAS,SAAS,qBAAqB,QAAQ,WAAW,GAC/E;AAIA,QAAI;AACF,YAAM,cAAc,MAAM;AAAA,QACxB,QAAQ,WAAW;AAAA,UACjB,WAAW,QAAQ;AAAA,UACnB,OAAO;AAAA,UACP,aAAa;AAAA,UACb,iBAAiB;AAAA,QACnB,CAAC;AAAA,MACH;AACA,YAAM,YAAY,WAAW;AAC7B,aAAO,kCAAkC,aAAa,QAAQ,mBAAmB,OAAO;AAAA,IAC1F,SAAS,OAAO;AACd,UAAI,2CAA2C,OAAO,WAAW,KAAK,GAAG;AACvE,YAAI,QAAQ,oBAAoB,WAAW,CAAC,QAAQ,oBAAoB;AACtE,iBAAO,EAAE,QAAQ,YAAY,QAAQ,2BAA2B;AAAA,QAClE;AACA,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,OAAO;AAAA,YACL,SAAS;AAAA,YACT,UAAU;AAAA,YACV,QAAQ;AAAA,UACV;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,QAAQ,UAAU,iCAAiC,qBAAqB;AAAA,MAC1E;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAIO,SAAS,kCACd,QACA,mBACA,QAC4B;AAC5B,MAAI,OAAO,WAAW,UAAU;AAC9B,WAAO,EAAE,QAAQ,YAAY,QAAQ,mBAAmB;AAAA,EAC1D;AACA,MAAI,wBAAwB,QAAQ,mBAAmB,MAAM,GAAG;AAC9D,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,OAAO;AAAA,QACL,SAAS;AAAA,QACT,UAAU;AAAA,QACV,QAAQ;AAAA,MACV;AAAA,IACF;AAAA,EACF;AACA,QAAM,WAAW,wBAAwB,MAAM;AAC/C,MAAI,aAAa,MAAM;AACrB,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,OAAO,EAAE,SAAS,UAAU,UAAU,QAAQ,uBAAuB;AAAA,IACvE;AAAA,EACF;AACA,SAAO;AAAA,IACL,QAAQ;AAAA,IACR,QAAQ,OAAO,SAAS,iCAAiC,IACrD,qBACA;AAAA,EACN;AACF;AAUA,eAAe,eACb,IACA,UACA,eACyD;AACzD,QAAM,eAAe,oBAAI,IAAY;AACrC,MAAI,UAAU;AACd,MAAI,SAA+B,CAAC;AACpC,MAAI;AACF,aAAS,MAAM,wBAAwB,EAAE;AAAA,EAC3C,SAAS,OAAO;AACd,kBAAc,KAAK,0CAA0C;AAAA,MAC3D,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,WAAO,EAAE,SAAS,aAAa;AAAA,EACjC;AACA,aAAW,SAAS,QAAQ;AAC1B,iBAAa,IAAI,MAAM,WAAW;AAAA,EACpC;AACA,QAAM,uBAAuB,QAAQ,sCAAsC,OAAO,UAAU;AAC1F,QAAI;AACF,YAAM,OACJ,SAAS,2BAA2B,eAChC,IACA,+BAA+B,UAAU,MAAM,OAAO;AAC5D,YAAM,SAAS,MAAM,kBAAkB,IAAI;AAAA,QACzC,WAAW,MAAM;AAAA,QACjB,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,eAAe,MAAM;AAAA,QACrB,yBAAyB;AAAA,QACzB,aAAa,SAAS;AAAA,QACtB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD,UAAI,OAAO,SAAS;AAClB,mBAAW;AAAA,MACb;AACA,UAAI,OAAO,aAAa,KAAK,SAAS,2BAA2B,WAAW;AAC1E,2BAAmB,eAAe,SAAS,OAAO,UAAU;AAAA,MAC9D;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,yDAAyD;AAAA,QAC1E,aAAa,MAAM;AAAA,QACnB,gBAAgB,MAAM;AAAA,QACtB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACD,SAAO,EAAE,SAAS,aAAa;AACjC;AAOA,eAAsB,oCAAoC,OAKxC;AAChB,QAAM;AAAA,IACJ,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,MAAM,+BAA+B;AAAA,EACvC;AACF;AAQA,eAAsB,4BAA4B,OAI7B;AACnB,QAAM,EAAE,UAAU,cAAc,IAAI;AACpC,MAAI,CAAC,2BAA2B,QAAQ,EAAG,QAAO;AAClD,MAAI,OAAO,MAAM,uBAAuB,6BAA6B,QAAQ,EAAG,QAAO;AACvF,oCAAkC,eAAe;AAAA,IAC/C,SAAS;AAAA,IACT,UAAU;AAAA,IACV,2BAA2B;AAAA,EAC7B,CAAC;AACD,0CAAwC,eAAe,gBAAgB;AACvE,SAAO;AACT;AAEA,SAAS,2BAA2B,UAAiD;AACnF,SACE,SAAS,mBAAmB,WAAW,CAAC,CAAC,SAAS,gBAAgB,CAAC,CAAC,SAAS;AAEjF;AAEA,eAAe,iCACb,IACA,UACA,eACA,6BACe;AACf,QAAM,QAAQ,IAAI;AAAA,KACf,YAAY;AACX,UAAI;AACF,+BAAuB,eAAe,MAAM,iBAAiB,EAAE,CAAC;AAAA,MAClE,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,IACH,6BAA6B,eAAe,UAAU,iBAAiB,YAAY;AACjF,+BAAyB,eAAe,MAAM,6BAA6B,EAAE,CAAC;AAAA,IAChF,CAAC;AAAA,IACD;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,uCAAuC,EAAE;AAAA,QACjD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,sCAAsC,EAAE;AAAA,QAChD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV,2CAAmC,eAAe,MAAM,2BAA2B,EAAE,CAAC;AAAA,MACxF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,+BAA+B,EAAE;AAAA,QACzC;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,yCAAyC,EAAE;AAAA,QACnD;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA;AAAA,MACA,YAAY;AACV;AAAA,UACE;AAAA,UACA,MAAM,qCAAqC,IAAI,MAAM,wBAAwB,EAAE,CAAC;AAAA,QAClF;AAAA,MACF;AAAA,IACF;AAAA,IACA,6BAA6B,eAAe,kBAAkB,oBAAoB,YAAY;AAC5F;AAAA,QACE;AAAA,QACA,MAAM,kCAAkC,EAAE;AAAA,MAC5C;AAAA,IACF,CAAC;AAAA,IACD,GAAI,SAAS,mBAAmB,iBAChC,SAAS,0CACL;AAAA,MACE;AAAA,QACE;AAAA,QACA;AAAA,QACA;AAAA,QACA,YAAY;AACV;AAAA,YACE;AAAA,YACA,MAAM,4BAA4B;AAAA,cAChC,WAAW,SAAS;AAAA,YACtB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAAA,IACF,IACA,CAAC;AAAA,KACJ,YAAY;AACX,UAAI;AACF,kCAA0B,eAAe,MAAM,4BAA4B,EAAE,CAAC;AAAA,MAChF,SAAS,OAAO;AACd,sBAAc,KAAK,uDAAuD;AAAA,UACxE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,+BAAuB,eAAe,MAAM,iBAAiB,EAAE,CAAC;AAAA,MAClE,SAAS,OAAO;AACd,sBAAc,KAAK,oDAAoD;AAAA,UACrE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,gCAAwB,eAAe,MAAM,sBAAsB,EAAE,CAAC;AAAA,MACxE,SAAS,OAAO;AACd,sBAAc,KAAK,qDAAqD;AAAA,UACtE,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX;AAAA,QACE;AAAA,QACA,SAAS;AAAA,MACX;AACA,UAAI;AACF,cAAM,cAAc,MAAM,8BAA8B,EAAE;AAC1D,6CAAqC,eAAe,aAAa,kBAAkB,IAAI;AAAA,MACzF,SAAS,OAAO;AACd,sBAAc,KAAK,4DAA4D;AAAA,UAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,KACF,YAAY;AACX,UAAI;AACF,cAAM,gBAAgB,MAAM,gCAAgC,EAAE;AAC9D;AAAA,UACE;AAAA,UACA,eAAe,mBAAmB;AAAA,QACpC;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,2DAA2D;AAAA,UAC5E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF,GAAG;AAAA,EACL,CAAC;AACH;AAEA,eAAe,6BACb,eACA,QACA,gBACA,SACe;AACf,MAAI;AACF,UAAM,QAAQ;AACd,4CAAwC,eAAe,MAAM;AAAA,EAC/D,SAAS,OAAO;AACd,4CAAwC,eAAe,MAAM;AAC7D,kBAAc,KAAK,mBAAmB,cAAc,yBAAyB;AAAA,MAC3E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH;AACF;AASA,eAAe,8BACb,IACA,UACA,cACA,eACiB;AACjB,QAAM,iBAAiB;AACvB,QAAM,MAAM,SAAS;AACrB,MAAI,eAAe;AACnB,QAAM;AAAA,IACJ,CAAC,GAAG,YAAY;AAAA,IAChB;AAAA,IACA,OAAO,gBAAgB;AACrB,UAAI;AACF,cAAM,SAAS,MAAM,mCAAmC,IAAI;AAAA,UAC1D;AAAA,UACA;AAAA,UACA,4BAA4B;AAAA,UAC5B,aAAa,SAAS;AAAA,QACxB,CAAC;AACD,YAAI,OAAO,aAAa,OAAO,QAAQ,SAAS,GAAG;AACjD,0BAAgB,OAAO,QAAQ;AAC/B,wBAAc,KAAK,gEAAgE;AAAA,YACjF;AAAA,YACA,QAAQ,OAAO;AAAA,YACf,SAAS,OAAO,QAAQ;AAAA,UAC1B,CAAC;AAAA,QACH;AAAA,MACF,SAAS,OAAO;AACd,sBAAc,KAAK,0DAA0D;AAAA,UAC3E;AAAA,UACA,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QAC9D,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAsB,8BACpB,IACA,UACA,eACA,WAAkD,gCACnC;AACf,MAAI,SAAS,mBAAmB,WAAW,CAAC,SAAS,gBAAgB,CAAC,SAAS;AAC7E;AACF,QAAM,aAAa,MAAM,gCAAgC,EAAE;AAC3D,QAAM,uBAAuB,YAAY,GAAG,OAAO,cAAc;AAC/D,QAAI;AACF,YAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,UAAU,WAAW;AAC5E,YAAM,WAAW,EAAE,GAAG,WAAW,UAAU;AAC3C,YAAM,UAAU,MAAM,iCAAiC,IAAI,QAAQ;AACnE,UAAI,CAAC,QAAS;AACd,YAAM,aAAa,MAAM,SAAS,UAAU,OAAO;AACnD,UAAI,CAAC,WAAY;AACjB,UAAI,MAAM,mCAAmC,IAAI,EAAE,GAAG,UAAU,SAAS,WAAW,CAAC,GAAG;AACtF,sBAAc,KAAK,kDAAkD;AAAA,UACnE,GAAG;AAAA,UACH,aAAa,QAAQ;AAAA,UACrB;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF,SAAS,OAAO;AACd,oBAAc,KAAK,iEAAiE;AAAA,QAClF,GAAG;AAAA,QACH,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF,CAAC;AACH;AAEO,SAAS,oCACd,QACA,WACS;AAKT,MAAI,OAAO,KAAK,CAAC,UAAU,MAAM,aAAa,aAAa,MAAM,eAAe,IAAI,GAAG;AACrF,WAAO;AAAA,EACT;AACA,SAAO,iCAAiC,QAAQ,SAAS;AAC3D;AAMO,SAAS,oCACd,QACA,oBACQ;AACR,SAAO,mBAAmB;AAAA,IAAO,CAAC,eAChC,OAAO,KAAK,CAAC,UAAU,MAAM,aAAa,UAAU,MAAM,eAAe,UAAU;AAAA,EACrF,EAAE;AACJ;AAKO,SAAS,iCACd,QACA,WACS;AACT,MAAI,OAAO,KAAK,CAAC,UAAU,MAAM,eAAe,UAAU,SAAS,GAAG;AACpE,WAAO;AAAA,EACT;AAEA,QAAM,UAAU,UAAU,KAAK;AAC/B,QAAM,cAAc,UAAU,KAAK;AACnC,QAAM,iBAAiB,UAAU,KAAK;AACtC,MAAI,CAAC,WAAW,CAAC,eAAe,CAAC,gBAAgB;AAC/C,WAAO;AAAA,EACT;AACA,QAAM,oBAAoB,OAAO;AAAA,IAC/B,CAAC,UACC,MAAM,YAAY,WAClB,MAAM,gBAAgB,eACtB,MAAM,mBAAmB;AAAA,EAC7B;AACA,SAAO;AAAA,IACL,CAAC,qBACA,kBAAkB,eAAe,QAAQ,kBAAkB,eAAe,UAAU;AAAA,EACvF;AACF;AAEA,eAAe,sCACb,UACA,IACA,eACiB;AAIjB,MAAI,CAAC,2BAA2B,QAAQ,GAAG;AACzC,WAAO;AAAA,EACT;AACA,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,sCAAsC,EAAE;AAAA,EAC7D,SAAS,OAAO;AACd,kBAAc,KAAK,4DAA4D;AAAA,MAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,MAAM,0BAA0B,UAAU,YAAY;AAAA,IACnE,uBAAuB,OAAO,cAAc;AAC1C,YAAM,SAAS,MAAM,sCAAsC,EAAE;AAC7D,aAAO,oCAAoC,QAAQ,SAAS;AAAA,IAC9D;AAAA,EACF,CAAC;AAKD,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI;AACF,YAAM,SAAS,MAAM,sCAAsC,EAAE;AAC7D,wCAAkC,eAAe;AAAA,QAC/C,SAAS,OAAO,UAAU;AAAA,QAC1B,UAAU,OAAO,UAAU,aAAa;AAAA,UAAO,CAAC,cAC9C,iCAAiC,QAAQ,SAAS;AAAA,QACpD,EAAE;AAAA,QACF,2BAA2B;AAAA,UACzB;AAAA,UACA,OAAO,UAAU;AAAA,QACnB;AAAA,MACF,CAAC;AACD,8CAAwC,eAAe,gBAAgB;AAAA,IACzE,SAAS,OAAO;AACd,8CAAwC,eAAe,gBAAgB;AACvE,oBAAc,KAAK,4DAA4D;AAAA,QAC7E,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AACA,aAAW,cAAc,OAAO,YAAY;AAC1C,kBAAc,KAAK,mDAAmD;AAAA,MACpE,WAAW,WAAW;AAAA,MACtB,QAAQ,WAAW;AAAA,MACnB,MAAM,KAAK,UAAU,WAAW,IAAI;AAAA,IACtC,CAAC;AAAA,EACH;AACA,MAAI,OAAO,WAAW,GAAG;AACvB,kBAAc,KAAK,gDAAgD;AAAA,MACjE,UAAU,OAAO;AAAA,MACjB,YAAY,OAAO,WAAW;AAAA,MAC9B,SAAS,OAAO;AAAA,MAChB,SAAS,SAAS;AAAA,IACpB,CAAC;AAAA,EACH;AACA,SAAO,OAAO,WAAW;AAC3B;AAWA,eAAe,gCACb,UACA,OAC8B;AAC9B,MAAI,CAAC,MAAM,cAAc,CAAC,MAAM,aAAa;AAC3C,UAAM,IAAI,MAAM,8DAA8D;AAAA,EAChF;AACA,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,MAAI;AACJ,MAAI;AACF,kBAAc,MAAM,oCAAoC,UAAU,MAAM,aAAa;AAAA,MACnF,WAAW,4BAA4B,MAAM,cAAc;AAAA,MAC3D,UAAU;AAAA,MACV,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH,SAAS,OAAO;AACd,QAAI,+BAA+B,SAAS,KAAK,EAAG,QAAO;AAC3D,UAAM;AAAA,EACR;AACA,MAAI,YAAY,eAAe,MAAM,YAAY;AAC/C,UAAM,IAAI,MAAM,iEAAiE;AAAA,EACnF;AACA,MAAI;AACF,QAAI,YAAY,cAAc,SAAS;AACrC,YAAM,2BAA2B,aAAa,0CAA0C;AACxF,aAAO;AAAA,IACT;AACA,UAAM,UAAU,YAAY;AAY5B,UAAM,MAAM,QAAQ,QAAQ,QAAQ;AACpC,QAAI,CAAC,IAAK,OAAM,IAAI,MAAM,6DAA6D;AACvF,QAAI;AACJ,UAAM,SAAS,MAAM,QAAQ,KAAK;AAAA,MAChC,IAAI,KAAK,SAAS;AAAA,QAChB,KAAK;AAAA,QACL,aAAa;AAAA,QACb,iBAAiB;AAAA,MACnB,CAAC;AAAA,MACD,IAAI,QAAe,CAAC,GAAG,WAAW;AAChC,gBAAQ;AAAA,UACN,MAAM,OAAO,IAAI,MAAM,qDAAqD,CAAC;AAAA,UAC7E;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH,CAAC,EAAE,QAAQ,MAAM;AACf,UAAI,MAAO,cAAa,KAAK;AAAA,IAC/B,CAAC;AACD,QAAI,2BAA2B,MAAM,KAAK,uBAAuB,MAAM,MAAM,GAAG;AAC9E,YAAM,IAAI,MAAM,yDAAyD;AAAA,IAC3E;AACA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,QAAI,+BAA+B,SAAS,KAAK,EAAG,QAAO;AAC3D,UAAM;AAAA,EACR;AACF;AAaO,SAAS,mBACd,OACoB;AACpB,MAAI,MAAM,wBAAwB,KAAM,QAAO;AAC/C,SAAO,MAAM,kBAAkB;AACjC;AAkBA,eAAe,sBACb,IACA,UACA,KACA,eACA,cACA,wBACA,SACA,eACA,MAAsC,MACpB;AAGlB,QAAM,EAAE,UAAU,IAAI,MAAM,uBAAuB,IAAI,IAAI,WAAW;AAGtE,QAAM,QAAQ,MAAMF,WAAU,IAAI,IAAI,aAAa,IAAI,cAAc;AACrE,MAAI,CAAC,OAAO;AAEV,WAAO;AAAA,EACT;AACA,MACE,MAAM,aAAa,cAClB,MAAM,aAAa,KAAK,CAAC,MAAM,6BAA6B,UAC7D,MAAM,eAAe,IAAI,YACzB;AAIA,WAAO;AAAA,EACT;AAEA,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,QAAM,kBAAkB,YAAY,UAAU,YAAY;AAC1D,QAAM,gBAAgB,+BAA+B,SAAS,MAAM,WAAW;AAC/E,MAAI,mBAAmB,CAAC,eAAe;AACrC,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,eAAe,iBAAiB,YAAY;AAC9C,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAEA,QAAM,mBAAmB,QAAQ;AACjC,MACE,mBACA,CAAC,MAAM,gBAAgB,eACvB,qCAAqC,KAAK,GAC1C;AAMA,UAAM,IAAI;AAAA,MACR,WAAW,IAAI,cAAc,UAAU,IAAI,UAAU;AAAA,IACvD;AAAA,EACF;AACA,MAAI,eAIO;AACX,MAAI,+BAA+B;AACnC,MAAI,qBAA8C;AAClD,MAAI,MAAM,cAAc,iBAAiB;AACvC,QAAI,MAAM,gBAAgB;AACxB,YAAM,eAAe,MAAM;AAC3B,YAAM,aAAa,MAAM;AACzB,UAAI,aAAa,aAAa;AAK5B,uBAAe;AAAA,UACb,GAAG;AAAA,UACH,SAAS,MAAM;AAAA,UACf,YAAY,MAAM;AAAA,UAClB;AAAA,UACA,mBAAmB,MAAM;AAAA,UACzB,iBAAiB,MAAM;AAAA,QACzB;AACA,6BAAqB;AAAA,MACvB,OAAO;AACL,YACE,aAAa,gBAAgB,QAAQ,eACrC,aAAa,WAAW,QAAQ,SAChC;AAKA,gBAAM,IAAI;AAAA,YACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE,wBAAwB,aAAa,OAAO;AAAA,UACtG;AAAA,QACF;AACA,cAAM,kBAAkB,MAAM,uCAAuC,IAAI;AAAA,UACvE;AAAA,UACA,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,WAAW,aAAa;AAAA,UACxB,eAAe,IAAI;AAAA,UACnB,oBAAoB;AAAA,QACtB,CAAC;AACD,cAAM,6BACJ,YAAY,WACZ,CAAC,sCAAsC,MAAM,aAAa,OAAO,KACjE,MAAM,SAAS,SAAS,WAAW,cACnC,MAAM,SAAS,SAAS,eAAe,cACvC,MAAM,SAAS,UAAU,WAAW;AAOtC,cAAM,UAAU,YACd,MAAM,yCAAyC,IAAI;AAAA,UACjD;AAAA,UACA,aAAa,IAAI;AAAA,UACjB,gBAAgB,IAAI;AAAA,UACpB,gBAAgB,aAAa;AAAA,UAC7B,WAAW,QAAQ;AAAA,UACnB,aAAa,QAAQ;AAAA,UACrB,SAAS,QAAQ;AAAA,UACjB,eAAe,IAAI;AAAA,UACnB,oBAAoB;AAAA,UACpB;AAAA,QACF,CAAC;AAMH,cAAM,sBACJ,kBAAkB,QAAQ,cAAc,aAAa;AACvD,YAAI,CAAC,MAAM,aAAa;AACtB,cAAI,8BAA8B,aAAa,gBAAgB,qBAAqB;AAClF,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2BAAe;AAAA,UACjB,OAAO;AACL,gBAAI,CAAC,iBAAiB;AACpB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,gBAAI,CAAC,4BAA4B;AAC/B,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AAIA,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2BAAe;AAAA,UACjB;AAAA,QACF,WAAW,aAAa,gBAAgB,qBAAqB;AAI3D,gBAAM,cAAc,MAAM,QAAQ;AAClC,cAAI,CAAC,aAAa;AAChB,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,0BAA0B,aAAa,EAAE;AAAA,YACxE;AAAA,UACF;AACA,yBAAe;AAAA,QACjB,OAAO;AACL,cAAI,CAAC,iBAAiB;AACpB,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,YAC1D;AAAA,UACF;AACA,gBAAM,gBAAgB,MAAM,uBAAuB,UAAU,KAAK;AAClE,cAAI,kBAAkB,WAAW;AAI/B,kBAAM,cAAc,MAAM,QAAQ;AAClC,gBAAI,CAAC,aAAa;AAChB,oBAAM,IAAI;AAAA,gBACR,WAAW,IAAI,cAAc,YAAY,aAAa,EAAE;AAAA,cAC1D;AAAA,YACF;AACA,2CAA+B;AAC/B,2BAAe;AAAA,UACjB,OAAO;AAML,kBAAM,IAAI;AAAA,cACR,WAAW,IAAI,cAAc,sBAAsB,aAAa,EAAE;AAAA,YACpE;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF,OAAO;AACL,YAAM,UAAU,MAAM,6BAA6B,IAAI;AAAA,QACrD;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,WAAW,QAAQ;AAAA,QACnB,aAAa,QAAQ;AAAA,QACrB,SAAS,QAAQ;AAAA,QACjB,eAAe,IAAI;AAAA,QACnB,oBAAoB,MAAM;AAAA,QAC1B,UAAU;AAAA,QACV;AAAA,QACA,eAAe;AAAA,QACf,oBAAoB,sCAAsC,SAAS,MAAM,WAAW;AAAA,QACpF,gCAAgC,sCAAsC,QAAQ;AAAA;AAAA;AAAA,QAG9E,8BAA8B,KAAK,IAAI,SAAS,oBAAoB,GAAM;AAAA,QAC1E,oBAAoB,eAAe,aAAa;AAAA,QAChD,cAAc,kBAAkB,QAAQ,cAAc,aAAa;AAAA,MACrE,CAAC;AACD,UAAI,QAAQ,WAAW,WAAW;AAChC,uBAAe,QAAQ;AAAA,MACzB,WAAW,QAAQ,WAAW,wBAAwB;AAOpD,cAAM,gBAAgB,MAAM,uBAAuB,UAAU,KAAK;AAClE,YAAI,kBAAkB,UAAW,QAAO;AACxC,uCAA+B;AAAA,MACjC,WAAW,QAAQ,WAAW,uBAAuB;AAInD,cAAM,IAAI;AAAA,UACR,WAAW,IAAI,cAAc;AAAA,QAC/B;AAAA,MACF,OAAO;AACL,eAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAUA,MAAI,YAAY,uBAAuB;AACvC,MAAI,kBACF,uBAAuB,sBAClB,MAAM,SAAS,QAAQ,SAAS,YAAY,OAC7C;AACN,QAAM,iBAAmC,OACvC,eACA,iBACA,kBACA,oBAGG;AAIH,QAAI,CAAC,MAAM,cAAc,CAAC,cAAc;AACtC,aAAO,EAAE,OAAO,OAAO,iBAAiB,KAAK;AAAA,IAC/C;AACA,UAAM,iBAAiB,oCAAoC,aAAa;AACxE,QAAI,uBAAsC;AAC1C,QACE,OAAO,kBAAkB,YACzB,iBACA,iBAAiB,YAAY,MAC5B,gBAAgB,aAAa,+BAC5B,gBAAgB,aAAa,6BAC/B;AACA,UAAI,CAAC,iBAAiB;AACpB,cAAM,IAAI,MAAM,8DAA8D;AAAA,MAChF;AACA,YAAM,YAAY,MAAM,kCAAkC,IAAI;AAAA,QAC5D;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,eAAe,MAAM;AAAA,QACrB,kBAAkB,IAAI;AAAA,QACtB,kBAAkB,MAAM;AAAA,QACxB,2BAA2B,aAAa;AAAA,QACxC,oBAAoB,gBAAgB;AAAA,QACpC,iBAAiB,gBAAgB;AAAA,QACjC,kBAAkB;AAAA,QAClB,sBAAsB;AAAA,MACxB,CAAC;AACD,6BAAuB,UAAU;AAAA,IACnC;AACA,UAAM,YAAY;AAAA,MAChB;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,iBAAiB,MAAM;AAAA,MACvB,eAAe,IAAI;AAAA,MACnB,oBAAoB,MAAM;AAAA,MAC1B,6BAA6B,aAAa;AAAA,MAC1C,WAAW,aAAa;AAAA,MACxB,mBAAmB,aAAa;AAAA,IAClC;AACA,QAAI;AACJ,QAAI,kBAAkB,MAAM;AAC1B,eAAS,MAAM,qBAAqB,IAAI;AAAA,QACtC,GAAG;AAAA,QACH,kBAAkB;AAAA,MACpB,CAAC;AAAA,IACH,OAAO;AACL,UAAI,CAAC,iBAAiB;AACpB,cAAM,IAAI,MAAM,qEAAqE;AAAA,MACvF;AACA,YAAM,YAAY,MAAM,8BAA8B;AAAA,QACpD,SAAS,MAAM;AAAA,QACf;AAAA,QACA;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,SACE,OAAO,kBAAkB,WACrB,gBACA;AAAA,UACE,OAAO,OAAO,KAAK,eAAe,QAAQ;AAAA,UAC1C,YAAY;AAAA,UACZ,QAAQ;AAAA,QACV;AAAA,QACN,SAAS;AAAA,MACX,CAAC;AACD,eAAS,MAAM,iCAAiC;AAAA,QAC9C;AAAA,QACA,GAAI,UAAU,sBAAsB,EAAE,KAAK,UAAU,oBAAoB,IAAI,CAAC;AAAA,QAC9E,SAAS,MACP,qBAAqB,IAAI;AAAA,UACvB,GAAG;AAAA,UACH,sBAAsB;AAAA,UACtB,GAAG;AAAA,UACH,GAAI,uBAAuB,EAAE,qBAAqB,IAAI,CAAC;AAAA,QACzD,CAAC;AAAA,MACL,CAAC;AAAA,IACH;AACA,QAAI,CAAC,OAAO,SAAS,sBAAsB;AACzC,YAAM,2CAA2C,IAAI;AAAA,QACnD;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,YAAY;AAAA,QACZ,QAAQ;AAAA,MACV,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,IAC1B;AACA,QAAI,OAAO,SAAS,kBAAkB,MAAM;AAC1C,kBAAY;AACZ,wBAAkB,OAAO;AAAA,IAC3B;AACA,WAAO;AAAA,EACT;AAcA,MACE,CAAC,gCACD,YAAY,WACZ,MAAM,cACN,CAAE,MAAM,qCAAqC,IAAI;AAAA,IAC/C;AAAA,IACA,aAAa,IAAI;AAAA,IACjB,gBAAgB,IAAI;AAAA,IACpB,eAAe,IAAI;AAAA,IACnB,oBAAoB,MAAM;AAAA,IAC1B,UAAU;AAAA,EACZ,CAAC,GACD;AACA,WAAO;AAAA,EACT;AAIA,MAAI,mBAAmB;AACvB,QAAM,qBAAqB,YAA2B;AACpD,UAAM,SAAS,MAAM,0BAA0B,IAAI;AAAA,MACjD;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,IACrB,CAAC;AACD,QAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,4DAA4D;AACzF,uBAAmB;AAAA,EACrB;AACA,QAAM,cAAoD,+BACtD,EAAE,YAAY,MAAM,8BAA8B,KAAK,IACvD,MAAM;AAAA,IACJ;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,cAAc,qBAAqB,QAAQ;AAAA,IAC3C;AAAA,IACA;AAAA,IACA,QAAQ,aAAa;AAAA,IACrB,YAAY;AACV,UAAI,CAAC,gBAAgB,CAAC,MAAM,WAAY;AACxC,YAAM,+BAA+B,IAAI;AAAA,QACvC;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,WAAW,aAAa;AAAA,QACxB,eAAe,IAAI;AAAA,QACnB,oBAAoB,MAAM;AAAA,MAC5B,CAAC;AAAA,IACH;AAAA,IACA,IAAI;AAAA,IACJ;AAAA,IACA,YAAY,WACR,YAAY;AACV,UAAI,CAAC,gBAAgB,CAAC,MAAM;AAC1B,cAAM,IAAI,MAAM,6CAA6C;AAC/D,aAAO,MAAM,8BAA8B,IAAI;AAAA,QAC7C;AAAA,QACA,aAAa,IAAI;AAAA,QACjB,gBAAgB,IAAI;AAAA,QACpB,SAAS,MAAM;AAAA,QACf,YAAY,IAAI;AAAA,QAChB,YAAY,MAAM;AAAA,QAClB,qBAAqB,aAAa;AAAA,QAClC,WAAW,aAAa;AAAA,QACxB,mBAAmB,aAAa;AAAA,MAClC,CAAC;AAAA,IACH,IACA;AAAA,EACN;AACJ,QAAM,aAAa,OAAO,gBAAgB,YAAY,cAAc,YAAY;AAChF,MAAI,CAAC,YAAY;AACf,WAAO;AAAA,EACT;AAKA,QAAM,4BACJ,gCACC,OAAO,gBAAgB,aAAa,YAAY;AACnD,MAAI,CAAC,oBAAoB,CAAC,2BAA2B;AACnD,QAAI,mBAAmB,MAAM,YAAY;AACvC,YAAM,IAAI,MAAM,qDAAqD;AAAA,IACvE;AACA,UAAM,mBAAmB;AAAA,EAC3B;AAMA,QAAM,YACJ,SAAS,2BAA2B,eAChC,IACA,+BAA+B,UAAU,OAAO;AACtD,MAAI,CAAC,2BAA2B;AAC9B,UAAM,YAAY,MAAM,kBAAkB,IAAI;AAAA,MAC5C;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,MACnB,yBAAyB;AAAA,MACzB,aAAa,SAAS;AAAA,MACtB,YAAY;AAAA,IACd,CAAC;AACD,QAAI,UAAU,aAAa,KAAK,SAAS,2BAA2B,WAAW;AAC7E,yBAAmB,eAAe,SAAS,UAAU,UAAU;AAAA,IACjE;AAAA,EACF;AAEA,QAAM,kBACJ,iCACC,OAAO,gBAAgB,YAAY,QAAQ,YAAY;AAK1D,QAAM,EAAE,UAAU,yBAAyB,wBAAwB,IAAI,MAAM;AAAA,IAC3E;AAAA,IACA;AAAA,MACE;AAAA,MACA,aAAa,IAAI;AAAA,MACjB,gBAAgB,IAAI;AAAA,MACpB,eAAe,IAAI;AAAA,MACnB,GAAI,eAAe,EAAE,mBAAmB,aAAa,GAAG,IAAI,CAAC;AAAA,MAC7D,8BAA8B;AAAA,MAC9B,0BAA0B,SAAS;AAAA;AAAA,MAEnC,iBACE,mBAAmB,MAAM,YAAY,gBAAgB,MAAM,oBAAoB;AAAA,IACnF;AAAA,EACF;AAGA,QAAM,4BAA4B,KAAK,IAAI,aAAa,yBAAyB,CAAC,UAAU;AAC1F,kBAAc,KAAK,yDAAyD;AAAA,MAC1E,gBAAgB,IAAI;AAAA,MACpB,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,IAC9D,CAAC;AAAA,EACH,CAAC;AACD,MAAI,UAAU;AAMZ,QAAI,mBAAmB,CAAC,yBAAyB;AAC/C,gDAA0C,eAAe,OAAO;AAAA,IAClE;AACA,QAAI,MAAM,6BAA6B,QAAQ;AAC7C;AAAA,QACE;AAAA,QACA,kBAAkB,qBAAqB;AAAA,MACzC;AAAA,IACF;AAKA,QAAI;AACF,YAAM,iCAAiC,IAAI,IAAI,aAAa,IAAI,gBAAgB;AAAA,QAC9E,MAAM;AAAA,QACN,SAAS;AAAA,UACP,OAAO;AAAA,UACP;AAAA,UACA;AAAA,UACA,YAAY,MAAM;AAAA,UAClB,8BAA8B;AAAA;AAAA;AAAA;AAAA;AAAA,UAK9B,aAAa,mBAAmB,KAAK;AAAA,QACvC;AAAA,MACF,CAAC;AAAA,IACH,SAAS,YAAY;AACnB,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,gBAAgB,IAAI;AAAA,UACpB,YAAY,IAAI;AAAA,UAChB,SAAS,MAAM;AAAA,UACf,YAAY,MAAM;AAAA,UAClB,OAAO,sBAAsB,QAAQ,WAAW,UAAU,OAAO,UAAU;AAAA,QAC7E;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAiBA,SAAS,yBACP,OACA,SAOA;AACA,SAAO;AAAA,IACL,SAAS,MAAM;AAAA,IACf,gBAAgB,MAAM;AAAA,IACtB,YAAY,MAAM;AAAA,IAClB;AAAA,IACA,YAAY,MAAM;AAAA,EACpB;AACF;AAmCA,eAAsB,8BACpB,IACA,OAWA,UAAU,EAAE,WAAAA,YAAW,qCAAqC,GACpB;AACxC,QAAM,UAAU,MAAM,QAAQ,UAAU,IAAI,MAAM,aAAa,MAAM,cAAc;AACnF,MACE,CAAC,WACD,QAAQ,OAAO,MAAM,WACrB,QAAQ,aAAa,cACrB,QAAQ,eAAe,MAAM,cAC7B,QAAQ,eAAe,MAAM,cAC7B,QAAQ,YAAY,YACpB,QAAQ,wBAAwB,MAAM,uBACtC,QAAQ,gBAAgB,OAAO,MAAM,aACrC,QAAQ,eAAe,sBAAsB,MAAM,qBACnD,QAAQ,eAAe,wBAAwB,MAAM;AAErD,UAAM,IAAI,MAAM,gDAAgD;AAClE,QAAM,YAAY,MAAM,QAAQ,qCAAqC,IAAI;AAAA,IACvE,WAAW,MAAM;AAAA,IACjB,aAAa,MAAM;AAAA,IACnB,gBAAgB,MAAM;AAAA,IACtB,eAAe,MAAM;AAAA,IACrB,oBAAoB,MAAM;AAAA,IAC1B,UAAU;AAAA,EACZ,CAAC;AACD,MAAI,CAAC,aAAa,UAAU,wBAAwB,MAAM;AACxD,UAAM,IAAI,MAAM,iEAAiE;AACnF,SAAO;AAAA,IACL,kBACE,QAAQ,eAAe,uBAAuB,QAC9C,OAAO,SAAS,QAAQ,eAAe,YAAY,QAAQ,CAAC,KAC5D,QAAQ,oBAAoB;AAAA,EAChC;AACF;AAEA,SAAS,8BACP,eACA,SACA,aACM;AACN,QAAM,UAAW,QACd;AACH,MAAI,CAAC,QAAS;AACd,sCAAoC,eAAe,QAAQ,MAAM;AACjE,MAAI,QAAQ,WAAW,YAAY;AACjC,kBAAc;AAAA,MACZ;AAAA,MACA,EAAE,GAAG,aAAa,QAAQ,QAAQ,OAAO;AAAA,IAC3C;AAAA,EACF;AACF;AAEA,eAAsB,qBACpB,UACA,OACA,eACA,gBACA,yBAA0DH,gCAC1D,qBAAuD,2BACvD,0BACA,qBAA8C,oBAC9C,sBACA,qBAAqB,OACrB,sBACA,aACA,oBACA,wBACqC;AACrC,QAAM,mBAAoB,MAAM,mBAAmB,MAAM;AACzD,QAAM,UAAU,8BAA8B,gBAAgB,KAAK;AACnE,QAAM,cAAc,yBAAyB,OAAO,OAAO;AAC3D,QAAM,qBAAqB,CAAC,SAAiB,UAC3C,yBAAyB,eAAe,SAAS,OAAO;AAAA,IACtD,gBAAgB,MAAM;AAAA,IACtB,YAAY,MAAM;AAAA,IAClB,GAAI,cAAc,EAAE,YAAY,IAAI,CAAC;AAAA,EACvC,CAAC;AACH,QAAM,6BAA6B,YAA2B;AAC5D,QAAI;AACF,YAAM,uBAAuB;AAAA,IAC/B,SAAS,OAAO;AACd,yBAAmB,uDAAuD,KAAK;AAAA,IACjF;AAAA,EACF;AAEA,MAAI,CAAC,WAAW,YAAY,QAAQ;AAClC,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAWA,MACE,YAAY,gBACZ,MAAM,YAAY,gBAClB,MAAM,oBAAoB,cAC1B;AACA,kBAAc;AAAA,MACZ;AAAA,MACA;AAAA,QACE,GAAG;AAAA,MACL;AAAA,IACF;AACA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAEA,QAAM,uBACJ,wBAAwB,+BAA+B,SAAS,MAAM,WAAW;AACnF,MAAI,CAAC,sBAAsB;AACzB,UAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,EACnF;AACA,MAAI,qBAAqB,iBAAiB,aAAa;AACrD,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAEA,QAAM,sBAAsB,wCAAwC,SAAS,MAAM,WAAW;AAO9F,MAAI,CAAC,MAAM,YAAY;AACrB,QAAI,MAAM,uBAAuB,eAAe,MAAM;AACpD,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AACA,QAAI,qBAAqB;AACvB,YAAM,IAAI;AAAA,QACR,mBAAmB,OAAO,oCAAoC,mBAAmB;AAAA,MACnF;AAAA,IACF;AACA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AACA,MAAI,uBAAuB,wBAAwB,MAAM,YAAY;AACnE,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO,mBAAmB,MAAM,UAAU,+CAA+C,mBAAmB;AAAA,IACjI;AAAA,EACF;AAMA,QAAM,+BACJ,YAAY,WACZ,CAAC,uBACD,MAAM,SAAS,SAAS,WAAW,cACnC,MAAM,SAAS,SAAS,eAAe,MAAM,cAC7C,MAAM,SAAS,UAAU,WAAW;AACtC,MAAI,8BAA8B;AAChC,UAAM,EAAE,MAAM,IAAI,MAAM,eAAe,IAAI;AAC3C,QAAI,CAAC,OAAO;AACV,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,GAAG;AAAA,QACL;AAAA,MACF;AACA,aAAO,EAAE,YAAY,OAAO,8BAA8B,MAAM;AAAA,IAClE;AACA,QAAI;AACF,YAAM,qBAAqB;AAC3B,YAAM;AAAA,QACJ;AAAA,QACA,MAAM;AAAA,QACN;AAAA,QACA,MAAM,uBAAuB;AAAA,MAC/B;AAAA,IACF,SAAS,OAAO;AACd,UAAI,CAAC,+BAA+B,SAAS,KAAK,GAAG;AACnD,cAAM;AAAA,MACR;AACA,oBAAc,KAAK,sEAAsE;AAAA,QACvF,GAAG;AAAA,MACL,CAAC;AAAA,IACH;AAIA,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAMA,MAAI,CAAC,uBAAuB,CAAC,MAAM,aAAa;AAC9C,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO,aAAa,MAAM,UAAU;AAAA,IACzD;AAAA,EACF;AAEA,QAAM,SAAS,uBAAuB,SAAkB,QAAQ;AAGhE,MAAI,CAAC,QAAQ;AAEX,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE;AAIA,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,QAAI,CAAC,OAAO,2BAA4B,YAAY,YAAY,CAAC,OAAO,QAAS;AAI/E,YAAM,IAAI,MAAM,mBAAmB,OAAO,yCAAyC;AAAA,IACrF;AAYA,UAAM,uBACH,MAAM,YAA2C,gBAAgB,MAAM;AAC1E,UAAM,eAAe,MAAMI;AAAA,MACzB;AAAA,MACA;AAAA,MACA,MAAM;AAAA,IACR;AACA,QAAI,iBAAiB,QAAW;AAC9B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,QAAI,YAAY,UAAU;AACxB,UAAI,CAAC,OAAO,2BAA2B,CAAC,wBAAwB;AAC9D,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,YAAM,uBAAuB;AAC7B,gBAAU,MAAM,OAAO,wBAAwB,cAAc,sBAAsB;AACnF,qBAAe;AAAA,IACjB,OAAO;AACL,YAAM,UAAU,MAAMC;AAAA,QACpB;AAAA,QACA;AAAA,QACA;AAAA,QACA,MAAM;AAAA,MACR;AACA,gBAAU,QAAQ;AAClB,qBAAe,QAAQ;AAAA,IACzB;AAAA,EACF,SAAS,OAAO;AACd,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,uEAAuE;AAAA,QACxF,GAAG;AAAA,MACL,CAAC;AACD,aAAO;AAAA,QACL,YAAY;AAAA,QACZ,8BAA8B,uBAAuB;AAAA,MACvD;AAAA,IACF;AAGA,UAAM;AAAA,EACR;AAEA,QAAM,oBAAoB,+BAA+B,SAAS,OAAO;AACzE,MACE,CAAC,qBACD,kBAAkB,aAAa,qBAAqB,YACpD,kBAAkB,aAAa,qBAAqB,YACpD,kBAAkB,iBAAiB,qBAAqB,cACxD;AACA,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAMA,MAAI;AACJ,QAAM,8BACJ,SACC,OAAO;AACV,QAAM,oBACJ,uBAAuB,sBACvB,YAAY,YACX,gCAAgC,yBAC/B,gCAAgC,wBAC9B,MAAM;AAAA,IACJ,gDAAgD,UAAU,OAAO;AAAA,EACnE,IACA;AAGN,MAAI,yBAAqC,MAAM;AAC/C,MAAI;AACF,QAAI,uBAAuB,uBAAuB,SAAS,kBAAkB;AAC3E,+BAAyB,0BAA0B,KAAK;AAIxD,YAAM,UAAU;AAAA,QACd;AAAA,QACA,KAAK,IAAI;AAAA,QACT;AAAA,UACE,WAAW,4BAA4B,WAAW;AAAA,UAClD,UAAU,kBAAkB;AAAA,QAC9B;AAAA,QACA;AAAA,MACF;AACA,wBAAkB,MAAM,wCAAwC;AAAA,QAC9D;AAAA,QACA,WAAW,SAAS;AAAA,QACpB,cAAc,IAAI;AAAA,UAChB,MAAM;AAAA,UACN;AAAA,UACA,SAAS;AAAA,UACT,MAAM;AAAA,UACN,MAAM;AAAA,QACR;AAAA,QACA,aAAa,OAAO,YAAY;AAC9B,cAAI;AACF,kBAAM,SAAS,MAAM;AAAA,cACnB,QAAQ,SAAS,eAAe,UAAU,QAAQ;AAAA,cAClD,QAAQ;AAAA,cACR;AAAA,cACA;AAAA,YACF;AACA,0BAAc;AAAA,cACZ,OAAO,QACH,yEACA;AAAA,cACJ;AAAA,gBACE,GAAG;AAAA,cACL;AAAA,YACF;AAAA,UACF,SAAS,OAAO;AACd,+BAAmB,6DAA6D,KAAK;AAAA,UACvF,UAAE;AACA,kBAAM,wBAAwB,OAAO;AAAA,UACvC;AAAA,QACF;AAAA,QACA,+BAA+B,CAAC,UAAU;AACxC,6BAAmB,2DAA2D,KAAK;AAAA,QACrF;AAAA,QACA,oBAAoB,CAAC,UAAU;AAI7B,eAAK,2BAA2B;AAChC,6BAAmB,2DAA2D,KAAK;AAAA,QACrF;AAAA,QACA,eAAe,MAAM,uBAAuB;AAAA,MAC9C,CAAC;AAAA,IACH;AAAA,EACF,SAAS,OAAO;AAGd,QAAI,EAAE,iBAAiB,oCAAqC,wBAAuB;AACnF,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,qDAAqD;AAAA,QACtE,GAAG;AAAA,MACL,CAAC;AACD,aAAO,EAAE,YAAY,MAAM,8BAA8B,KAAK;AAAA,IAChE;AACA;AAAA,MACE;AAAA,MACA;AAAA,IACF;AAIA,QAAI,EAAE,iBAAiB,qCAAqC;AAC1D,YAAM,2BAA2B;AAAA,IACnC;AAEA,UAAM;AAAA,EACR;AASA,MAAI,uBAAuB,qBAAqB;AAC9C,kBAAc,KAAK,sEAAsE;AAAA,MACvF,GAAG;AAAA,IACL,CAAC;AAAA,EACH,WAAW,iBAAiB;AAC1B,QAAI;AACJ,QAAI;AACF,OAAC,EAAE,MAAM,IAAI,MAAM;AAAA,QACjB,gBAAgB,SAAS,eAAe,kBAAkB,gBAAgB;AAAA,QAC1E,gBAAgB;AAAA,QAChB;AAAA,QACA;AAAA,MACF;AAAA,IACF,UAAE;AACA,UAAI;AACF,cAAM,wBAAwB,eAAe;AAAA,MAC/C,UAAE;AACA,+BAAuB;AAAA,MACzB;AAAA,IACF;AACA,QAAI,CAAC,OAAO;AACV,oBAAc;AAAA,QACZ;AAAA,QACA;AAAA,UACE,GAAG;AAAA,QACL;AAAA,MACF;AACA,aAAO,EAAE,YAAY,OAAO,8BAA8B,MAAM;AAAA,IAClE;AAAA,EACF,OAAO;AAIL,UAAM,IAAI;AAAA,MACR,mBAAmB,OAAO;AAAA,IAC5B;AAAA,EACF;AAIA,MAAI;AACF,QAAI,CAAC,WAAW,CAAC,cAAc;AAC7B,YAAM,IAAI,MAAM,mBAAmB,OAAO,uCAAuC;AAAA,IACnF;AACA,2CAAuC,SAAS,OAAO;AACvD,UAAM,qBAAqB;AAC3B,UAAM,+BAA+B,QAAQ,cAAc,OAAO;AAClE,QAAI,YAAY,SAAU,+BAA8B,eAAe,SAAS,WAAW;AAC3F,WAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,EACjE,SAAS,OAAO;AACd,QAAI,+BAA+B,OAAO,WAAW,KAAK,GAAG;AAC3D,oBAAc,KAAK,0DAA0D;AAAA,QAC3E,GAAG;AAAA,MACL,CAAC;AAGD,aAAO,EAAE,YAAY,MAAM,8BAA8B,MAAM;AAAA,IACjE;AACA,UAAM;AAAA,EACR;AACF;","names":["Context","readLease","createSandboxClientForBackend","deserializeSandboxSessionStateEnvelope","resumeExactSandboxSession","sandboxLeaseTelemetryKey","Context","heartbeat","createSandboxClientForBackend","drainableInventory","drainable","readLease","deserializeSandboxSessionStateEnvelope","resumeExactSandboxSession"]}