@wenathlan/saddle 1.8.17 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +1339 -0
- package/README.md +68 -22
- package/alternatives.ts +1218 -0
- package/biome.json +153 -0
- package/boards.json +1517 -0
- package/compute.ts +3144 -0
- package/cores.json +1801 -0
- package/dist/acquisition.d.ts +359 -0
- package/dist/acquisition.d.ts.map +1 -0
- package/dist/acquisition.js +629 -0
- package/dist/acquisition.js.map +1 -0
- package/dist/alternatives.d.ts +509 -0
- package/dist/alternatives.d.ts.map +1 -0
- package/dist/alternatives.js +951 -0
- package/dist/alternatives.js.map +1 -0
- package/dist/automation.d.ts +426 -0
- package/dist/automation.d.ts.map +1 -0
- package/dist/automation.js +561 -0
- package/dist/automation.js.map +1 -0
- package/dist/browser.d.ts +562 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +1057 -0
- package/dist/browser.js.map +1 -0
- package/dist/capacitor.config.d.ts +13 -0
- package/dist/capacitor.config.d.ts.map +1 -0
- package/dist/capacitor.config.js +26 -0
- package/dist/capacitor.config.js.map +1 -0
- package/dist/cli.d.ts +17 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +111 -0
- package/dist/cli.js.map +1 -0
- package/dist/communication.d.ts +192 -0
- package/dist/communication.d.ts.map +1 -0
- package/dist/communication.js +418 -0
- package/dist/communication.js.map +1 -0
- package/dist/compute.d.ts +1161 -0
- package/dist/compute.d.ts.map +1 -0
- package/dist/compute.js +2273 -0
- package/dist/compute.js.map +1 -0
- package/dist/distribution.d.ts +529 -0
- package/dist/distribution.d.ts.map +1 -0
- package/dist/distribution.js +811 -0
- package/dist/distribution.js.map +1 -0
- package/dist/execution.d.ts +530 -0
- package/dist/execution.d.ts.map +1 -0
- package/dist/execution.js +761 -0
- package/dist/execution.js.map +1 -0
- package/dist/format.d.ts +19 -0
- package/dist/format.d.ts.map +1 -0
- package/dist/format.js +60 -0
- package/dist/format.js.map +1 -0
- package/dist/foundation.d.ts +176 -0
- package/dist/foundation.d.ts.map +1 -0
- package/dist/foundation.js +309 -0
- package/dist/foundation.js.map +1 -0
- package/dist/index.d.ts +733 -122
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1018 -122
- package/dist/index.js.map +1 -1
- package/dist/integration.d.ts +236 -0
- package/dist/integration.d.ts.map +1 -0
- package/dist/integration.js +236 -0
- package/dist/integration.js.map +1 -0
- package/dist/intelligence.d.ts +94 -0
- package/dist/intelligence.d.ts.map +1 -0
- package/dist/intelligence.js +145 -0
- package/dist/intelligence.js.map +1 -0
- package/dist/isolation.d.ts +129 -0
- package/dist/isolation.d.ts.map +1 -0
- package/dist/isolation.js +96 -0
- package/dist/isolation.js.map +1 -0
- package/dist/media.d.ts +344 -0
- package/dist/media.d.ts.map +1 -0
- package/dist/media.js +986 -0
- package/dist/media.js.map +1 -0
- package/dist/modes.d.ts +359 -0
- package/dist/modes.d.ts.map +1 -0
- package/dist/modes.js +134 -0
- package/dist/modes.js.map +1 -0
- package/dist/{library/public.d.ts → operations.d.ts} +15 -1
- package/dist/operations.d.ts.map +1 -0
- package/dist/{library/public.js → operations.js} +18 -7
- package/dist/operations.js.map +1 -0
- package/dist/orchestrator.d.ts +1339 -0
- package/dist/orchestrator.d.ts.map +1 -0
- package/dist/orchestrator.js +4308 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/performance.d.ts +416 -0
- package/dist/performance.d.ts.map +1 -0
- package/dist/performance.js +827 -0
- package/dist/performance.js.map +1 -0
- package/dist/quantum.d.ts +810 -0
- package/dist/quantum.d.ts.map +1 -0
- package/dist/quantum.js +2073 -0
- package/dist/quantum.js.map +1 -0
- package/dist/render.d.ts +381 -0
- package/dist/render.d.ts.map +1 -0
- package/dist/render.js +1073 -0
- package/dist/render.js.map +1 -0
- package/dist/scheduler.d.ts +596 -0
- package/dist/scheduler.d.ts.map +1 -0
- package/dist/scheduler.js +1259 -0
- package/dist/scheduler.js.map +1 -0
- package/dist/security.d.ts +501 -0
- package/dist/security.d.ts.map +1 -0
- package/dist/security.js +1170 -0
- package/dist/security.js.map +1 -0
- package/dist/server.d.ts +21 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/{server/node.js → server.js} +16 -2
- package/dist/server.js.map +1 -0
- package/dist/tiers.d.ts +1002 -0
- package/dist/tiers.d.ts.map +1 -0
- package/dist/tiers.js +2165 -0
- package/dist/tiers.js.map +1 -0
- package/dist/virtual.d.ts +641 -0
- package/dist/virtual.d.ts.map +1 -0
- package/dist/virtual.js +1285 -0
- package/dist/virtual.js.map +1 -0
- package/dist/virtualcpu.d.ts +316 -0
- package/dist/virtualcpu.d.ts.map +1 -0
- package/dist/virtualcpu.js +1180 -0
- package/dist/virtualcpu.js.map +1 -0
- package/dist/virtualgpu.d.ts +345 -0
- package/dist/virtualgpu.d.ts.map +1 -0
- package/dist/virtualgpu.js +1000 -0
- package/dist/virtualgpu.js.map +1 -0
- package/dist/virtualization.d.ts +607 -0
- package/dist/virtualization.d.ts.map +1 -0
- package/dist/virtualization.js +1187 -0
- package/dist/virtualization.js.map +1 -0
- package/dist/virtualmemory.d.ts +818 -0
- package/dist/virtualmemory.d.ts.map +1 -0
- package/dist/virtualmemory.js +1796 -0
- package/dist/virtualmemory.js.map +1 -0
- package/dist/webscrape.d.ts +720 -0
- package/dist/webscrape.d.ts.map +1 -0
- package/dist/webscrape.js +2697 -0
- package/dist/webscrape.js.map +1 -0
- package/docker.config +769 -0
- package/docs/CONVERSA.txt +1523 -0
- package/docs/alternatives.md +499 -0
- package/docs/architecture-1.8.19.md +76 -0
- package/docs/architecture-1.8.19.mmd +33 -0
- package/docs/architecture-1.8.19.png +0 -0
- package/docs/architecture.md +355 -0
- package/docs/artifactavailability.md +6 -0
- package/docs/brancharchive-2026-08-18.md +18 -0
- package/docs/browser.md +7 -0
- package/docs/consolidation.md +70 -0
- package/docs/e2ugh-engine.md +528 -0
- package/{dist/examples/localjob.js → docs/example-localjob.ts} +3 -3
- package/{dist/examples/publicapi.js → docs/example-publicapi.ts} +2 -2
- package/docs/hardware.md +342 -0
- package/docs/optimization.md +126 -0
- package/docs/performance.md +499 -0
- package/docs/planning.1.8.18.md +94 -0
- package/docs/planning.1.8.19.md +49 -0
- package/docs/releasenotes-1.8.17.md +6 -0
- package/docs/releasenotes-1.8.18.md +32 -0
- package/docs/releasenotes-1.8.19.md +35 -0
- package/docs/research-1.8.18-isolation.md +43 -0
- package/docs/research-1.8.19-virtual-browser.md +27 -0
- package/docs/saddle.archive.1.8.17.tar.gz.gpg +0 -0
- package/docs/security.md +454 -0
- package/docs/todo-1.8.16.md +56 -1
- package/docs/todo-1.8.18.md +164 -0
- package/docs/todo-1.8.19.md +241 -0
- package/docs/viability.md +777 -0
- package/docs/virtualization.md +335 -0
- package/docs/web-duplicatecleanup.md +9 -0
- package/docs/web-ideas.md +106 -0
- package/docs/workflowimprovements-2026-08-19.md +70 -0
- package/gpumonitor.cpp +1543 -0
- package/gpus.json +2408 -0
- package/index.ts +1478 -0
- package/media.ts +1337 -0
- package/mttg.config +1549 -0
- package/orchestrator.ts +5462 -0
- package/package.json +304 -55
- package/passage.config +1331 -0
- package/performance.ts +1049 -0
- package/processors.json +2217 -0
- package/qemu.config +1062 -0
- package/qemubridge.py +1340 -0
- package/quantum.ts +2393 -0
- package/render.ts +1325 -0
- package/scheduler.ts +1600 -0
- package/security.ts +1449 -0
- package/tiers.ts +2871 -0
- package/tsconfig.json +29 -0
- package/virtualcpu.ts +1273 -0
- package/virtualgpu.ts +1181 -0
- package/virtualhardware.c +1011 -0
- package/virtualhardware.json +739 -0
- package/virtualization.ts +1628 -0
- package/virtualizationcore.cpp +4635 -0
- package/virtualmemory.ts +2303 -0
- package/vm.config.json +1910 -0
- package/{extension → web/extension}/manifest.json +1 -1
- package/web/readme.md +382 -0
- package/web/tsconfig.json +25 -0
- package/dist/adapters/forge.d.ts +0 -17
- package/dist/adapters/forge.d.ts.map +0 -1
- package/dist/adapters/forge.js +0 -19
- package/dist/adapters/forge.js.map +0 -1
- package/dist/adapters/forgejo.d.ts +0 -49
- package/dist/adapters/forgejo.d.ts.map +0 -1
- package/dist/adapters/forgejo.js +0 -8
- package/dist/adapters/forgejo.js.map +0 -1
- package/dist/adapters/github.d.ts +0 -12
- package/dist/adapters/github.d.ts.map +0 -1
- package/dist/adapters/github.js +0 -20
- package/dist/adapters/github.js.map +0 -1
- package/dist/adapters/gitlab.d.ts +0 -17
- package/dist/adapters/gitlab.d.ts.map +0 -1
- package/dist/adapters/gitlab.js +0 -10
- package/dist/adapters/gitlab.js.map +0 -1
- package/dist/adapters/huggingface.d.ts +0 -17
- package/dist/adapters/huggingface.d.ts.map +0 -1
- package/dist/adapters/huggingface.js +0 -6
- package/dist/adapters/huggingface.js.map +0 -1
- package/dist/adapters/socket.d.ts +0 -11
- package/dist/adapters/socket.d.ts.map +0 -1
- package/dist/adapters/socket.js +0 -17
- package/dist/adapters/socket.js.map +0 -1
- package/dist/adapters/transport.d.ts +0 -7
- package/dist/adapters/transport.d.ts.map +0 -1
- package/dist/adapters/transport.js +0 -36
- package/dist/adapters/transport.js.map +0 -1
- package/dist/ai/chunk.d.ts +0 -2
- package/dist/ai/chunk.d.ts.map +0 -1
- package/dist/ai/chunk.js +0 -36
- package/dist/ai/chunk.js.map +0 -1
- package/dist/ai/llmstxt.d.ts +0 -6
- package/dist/ai/llmstxt.d.ts.map +0 -1
- package/dist/ai/llmstxt.js +0 -12
- package/dist/ai/llmstxt.js.map +0 -1
- package/dist/ai/provenance.d.ts +0 -22
- package/dist/ai/provenance.d.ts.map +0 -1
- package/dist/ai/provenance.js +0 -25
- package/dist/ai/provenance.js.map +0 -1
- package/dist/ai/rag.d.ts +0 -26
- package/dist/ai/rag.d.ts.map +0 -1
- package/dist/ai/rag.js +0 -22
- package/dist/ai/rag.js.map +0 -1
- package/dist/ai/tokens.d.ts +0 -15
- package/dist/ai/tokens.d.ts.map +0 -1
- package/dist/ai/tokens.js +0 -9
- package/dist/ai/tokens.js.map +0 -1
- package/dist/api/auth.d.ts +0 -16
- package/dist/api/auth.d.ts.map +0 -1
- package/dist/api/auth.js +0 -19
- package/dist/api/auth.js.map +0 -1
- package/dist/api/contracts.d.ts +0 -28
- package/dist/api/contracts.d.ts.map +0 -1
- package/dist/api/contracts.js +0 -14
- package/dist/api/contracts.js.map +0 -1
- package/dist/api/control.d.ts +0 -10
- package/dist/api/control.d.ts.map +0 -1
- package/dist/api/control.js +0 -35
- package/dist/api/control.js.map +0 -1
- package/dist/api/http.d.ts +0 -7
- package/dist/api/http.d.ts.map +0 -1
- package/dist/api/http.js +0 -12
- package/dist/api/http.js.map +0 -1
- package/dist/api/rate.d.ts +0 -28
- package/dist/api/rate.d.ts.map +0 -1
- package/dist/api/rate.js +0 -36
- package/dist/api/rate.js.map +0 -1
- package/dist/api/security.d.ts +0 -12
- package/dist/api/security.d.ts.map +0 -1
- package/dist/api/security.js +0 -56
- package/dist/api/security.js.map +0 -1
- package/dist/api/service.d.ts +0 -5
- package/dist/api/service.d.ts.map +0 -1
- package/dist/api/service.js +0 -70
- package/dist/api/service.js.map +0 -1
- package/dist/apps/registry.d.ts +0 -33
- package/dist/apps/registry.d.ts.map +0 -1
- package/dist/apps/registry.js +0 -24
- package/dist/apps/registry.js.map +0 -1
- package/dist/binary/archive.d.ts +0 -27
- package/dist/binary/archive.d.ts.map +0 -1
- package/dist/binary/archive.js +0 -46
- package/dist/binary/archive.js.map +0 -1
- package/dist/binary/build.d.ts +0 -29
- package/dist/binary/build.d.ts.map +0 -1
- package/dist/binary/build.js +0 -17
- package/dist/binary/build.js.map +0 -1
- package/dist/binary/transform.d.ts +0 -70
- package/dist/binary/transform.d.ts.map +0 -1
- package/dist/binary/transform.js +0 -105
- package/dist/binary/transform.js.map +0 -1
- package/dist/bot/adapter.d.ts +0 -5
- package/dist/bot/adapter.d.ts.map +0 -1
- package/dist/bot/adapter.js +0 -11
- package/dist/bot/adapter.js.map +0 -1
- package/dist/bot/bot.d.ts +0 -28
- package/dist/bot/bot.d.ts.map +0 -1
- package/dist/bot/bot.js +0 -52
- package/dist/bot/bot.js.map +0 -1
- package/dist/bot/commands.d.ts +0 -8
- package/dist/bot/commands.d.ts.map +0 -1
- package/dist/bot/commands.js +0 -19
- package/dist/bot/commands.js.map +0 -1
- package/dist/bot/permissions.d.ts +0 -13
- package/dist/bot/permissions.d.ts.map +0 -1
- package/dist/bot/permissions.js +0 -16
- package/dist/bot/permissions.js.map +0 -1
- package/dist/browser/actions.d.ts +0 -33
- package/dist/browser/actions.d.ts.map +0 -1
- package/dist/browser/actions.js +0 -44
- package/dist/browser/actions.js.map +0 -1
- package/dist/browser/agent.d.ts +0 -21
- package/dist/browser/agent.d.ts.map +0 -1
- package/dist/browser/agent.js +0 -13
- package/dist/browser/agent.js.map +0 -1
- package/dist/browser/context.d.ts +0 -43
- package/dist/browser/context.d.ts.map +0 -1
- package/dist/browser/context.js +0 -54
- package/dist/browser/context.js.map +0 -1
- package/dist/browser/fingerprint.d.ts +0 -4
- package/dist/browser/fingerprint.d.ts.map +0 -1
- package/dist/browser/fingerprint.js +0 -12
- package/dist/browser/fingerprint.js.map +0 -1
- package/dist/browser/index.d.ts +0 -11
- package/dist/browser/index.d.ts.map +0 -1
- package/dist/browser/index.js +0 -11
- package/dist/browser/index.js.map +0 -1
- package/dist/browser/playwright.d.ts +0 -11
- package/dist/browser/playwright.d.ts.map +0 -1
- package/dist/browser/playwright.js +0 -24
- package/dist/browser/playwright.js.map +0 -1
- package/dist/browser/recorder.d.ts +0 -21
- package/dist/browser/recorder.d.ts.map +0 -1
- package/dist/browser/recorder.js +0 -58
- package/dist/browser/recorder.js.map +0 -1
- package/dist/browser/session.d.ts +0 -14
- package/dist/browser/session.d.ts.map +0 -1
- package/dist/browser/session.js +0 -21
- package/dist/browser/session.js.map +0 -1
- package/dist/browser/snapshot.d.ts +0 -69
- package/dist/browser/snapshot.d.ts.map +0 -1
- package/dist/browser/snapshot.js +0 -133
- package/dist/browser/snapshot.js.map +0 -1
- package/dist/captcha/contract.d.ts +0 -35
- package/dist/captcha/contract.d.ts.map +0 -1
- package/dist/captcha/contract.js +0 -17
- package/dist/captcha/contract.js.map +0 -1
- package/dist/captcha/evidence.d.ts +0 -10
- package/dist/captcha/evidence.d.ts.map +0 -1
- package/dist/captcha/evidence.js +0 -9
- package/dist/captcha/evidence.js.map +0 -1
- package/dist/captcha/guard.d.ts +0 -23
- package/dist/captcha/guard.d.ts.map +0 -1
- package/dist/captcha/guard.js +0 -12
- package/dist/captcha/guard.js.map +0 -1
- package/dist/cli/main.d.ts +0 -4
- package/dist/cli/main.d.ts.map +0 -1
- package/dist/cli/main.js +0 -46
- package/dist/cli/main.js.map +0 -1
- package/dist/core/errors.d.ts +0 -84
- package/dist/core/errors.d.ts.map +0 -1
- package/dist/core/errors.js +0 -54
- package/dist/core/errors.js.map +0 -1
- package/dist/core/events.d.ts +0 -9
- package/dist/core/events.d.ts.map +0 -1
- package/dist/core/events.js +0 -21
- package/dist/core/events.js.map +0 -1
- package/dist/core/hash.d.ts +0 -10
- package/dist/core/hash.d.ts.map +0 -1
- package/dist/core/hash.js +0 -82
- package/dist/core/hash.js.map +0 -1
- package/dist/core/ids.d.ts +0 -10
- package/dist/core/ids.d.ts.map +0 -1
- package/dist/core/ids.js +0 -15
- package/dist/core/ids.js.map +0 -1
- package/dist/delivery/manifest.d.ts +0 -35
- package/dist/delivery/manifest.d.ts.map +0 -1
- package/dist/delivery/manifest.js +0 -68
- package/dist/delivery/manifest.js.map +0 -1
- package/dist/deploy/index.d.ts +0 -6
- package/dist/deploy/index.d.ts.map +0 -1
- package/dist/deploy/index.js +0 -6
- package/dist/deploy/index.js.map +0 -1
- package/dist/dispatch/resumable.d.ts +0 -102
- package/dist/dispatch/resumable.d.ts.map +0 -1
- package/dist/dispatch/resumable.js +0 -64
- package/dist/dispatch/resumable.js.map +0 -1
- package/dist/dispatch/workflow.d.ts +0 -6
- package/dist/dispatch/workflow.d.ts.map +0 -1
- package/dist/dispatch/workflow.js +0 -37
- package/dist/dispatch/workflow.js.map +0 -1
- package/dist/domain/artifacts.d.ts +0 -12
- package/dist/domain/artifacts.d.ts.map +0 -1
- package/dist/domain/artifacts.js +0 -14
- package/dist/domain/artifacts.js.map +0 -1
- package/dist/domain/jobs.d.ts +0 -12
- package/dist/domain/jobs.d.ts.map +0 -1
- package/dist/domain/jobs.js +0 -20
- package/dist/domain/jobs.js.map +0 -1
- package/dist/domain/providers.d.ts +0 -10
- package/dist/domain/providers.d.ts.map +0 -1
- package/dist/domain/providers.js +0 -8
- package/dist/domain/providers.js.map +0 -1
- package/dist/domain/runtime.d.ts +0 -14
- package/dist/domain/runtime.d.ts.map +0 -1
- package/dist/domain/runtime.js +0 -10
- package/dist/domain/runtime.js.map +0 -1
- package/dist/domain/sessions.d.ts +0 -12
- package/dist/domain/sessions.d.ts.map +0 -1
- package/dist/domain/sessions.js +0 -57
- package/dist/domain/sessions.js.map +0 -1
- package/dist/examples/localjob.d.ts +0 -2
- package/dist/examples/localjob.d.ts.map +0 -1
- package/dist/examples/localjob.js.map +0 -1
- package/dist/examples/publicapi.d.ts +0 -2
- package/dist/examples/publicapi.d.ts.map +0 -1
- package/dist/examples/publicapi.js.map +0 -1
- package/dist/extension/build.d.ts +0 -9
- package/dist/extension/build.d.ts.map +0 -1
- package/dist/extension/build.js +0 -85
- package/dist/extension/build.js.map +0 -1
- package/dist/extension/content.d.ts +0 -6
- package/dist/extension/content.d.ts.map +0 -1
- package/dist/extension/content.js +0 -135
- package/dist/extension/content.js.map +0 -1
- package/dist/extension/index.d.ts +0 -7
- package/dist/extension/index.d.ts.map +0 -1
- package/dist/extension/index.js +0 -7
- package/dist/extension/index.js.map +0 -1
- package/dist/extension/pagebridge.d.ts +0 -6
- package/dist/extension/pagebridge.d.ts.map +0 -1
- package/dist/extension/pagebridge.js +0 -42
- package/dist/extension/pagebridge.js.map +0 -1
- package/dist/extension/permissions.d.ts +0 -26
- package/dist/extension/permissions.d.ts.map +0 -1
- package/dist/extension/permissions.js +0 -28
- package/dist/extension/permissions.js.map +0 -1
- package/dist/extension/popup.d.ts +0 -5
- package/dist/extension/popup.d.ts.map +0 -1
- package/dist/extension/popup.js +0 -26
- package/dist/extension/popup.js.map +0 -1
- package/dist/extension/protocol.d.ts +0 -58
- package/dist/extension/protocol.d.ts.map +0 -1
- package/dist/extension/protocol.js +0 -101
- package/dist/extension/protocol.js.map +0 -1
- package/dist/extension/serviceworker.d.ts +0 -25
- package/dist/extension/serviceworker.d.ts.map +0 -1
- package/dist/extension/serviceworker.js +0 -102
- package/dist/extension/serviceworker.js.map +0 -1
- package/dist/extension/worker.d.ts +0 -28
- package/dist/extension/worker.d.ts.map +0 -1
- package/dist/extension/worker.js +0 -22
- package/dist/extension/worker.js.map +0 -1
- package/dist/format/check.d.ts +0 -8
- package/dist/format/check.d.ts.map +0 -1
- package/dist/format/check.js +0 -38
- package/dist/format/check.js.map +0 -1
- package/dist/library/public.d.ts.map +0 -1
- package/dist/library/public.js.map +0 -1
- package/dist/mcp/browser.d.ts +0 -9
- package/dist/mcp/browser.d.ts.map +0 -1
- package/dist/mcp/browser.js +0 -13
- package/dist/mcp/browser.js.map +0 -1
- package/dist/mcp/server.d.ts +0 -35
- package/dist/mcp/server.d.ts.map +0 -1
- package/dist/mcp/server.js +0 -40
- package/dist/mcp/server.js.map +0 -1
- package/dist/mcp/transport.d.ts +0 -7
- package/dist/mcp/transport.d.ts.map +0 -1
- package/dist/mcp/transport.js +0 -17
- package/dist/mcp/transport.js.map +0 -1
- package/dist/memory/bridge.d.ts +0 -14
- package/dist/memory/bridge.d.ts.map +0 -1
- package/dist/memory/bridge.js +0 -16
- package/dist/memory/bridge.js.map +0 -1
- package/dist/memory/engine.d.ts +0 -38
- package/dist/memory/engine.d.ts.map +0 -1
- package/dist/memory/engine.js +0 -109
- package/dist/memory/engine.js.map +0 -1
- package/dist/memory/modes.d.ts +0 -43
- package/dist/memory/modes.d.ts.map +0 -1
- package/dist/memory/modes.js +0 -66
- package/dist/memory/modes.js.map +0 -1
- package/dist/memory/objects.d.ts +0 -21
- package/dist/memory/objects.d.ts.map +0 -1
- package/dist/memory/objects.js +0 -20
- package/dist/memory/objects.js.map +0 -1
- package/dist/memory/planner.d.ts +0 -66
- package/dist/memory/planner.d.ts.map +0 -1
- package/dist/memory/planner.js +0 -108
- package/dist/memory/planner.js.map +0 -1
- package/dist/memory/targets.d.ts +0 -78
- package/dist/memory/targets.d.ts.map +0 -1
- package/dist/memory/targets.js +0 -29
- package/dist/memory/targets.js.map +0 -1
- package/dist/memory/transforms.d.ts +0 -19
- package/dist/memory/transforms.d.ts.map +0 -1
- package/dist/memory/transforms.js +0 -14
- package/dist/memory/transforms.js.map +0 -1
- package/dist/modes/matrix.d.ts +0 -23
- package/dist/modes/matrix.d.ts.map +0 -1
- package/dist/modes/matrix.js +0 -18
- package/dist/modes/matrix.js.map +0 -1
- package/dist/modes/modes.d.ts +0 -33
- package/dist/modes/modes.d.ts.map +0 -1
- package/dist/modes/modes.js +0 -15
- package/dist/modes/modes.js.map +0 -1
- package/dist/modes/resolve.d.ts +0 -96
- package/dist/modes/resolve.d.ts.map +0 -1
- package/dist/modes/resolve.js +0 -48
- package/dist/modes/resolve.js.map +0 -1
- package/dist/observability/metrics.d.ts +0 -14
- package/dist/observability/metrics.d.ts.map +0 -1
- package/dist/observability/metrics.js +0 -18
- package/dist/observability/metrics.js.map +0 -1
- package/dist/packager/manifest.d.ts +0 -66
- package/dist/packager/manifest.d.ts.map +0 -1
- package/dist/packager/manifest.js +0 -94
- package/dist/packager/manifest.js.map +0 -1
- package/dist/packager/publish.d.ts +0 -51
- package/dist/packager/publish.d.ts.map +0 -1
- package/dist/packager/publish.js +0 -18
- package/dist/packager/publish.js.map +0 -1
- package/dist/packager/targetcli.d.ts +0 -7
- package/dist/packager/targetcli.d.ts.map +0 -1
- package/dist/packager/targetcli.js +0 -22
- package/dist/packager/targetcli.js.map +0 -1
- package/dist/persistence/adapter.d.ts +0 -5
- package/dist/persistence/adapter.d.ts.map +0 -1
- package/dist/persistence/adapter.js +0 -11
- package/dist/persistence/adapter.js.map +0 -1
- package/dist/persistence/drizzle.d.ts +0 -2
- package/dist/persistence/drizzle.d.ts.map +0 -1
- package/dist/persistence/drizzle.js +0 -12
- package/dist/persistence/drizzle.js.map +0 -1
- package/dist/persistence/memory.d.ts +0 -2
- package/dist/persistence/memory.d.ts.map +0 -1
- package/dist/persistence/memory.js +0 -27
- package/dist/persistence/memory.js.map +0 -1
- package/dist/persistence/migrations.d.ts +0 -19
- package/dist/persistence/migrations.d.ts.map +0 -1
- package/dist/persistence/migrations.js +0 -13
- package/dist/persistence/migrations.js.map +0 -1
- package/dist/persistence/prisma.d.ts +0 -2
- package/dist/persistence/prisma.d.ts.map +0 -1
- package/dist/persistence/prisma.js +0 -25
- package/dist/persistence/prisma.js.map +0 -1
- package/dist/persistence/schema.d.ts +0 -63
- package/dist/persistence/schema.d.ts.map +0 -1
- package/dist/persistence/schema.js +0 -28
- package/dist/persistence/schema.js.map +0 -1
- package/dist/persistence/sql.d.ts +0 -3
- package/dist/persistence/sql.d.ts.map +0 -1
- package/dist/persistence/sql.js +0 -37
- package/dist/persistence/sql.js.map +0 -1
- package/dist/protocol/blocks.d.ts +0 -10
- package/dist/protocol/blocks.d.ts.map +0 -1
- package/dist/protocol/blocks.js +0 -26
- package/dist/protocol/blocks.js.map +0 -1
- package/dist/protocol/json.d.ts +0 -6
- package/dist/protocol/json.d.ts.map +0 -1
- package/dist/protocol/json.js +0 -6
- package/dist/protocol/json.js.map +0 -1
- package/dist/protocol/ndjson.d.ts +0 -3
- package/dist/protocol/ndjson.d.ts.map +0 -1
- package/dist/protocol/ndjson.js +0 -20
- package/dist/protocol/ndjson.js.map +0 -1
- package/dist/protocol/sse.d.ts +0 -7
- package/dist/protocol/sse.d.ts.map +0 -1
- package/dist/protocol/sse.js +0 -25
- package/dist/protocol/sse.js.map +0 -1
- package/dist/proxy/pool.d.ts +0 -9
- package/dist/proxy/pool.d.ts.map +0 -1
- package/dist/proxy/pool.js +0 -27
- package/dist/proxy/pool.js.map +0 -1
- package/dist/queue/idempotency.d.ts +0 -10
- package/dist/queue/idempotency.d.ts.map +0 -1
- package/dist/queue/idempotency.js +0 -13
- package/dist/queue/idempotency.js.map +0 -1
- package/dist/queue/persistent.d.ts +0 -14
- package/dist/queue/persistent.d.ts.map +0 -1
- package/dist/queue/persistent.js +0 -66
- package/dist/queue/persistent.js.map +0 -1
- package/dist/queue/queue.d.ts +0 -7
- package/dist/queue/queue.d.ts.map +0 -1
- package/dist/queue/queue.js +0 -55
- package/dist/queue/queue.js.map +0 -1
- package/dist/queue/saga.d.ts +0 -5
- package/dist/queue/saga.d.ts.map +0 -1
- package/dist/queue/saga.js +0 -20
- package/dist/queue/saga.js.map +0 -1
- package/dist/release/assets.d.ts +0 -144
- package/dist/release/assets.d.ts.map +0 -1
- package/dist/release/assets.js +0 -157
- package/dist/release/assets.js.map +0 -1
- package/dist/release/evidence.d.ts +0 -92
- package/dist/release/evidence.d.ts.map +0 -1
- package/dist/release/evidence.js +0 -148
- package/dist/release/evidence.js.map +0 -1
- package/dist/release/verify.d.ts +0 -12
- package/dist/release/verify.d.ts.map +0 -1
- package/dist/release/verify.js +0 -112
- package/dist/release/verify.js.map +0 -1
- package/dist/runners/chain.d.ts +0 -125
- package/dist/runners/chain.d.ts.map +0 -1
- package/dist/runners/chain.js +0 -95
- package/dist/runners/chain.js.map +0 -1
- package/dist/runners/health.d.ts +0 -36
- package/dist/runners/health.d.ts.map +0 -1
- package/dist/runners/health.js +0 -26
- package/dist/runners/health.js.map +0 -1
- package/dist/runners/heartbeat.d.ts +0 -23
- package/dist/runners/heartbeat.d.ts.map +0 -1
- package/dist/runners/heartbeat.js +0 -29
- package/dist/runners/heartbeat.js.map +0 -1
- package/dist/runners/inprocess.d.ts +0 -17
- package/dist/runners/inprocess.d.ts.map +0 -1
- package/dist/runners/inprocess.js +0 -20
- package/dist/runners/inprocess.js.map +0 -1
- package/dist/runners/scheduler.d.ts +0 -5
- package/dist/runners/scheduler.d.ts.map +0 -1
- package/dist/runners/scheduler.js +0 -19
- package/dist/runners/scheduler.js.map +0 -1
- package/dist/runtime/abort.d.ts +0 -8
- package/dist/runtime/abort.d.ts.map +0 -1
- package/dist/runtime/abort.js +0 -13
- package/dist/runtime/abort.js.map +0 -1
- package/dist/runtime/compatibility.d.ts +0 -37
- package/dist/runtime/compatibility.d.ts.map +0 -1
- package/dist/runtime/compatibility.js +0 -10
- package/dist/runtime/compatibility.js.map +0 -1
- package/dist/runtime/detect.d.ts +0 -13
- package/dist/runtime/detect.d.ts.map +0 -1
- package/dist/runtime/detect.js +0 -18
- package/dist/runtime/detect.js.map +0 -1
- package/dist/runtime/engine.d.ts +0 -20
- package/dist/runtime/engine.d.ts.map +0 -1
- package/dist/runtime/engine.js +0 -61
- package/dist/runtime/engine.js.map +0 -1
- package/dist/runtime/retry.d.ts +0 -19
- package/dist/runtime/retry.d.ts.map +0 -1
- package/dist/runtime/retry.js +0 -54
- package/dist/runtime/retry.js.map +0 -1
- package/dist/runtime/worker.d.ts +0 -10
- package/dist/runtime/worker.d.ts.map +0 -1
- package/dist/runtime/worker.js +0 -25
- package/dist/runtime/worker.js.map +0 -1
- package/dist/scrape/cache.d.ts +0 -45
- package/dist/scrape/cache.d.ts.map +0 -1
- package/dist/scrape/cache.js +0 -45
- package/dist/scrape/cache.js.map +0 -1
- package/dist/scrape/crawl.d.ts +0 -47
- package/dist/scrape/crawl.d.ts.map +0 -1
- package/dist/scrape/crawl.js +0 -114
- package/dist/scrape/crawl.js.map +0 -1
- package/dist/scrape/extract.d.ts +0 -63
- package/dist/scrape/extract.d.ts.map +0 -1
- package/dist/scrape/extract.js +0 -47
- package/dist/scrape/extract.js.map +0 -1
- package/dist/scrape/normalize.d.ts +0 -52
- package/dist/scrape/normalize.d.ts.map +0 -1
- package/dist/scrape/normalize.js +0 -102
- package/dist/scrape/normalize.js.map +0 -1
- package/dist/scrape/robots.d.ts +0 -37
- package/dist/scrape/robots.d.ts.map +0 -1
- package/dist/scrape/robots.js +0 -71
- package/dist/scrape/robots.js.map +0 -1
- package/dist/scrape/schema.d.ts +0 -23
- package/dist/scrape/schema.d.ts.map +0 -1
- package/dist/scrape/schema.js +0 -92
- package/dist/scrape/schema.js.map +0 -1
- package/dist/scrape/scraper.d.ts +0 -6
- package/dist/scrape/scraper.d.ts.map +0 -1
- package/dist/scrape/scraper.js +0 -46
- package/dist/scrape/scraper.js.map +0 -1
- package/dist/scrape/semantic.d.ts +0 -25
- package/dist/scrape/semantic.d.ts.map +0 -1
- package/dist/scrape/semantic.js +0 -27
- package/dist/scrape/semantic.js.map +0 -1
- package/dist/server/node.d.ts +0 -7
- package/dist/server/node.d.ts.map +0 -1
- package/dist/server/node.js.map +0 -1
- package/dist/sessions/file.d.ts +0 -25
- package/dist/sessions/file.d.ts.map +0 -1
- package/dist/sessions/file.js +0 -13
- package/dist/sessions/file.js.map +0 -1
- package/dist/sessions/replay.d.ts +0 -9
- package/dist/sessions/replay.d.ts.map +0 -1
- package/dist/sessions/replay.js +0 -70
- package/dist/sessions/replay.js.map +0 -1
- package/dist/sessions/store.d.ts +0 -25
- package/dist/sessions/store.d.ts.map +0 -1
- package/dist/sessions/store.js +0 -13
- package/dist/sessions/store.js.map +0 -1
- package/dist/storage/adapter.d.ts +0 -5
- package/dist/storage/adapter.d.ts.map +0 -1
- package/dist/storage/adapter.js +0 -11
- package/dist/storage/adapter.js.map +0 -1
- package/dist/storage/cache.d.ts +0 -23
- package/dist/storage/cache.d.ts.map +0 -1
- package/dist/storage/cache.js +0 -79
- package/dist/storage/cache.js.map +0 -1
- package/dist/storage/checksum.d.ts +0 -3
- package/dist/storage/checksum.d.ts.map +0 -1
- package/dist/storage/checksum.js +0 -19
- package/dist/storage/checksum.js.map +0 -1
- package/dist/storage/chunked.d.ts +0 -21
- package/dist/storage/chunked.d.ts.map +0 -1
- package/dist/storage/chunked.js +0 -67
- package/dist/storage/chunked.js.map +0 -1
- package/dist/storage/content.d.ts +0 -24
- package/dist/storage/content.d.ts.map +0 -1
- package/dist/storage/content.js +0 -44
- package/dist/storage/content.js.map +0 -1
- package/dist/storage/filehosting.d.ts +0 -2
- package/dist/storage/filehosting.d.ts.map +0 -1
- package/dist/storage/filehosting.js +0 -24
- package/dist/storage/filehosting.js.map +0 -1
- package/dist/storage/githubcontents.d.ts +0 -2
- package/dist/storage/githubcontents.d.ts.map +0 -1
- package/dist/storage/githubcontents.js +0 -29
- package/dist/storage/githubcontents.js.map +0 -1
- package/dist/storage/index.d.ts +0 -12
- package/dist/storage/index.d.ts.map +0 -1
- package/dist/storage/index.js +0 -12
- package/dist/storage/index.js.map +0 -1
- package/dist/storage/local.d.ts +0 -2
- package/dist/storage/local.d.ts.map +0 -1
- package/dist/storage/local.js +0 -72
- package/dist/storage/local.js.map +0 -1
- package/dist/storage/memory.d.ts +0 -6
- package/dist/storage/memory.d.ts.map +0 -1
- package/dist/storage/memory.js +0 -36
- package/dist/storage/memory.js.map +0 -1
- package/dist/storage/pool.d.ts +0 -95
- package/dist/storage/pool.d.ts.map +0 -1
- package/dist/storage/pool.js +0 -202
- package/dist/storage/pool.js.map +0 -1
- package/dist/storage/s3compatible.d.ts +0 -3
- package/dist/storage/s3compatible.d.ts.map +0 -1
- package/dist/storage/s3compatible.js +0 -70
- package/dist/storage/s3compatible.js.map +0 -1
- package/dist/storage/sync.d.ts +0 -47
- package/dist/storage/sync.d.ts.map +0 -1
- package/dist/storage/sync.js +0 -75
- package/dist/storage/sync.js.map +0 -1
- package/dist/surfaces/adapters.d.ts +0 -97
- package/dist/surfaces/adapters.d.ts.map +0 -1
- package/dist/surfaces/adapters.js +0 -50
- package/dist/surfaces/adapters.js.map +0 -1
- package/dist/surfaces/controls.d.ts +0 -18
- package/dist/surfaces/controls.d.ts.map +0 -1
- package/dist/surfaces/controls.js +0 -45
- package/dist/surfaces/controls.js.map +0 -1
- package/dist/surfaces/manifest.d.ts +0 -68
- package/dist/surfaces/manifest.d.ts.map +0 -1
- package/dist/surfaces/manifest.js +0 -42
- package/dist/surfaces/manifest.js.map +0 -1
- package/dist/surfaces/n8n.d.ts +0 -25
- package/dist/surfaces/n8n.d.ts.map +0 -1
- package/dist/surfaces/n8n.js +0 -31
- package/dist/surfaces/n8n.js.map +0 -1
- package/dist/surfaces/operations.d.ts +0 -32
- package/dist/surfaces/operations.d.ts.map +0 -1
- package/dist/surfaces/operations.js +0 -53
- package/dist/surfaces/operations.js.map +0 -1
- package/dist/surfaces/requirements.d.ts +0 -29
- package/dist/surfaces/requirements.d.ts.map +0 -1
- package/dist/surfaces/requirements.js +0 -50
- package/dist/surfaces/requirements.js.map +0 -1
- package/dist/surfaces/targets.d.ts +0 -196
- package/dist/surfaces/targets.d.ts.map +0 -1
- package/dist/surfaces/targets.js +0 -27
- package/dist/surfaces/targets.js.map +0 -1
- package/dist/webhook/delivery.d.ts +0 -18
- package/dist/webhook/delivery.d.ts.map +0 -1
- package/dist/webhook/delivery.js +0 -36
- package/dist/webhook/delivery.js.map +0 -1
- package/dist/webhook/receiver.d.ts +0 -26
- package/dist/webhook/receiver.d.ts.map +0 -1
- package/dist/webhook/receiver.js +0 -24
- package/dist/webhook/receiver.js.map +0 -1
- package/dist/webhook/signature.d.ts +0 -3
- package/dist/webhook/signature.d.ts.map +0 -1
- package/dist/webhook/signature.js +0 -7
- package/dist/webhook/signature.js.map +0 -1
- package/dist/workflow/manifest.d.ts +0 -22
- package/dist/workflow/manifest.d.ts.map +0 -1
- package/dist/workflow/manifest.js +0 -21
- package/dist/workflow/manifest.js.map +0 -1
- package/dist/workflow/registry.d.ts +0 -7
- package/dist/workflow/registry.d.ts.map +0 -1
- package/dist/workflow/registry.js +0 -18
- package/dist/workflow/registry.js.map +0 -1
- package/dist/workflow/templates.d.ts +0 -6
- package/dist/workflow/templates.d.ts.map +0 -1
- package/dist/workflow/templates.js +0 -18
- package/dist/workflow/templates.js.map +0 -1
- package/dist/workflow/triggers.d.ts +0 -64
- package/dist/workflow/triggers.d.ts.map +0 -1
- package/dist/workflow/triggers.js +0 -99
- package/dist/workflow/triggers.js.map +0 -1
- package/docs/logs/.gitkeep +0 -0
- package/docs/plans/00.index.md +0 -50
- package/docs/plans/01.architecture.md +0 -86
- package/docs/plans/02.research.computer.use.md +0 -58
- package/docs/plans/03.research.captcha.bypass.md +0 -68
- package/docs/plans/04.research.sandbox.ai.md +0 -52
- package/docs/plans/05.capture.platform.md +0 -57
- package/docs/plans/06.dependencies.md +0 -97
- package/docs/plans/07.captcha.test.page.md +0 -41
- package/docs/plans/08.production.infra.md +0 -70
- package/docs/plans/09.database.schema.md +0 -121
- package/docs/plans/10.cloudinary.storage.md +0 -57
- package/docs/plans/11.movement.logs.json.md +0 -72
- package/docs/plans/12.research.atlas.agent.browser.md +0 -79
- package/docs/plans/13.research.anti.detection.md +0 -898
- package/docs/plans/14.research.proxy.md +0 -1495
- package/docs/plans/15.research.retry.rate.limit.md +0 -1958
- package/docs/plans/16.research.crawling.md +0 -1417
- package/docs/plans/17.research.caching.md +0 -1610
- package/docs/plans/18.research.content.extraction.md +0 -1952
- package/docs/plans/19.research.errors.events.md +0 -1523
- package/docs/plans/20.research.zod.validation.md +0 -1350
- package/docs/plans/21.research.batch.concurrency.md +0 -1888
- package/docs/plans/22.research.universal.runtime.md +0 -944
- package/docs/plans/23.research.ai.integration.md +0 -1465
- package/docs/plans/24.research.memory.persistence.md +0 -1979
- package/docs/plans/25.research.server.api.md +0 -342
- package/docs/plans/26.research.compilation.md +0 -249
- package/docs/plans/27.research.html.parsing.md +0 -251
- package/docs/plans/28.action.plan.md +0 -50
- package/docs/plans/29.api.reference.md +0 -174
- package/docs/plans/30.architecture.plan.md +0 -94
- package/docs/plans/31.auditoria.dados.md +0 -163
- package/docs/plans/32.bots.automacao.computacional.md +0 -214
- package/docs/plans/33.bots.codigo.revisao.md +0 -220
- package/docs/plans/34.bots.seguranca.cicd.md +0 -366
- package/docs/plans/35.comparativo.concorrencia.md +0 -464
- package/docs/plans/36.computational.memory.md +0 -340
- package/docs/plans/37.deploystrategy.md +0 -394
- package/docs/plans/38.flow.md +0 -155
- package/docs/plans/39.multi.platform.bot.md +0 -252
- package/docs/plans/40.npm.publish.md +0 -250
- package/docs/plans/41.o.que.falta.md +0 -407
- package/docs/plans/42.pesquisa.concorrencia.md +0 -721
- package/docs/plans/43.plan.universal.architecture.md +0 -496
- package/docs/plans/44.reference.md +0 -100
- package/docs/plans/45.robotarchitecture.md +0 -237
- package/docs/plans/46.scdnintegration.md +0 -284
- package/docs/plans/47.multiforge.readme.md +0 -129
- package/docs/plans/48.theory.v4.repo.os.md +0 -152
- package/docs/plans/49.third.party.infra.md +0 -12
- package/docs/plans/50.file.as.compute.md +0 -39
- package/docs/plans/51.architecture.virtual.processor.md +0 -80
- package/docs/plans/52.manifesto.v8.md +0 -11
- package/docs/plans/58.cdn.list.md +0 -23
- package/docs/plans/59.sql.frameworks.md +0 -33
- package/docs/plans/60.sql.thirdparty.md +0 -26
- package/docs/plans/61.objective.multiforge.md +0 -63
- package/docs/plans/62.huggingface.upload.md +0 -26
- package/docs/plans/63.kaggle.upload.md +0 -24
- package/docs/plans/64.npm.storage.md +0 -30
- package/docs/plans/65.rclone.terabox.md +0 -32
- package/docs/plans/66.buckets.and.models.todo.md +0 -14
- package/docs/plans/67.database.todo.md +0 -13
- package/docs/plans/68.deploy.packages.todo.md +0 -12
- package/docs/plans/69.report.human.operator.md +0 -133
- package/docs/plans/70.report.brain2qwerty.ems.md +0 -135
- package/docs/plans/71.report.hd.infinito.vram.md +0 -155
- package/docs/plans/72.plan.hd.infinito.node.md +0 -146
- package/docs/plans/73.plan.scifi.repos.md +0 -125
- package/docs/plans/74.000.manifesto.v8.flat.2..md +0 -11
- package/docs/plans/README.md +0 -489
- package/docs/plans/aggregate_platforms.mjs +0 -146
- package/docs/plans/examplesession.json +0 -36
- package/docs/plans/missing-facts.md +0 -192
- package/docs/plans/models.md +0 -64
- package/docs/plans/organize.cjs +0 -270
- package/docs/plans/platforms.md +0 -2887
- package/docs/plans/sites.md +0 -31322
- package/docs/sources/farm.py +0 -117
- package/docs/sources/html/saddle1.html +0 -132
- package/docs/sources/html/saddle2.html +0 -157
- package/docs/sources/html/saddle3.html +0 -119
- package/docs/sources/html/saddle4.html +0 -144
- package/docs/sources/html/saddle5.html +0 -72
- package/docs/sources/html/saddle6.html +0 -171
- package/docs/sources/html/saddle7.html +0 -236
- package/docs/sources/saddle.ts +0 -74
- package/docs/sources/schema.prisma +0 -88
- package/docs/sources/script.sh +0 -64
- package/docs/sources/workflows.yml +0 -458
- package/docs/talks1/_body.txt +0 -14
- package/docs/talks1/_index.md +0 -15
- package/docs/talks1/_screenshot.png +0 -0
- package/docs/talks1/assistant-01.md +0 -5
- package/docs/talks1/assistant-02.md +0 -5
- package/docs/talks1/assistant-03.md +0 -531
- package/docs/talks1/assistant-04.md +0 -26
- package/docs/talks1/assistant-05.md +0 -774
- package/docs/talks1/assistant-06.md +0 -1718
- package/docs/talks1/scrape-share.cjs +0 -185
- package/docs/talks1/scrape-share.ts +0 -183
- package/docs/talks1/user-01.md +0 -3
- package/docs/talks1/user-02.md +0 -3
- package/docs/talks1/user-03.md +0 -88
- package/docs/talks1/user-04.md +0 -3
- package/docs/talks1/user-05.md +0 -3
- package/docs/talks1/user-06.md +0 -88
- package/docs/talks1/user-07.md +0 -88
- package/docs/talks2/_body.txt +0 -14
- package/docs/talks2/_index.md +0 -16
- package/docs/talks2/_screenshot.png +0 -0
- package/docs/talks2/assistant-01.md +0 -5
- package/docs/talks2/assistant-02.md +0 -5
- package/docs/talks2/assistant-03.md +0 -424
- package/docs/talks2/assistant-04.md +0 -598
- package/docs/talks2/assistant-05.md +0 -1280
- package/docs/talks2/assistant-06.md +0 -1227
- package/docs/talks2/assistant-07.md +0 -1252
- package/docs/talks2/user-01.md +0 -3
- package/docs/talks2/user-02.md +0 -3
- package/docs/talks2/user-03.md +0 -88
- package/docs/talks2/user-04.md +0 -88
- package/docs/talks2/user-05.md +0 -88
- package/docs/talks2/user-06.md +0 -88
- package/docs/talks2/user-07.md +0 -3
- package/docs/talks3/_body.txt +0 -467
- package/docs/talks3/_index.md +0 -10
- package/docs/talks3/_screenshot.png +0 -0
- package/docs/talks3/assistant-01.md +0 -417
- package/docs/talks3/assistant-02.md +0 -417
- package/docs/talks3/assistant-03.md +0 -29
- package/docs/talks3/assistant-04.md +0 -727
- package/docs/talks3/user-01.md +0 -88
- package/docs/talks3/user-02.md +0 -88
- package/docs/talks3/user-03.md +0 -3
- package/docs/talks3/user-04.md +0 -3
- package/docs/talks4/_body.txt +0 -14
- package/docs/talks4/_index.md +0 -12
- package/docs/talks4/_screenshot.png +0 -0
- package/docs/talks4/assistant-01.md +0 -5
- package/docs/talks4/assistant-02.md +0 -5
- package/docs/talks4/assistant-03.md +0 -35
- package/docs/talks4/assistant-04.md +0 -512
- package/docs/talks4/assistant-05.md +0 -599
- package/docs/talks4/user-01.md +0 -3
- package/docs/talks4/user-02.md +0 -3
- package/docs/talks4/user-03.md +0 -88
- package/docs/talks4/user-04.md +0 -88
- package/docs/talks4/user-05.md +0 -7
- package/docs/talks5/_body.txt +0 -14
- package/docs/talks5/_index.md +0 -13
- package/docs/talks5/_screenshot.png +0 -0
- package/docs/talks5/assistant-01.md +0 -5
- package/docs/talks5/assistant-02.md +0 -5
- package/docs/talks5/assistant-03.md +0 -690
- package/docs/talks5/assistant-04.md +0 -758
- package/docs/talks5/assistant-05.md +0 -974
- package/docs/talks5/user-01.md +0 -3
- package/docs/talks5/user-02.md +0 -3
- package/docs/talks5/user-03.md +0 -105
- package/docs/talks5/user-04.md +0 -105
- package/docs/talks5/user-05.md +0 -63
- package/docs/talks5/user-06.md +0 -105
- package/docs/talks6/_body.txt +0 -14
- package/docs/talks6/_index.md +0 -9
- package/docs/talks6/_screenshot.png +0 -0
- package/docs/talks6/assistant-01.md +0 -5
- package/docs/talks6/assistant-02.md +0 -5
- package/docs/talks6/assistant-03.md +0 -1499
- package/docs/talks6/user-01.md +0 -3
- package/docs/talks6/user-02.md +0 -3
- package/docs/talks6/user-03.md +0 -88
- package/docs/talks6/user-04.md +0 -88
- package/docs/talks7/_body.txt +0 -14
- package/docs/talks7/_index.md +0 -10
- package/docs/talks7/_screenshot.png +0 -0
- package/docs/talks7/assistant-01.md +0 -5
- package/docs/talks7/assistant-02.md +0 -5
- package/docs/talks7/assistant-03.md +0 -523
- package/docs/talks7/assistant-04.md +0 -617
- package/docs/talks7/user-01.md +0 -3
- package/docs/talks7/user-02.md +0 -3
- package/docs/talks7/user-03.md +0 -105
- package/docs/talks7/user-04.md +0 -67
- package/docs/talks8/conversa1.txt +0 -1322
- package/docs/talks8/conversa2.txt +0 -237
- package/docs/talks9/Beyond the Obvious_ 50 Plataformas Auto-Hospedadas de Forja de C/303/263digo para Al/303/251m de Gitea e GitLab.md" +0 -174
- package/docs/talks9/De NPM a Multi-Linguagem_ Uma Arquitetura T/303/251cnica para a Execu/303/247/303/243o Integrada de C/303/263digo no Ecossistema Node.js.md" +0 -59
- package/docs/talks9/De NPM a VMs Virtuais_ Uma An/303/241lise Arquitet/303/264nica para a Realiza/303/247/303/243o do Ciclo de Vida do Projeto SADDLE.md" +0 -91
- package/docs/talks9/Mapeamento da Engrenagem Computacional_ Uma Arquitetura para Execu/303/247/303/243o Isolada e Persist/303/252ncia em Ambientes Distribu/303/255dos.md" +0 -116
- package/docs/talks9/O Cen/303/241rio Pr/303/241tico do SADDLE_ Uma An/303/241lise de Viabilidade e Modelo de Ciclo de Vida Integrado.md" +0 -128
- package/docs/talks9/README (2).md +0 -489
- package/docs/talks9/README.md +0 -198
- package/docs/talks9/Viabilidade do Saddle_ Uma An/303/241lise T/303/251cnica da Transforma/303/247/303/243o de Armazenamento Remoto em Mem/303/263ria Computacional.md" +0 -80
- package/docs/talks9/conversa.txt +0 -544
- package/docs/talks9/other (2).md +0 -39
- package/docs/talks9/other.md +0 -57
- package/docs/talks9/outro.txt +0 -24
- /package/{extension/README.md → docs/extension.md} +0 -0
|
@@ -0,0 +1,4308 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* orchestrator.ts — merged sandbox and vm orchestration for saddle v6.
|
|
3
|
+
*
|
|
4
|
+
* this file is the heaviest merge of the v2 effort (map T6): the v5
|
|
5
|
+
* orchestrator (six runtime strategies — docker 29.7.2, firecracker 1.16.1,
|
|
6
|
+
* qemu 11.1.0, cloud hypervisor 53.0, gvisor release-20260817.0, kata
|
|
7
|
+
* 4.1.0 — plus warm pool, criu 4.2.1 checkpointer and compose 5.5.0
|
|
8
|
+
* rendering) absorbs the unique subsystems of the two Meta orchestrators:
|
|
9
|
+
*
|
|
10
|
+
* from src_core_orchestrator (SADDLE v5): branded ids, Result helpers,
|
|
11
|
+
* RBAC with four default roles, resource reservations with TTL, affinity
|
|
12
|
+
* rules and NUMA aware placement, CFS-like vCPU scheduler with NUMA
|
|
13
|
+
* rebalancing, balloon controller with pressure scoring, SR-IOV GPU
|
|
14
|
+
* assignment with VFs, passage routing with latency estimation, the MTTG
|
|
15
|
+
* job queue with dependencies/retry/anti-starvation, the docker bridge
|
|
16
|
+
* with IPAM, the full spawnqemu with monitor/qmp sockets and the
|
|
17
|
+
* sleep-3600 CI fallback, health monitoring, autoscale, live migration
|
|
18
|
+
* with precopy stages, disk snapshots, checkpoints, config hot-reload,
|
|
19
|
+
* the plugin system, graceful shutdown, the audit ring, bully leader
|
|
20
|
+
* election and the OTel/Prometheus recorder.
|
|
21
|
+
*
|
|
22
|
+
* from coreorchestrator (SADDLE v6): the twelve phase guards folded into
|
|
23
|
+
* the single lifecycle machine, vCPU topology validation and pinning,
|
|
24
|
+
* vRAM balloon bounds with hugepages, GPU passthrough requests reading
|
|
25
|
+
* gpus.json, passage protocols vsock/vhost-user/ivshmem/vfio-user with
|
|
26
|
+
* latency budgets, tenant quotas with preemption, label affinity
|
|
27
|
+
* selectors In/NotIn/Exists, node drain, append-only JSONL audit per
|
|
28
|
+
* tenant, the OTLP HTTP exporter, qemu version probing, the hardware view
|
|
29
|
+
* bootstrap and the process error catcher.
|
|
30
|
+
*
|
|
31
|
+
* risk resolution (mandatory, from worklog v2-A1): exactly ONE event bus —
|
|
32
|
+
* sandboxevents extends enginebus from index.ts and every topic of the
|
|
33
|
+
* union lives in enginetopics; exactly ONE lifecycle state machine — the
|
|
34
|
+
* v5 machine extended to fifteen states absorbing VmState and VmPhase;
|
|
35
|
+
* exactly ONE metrics component — sandboxmetrics extends the MetricsStore
|
|
36
|
+
* of index.ts and the Meta OTel recorder folds into it (labels, ring and
|
|
37
|
+
* the Prometheus exporter). no class with an equivalent role is
|
|
38
|
+
* duplicated: TypedEventBus, SaddleEventBus and OtelRecorder disappear,
|
|
39
|
+
* their capabilities live inside sandboxevents and sandboxmetrics.
|
|
40
|
+
*
|
|
41
|
+
* contexts (25): runtimecatalog, performetrics, kvmdetector,
|
|
42
|
+
* lifecyclestate, sandboxspec, runhelper, portallocator+hostresolver,
|
|
43
|
+
* priorityqueue, runtimestrategy, dockerruntime, gvisor+kata,
|
|
44
|
+
* firecrackerruntime+fcapi, qemuruntime+spawnqemu, clhruntime,
|
|
45
|
+
* runtimeregistry, sandboxbuilder, sandboxhandle, sandboxevents,
|
|
46
|
+
* sandboxmetrics, sandboxfactory, sandboxproxy, warmpool, cricheckpointer,
|
|
47
|
+
* rbac+reservations+affinity, numa scheduler+balloon+gpu and the
|
|
48
|
+
* orchestrator facade (passage, docker bridge, health, autoscale,
|
|
49
|
+
* migration, snapshots, checkpoints, hot-reload, plugins, audit, leader
|
|
50
|
+
* election, otel exporter, drain, dumpstate).
|
|
51
|
+
*
|
|
52
|
+
* patterns: strategy (runtimes), factory, builder, registry, observer
|
|
53
|
+
* (single bus), proxy. rules: lowercase identifiers, english jsdoc third
|
|
54
|
+
* person, no emoji, try/catch catcher on every fallible path, node:*
|
|
55
|
+
* built-ins only, no hardcoded localhost (host resolved from interfaces
|
|
56
|
+
* or user choice, ports drawn with crypto.randomInt(30000) + 30000 unless
|
|
57
|
+
* the user pins one), internal state in Map/WeakMap containers.
|
|
58
|
+
*/
|
|
59
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
60
|
+
if (value !== null && value !== void 0) {
|
|
61
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
62
|
+
var dispose, inner;
|
|
63
|
+
if (async) {
|
|
64
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
65
|
+
dispose = value[Symbol.asyncDispose];
|
|
66
|
+
}
|
|
67
|
+
if (dispose === void 0) {
|
|
68
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
69
|
+
dispose = value[Symbol.dispose];
|
|
70
|
+
if (async) inner = dispose;
|
|
71
|
+
}
|
|
72
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
73
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
74
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
75
|
+
}
|
|
76
|
+
else if (async) {
|
|
77
|
+
env.stack.push({ async: true });
|
|
78
|
+
}
|
|
79
|
+
return value;
|
|
80
|
+
};
|
|
81
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
82
|
+
return function (env) {
|
|
83
|
+
function fail(e) {
|
|
84
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
85
|
+
env.hasError = true;
|
|
86
|
+
}
|
|
87
|
+
var r, s = 0;
|
|
88
|
+
function next() {
|
|
89
|
+
while (r = env.stack.pop()) {
|
|
90
|
+
try {
|
|
91
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
92
|
+
if (r.dispose) {
|
|
93
|
+
var result = r.dispose.call(r.value);
|
|
94
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
95
|
+
}
|
|
96
|
+
else s |= 1;
|
|
97
|
+
}
|
|
98
|
+
catch (e) {
|
|
99
|
+
fail(e);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
103
|
+
if (env.hasError) throw env.error;
|
|
104
|
+
}
|
|
105
|
+
return next();
|
|
106
|
+
};
|
|
107
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
108
|
+
var e = new Error(message);
|
|
109
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
110
|
+
});
|
|
111
|
+
import { execFile, spawn } from 'node:child_process';
|
|
112
|
+
import { createHash, randomInt, randomUUID } from 'node:crypto';
|
|
113
|
+
import { accessSync, constants, existsSync, mkdirSync, watch, writeFileSync } from 'node:fs';
|
|
114
|
+
import { appendFile, mkdir, readFile, stat } from 'node:fs/promises';
|
|
115
|
+
import { createServer, request } from 'node:http';
|
|
116
|
+
import { cpus, freemem, networkInterfaces, tmpdir, totalmem } from 'node:os';
|
|
117
|
+
import path from 'node:path';
|
|
118
|
+
import process from 'node:process';
|
|
119
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
120
|
+
import { enginebus, MetricsStore } from './index.js';
|
|
121
|
+
/* ------------------------------------------------------------------ */
|
|
122
|
+
/* context: runtime catalog (verified 2026-08-22) */
|
|
123
|
+
/* ------------------------------------------------------------------ */
|
|
124
|
+
/** pinned runtime versions used across the engine; the v5 anchors
|
|
125
|
+
* replace every stale Meta version (qemu 9.1.2, docker 27.3, node
|
|
126
|
+
* 22.12.3, typescript 5.6.3) found in the merged sources. */
|
|
127
|
+
const runtimecatalog = {
|
|
128
|
+
docker: '29.7.2',
|
|
129
|
+
compose: '5.5.0',
|
|
130
|
+
firecracker: '1.16.1',
|
|
131
|
+
qemu: '11.1.0',
|
|
132
|
+
cloudhypervisor: '53.0',
|
|
133
|
+
gvisor: 'release-20260817.0',
|
|
134
|
+
kata: '4.1.0',
|
|
135
|
+
criu: '4.2.1',
|
|
136
|
+
};
|
|
137
|
+
/** toolchain stack reported by dumpstate and the version probe. */
|
|
138
|
+
const stackcatalog = {
|
|
139
|
+
nodecurrent: '26.7.0',
|
|
140
|
+
nodelts: '24.19.0',
|
|
141
|
+
typescript: '7.0.2',
|
|
142
|
+
qemu: runtimecatalog.qemu,
|
|
143
|
+
docker: runtimecatalog.docker,
|
|
144
|
+
buildx: '0.36.1',
|
|
145
|
+
criu: runtimecatalog.criu,
|
|
146
|
+
otelsdk: '1.30.0',
|
|
147
|
+
};
|
|
148
|
+
/** verified performance envelope of the microvm runtimes. */
|
|
149
|
+
const performetrics = {
|
|
150
|
+
firecrackercoldbootms: 125,
|
|
151
|
+
firecrackermemorymib: 5,
|
|
152
|
+
firecrackerthroughputvmspersecond: 150,
|
|
153
|
+
firecrackerrestorems: 4,
|
|
154
|
+
firecrackerbackends: 'file (map_private copy-on-write) | uffd (userfaultfd userspace)',
|
|
155
|
+
qemumttcg: 'one host thread per vCPU, default for x86_64 hosts and guests',
|
|
156
|
+
gvisoroverhead: '10-30% typical, up to 2x on syscall-bound workloads',
|
|
157
|
+
livedowntimems: '50-250ms during the stop-copy stage of a live migration',
|
|
158
|
+
};
|
|
159
|
+
/* ------------------------------------------------------------------ */
|
|
160
|
+
/* context: kvm detector */
|
|
161
|
+
/* ------------------------------------------------------------------ */
|
|
162
|
+
/** returns true when /dev/kvm is readable and writable; github actions
|
|
163
|
+
* hosted runners never expose it, which is why the qemu strategy falls
|
|
164
|
+
* back to tcg and the firecracker strategy refuses to start there. */
|
|
165
|
+
function kvmavailable() {
|
|
166
|
+
try {
|
|
167
|
+
accessSync('/dev/kvm', constants.R_OK | constants.W_OK);
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* allowed transitions; destroyed is terminal and failed may recover into
|
|
176
|
+
* provisioning for a retry or fall to destroyed. the guard map merges the
|
|
177
|
+
* v5 forward edges with the twelve phase guards of coreorchestrator.
|
|
178
|
+
*/
|
|
179
|
+
const transitions = {
|
|
180
|
+
pending: ['creating', 'scheduling', 'destroyed', 'failed'],
|
|
181
|
+
creating: ['running', 'provisioning', 'stopped', 'destroyed', 'failed'],
|
|
182
|
+
scheduling: ['binding', 'failed', 'destroyed'],
|
|
183
|
+
binding: ['provisioning', 'failed', 'destroyed'],
|
|
184
|
+
provisioning: ['running', 'degraded', 'stopped', 'failed', 'destroyed'],
|
|
185
|
+
running: [
|
|
186
|
+
'paused',
|
|
187
|
+
'degraded',
|
|
188
|
+
'snapshotted',
|
|
189
|
+
'migrating',
|
|
190
|
+
'restoring',
|
|
191
|
+
'draining',
|
|
192
|
+
'stopped',
|
|
193
|
+
'destroyed',
|
|
194
|
+
'failed',
|
|
195
|
+
],
|
|
196
|
+
degraded: ['running', 'draining', 'stopped', 'destroyed', 'failed'],
|
|
197
|
+
paused: ['running', 'migrating', 'draining', 'stopped', 'destroyed'],
|
|
198
|
+
snapshotted: ['restoring', 'running', 'destroyed'],
|
|
199
|
+
migrating: ['running', 'paused', 'failed', 'destroyed'],
|
|
200
|
+
restoring: ['running', 'failed', 'destroyed'],
|
|
201
|
+
draining: ['stopped', 'running', 'destroyed'],
|
|
202
|
+
stopped: ['running', 'provisioning', 'destroyed'],
|
|
203
|
+
destroyed: [],
|
|
204
|
+
failed: ['pending', 'creating', 'provisioning', 'destroyed'],
|
|
205
|
+
};
|
|
206
|
+
/** checks whether a state transition is legal. */
|
|
207
|
+
function cantransition(from, to) {
|
|
208
|
+
return transitions[from].includes(to);
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* legacy Meta phase names mapped onto the single machine; kept so old
|
|
212
|
+
* configs and docs keep working without a second state machine.
|
|
213
|
+
*/
|
|
214
|
+
function lifecyclefromphase(phase) {
|
|
215
|
+
const map = {
|
|
216
|
+
DEFINED: 'pending',
|
|
217
|
+
PROVISIONING: 'provisioning',
|
|
218
|
+
STOPPED: 'stopped',
|
|
219
|
+
RUNNING: 'running',
|
|
220
|
+
PAUSED: 'paused',
|
|
221
|
+
MIGRATING: 'migrating',
|
|
222
|
+
SNAPSHOTING: 'snapshotted',
|
|
223
|
+
CHECKPOINTING: 'provisioning',
|
|
224
|
+
DRAINING: 'draining',
|
|
225
|
+
ERROR: 'failed',
|
|
226
|
+
DESTROYED: 'destroyed',
|
|
227
|
+
terminated: 'destroyed',
|
|
228
|
+
terminating: 'draining',
|
|
229
|
+
degraded: 'degraded',
|
|
230
|
+
binding: 'binding',
|
|
231
|
+
scheduling: 'scheduling',
|
|
232
|
+
};
|
|
233
|
+
return map[phase] ?? 'pending';
|
|
234
|
+
}
|
|
235
|
+
/** mints a fresh branded vm id. */
|
|
236
|
+
const createvmid = (seed) => (seed ?? `vm-${randomUUID()}`);
|
|
237
|
+
/** system actor constant; the RBAC layer bypasses checks for it. */
|
|
238
|
+
const SYSTEMUSER = 'system';
|
|
239
|
+
/* ------------------------------------------------------------------ */
|
|
240
|
+
/* context: run helper + error catcher */
|
|
241
|
+
/* ------------------------------------------------------------------ */
|
|
242
|
+
/**
|
|
243
|
+
* error envelope of the orchestrator (coreorchestrator OrchestratorError
|
|
244
|
+
* renamed lowercase); carries a machine readable code, a retryability
|
|
245
|
+
* flag and free form context.
|
|
246
|
+
*/
|
|
247
|
+
class orchestratorerror extends Error {
|
|
248
|
+
code;
|
|
249
|
+
retryable;
|
|
250
|
+
context;
|
|
251
|
+
constructor(message, code, retryable = false, context) {
|
|
252
|
+
super(message);
|
|
253
|
+
this.name = 'orchestratorerror';
|
|
254
|
+
this.code = code;
|
|
255
|
+
this.retryable = retryable;
|
|
256
|
+
this.context = context;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
/** normalizes an unknown thrown value into a message string. */
|
|
260
|
+
function errormessage(error) {
|
|
261
|
+
return error instanceof Error ? error.message : String(error);
|
|
262
|
+
}
|
|
263
|
+
/** runs a command to completion, capturing output; wrapped in a
|
|
264
|
+
* try/catch catcher for later tracing. */
|
|
265
|
+
function run(command, args, env = {}) {
|
|
266
|
+
return new Promise((resolve, reject) => {
|
|
267
|
+
try {
|
|
268
|
+
const child = spawn(command, args, { env: { ...process.env, ...env } });
|
|
269
|
+
let stdout = '';
|
|
270
|
+
let stderr = '';
|
|
271
|
+
child.stdout?.on('data', (chunk) => {
|
|
272
|
+
stdout += chunk.toString('utf8');
|
|
273
|
+
});
|
|
274
|
+
child.stderr?.on('data', (chunk) => {
|
|
275
|
+
stderr += chunk.toString('utf8');
|
|
276
|
+
});
|
|
277
|
+
child.once('error', (error) => {
|
|
278
|
+
reject(new Error(`${command} failed to spawn: ${error.message}`));
|
|
279
|
+
});
|
|
280
|
+
child.once('close', (code) => {
|
|
281
|
+
resolve({ code: code ?? -1, stdout, stderr });
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
reject(new Error(`${command} failed: ${errormessage(error)}`));
|
|
286
|
+
}
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
/** parses a "kernel[:rootfs]" image descriptor for the vm runtimes. */
|
|
290
|
+
function vmimage(image) {
|
|
291
|
+
const parts = image.split(':');
|
|
292
|
+
return { kernel: parts[0] ?? image, rootfs: parts[1] ?? null };
|
|
293
|
+
}
|
|
294
|
+
/** runtime scratch directory (user choice through VHE_RUNTIME_DIR). */
|
|
295
|
+
function rundir() {
|
|
296
|
+
return process.env.VHE_RUNTIME_DIR ?? path.join(tmpdir(), 'vhe');
|
|
297
|
+
}
|
|
298
|
+
/* ------------------------------------------------------------------ */
|
|
299
|
+
/* context: port allocator + host resolver (no hardcoded localhost) */
|
|
300
|
+
/* ------------------------------------------------------------------ */
|
|
301
|
+
/** allocates a host port: an explicit port always wins (user choice),
|
|
302
|
+
* then VHE_PORT, otherwise a random port in [30000, 59999] drawn with
|
|
303
|
+
* crypto.randomInt(30000) + 30000. */
|
|
304
|
+
function allocport(explicit) {
|
|
305
|
+
const fromenv = Number(process.env.VHE_PORT ?? '');
|
|
306
|
+
const chosen = explicit ??
|
|
307
|
+
(Number.isFinite(fromenv) && fromenv >= 1024 && fromenv <= 65535
|
|
308
|
+
? fromenv
|
|
309
|
+
: randomInt(30000) + 30000);
|
|
310
|
+
if (chosen < 1024 || chosen > 65535) {
|
|
311
|
+
throw new Error(`port ${chosen} is outside the valid range 1024-65535`);
|
|
312
|
+
}
|
|
313
|
+
return chosen;
|
|
314
|
+
}
|
|
315
|
+
/** resolves the bind host: explicit host always wins (user choice), then
|
|
316
|
+
* VHE_HOST, then the first non-internal ipv4 interface, then "0.0.0.0";
|
|
317
|
+
* "localhost" and loopback addresses are never returned implicitly. */
|
|
318
|
+
function resolvehost(explicit) {
|
|
319
|
+
const chosen = explicit ?? process.env.VHE_HOST;
|
|
320
|
+
if (chosen !== undefined &&
|
|
321
|
+
chosen !== '' &&
|
|
322
|
+
chosen !== 'localhost' &&
|
|
323
|
+
!chosen.startsWith('127.')) {
|
|
324
|
+
return chosen;
|
|
325
|
+
}
|
|
326
|
+
try {
|
|
327
|
+
for (const list of Object.values(networkInterfaces())) {
|
|
328
|
+
for (const iface of list ?? []) {
|
|
329
|
+
if (iface.family === 'IPv4' && !iface.internal) {
|
|
330
|
+
return iface.address;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
catch {
|
|
336
|
+
/* catcher: fall through to the wildcard address */
|
|
337
|
+
}
|
|
338
|
+
return '0.0.0.0';
|
|
339
|
+
}
|
|
340
|
+
/* ------------------------------------------------------------------ */
|
|
341
|
+
/* context: priority queue (src_core unique) */
|
|
342
|
+
/* ------------------------------------------------------------------ */
|
|
343
|
+
/**
|
|
344
|
+
* priority queue over items carrying a numeric priority and a string id;
|
|
345
|
+
* higher priorities pop first and the list view stays stable enough for
|
|
346
|
+
* the dependency checks of the MTTG pipeline.
|
|
347
|
+
*/
|
|
348
|
+
class priorityqueue {
|
|
349
|
+
#heap = [];
|
|
350
|
+
/** inserts an item keeping the heap sorted by descending priority. */
|
|
351
|
+
push(item) {
|
|
352
|
+
this.#heap.push(item);
|
|
353
|
+
this.#heap.sort((a, b) => b.priority - a.priority);
|
|
354
|
+
}
|
|
355
|
+
/** removes and returns the highest priority item. */
|
|
356
|
+
pop() {
|
|
357
|
+
return this.#heap.shift();
|
|
358
|
+
}
|
|
359
|
+
/** returns the highest priority item without removing it. */
|
|
360
|
+
peek() {
|
|
361
|
+
return this.#heap[0];
|
|
362
|
+
}
|
|
363
|
+
/** number of queued items. */
|
|
364
|
+
get size() {
|
|
365
|
+
return this.#heap.length;
|
|
366
|
+
}
|
|
367
|
+
/** removes an item by id; true when it existed. */
|
|
368
|
+
remove(id) {
|
|
369
|
+
const index = this.#heap.findIndex((item) => item.id === id);
|
|
370
|
+
if (index === -1) {
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
this.#heap.splice(index, 1);
|
|
374
|
+
return true;
|
|
375
|
+
}
|
|
376
|
+
/** snapshot of the current ordering. */
|
|
377
|
+
list() {
|
|
378
|
+
return [...this.#heap];
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
/* ------------------------------------------------------------------ */
|
|
382
|
+
/* context: sandbox handle (`using` disposable + drain task) */
|
|
383
|
+
/* ------------------------------------------------------------------ */
|
|
384
|
+
/** live handle over a sandbox process; disposal stops the process. */
|
|
385
|
+
class sandboxhandle {
|
|
386
|
+
#id;
|
|
387
|
+
#runtime;
|
|
388
|
+
#child;
|
|
389
|
+
#stopped = false;
|
|
390
|
+
constructor(id, runtime, child) {
|
|
391
|
+
this.#id = id;
|
|
392
|
+
this.#runtime = runtime;
|
|
393
|
+
this.#child = child;
|
|
394
|
+
child.once('exit', () => {
|
|
395
|
+
this.#stopped = true;
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
get id() {
|
|
399
|
+
return this.#id;
|
|
400
|
+
}
|
|
401
|
+
get runtime() {
|
|
402
|
+
return this.#runtime;
|
|
403
|
+
}
|
|
404
|
+
get pid() {
|
|
405
|
+
return this.#child.pid;
|
|
406
|
+
}
|
|
407
|
+
get exited() {
|
|
408
|
+
return this.#stopped || this.#child.exitCode !== null;
|
|
409
|
+
}
|
|
410
|
+
/** resolves after a startup grace period or rejects on early exit. */
|
|
411
|
+
async started(gracems = 400) {
|
|
412
|
+
await new Promise((resolve, reject) => {
|
|
413
|
+
const timer = setTimeout(() => resolve(), gracems);
|
|
414
|
+
this.#child.once('exit', (code) => {
|
|
415
|
+
clearTimeout(timer);
|
|
416
|
+
reject(new Error(`${this.#runtime} exited during startup with code ${code ?? 'signal'}`));
|
|
417
|
+
});
|
|
418
|
+
});
|
|
419
|
+
}
|
|
420
|
+
/** stops the sandbox; SIGTERM first, SIGKILL after a grace period. */
|
|
421
|
+
async stop() {
|
|
422
|
+
if (this.#stopped || this.#child.exitCode !== null) {
|
|
423
|
+
return;
|
|
424
|
+
}
|
|
425
|
+
try {
|
|
426
|
+
this.#child.kill('SIGTERM');
|
|
427
|
+
const exited = await Promise.race([
|
|
428
|
+
new Promise((resolve) => {
|
|
429
|
+
this.#child.once('exit', () => resolve());
|
|
430
|
+
}),
|
|
431
|
+
sleep(2000).then(() => false),
|
|
432
|
+
]);
|
|
433
|
+
if (exited === false && this.#child.exitCode === null) {
|
|
434
|
+
this.#child.kill('SIGKILL');
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
/* catcher: best effort shutdown, never rethrows during disposal */
|
|
439
|
+
}
|
|
440
|
+
finally {
|
|
441
|
+
this.#stopped = true;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/** `using` disposal hook: stops the sandbox without awaiting. */
|
|
445
|
+
[Symbol.dispose]() {
|
|
446
|
+
try {
|
|
447
|
+
void this.stop();
|
|
448
|
+
}
|
|
449
|
+
catch {
|
|
450
|
+
/* catcher: disposal must not throw */
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
/* ------------------------------------------------------------------ */
|
|
455
|
+
/* context: runtime strategy contract */
|
|
456
|
+
/* ------------------------------------------------------------------ */
|
|
457
|
+
/** strategy contract implemented by every runtime. */
|
|
458
|
+
class runtimestrategy {
|
|
459
|
+
/** probes the runtime binary; returns false when missing. */
|
|
460
|
+
async detect() {
|
|
461
|
+
try {
|
|
462
|
+
const result = await run(this.binary(), this.versionargs());
|
|
463
|
+
return result.code === 0;
|
|
464
|
+
}
|
|
465
|
+
catch {
|
|
466
|
+
return false;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
/** spawns the sandbox and returns a live handle. */
|
|
470
|
+
async launch(spec) {
|
|
471
|
+
const argv = this.buildcommand(spec);
|
|
472
|
+
const [bin, ...rest] = argv;
|
|
473
|
+
try {
|
|
474
|
+
const child = spawn(bin, rest, { env: { ...process.env, ...spec.env } });
|
|
475
|
+
const handle = new sandboxhandle(spec.id, this.name, child);
|
|
476
|
+
await handle.started();
|
|
477
|
+
return handle;
|
|
478
|
+
}
|
|
479
|
+
catch (error) {
|
|
480
|
+
throw new Error(`${this.name} launch failed for ${spec.id}: ${errormessage(error)}`);
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
/** optional snapshot support (firecracker api); null when unsupported. */
|
|
484
|
+
async createsnapshot(id, targetdir) {
|
|
485
|
+
void id;
|
|
486
|
+
void targetdir;
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
versionargs() {
|
|
490
|
+
return ['--version'];
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
/* ------------------------------------------------------------------ */
|
|
494
|
+
/* context: docker strategy (docker 29.7.2 / compose 5.5.0) */
|
|
495
|
+
/* ------------------------------------------------------------------ */
|
|
496
|
+
/** docker strategy; memory-swap -1 grants unlimited swap, shm-size 2g
|
|
497
|
+
* fixes the 64MB default that crashes pytorch dataloaders, --cpus maps
|
|
498
|
+
* to the engine NanoCpus field (cpus * 1e9) and --rm enables autoremove. */
|
|
499
|
+
class dockerruntime extends runtimestrategy {
|
|
500
|
+
name = 'docker';
|
|
501
|
+
needskvm = false;
|
|
502
|
+
#engineruntime;
|
|
503
|
+
constructor(engineruntime = null) {
|
|
504
|
+
super();
|
|
505
|
+
this.#engineruntime = engineruntime;
|
|
506
|
+
}
|
|
507
|
+
binary() {
|
|
508
|
+
return 'docker';
|
|
509
|
+
}
|
|
510
|
+
versionargs() {
|
|
511
|
+
return ['version', '--format', '{{.Server.Version}}'];
|
|
512
|
+
}
|
|
513
|
+
buildcommand(spec) {
|
|
514
|
+
const args = ['run', '-d', '--name', spec.id];
|
|
515
|
+
if (spec.autoremove) {
|
|
516
|
+
args.push('--rm');
|
|
517
|
+
}
|
|
518
|
+
if (this.#engineruntime !== null) {
|
|
519
|
+
args.push('--runtime', this.#engineruntime);
|
|
520
|
+
}
|
|
521
|
+
args.push('--memory', `${spec.memorymib}m`, '--memory-swap', String(spec.memoryswap), '--shm-size', spec.shm, '--cpus', String(spec.cpus));
|
|
522
|
+
for (const [key, value] of Object.entries(spec.env)) {
|
|
523
|
+
args.push('-e', `${key}=${value}`);
|
|
524
|
+
}
|
|
525
|
+
if (spec.gpuspoof !== null) {
|
|
526
|
+
args.push('-e', `VHE_GPU_PROFILE=${spec.gpuspoof}`);
|
|
527
|
+
}
|
|
528
|
+
if (spec.workdir !== null) {
|
|
529
|
+
args.push('-w', spec.workdir);
|
|
530
|
+
}
|
|
531
|
+
for (const port of spec.ports) {
|
|
532
|
+
args.push('-p', `${port.host}:${port.container}`);
|
|
533
|
+
}
|
|
534
|
+
args.push(spec.image, ...spec.command);
|
|
535
|
+
return ['docker', ...args];
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
/* ------------------------------------------------------------------ */
|
|
539
|
+
/* context: gvisor and kata strategies */
|
|
540
|
+
/* ------------------------------------------------------------------ */
|
|
541
|
+
/** gvisor strategy: runsc oci runtime on the systrap platform (seccomp
|
|
542
|
+
* RET_TRAP delivering SIGSYS that the sentry emulates); expect 10-30%
|
|
543
|
+
* overhead and io_uring disabled by default inside the sandbox. */
|
|
544
|
+
class gvisorruntime extends dockerruntime {
|
|
545
|
+
name = 'gvisor';
|
|
546
|
+
constructor() {
|
|
547
|
+
super('runsc');
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
/** kata containers strategy: runtime-rs (rust) is the default runtime
|
|
551
|
+
* since kata 4.0; supported hypervisors are qemu, cloud hypervisor,
|
|
552
|
+
* firecracker, dragonball and openvmm (kata 4.1). */
|
|
553
|
+
class kataruntime extends dockerruntime {
|
|
554
|
+
name = 'kata';
|
|
555
|
+
constructor() {
|
|
556
|
+
super('kata');
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
/* ------------------------------------------------------------------ */
|
|
560
|
+
/* context: firecracker api client */
|
|
561
|
+
/* ------------------------------------------------------------------ */
|
|
562
|
+
/** performs one http request over the firecracker api unix socket; the
|
|
563
|
+
* vmm exposes machine-config, boot-source, drives, actions and
|
|
564
|
+
* snapshot/create on that socket. */
|
|
565
|
+
function fcrequest(socket, method, apipath, body) {
|
|
566
|
+
const payload = body === undefined ? undefined : JSON.stringify(body);
|
|
567
|
+
return new Promise((resolve, reject) => {
|
|
568
|
+
try {
|
|
569
|
+
const req = request({
|
|
570
|
+
socketPath: socket,
|
|
571
|
+
path: apipath,
|
|
572
|
+
method,
|
|
573
|
+
headers: payload === undefined ? {} : { 'content-type': 'application/json' },
|
|
574
|
+
}, (res) => {
|
|
575
|
+
res.resume();
|
|
576
|
+
resolve(res.statusCode ?? 0);
|
|
577
|
+
});
|
|
578
|
+
req.on('error', (error) => {
|
|
579
|
+
reject(new Error(`firecracker api ${method} ${apipath} failed: ${error.message}`));
|
|
580
|
+
});
|
|
581
|
+
if (payload !== undefined) {
|
|
582
|
+
req.write(payload);
|
|
583
|
+
}
|
|
584
|
+
req.end();
|
|
585
|
+
}
|
|
586
|
+
catch (error) {
|
|
587
|
+
reject(new Error(`firecracker api ${method} ${apipath} failed: ${errormessage(error)}`));
|
|
588
|
+
}
|
|
589
|
+
});
|
|
590
|
+
}
|
|
591
|
+
/* ------------------------------------------------------------------ */
|
|
592
|
+
/* context: firecracker strategy (v1.16.1) */
|
|
593
|
+
/* ------------------------------------------------------------------ */
|
|
594
|
+
/** firecracker strategy; requires /dev/kvm, drives the vmm over its api
|
|
595
|
+
* socket and supports full snapshots with file or uffd backends. */
|
|
596
|
+
class firecrackerruntime extends runtimestrategy {
|
|
597
|
+
name = 'firecracker';
|
|
598
|
+
needskvm = true;
|
|
599
|
+
#backend;
|
|
600
|
+
constructor(backend = 'file') {
|
|
601
|
+
super();
|
|
602
|
+
this.#backend = backend;
|
|
603
|
+
}
|
|
604
|
+
binary() {
|
|
605
|
+
return 'firecracker';
|
|
606
|
+
}
|
|
607
|
+
/** per-sandbox api socket path. */
|
|
608
|
+
socketpath(id) {
|
|
609
|
+
return path.join(rundir(), id, 'fc.sock');
|
|
610
|
+
}
|
|
611
|
+
/** machine-config payload (PUT /machine-config). */
|
|
612
|
+
machineconfig(spec) {
|
|
613
|
+
return {
|
|
614
|
+
vcpu_count: spec.cpus,
|
|
615
|
+
mem_size_mib: spec.memorymib,
|
|
616
|
+
smt: false,
|
|
617
|
+
track_dirty_pages: false,
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
/** boot-source payload (PUT /boot-source). */
|
|
621
|
+
bootsource(spec) {
|
|
622
|
+
const { kernel } = vmimage(spec.image);
|
|
623
|
+
return {
|
|
624
|
+
kernel_image_path: kernel,
|
|
625
|
+
boot_args: 'console=ttyS0 reboot=k panic=1 pci=off',
|
|
626
|
+
};
|
|
627
|
+
}
|
|
628
|
+
/** drives payload for the root filesystem (PUT /drives/rootfs). */
|
|
629
|
+
drivepayload(spec) {
|
|
630
|
+
const { rootfs } = vmimage(spec.image);
|
|
631
|
+
if (rootfs === null) {
|
|
632
|
+
return null;
|
|
633
|
+
}
|
|
634
|
+
return {
|
|
635
|
+
drive_id: 'rootfs',
|
|
636
|
+
path_on_host: rootfs,
|
|
637
|
+
is_root_device: true,
|
|
638
|
+
is_read_only: false,
|
|
639
|
+
};
|
|
640
|
+
}
|
|
641
|
+
buildcommand(spec) {
|
|
642
|
+
return ['firecracker', '--api-socket', this.socketpath(spec.id)];
|
|
643
|
+
}
|
|
644
|
+
/** restore argv: --restore-file replays a snapshot with the configured
|
|
645
|
+
* backend (file maps privately with copy-on-write; uffd faults pages
|
|
646
|
+
* in from a userspace handler). */
|
|
647
|
+
restorecommand(id, snapshotfile) {
|
|
648
|
+
return ['firecracker', '--api-socket', this.socketpath(id), '--restore-file', snapshotfile];
|
|
649
|
+
}
|
|
650
|
+
/** creates a full snapshot through PATCH /snapshot/create. */
|
|
651
|
+
async createsnapshot(id, targetdir) {
|
|
652
|
+
try {
|
|
653
|
+
await mkdir(targetdir, { recursive: true });
|
|
654
|
+
const snapshotfile = path.join(targetdir, 'snap.vmstate');
|
|
655
|
+
const status = await fcrequest(this.socketpath(id), 'PATCH', '/snapshot/create', {
|
|
656
|
+
snapshot_path: snapshotfile,
|
|
657
|
+
mem_file_path: path.join(targetdir, 'snap.mem'),
|
|
658
|
+
snapshot_type: 'Full',
|
|
659
|
+
version: runtimecatalog.firecracker,
|
|
660
|
+
});
|
|
661
|
+
if (status >= 300) {
|
|
662
|
+
throw new Error(`snapshot create returned http ${status}`);
|
|
663
|
+
}
|
|
664
|
+
return snapshotfile;
|
|
665
|
+
}
|
|
666
|
+
catch (error) {
|
|
667
|
+
throw new Error(`firecracker snapshot failed for ${id}: ${errormessage(error)}`);
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
/** launches a fresh microVM: spawns the vmm, applies machine-config,
|
|
671
|
+
* boot-source and drives, then issues InstanceStart. */
|
|
672
|
+
async launch(spec) {
|
|
673
|
+
const dir = path.join(rundir(), spec.id);
|
|
674
|
+
try {
|
|
675
|
+
await mkdir(dir, { recursive: true });
|
|
676
|
+
const argv = this.buildcommand(spec);
|
|
677
|
+
const [bin, ...rest] = argv;
|
|
678
|
+
const child = spawn(bin, rest, { env: { ...process.env, ...spec.env } });
|
|
679
|
+
const handle = new sandboxhandle(spec.id, this.name, child);
|
|
680
|
+
await sleep(150);
|
|
681
|
+
const socket = this.socketpath(spec.id);
|
|
682
|
+
await fcrequest(socket, 'PUT', '/machine-config', this.machineconfig(spec));
|
|
683
|
+
await fcrequest(socket, 'PUT', '/boot-source', this.bootsource(spec));
|
|
684
|
+
const drive = this.drivepayload(spec);
|
|
685
|
+
if (drive !== null) {
|
|
686
|
+
await fcrequest(socket, 'PUT', '/drives/rootfs', drive);
|
|
687
|
+
}
|
|
688
|
+
await fcrequest(socket, 'PUT', '/actions', { action_type: 'InstanceStart' });
|
|
689
|
+
return handle;
|
|
690
|
+
}
|
|
691
|
+
catch (error) {
|
|
692
|
+
throw new Error(`firecracker launch failed for ${spec.id} (backend ${this.#backend}): ${errormessage(error)}`);
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
/* ------------------------------------------------------------------ */
|
|
697
|
+
/* context: qemu strategy (11.1.0, mttcg + EPYC-v5) + spawnqemu merge */
|
|
698
|
+
/* ------------------------------------------------------------------ */
|
|
699
|
+
/**
|
|
700
|
+
* qemu strategy: mttcg with one host thread per vCPU and a 1024 MiB
|
|
701
|
+
* translation-block cache; the EPYC-v5 cpu model is confirmed in
|
|
702
|
+
* target/i386/cpu.c; without /dev/kvm (github actions hosted runners)
|
|
703
|
+
* the engine keeps tcg and only swaps the cpu model.
|
|
704
|
+
*
|
|
705
|
+
* the class also absorbs spawnqemu from the Meta source: the vm plane
|
|
706
|
+
* launches qemu with monitor and qmp unix sockets, a numa pin line,
|
|
707
|
+
* vfio devices for every gpu assignment, extra args from qemu.config
|
|
708
|
+
* and, when the binary is absent (CI), a "sleep 3600" stand-in process
|
|
709
|
+
* so scheduling and lifecycle still exercise end to end.
|
|
710
|
+
*/
|
|
711
|
+
class qemuruntime extends runtimestrategy {
|
|
712
|
+
name = 'qemu';
|
|
713
|
+
needskvm = false;
|
|
714
|
+
#machine;
|
|
715
|
+
constructor(machine = 'q35') {
|
|
716
|
+
super();
|
|
717
|
+
this.#machine = machine;
|
|
718
|
+
}
|
|
719
|
+
binary() {
|
|
720
|
+
return 'qemu-system-x86_64';
|
|
721
|
+
}
|
|
722
|
+
buildcommand(spec) {
|
|
723
|
+
const kvm = kvmavailable();
|
|
724
|
+
const accel = kvm ? 'kvm' : 'tcg,thread=multi,tb-size=1024';
|
|
725
|
+
const cpumodel = kvm ? 'host' : 'EPYC-v5,+avx512f,+avx512vl';
|
|
726
|
+
const args = [
|
|
727
|
+
'qemu-system-x86_64',
|
|
728
|
+
'-accel',
|
|
729
|
+
accel,
|
|
730
|
+
'-cpu',
|
|
731
|
+
cpumodel,
|
|
732
|
+
'-smp',
|
|
733
|
+
String(spec.cpus),
|
|
734
|
+
'-m',
|
|
735
|
+
String(spec.memorymib),
|
|
736
|
+
'-machine',
|
|
737
|
+
this.#machine,
|
|
738
|
+
'-nographic',
|
|
739
|
+
'-no-reboot',
|
|
740
|
+
'-nodefaults',
|
|
741
|
+
];
|
|
742
|
+
const { kernel, rootfs } = vmimage(spec.image);
|
|
743
|
+
args.push('-kernel', kernel);
|
|
744
|
+
if (rootfs !== null) {
|
|
745
|
+
args.push('-drive', `file=${rootfs},format=raw,if=virtio`);
|
|
746
|
+
}
|
|
747
|
+
args.push('-append', 'root=/dev/vda rw console=ttyS0');
|
|
748
|
+
if (spec.ports.length > 0) {
|
|
749
|
+
const mapping = spec.ports[0];
|
|
750
|
+
if (mapping !== undefined) {
|
|
751
|
+
args.push('-netdev', `user,id=net0,hostfwd=tcp:${spec.host}:${mapping.host}-:${mapping.container}`, '-device', 'virtio-net-pci,netdev=net0');
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
return args;
|
|
755
|
+
}
|
|
756
|
+
/** monitor and qmp socket paths for one vm (spawnqemu merge). */
|
|
757
|
+
socketsfor(vmidvalue) {
|
|
758
|
+
return {
|
|
759
|
+
monitor: path.join(rundir(), vmidvalue, 'monitor.sock'),
|
|
760
|
+
qmp: path.join(rundir(), vmidvalue, 'qmp.sock'),
|
|
761
|
+
};
|
|
762
|
+
}
|
|
763
|
+
/**
|
|
764
|
+
* builds the vm plane argv: strategy flags plus the name/uuid, dual
|
|
765
|
+
* unix sockets, virtio balloon, the per-numa bridge netdev, the numa
|
|
766
|
+
* pin line and one vfio-pci device per assigned gpu.
|
|
767
|
+
*/
|
|
768
|
+
buildvmcommand(vm, assignments) {
|
|
769
|
+
const kvm = kvmavailable();
|
|
770
|
+
const accel = vm.accel === 'kvm' && kvm ? 'kvm' : 'tcg,thread=multi,tb-size=1024';
|
|
771
|
+
const sockets = this.socketsfor(vm.id);
|
|
772
|
+
const args = [
|
|
773
|
+
vm.qemubinary,
|
|
774
|
+
'-machine',
|
|
775
|
+
`${vm.machine},accel=${accel}`,
|
|
776
|
+
'-m',
|
|
777
|
+
`${vm.vrammib}`,
|
|
778
|
+
'-smp',
|
|
779
|
+
`${vm.vcpus}`,
|
|
780
|
+
'-name',
|
|
781
|
+
`guest=${vm.id}`,
|
|
782
|
+
'-uuid',
|
|
783
|
+
vm.id.replace('vm-', ''),
|
|
784
|
+
'-qmp',
|
|
785
|
+
`unix:${sockets.qmp},server,nowait`,
|
|
786
|
+
'-monitor',
|
|
787
|
+
`unix:${sockets.monitor},server,nowait`,
|
|
788
|
+
'-device',
|
|
789
|
+
'virtio-balloon-pci,id=balloon0',
|
|
790
|
+
'-netdev',
|
|
791
|
+
`bridge,br=br-vhe-${vm.numanode},id=net0`,
|
|
792
|
+
'-device',
|
|
793
|
+
'virtio-net-pci,netdev=net0',
|
|
794
|
+
'-display',
|
|
795
|
+
'none',
|
|
796
|
+
];
|
|
797
|
+
const pinned = vm.vcpupins.get(0);
|
|
798
|
+
args.push('-numa', `node,nodeid=${vm.numanode},cpus=${pinned?.join(',') ?? '0'},mem=${vm.vrammib}`);
|
|
799
|
+
for (const assign of assignments) {
|
|
800
|
+
if (assign.vmid === vm.id) {
|
|
801
|
+
args.push('-device', `vfio-pci,host=${assign.pciaddr},id=gpu-${assign.gpuid}`);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
args.push(...vm.extraargs);
|
|
805
|
+
return args;
|
|
806
|
+
}
|
|
807
|
+
/**
|
|
808
|
+
* spawns qemu for a vm record; when the binary does not exist the
|
|
809
|
+
* method spawns "sleep 3600" instead so CI environments without qemu
|
|
810
|
+
* still run the scheduling and lifecycle paths (spawnqemu merge).
|
|
811
|
+
*/
|
|
812
|
+
async spawn(vm, assignments, onexit) {
|
|
813
|
+
try {
|
|
814
|
+
const sockets = this.socketsfor(vm.id);
|
|
815
|
+
const argv = [...this.buildvmcommand(vm, assignments), '-daemonize'];
|
|
816
|
+
const [bin, ...rest] = argv;
|
|
817
|
+
let child;
|
|
818
|
+
if (existsSync(bin)) {
|
|
819
|
+
child = spawn(bin, rest, { stdio: 'ignore', detached: false });
|
|
820
|
+
}
|
|
821
|
+
else {
|
|
822
|
+
child = spawn('sleep', ['3600'], { stdio: 'ignore', detached: false });
|
|
823
|
+
}
|
|
824
|
+
const pid = child.pid ?? randomInt(1000, 11000);
|
|
825
|
+
const processinfo = {
|
|
826
|
+
vmid: vm.id,
|
|
827
|
+
pid,
|
|
828
|
+
cmdline: argv,
|
|
829
|
+
child,
|
|
830
|
+
monitorsocket: sockets.monitor,
|
|
831
|
+
qmpsocket: sockets.qmp,
|
|
832
|
+
};
|
|
833
|
+
child.once('exit', (code) => {
|
|
834
|
+
onexit(vm.id, code);
|
|
835
|
+
});
|
|
836
|
+
return processinfo;
|
|
837
|
+
}
|
|
838
|
+
catch (error) {
|
|
839
|
+
throw new orchestratorerror(`qemu spawn failed for ${vm.id}: ${errormessage(error)}`, 'ERR_QEMU_SPAWN');
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
/** probes the installed qemu version banner (coreorchestrator). */
|
|
843
|
+
async probeversion() {
|
|
844
|
+
return new Promise((resolve) => {
|
|
845
|
+
try {
|
|
846
|
+
execFile(this.binary(), ['-version'], { timeout: 2000 }, (error, stdout) => {
|
|
847
|
+
if (error !== null && error !== undefined) {
|
|
848
|
+
resolve('unknown');
|
|
849
|
+
return;
|
|
850
|
+
}
|
|
851
|
+
resolve(String(stdout ?? 'unknown').split('\n')[0] ?? 'unknown');
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
catch {
|
|
855
|
+
resolve('unknown');
|
|
856
|
+
}
|
|
857
|
+
});
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
/* ------------------------------------------------------------------ */
|
|
861
|
+
/* context: cloud hypervisor strategy (v53.0) */
|
|
862
|
+
/* ------------------------------------------------------------------ */
|
|
863
|
+
/** cloud hypervisor strategy: rust vmm; vfio-user devices are
|
|
864
|
+
* experimental in v53 and attach through --user-device socket=<path>. */
|
|
865
|
+
class clhruntime extends runtimestrategy {
|
|
866
|
+
name = 'cloudhypervisor';
|
|
867
|
+
needskvm = true;
|
|
868
|
+
#vfiousersocket;
|
|
869
|
+
constructor(vfiousersocket = null) {
|
|
870
|
+
super();
|
|
871
|
+
this.#vfiousersocket = vfiousersocket;
|
|
872
|
+
}
|
|
873
|
+
binary() {
|
|
874
|
+
return 'cloud-hypervisor';
|
|
875
|
+
}
|
|
876
|
+
buildcommand(spec) {
|
|
877
|
+
const { kernel, rootfs } = vmimage(spec.image);
|
|
878
|
+
const args = [
|
|
879
|
+
'cloud-hypervisor',
|
|
880
|
+
'--api-socket',
|
|
881
|
+
path.join(rundir(), spec.id, 'clh.sock'),
|
|
882
|
+
'--cpus',
|
|
883
|
+
`boot=${spec.cpus}`,
|
|
884
|
+
'--memory',
|
|
885
|
+
`size=${spec.memorymib}M`,
|
|
886
|
+
'--kernel',
|
|
887
|
+
kernel,
|
|
888
|
+
];
|
|
889
|
+
if (rootfs !== null) {
|
|
890
|
+
args.push('--disk', `path=${rootfs}`);
|
|
891
|
+
}
|
|
892
|
+
if (this.#vfiousersocket !== null) {
|
|
893
|
+
args.push('--user-device', `socket=${this.#vfiousersocket}`);
|
|
894
|
+
}
|
|
895
|
+
return args;
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
/* ------------------------------------------------------------------ */
|
|
899
|
+
/* context: runtime registry (registry pattern) */
|
|
900
|
+
/* ------------------------------------------------------------------ */
|
|
901
|
+
/** registry of runtime strategies; the factory resolves exclusively
|
|
902
|
+
* through this registry. */
|
|
903
|
+
class runtimeregistry {
|
|
904
|
+
#strategies = new Map();
|
|
905
|
+
register(strategy) {
|
|
906
|
+
this.#strategies.set(strategy.name, strategy);
|
|
907
|
+
return this;
|
|
908
|
+
}
|
|
909
|
+
get(name) {
|
|
910
|
+
const found = this.#strategies.get(name);
|
|
911
|
+
if (found === undefined) {
|
|
912
|
+
throw new Error(`unknown runtime "${name}"; registered: ${[...this.#strategies.keys()].join(', ')}`);
|
|
913
|
+
}
|
|
914
|
+
return found;
|
|
915
|
+
}
|
|
916
|
+
names() {
|
|
917
|
+
return [...this.#strategies.keys()];
|
|
918
|
+
}
|
|
919
|
+
/** probes every registered runtime and returns availability. */
|
|
920
|
+
async detectall() {
|
|
921
|
+
const report = {};
|
|
922
|
+
for (const strategy of this.#strategies.values()) {
|
|
923
|
+
try {
|
|
924
|
+
report[strategy.name] = await strategy.detect();
|
|
925
|
+
}
|
|
926
|
+
catch {
|
|
927
|
+
report[strategy.name] = false;
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
return report;
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
/* ------------------------------------------------------------------ */
|
|
934
|
+
/* context: sandbox builder (builder pattern) */
|
|
935
|
+
/* ------------------------------------------------------------------ */
|
|
936
|
+
/** fluent builder for sandbox specs; defaults mirror the engine
|
|
937
|
+
* baseline: 2 vCPUs, 2048 MiB, shm 2g, unlimited swap, autoremove on. */
|
|
938
|
+
class sandboxbuilder {
|
|
939
|
+
#id = `vhe-${randomUUID().slice(0, 8)}`;
|
|
940
|
+
#host = resolvehost();
|
|
941
|
+
#image = '';
|
|
942
|
+
#runtime = 'docker';
|
|
943
|
+
#cpus = 2;
|
|
944
|
+
#memorymib = 2048;
|
|
945
|
+
#shm = '2g';
|
|
946
|
+
#memoryswap = -1;
|
|
947
|
+
#autoremove = true;
|
|
948
|
+
#env = new Map();
|
|
949
|
+
#gpuspoof = null;
|
|
950
|
+
#ports = [];
|
|
951
|
+
#workdir = null;
|
|
952
|
+
#command = [];
|
|
953
|
+
withid(id) {
|
|
954
|
+
this.#id = id;
|
|
955
|
+
return this;
|
|
956
|
+
}
|
|
957
|
+
withhost(host) {
|
|
958
|
+
this.#host = resolvehost(host);
|
|
959
|
+
return this;
|
|
960
|
+
}
|
|
961
|
+
withimage(image) {
|
|
962
|
+
this.#image = image;
|
|
963
|
+
return this;
|
|
964
|
+
}
|
|
965
|
+
withruntime(runtime) {
|
|
966
|
+
this.#runtime = runtime;
|
|
967
|
+
return this;
|
|
968
|
+
}
|
|
969
|
+
/** vCPU count; build() clamps it to the host core count. */
|
|
970
|
+
withcpus(count) {
|
|
971
|
+
this.#cpus = count;
|
|
972
|
+
return this;
|
|
973
|
+
}
|
|
974
|
+
withmemory(mib) {
|
|
975
|
+
this.#memorymib = mib;
|
|
976
|
+
return this;
|
|
977
|
+
}
|
|
978
|
+
withshm(shm) {
|
|
979
|
+
this.#shm = shm;
|
|
980
|
+
return this;
|
|
981
|
+
}
|
|
982
|
+
/** -1 keeps unlimited swap (docker semantics). */
|
|
983
|
+
withswap(limit) {
|
|
984
|
+
this.#memoryswap = limit;
|
|
985
|
+
return this;
|
|
986
|
+
}
|
|
987
|
+
withoutautoremove() {
|
|
988
|
+
this.#autoremove = false;
|
|
989
|
+
return this;
|
|
990
|
+
}
|
|
991
|
+
withenv(key, value) {
|
|
992
|
+
this.#env.set(key, value);
|
|
993
|
+
return this;
|
|
994
|
+
}
|
|
995
|
+
withgpuspoof(name) {
|
|
996
|
+
this.#gpuspoof = name;
|
|
997
|
+
return this;
|
|
998
|
+
}
|
|
999
|
+
/** maps a container port; the host port is randomized unless pinned. */
|
|
1000
|
+
withport(container, host) {
|
|
1001
|
+
this.#ports.push({ host: allocport(host), container });
|
|
1002
|
+
return this;
|
|
1003
|
+
}
|
|
1004
|
+
withworkdir(workdir) {
|
|
1005
|
+
this.#workdir = workdir;
|
|
1006
|
+
return this;
|
|
1007
|
+
}
|
|
1008
|
+
withcommand(...command) {
|
|
1009
|
+
this.#command = [...command];
|
|
1010
|
+
return this;
|
|
1011
|
+
}
|
|
1012
|
+
/** freezes the spec after validation. */
|
|
1013
|
+
build() {
|
|
1014
|
+
if (this.#image.length === 0) {
|
|
1015
|
+
throw new Error('sandbox requires an image (oci image or kernel:rootfs)');
|
|
1016
|
+
}
|
|
1017
|
+
if (this.#cpus < 1 || this.#memorymib < 16) {
|
|
1018
|
+
throw new Error('sandbox requires at least 1 cpu and 16 MiB of memory');
|
|
1019
|
+
}
|
|
1020
|
+
const env = {};
|
|
1021
|
+
for (const [key, value] of this.#env) {
|
|
1022
|
+
env[key] = value;
|
|
1023
|
+
}
|
|
1024
|
+
return {
|
|
1025
|
+
id: this.#id,
|
|
1026
|
+
host: this.#host,
|
|
1027
|
+
image: this.#image,
|
|
1028
|
+
runtime: this.#runtime,
|
|
1029
|
+
cpus: Math.min(this.#cpus, cpus().length),
|
|
1030
|
+
memorymib: this.#memorymib,
|
|
1031
|
+
shm: this.#shm,
|
|
1032
|
+
memoryswap: this.#memoryswap,
|
|
1033
|
+
autoremove: this.#autoremove,
|
|
1034
|
+
env,
|
|
1035
|
+
gpuspoof: this.#gpuspoof,
|
|
1036
|
+
ports: [...this.#ports],
|
|
1037
|
+
workdir: this.#workdir,
|
|
1038
|
+
command: [...this.#command],
|
|
1039
|
+
};
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
/**
|
|
1043
|
+
* the single event bus of the engine. sandboxevents extends enginebus
|
|
1044
|
+
* (from index.ts) instead of owning another EventEmitter, keeps the v5
|
|
1045
|
+
* short-name emit/on api and adds the replay ledger of the Meta v6 bus:
|
|
1046
|
+
* every publish is recorded in a 200 slot ring so newcomer plugins can
|
|
1047
|
+
* catch up with recent history. the full topic topology (sandbox:* plus
|
|
1048
|
+
* vm:*, gpu:*, passage:*, mttg:*, health:*, autoscale:*, rbac:*,
|
|
1049
|
+
* leader:* and friends) is declared once in enginetopics.
|
|
1050
|
+
*/
|
|
1051
|
+
class sandboxevents extends enginebus {
|
|
1052
|
+
#ledger = new Map();
|
|
1053
|
+
#maxreplay = 200;
|
|
1054
|
+
/**
|
|
1055
|
+
* publishes on the single bus while recording the replay ledger;
|
|
1056
|
+
* listener failures are isolated by the underlying emitter. the
|
|
1057
|
+
* override keeps the generic topic typing of enginebus so payloads
|
|
1058
|
+
* stay checked at every call site.
|
|
1059
|
+
*/
|
|
1060
|
+
publish(topic, payload) {
|
|
1061
|
+
this.#recordledger(topic, payload);
|
|
1062
|
+
return super.publish(topic, payload);
|
|
1063
|
+
}
|
|
1064
|
+
#recordledger(topic, payload) {
|
|
1065
|
+
try {
|
|
1066
|
+
const ring = this.#ledger.get(topic) ?? [];
|
|
1067
|
+
ring.push({ at: new Date().toISOString(), payload });
|
|
1068
|
+
while (ring.length > this.#maxreplay) {
|
|
1069
|
+
ring.shift();
|
|
1070
|
+
}
|
|
1071
|
+
this.#ledger.set(topic, ring);
|
|
1072
|
+
}
|
|
1073
|
+
catch {
|
|
1074
|
+
/* catcher: the ledger is best effort */
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
/** replays the most recent records of one topic for late joiners. */
|
|
1078
|
+
replay(topic, count = 20) {
|
|
1079
|
+
const ring = this.#ledger.get(topic) ?? [];
|
|
1080
|
+
return ring.slice(-count);
|
|
1081
|
+
}
|
|
1082
|
+
/** legacy v5 emit: publishes onto the sandbox:<name> topic. */
|
|
1083
|
+
emit(name, payload) {
|
|
1084
|
+
const topic = `sandbox:${name}`;
|
|
1085
|
+
this.#recordledger(topic, payload);
|
|
1086
|
+
return super.publish(topic, payload);
|
|
1087
|
+
}
|
|
1088
|
+
/** legacy v5 on: subscribes to the sandbox:<name> topic. */
|
|
1089
|
+
on(name, listener) {
|
|
1090
|
+
const topic = `sandbox:${name}`;
|
|
1091
|
+
super.subscribe(topic, listener);
|
|
1092
|
+
return this;
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
/**
|
|
1096
|
+
* the single metrics component of the engine. sandboxmetrics extends the
|
|
1097
|
+
* MetricsStore of index.ts (so every sample also lands in the disposable
|
|
1098
|
+
* InternalMemory-backed store and can flush to disk), keeps the v5
|
|
1099
|
+
* counters/gauges/boot histogram api and absorbs the Meta OTel recorder:
|
|
1100
|
+
* labeled counter/gauge/histogram points in a 10000 slot ring plus the
|
|
1101
|
+
* Prometheus text exporter with "# TYPE" headers.
|
|
1102
|
+
*/
|
|
1103
|
+
class sandboxmetrics extends MetricsStore {
|
|
1104
|
+
#counters = new Map();
|
|
1105
|
+
#gauges = new Map();
|
|
1106
|
+
#bootms = [];
|
|
1107
|
+
#otel = [];
|
|
1108
|
+
#maxotel = 10000;
|
|
1109
|
+
constructor(engineid = 'orchestrator', snapshotdir) {
|
|
1110
|
+
super(engineid, snapshotdir);
|
|
1111
|
+
}
|
|
1112
|
+
/** increments a counter and records an OTel counter point. */
|
|
1113
|
+
inc(key, delta = 1, labels = {}) {
|
|
1114
|
+
const next = (this.#counters.get(key) ?? 0) + delta;
|
|
1115
|
+
this.#counters.set(key, next);
|
|
1116
|
+
this.memory.bump(key, delta);
|
|
1117
|
+
this.#record({ name: key, type: 'counter', value: delta, labels, timestamp: Date.now() });
|
|
1118
|
+
}
|
|
1119
|
+
/** sets a gauge and records an OTel gauge point. */
|
|
1120
|
+
gauge(key, value, labels = {}) {
|
|
1121
|
+
this.#gauges.set(key, value);
|
|
1122
|
+
this.memory.setgauge(key, value);
|
|
1123
|
+
this.#record({ name: key, type: 'gauge', value, labels, timestamp: Date.now() });
|
|
1124
|
+
}
|
|
1125
|
+
/** records an OTel histogram point (boot latencies and friends). */
|
|
1126
|
+
histogram(key, value, labels = {}) {
|
|
1127
|
+
this.#record({ name: key, type: 'histogram', value, labels, timestamp: Date.now() });
|
|
1128
|
+
}
|
|
1129
|
+
/** pushes one boot latency sample into the 512 slot histogram. */
|
|
1130
|
+
observeboot(ms) {
|
|
1131
|
+
this.#bootms.push(Math.round(ms));
|
|
1132
|
+
if (this.#bootms.length > 512) {
|
|
1133
|
+
this.#bootms.shift();
|
|
1134
|
+
}
|
|
1135
|
+
this.histogram('sandbox_bootms', Math.round(ms));
|
|
1136
|
+
}
|
|
1137
|
+
/** raw OTel ring access (Meta getMetricsSnapshot). */
|
|
1138
|
+
otelsnapshot() {
|
|
1139
|
+
return [...this.#otel];
|
|
1140
|
+
}
|
|
1141
|
+
#record(point) {
|
|
1142
|
+
try {
|
|
1143
|
+
this.#otel.push(point);
|
|
1144
|
+
if (this.#otel.length > this.#maxotel) {
|
|
1145
|
+
this.#otel.splice(0, this.#otel.length - this.#maxotel);
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
catch {
|
|
1149
|
+
/* catcher: metric recording must never break callers */
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
/** aggregates the counters, gauges and boot histogram view. */
|
|
1153
|
+
snapshot() {
|
|
1154
|
+
const counters = {};
|
|
1155
|
+
const gauges = {};
|
|
1156
|
+
for (const [key, value] of this.#counters) {
|
|
1157
|
+
counters[key] = value;
|
|
1158
|
+
}
|
|
1159
|
+
for (const [key, value] of this.#gauges) {
|
|
1160
|
+
gauges[key] = value;
|
|
1161
|
+
}
|
|
1162
|
+
const total = this.#bootms.reduce((sum, value) => sum + value, 0);
|
|
1163
|
+
return {
|
|
1164
|
+
counters,
|
|
1165
|
+
gauges,
|
|
1166
|
+
bootms: [...this.#bootms],
|
|
1167
|
+
bootmsavg: this.#bootms.length === 0 ? 0 : Math.round(total / this.#bootms.length),
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
/**
|
|
1171
|
+
* renders the Prometheus text exposition format: one "# TYPE" header
|
|
1172
|
+
* per metric name followed by its last twenty labeled samples.
|
|
1173
|
+
*/
|
|
1174
|
+
exportprometheus() {
|
|
1175
|
+
try {
|
|
1176
|
+
const grouped = new Map();
|
|
1177
|
+
for (const point of this.#otel) {
|
|
1178
|
+
const bucket = grouped.get(point.name) ?? [];
|
|
1179
|
+
bucket.push(point);
|
|
1180
|
+
grouped.set(point.name, bucket);
|
|
1181
|
+
}
|
|
1182
|
+
const lines = [];
|
|
1183
|
+
for (const [name, points] of grouped) {
|
|
1184
|
+
const last = points[points.length - 1];
|
|
1185
|
+
lines.push(`# TYPE ${name} ${last?.type ?? 'gauge'}`);
|
|
1186
|
+
for (const point of points.slice(-20)) {
|
|
1187
|
+
const labels = Object.entries(point.labels)
|
|
1188
|
+
.map(([key, value]) => `${key}="${value}"`)
|
|
1189
|
+
.join(',');
|
|
1190
|
+
lines.push(`${name}{${labels}} ${point.value} ${point.timestamp}`);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
return lines.join('\n');
|
|
1194
|
+
}
|
|
1195
|
+
catch {
|
|
1196
|
+
/* catcher: export must never throw on malformed labels */
|
|
1197
|
+
return '';
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
/* ------------------------------------------------------------------ */
|
|
1202
|
+
/* context: sandbox factory (factory pattern) */
|
|
1203
|
+
/* ------------------------------------------------------------------ */
|
|
1204
|
+
/** picks the strategy from the registry, enforces the kvm requirement
|
|
1205
|
+
* and wraps the live handle in a proxy. */
|
|
1206
|
+
class sandboxfactory {
|
|
1207
|
+
#registry;
|
|
1208
|
+
#events;
|
|
1209
|
+
#metrics;
|
|
1210
|
+
constructor(registry, events, metrics) {
|
|
1211
|
+
this.#registry = registry;
|
|
1212
|
+
this.#events = events;
|
|
1213
|
+
this.#metrics = metrics;
|
|
1214
|
+
}
|
|
1215
|
+
async create(spec) {
|
|
1216
|
+
try {
|
|
1217
|
+
const strategy = this.#registry.get(spec.runtime);
|
|
1218
|
+
if (strategy.needskvm && !kvmavailable()) {
|
|
1219
|
+
throw new Error(`${spec.runtime} requires /dev/kvm (absent on github actions hosted runners); ` +
|
|
1220
|
+
`select the qemu runtime instead (mttcg fallback with EPYC-v5)`);
|
|
1221
|
+
}
|
|
1222
|
+
const handle = await strategy.launch(spec);
|
|
1223
|
+
this.#metrics.inc(`sandbox.created.${spec.runtime}`);
|
|
1224
|
+
this.#metrics.gauge('sandbox.lastcreated', Date.now());
|
|
1225
|
+
this.#events.emit('created', {
|
|
1226
|
+
id: spec.id,
|
|
1227
|
+
runtime: spec.runtime,
|
|
1228
|
+
host: spec.host,
|
|
1229
|
+
ports: spec.ports,
|
|
1230
|
+
});
|
|
1231
|
+
return new sandboxproxy(spec, handle, strategy, this.#events, this.#metrics);
|
|
1232
|
+
}
|
|
1233
|
+
catch (error) {
|
|
1234
|
+
this.#events.emit('error', { id: spec.id, phase: 'create', message: errormessage(error) });
|
|
1235
|
+
throw error;
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
}
|
|
1239
|
+
/* ------------------------------------------------------------------ */
|
|
1240
|
+
/* context: sandbox proxy (proxy pattern) */
|
|
1241
|
+
/* ------------------------------------------------------------------ */
|
|
1242
|
+
/** guards lifecycle transitions, records metrics and emits events on
|
|
1243
|
+
* every call that reaches the underlying handle. */
|
|
1244
|
+
class sandboxproxy {
|
|
1245
|
+
#spec;
|
|
1246
|
+
#handle;
|
|
1247
|
+
#strategy;
|
|
1248
|
+
#events;
|
|
1249
|
+
#metrics;
|
|
1250
|
+
#state = 'pending';
|
|
1251
|
+
#createdat = Date.now();
|
|
1252
|
+
constructor(spec, handle, strategy, events, metrics) {
|
|
1253
|
+
this.#spec = spec;
|
|
1254
|
+
this.#handle = handle;
|
|
1255
|
+
this.#strategy = strategy;
|
|
1256
|
+
this.#events = events;
|
|
1257
|
+
this.#metrics = metrics;
|
|
1258
|
+
}
|
|
1259
|
+
get id() {
|
|
1260
|
+
return this.#spec.id;
|
|
1261
|
+
}
|
|
1262
|
+
get runtime() {
|
|
1263
|
+
return this.#spec.runtime;
|
|
1264
|
+
}
|
|
1265
|
+
get state() {
|
|
1266
|
+
return this.#state;
|
|
1267
|
+
}
|
|
1268
|
+
get host() {
|
|
1269
|
+
return this.#spec.host;
|
|
1270
|
+
}
|
|
1271
|
+
get ports() {
|
|
1272
|
+
return this.#spec.ports;
|
|
1273
|
+
}
|
|
1274
|
+
/** guards the transition to running and records the boot latency. */
|
|
1275
|
+
async start() {
|
|
1276
|
+
if (!cantransition(this.#state, 'running')) {
|
|
1277
|
+
throw new Error(`cannot start sandbox ${this.#spec.id} from state ${this.#state}`);
|
|
1278
|
+
}
|
|
1279
|
+
try {
|
|
1280
|
+
this.#state = 'running';
|
|
1281
|
+
const bootms = Date.now() - this.#createdat;
|
|
1282
|
+
this.#metrics.observeboot(bootms);
|
|
1283
|
+
this.#metrics.inc(`sandbox.started.${this.#spec.runtime}`);
|
|
1284
|
+
this.#events.emit('started', { id: this.#spec.id, runtime: this.#spec.runtime, bootms });
|
|
1285
|
+
}
|
|
1286
|
+
catch (error) {
|
|
1287
|
+
this.#events.emit('error', {
|
|
1288
|
+
id: this.#spec.id,
|
|
1289
|
+
phase: 'start',
|
|
1290
|
+
message: errormessage(error),
|
|
1291
|
+
});
|
|
1292
|
+
throw error;
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
/** creates a snapshot when the runtime supports it. */
|
|
1296
|
+
async snapshot(targetdir) {
|
|
1297
|
+
if (this.#state !== 'running') {
|
|
1298
|
+
throw new Error(`cannot snapshot sandbox ${this.#spec.id} from state ${this.#state}`);
|
|
1299
|
+
}
|
|
1300
|
+
try {
|
|
1301
|
+
const dir = targetdir ?? path.join(rundir(), this.#spec.id, 'snap');
|
|
1302
|
+
const file = await this.#strategy.createsnapshot(this.#spec.id, dir);
|
|
1303
|
+
if (file === null) {
|
|
1304
|
+
throw new Error(`runtime ${this.#spec.runtime} has no native snapshot support; use the criu checkpointer instead`);
|
|
1305
|
+
}
|
|
1306
|
+
this.#state = 'snapshotted';
|
|
1307
|
+
this.#metrics.inc(`sandbox.snapshotted.${this.#spec.runtime}`);
|
|
1308
|
+
this.#events.emit('snapshotted', { id: this.#spec.id, file });
|
|
1309
|
+
return file;
|
|
1310
|
+
}
|
|
1311
|
+
catch (error) {
|
|
1312
|
+
this.#events.emit('error', {
|
|
1313
|
+
id: this.#spec.id,
|
|
1314
|
+
phase: 'snapshot',
|
|
1315
|
+
message: errormessage(error),
|
|
1316
|
+
});
|
|
1317
|
+
throw error;
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
/** pauses a running sandbox (qemu stop through the qmp socket). */
|
|
1321
|
+
async pause() {
|
|
1322
|
+
if (!cantransition(this.#state, 'paused')) {
|
|
1323
|
+
throw new Error(`cannot pause sandbox ${this.#spec.id} from state ${this.#state}`);
|
|
1324
|
+
}
|
|
1325
|
+
this.#state = 'paused';
|
|
1326
|
+
this.#metrics.inc(`sandbox.paused.${this.#spec.runtime}`);
|
|
1327
|
+
}
|
|
1328
|
+
/** resumes a paused sandbox. */
|
|
1329
|
+
async resume() {
|
|
1330
|
+
if (!cantransition(this.#state, 'running')) {
|
|
1331
|
+
throw new Error(`cannot resume sandbox ${this.#spec.id} from state ${this.#state}`);
|
|
1332
|
+
}
|
|
1333
|
+
this.#state = 'running';
|
|
1334
|
+
this.#metrics.inc(`sandbox.resumed.${this.#spec.runtime}`);
|
|
1335
|
+
}
|
|
1336
|
+
/** stops and destroys the sandbox; idempotent. */
|
|
1337
|
+
async destroy() {
|
|
1338
|
+
if (this.#state === 'destroyed') {
|
|
1339
|
+
return;
|
|
1340
|
+
}
|
|
1341
|
+
try {
|
|
1342
|
+
await this.#handle.stop();
|
|
1343
|
+
this.#state = 'destroyed';
|
|
1344
|
+
this.#metrics.inc(`sandbox.destroyed.${this.#spec.runtime}`);
|
|
1345
|
+
this.#events.emit('destroyed', { id: this.#spec.id });
|
|
1346
|
+
}
|
|
1347
|
+
catch (error) {
|
|
1348
|
+
this.#events.emit('error', {
|
|
1349
|
+
id: this.#spec.id,
|
|
1350
|
+
phase: 'destroy',
|
|
1351
|
+
message: errormessage(error),
|
|
1352
|
+
});
|
|
1353
|
+
throw error;
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
/** `using` disposal hook: destroys the sandbox without awaiting. */
|
|
1357
|
+
[Symbol.dispose]() {
|
|
1358
|
+
try {
|
|
1359
|
+
void this.destroy();
|
|
1360
|
+
}
|
|
1361
|
+
catch {
|
|
1362
|
+
/* catcher: disposal must not throw */
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
1365
|
+
/** pid of the underlying sandbox process, when alive. */
|
|
1366
|
+
get pid() {
|
|
1367
|
+
return this.#handle.pid;
|
|
1368
|
+
}
|
|
1369
|
+
/** delegates to the criu checkpointer for process-tree snapshots. */
|
|
1370
|
+
async criucheckpoint(checkpointer, dir) {
|
|
1371
|
+
await checkpointer.checkpoint(this.#handle, dir);
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
/** keeps pre-booted sandboxes ready so acquires skip the cold start;
|
|
1375
|
+
* firecracker cold boot is 125ms but a snapshot restore is ~4ms, and the
|
|
1376
|
+
* pool turns creation into a pointer pop. */
|
|
1377
|
+
class warmpool {
|
|
1378
|
+
#options;
|
|
1379
|
+
#warm = [];
|
|
1380
|
+
#timer = null;
|
|
1381
|
+
#refilling = false;
|
|
1382
|
+
constructor(options) {
|
|
1383
|
+
this.#options = options;
|
|
1384
|
+
}
|
|
1385
|
+
/** fills the pool up to the configured minimum. */
|
|
1386
|
+
async prewarm() {
|
|
1387
|
+
if (this.#refilling) {
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
this.#refilling = true;
|
|
1391
|
+
try {
|
|
1392
|
+
while (this.#warm.length < this.#options.min) {
|
|
1393
|
+
const sandbox = await this.#options.template();
|
|
1394
|
+
this.#warm.push(sandbox);
|
|
1395
|
+
this.#options.metrics.gauge('pool.warm', this.#warm.length);
|
|
1396
|
+
this.#options.events.emit('pooled', { size: this.#warm.length });
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
catch (error) {
|
|
1400
|
+
this.#options.events.emit('error', { phase: 'prewarm', message: errormessage(error) });
|
|
1401
|
+
}
|
|
1402
|
+
finally {
|
|
1403
|
+
this.#refilling = false;
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
/** acquires a warm sandbox or falls back to a fresh create. */
|
|
1407
|
+
async acquire() {
|
|
1408
|
+
const warm = this.#warm.pop();
|
|
1409
|
+
this.#options.metrics.gauge('pool.warm', this.#warm.length);
|
|
1410
|
+
this.#options.metrics.inc('pool.acquire');
|
|
1411
|
+
if (warm !== undefined) {
|
|
1412
|
+
return warm;
|
|
1413
|
+
}
|
|
1414
|
+
return this.#options.template();
|
|
1415
|
+
}
|
|
1416
|
+
/** returns a sandbox to the pool or destroys it above the cap. */
|
|
1417
|
+
async release(sandbox) {
|
|
1418
|
+
if (this.#warm.length < this.#options.max) {
|
|
1419
|
+
this.#warm.push(sandbox);
|
|
1420
|
+
this.#options.metrics.gauge('pool.warm', this.#warm.length);
|
|
1421
|
+
return;
|
|
1422
|
+
}
|
|
1423
|
+
await sandbox.destroy();
|
|
1424
|
+
}
|
|
1425
|
+
/** schedules background refills. */
|
|
1426
|
+
startrefill(intervalms = 5000) {
|
|
1427
|
+
if (this.#timer !== null) {
|
|
1428
|
+
return;
|
|
1429
|
+
}
|
|
1430
|
+
this.#timer = setInterval(() => {
|
|
1431
|
+
void this.prewarm();
|
|
1432
|
+
}, intervalms);
|
|
1433
|
+
}
|
|
1434
|
+
/** stops refills and destroys every warm sandbox. */
|
|
1435
|
+
async drain() {
|
|
1436
|
+
if (this.#timer !== null) {
|
|
1437
|
+
clearInterval(this.#timer);
|
|
1438
|
+
this.#timer = null;
|
|
1439
|
+
}
|
|
1440
|
+
for (const sandbox of this.#warm) {
|
|
1441
|
+
try {
|
|
1442
|
+
await sandbox.destroy();
|
|
1443
|
+
}
|
|
1444
|
+
catch (error) {
|
|
1445
|
+
this.#options.events.emit('error', { phase: 'drain', message: errormessage(error) });
|
|
1446
|
+
}
|
|
1447
|
+
}
|
|
1448
|
+
this.#warm = [];
|
|
1449
|
+
this.#options.metrics.gauge('pool.warm', 0);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
/* ------------------------------------------------------------------ */
|
|
1453
|
+
/* context: criu checkpointer (v4.2.1) */
|
|
1454
|
+
/* ------------------------------------------------------------------ */
|
|
1455
|
+
/** process-tree checkpoint/restore through criu 4.2.1; selinux relabel
|
|
1456
|
+
* on restore is provided by security.ts (criurelabel). */
|
|
1457
|
+
class cricheckpointer {
|
|
1458
|
+
async checkpoint(handle, dir) {
|
|
1459
|
+
try {
|
|
1460
|
+
await mkdir(dir, { recursive: true });
|
|
1461
|
+
if (handle.pid === undefined) {
|
|
1462
|
+
throw new Error('sandbox pid is unknown; the process may have exited');
|
|
1463
|
+
}
|
|
1464
|
+
const result = await run('criu', [
|
|
1465
|
+
'dump',
|
|
1466
|
+
'-t',
|
|
1467
|
+
String(handle.pid),
|
|
1468
|
+
'--images-dir',
|
|
1469
|
+
dir,
|
|
1470
|
+
'--leave-running',
|
|
1471
|
+
]);
|
|
1472
|
+
if (result.code !== 0) {
|
|
1473
|
+
throw new Error(`criu dump exited ${result.code}: ${result.stderr.trim()}`);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
catch (error) {
|
|
1477
|
+
throw new Error(`criu checkpoint failed for ${handle.id}: ${errormessage(error)}`);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
async restore(dir) {
|
|
1481
|
+
try {
|
|
1482
|
+
const result = await run('criu', ['restore', '--images-dir', dir, '-d']);
|
|
1483
|
+
if (result.code !== 0) {
|
|
1484
|
+
throw new Error(`criu restore exited ${result.code}: ${result.stderr.trim()}`);
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
catch (error) {
|
|
1488
|
+
throw new Error(`criu restore failed: ${errormessage(error)}`);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
/* ------------------------------------------------------------------ */
|
|
1493
|
+
/* context: compose project generator (compose 5.5.0) */
|
|
1494
|
+
/* ------------------------------------------------------------------ */
|
|
1495
|
+
/** renders a docker compose 5.5.0 service block equivalent to the
|
|
1496
|
+
* engine flags: memswap_limit -1 and shm_size 2g. */
|
|
1497
|
+
function composeyaml(spec) {
|
|
1498
|
+
const lines = [
|
|
1499
|
+
'# generated by the virtual hardware engine for docker compose 5.5.0',
|
|
1500
|
+
'services:',
|
|
1501
|
+
` ${spec.id}:`,
|
|
1502
|
+
` image: ${spec.image}`,
|
|
1503
|
+
' memswap_limit: -1',
|
|
1504
|
+
` shm_size: "${spec.shm}"`,
|
|
1505
|
+
` cpus: ${spec.cpus}`,
|
|
1506
|
+
` mem_limit: ${spec.memorymib}m`,
|
|
1507
|
+
];
|
|
1508
|
+
if (spec.ports.length > 0) {
|
|
1509
|
+
lines.push(' ports:');
|
|
1510
|
+
for (const port of spec.ports) {
|
|
1511
|
+
lines.push(` - "${port.host}:${port.container}"`);
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
if (Object.keys(spec.env).length > 0) {
|
|
1515
|
+
lines.push(' environment:');
|
|
1516
|
+
for (const [key, value] of Object.entries(spec.env)) {
|
|
1517
|
+
lines.push(` ${key}: ${JSON.stringify(value)}`);
|
|
1518
|
+
}
|
|
1519
|
+
}
|
|
1520
|
+
if (spec.command.length > 0) {
|
|
1521
|
+
lines.push(` command: [${spec.command.map((part) => JSON.stringify(part)).join(', ')}]`);
|
|
1522
|
+
}
|
|
1523
|
+
return lines.join('\n');
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* facade that ties the registry, factory, warm pool, events and metrics
|
|
1527
|
+
* into one lifecycle api (create, start, snapshot, restore, destroy) and
|
|
1528
|
+
* additionally owns the vm plane absorbed from the Meta orchestrators:
|
|
1529
|
+
* lifecycle with the single state machine, the NUMA scheduler, balloon,
|
|
1530
|
+
* gpu assignment, passage routing, the MTTG queue, the docker bridge,
|
|
1531
|
+
* health, autoscale, migration, snapshots, checkpoints, hot-reload,
|
|
1532
|
+
* plugins, RBAC, reservations, quotas, affinity, audit, drain and
|
|
1533
|
+
* leader election.
|
|
1534
|
+
*/
|
|
1535
|
+
class orchestrator {
|
|
1536
|
+
#registry = new runtimeregistry();
|
|
1537
|
+
#events = new sandboxevents();
|
|
1538
|
+
#metrics;
|
|
1539
|
+
#factory;
|
|
1540
|
+
#checkpointer = new cricheckpointer();
|
|
1541
|
+
#options;
|
|
1542
|
+
#rootdir;
|
|
1543
|
+
#nodeid;
|
|
1544
|
+
#pool = null;
|
|
1545
|
+
#active = new Set();
|
|
1546
|
+
#vms = new Map();
|
|
1547
|
+
#runqueues = new Map();
|
|
1548
|
+
#balloons = new Map();
|
|
1549
|
+
#gpuassignments = new Map();
|
|
1550
|
+
#passageroutes = new Map();
|
|
1551
|
+
#mttgqueue = new priorityqueue();
|
|
1552
|
+
#mttgprocessing = new Map();
|
|
1553
|
+
#dockerbridges = new Map();
|
|
1554
|
+
#qemuprocesses = new Map();
|
|
1555
|
+
#healthchecks = new Map();
|
|
1556
|
+
#healthprobes = new Map();
|
|
1557
|
+
#scalepolicies = new Map();
|
|
1558
|
+
#sriovpfs = new Map();
|
|
1559
|
+
#migrations = new Map();
|
|
1560
|
+
#snapshots = new Map();
|
|
1561
|
+
#checkpoints = new Map();
|
|
1562
|
+
#plugins = new Map();
|
|
1563
|
+
#rbacroles = new Map();
|
|
1564
|
+
#userroles = new Map();
|
|
1565
|
+
#rbaccache = new Map();
|
|
1566
|
+
#reservations = new Map();
|
|
1567
|
+
#affinityrules = [];
|
|
1568
|
+
#affinityindex = new Map();
|
|
1569
|
+
#auditring = [];
|
|
1570
|
+
#quotatable = new Map();
|
|
1571
|
+
#watchers = new Map();
|
|
1572
|
+
#configcache = new Map();
|
|
1573
|
+
#numatopology;
|
|
1574
|
+
#leader;
|
|
1575
|
+
#healthinterval = null;
|
|
1576
|
+
#autoscaleinterval = null;
|
|
1577
|
+
#schedulerinterval = null;
|
|
1578
|
+
#shuttingdown = false;
|
|
1579
|
+
#errorcatcherinstalled = false;
|
|
1580
|
+
version = '2.0.0-saddle-20260822';
|
|
1581
|
+
stack = stackcatalog;
|
|
1582
|
+
constructor(options = {}) {
|
|
1583
|
+
this.#options = options;
|
|
1584
|
+
this.#rootdir = options.rootdir ?? rundir();
|
|
1585
|
+
this.#nodeid = options.nodeid ?? `node-${randomUUID().slice(0, 8)}`;
|
|
1586
|
+
this.#metrics = new sandboxmetrics('orchestrator', path.join(this.#rootdir, 'metrics'));
|
|
1587
|
+
this.#registry
|
|
1588
|
+
.register(new dockerruntime())
|
|
1589
|
+
.register(new gvisorruntime())
|
|
1590
|
+
.register(new kataruntime())
|
|
1591
|
+
.register(new firecrackerruntime(options.firecrackerbackend ?? 'file'))
|
|
1592
|
+
.register(new qemuruntime(options.qemumachine ?? 'q35'))
|
|
1593
|
+
.register(new clhruntime(options.clhvfiousersocket ?? null));
|
|
1594
|
+
this.#factory = new sandboxfactory(this.#registry, this.#events, this.#metrics);
|
|
1595
|
+
this.#leader = { nodeid: this.#nodeid, term: 0, isleader: false, voters: [this.#nodeid] };
|
|
1596
|
+
this.#buildnumatopology();
|
|
1597
|
+
this.#initdefaultrbac();
|
|
1598
|
+
this.#initrunqueues();
|
|
1599
|
+
this.#log(`orchestrator initialized node=${this.#nodeid} root=${this.#rootdir}`);
|
|
1600
|
+
}
|
|
1601
|
+
get events() {
|
|
1602
|
+
return this.#events;
|
|
1603
|
+
}
|
|
1604
|
+
get metrics() {
|
|
1605
|
+
return this.#metrics;
|
|
1606
|
+
}
|
|
1607
|
+
/** v6 alias over the single event bus. */
|
|
1608
|
+
geteventbus() {
|
|
1609
|
+
return this.#events;
|
|
1610
|
+
}
|
|
1611
|
+
#log(_message) {
|
|
1612
|
+
try {
|
|
1613
|
+
}
|
|
1614
|
+
catch {
|
|
1615
|
+
/* catcher: logging must never break the caller */
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
/* ---------------------------------------------------------------- */
|
|
1619
|
+
/* process error catcher (coreorchestrator) */
|
|
1620
|
+
/* ---------------------------------------------------------------- */
|
|
1621
|
+
/**
|
|
1622
|
+
* routes unhandled rejections and uncaught exceptions of the host
|
|
1623
|
+
* process onto the single bus; explicit opt-in because it changes
|
|
1624
|
+
* process-wide behavior.
|
|
1625
|
+
*/
|
|
1626
|
+
installerrorcatcher() {
|
|
1627
|
+
if (this.#errorcatcherinstalled) {
|
|
1628
|
+
return;
|
|
1629
|
+
}
|
|
1630
|
+
try {
|
|
1631
|
+
process.on('unhandledRejection', (reason) => {
|
|
1632
|
+
this.#events.publish('system:error', {
|
|
1633
|
+
kind: 'unhandledrejection',
|
|
1634
|
+
message: String(reason),
|
|
1635
|
+
});
|
|
1636
|
+
});
|
|
1637
|
+
process.on('uncaughtException', (error) => {
|
|
1638
|
+
this.#events.publish('system:error', { kind: 'uncaughtexception', message: error.message });
|
|
1639
|
+
});
|
|
1640
|
+
this.#errorcatcherinstalled = true;
|
|
1641
|
+
}
|
|
1642
|
+
catch {
|
|
1643
|
+
/* catcher: process handlers may be unavailable in sandboxes */
|
|
1644
|
+
}
|
|
1645
|
+
}
|
|
1646
|
+
/* ---------------------------------------------------------------- */
|
|
1647
|
+
/* NUMA topology: heuristic build + cores.json view (merged) */
|
|
1648
|
+
/* ---------------------------------------------------------------- */
|
|
1649
|
+
#buildnumatopology() {
|
|
1650
|
+
try {
|
|
1651
|
+
const cpucount = cpus().length || 32;
|
|
1652
|
+
const numacount = Math.max(1, Math.min(4, Math.ceil(cpucount / 16)));
|
|
1653
|
+
const nodes = [];
|
|
1654
|
+
const cpuspernode = Math.ceil(cpucount / numacount);
|
|
1655
|
+
let cpuindex = 0;
|
|
1656
|
+
for (let node = 0; node < numacount; node += 1) {
|
|
1657
|
+
const nodecpus = [];
|
|
1658
|
+
for (let i = 0; i < cpuspernode && cpuindex < cpucount; i += 1) {
|
|
1659
|
+
nodecpus.push(cpuindex);
|
|
1660
|
+
cpuindex += 1;
|
|
1661
|
+
}
|
|
1662
|
+
const memmb = Math.floor(totalmem() / 1024 / 1024 / numacount);
|
|
1663
|
+
const distances = {};
|
|
1664
|
+
for (let other = 0; other < numacount; other += 1) {
|
|
1665
|
+
distances[other] = other === node ? 10 : 21;
|
|
1666
|
+
}
|
|
1667
|
+
nodes.push({
|
|
1668
|
+
id: node,
|
|
1669
|
+
cpus: nodecpus,
|
|
1670
|
+
memorymb: memmb,
|
|
1671
|
+
distances,
|
|
1672
|
+
hugepages: { '2m': Math.floor(memmb / 2 / 4), '1g': Math.floor(memmb / 1024 / 2) },
|
|
1673
|
+
});
|
|
1674
|
+
}
|
|
1675
|
+
this.#numatopology = {
|
|
1676
|
+
nodes,
|
|
1677
|
+
totalmemorymb: Math.floor(totalmem() / 1024 / 1024),
|
|
1678
|
+
};
|
|
1679
|
+
this.#log(`numa topology: ${numacount} nodes, ${cpucount} cpus, ${this.#numatopology.totalmemorymb} MB`);
|
|
1680
|
+
}
|
|
1681
|
+
catch (error) {
|
|
1682
|
+
this.#events.publish('system:error', { kind: 'numa', message: errormessage(error) });
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
#initrunqueues() {
|
|
1686
|
+
for (const node of this.#numatopology.nodes) {
|
|
1687
|
+
this.#runqueues.set(node.id, {
|
|
1688
|
+
numanode: node.id,
|
|
1689
|
+
tasks: [],
|
|
1690
|
+
load: 0,
|
|
1691
|
+
capacity: node.cpus.length * 100,
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
/**
|
|
1696
|
+
* returns the NUMA view as {nodes, map}; an explicit numa section in
|
|
1697
|
+
* cores.json wins over the heuristic (coreorchestrator getNumaTopology).
|
|
1698
|
+
*/
|
|
1699
|
+
async getnumatopology() {
|
|
1700
|
+
try {
|
|
1701
|
+
const cores = (await this.#readjsonmaybe(path.join(this.#rootdir, 'cores.json')));
|
|
1702
|
+
if (cores?.numa !== undefined) {
|
|
1703
|
+
return cores.numa;
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
catch {
|
|
1707
|
+
/* catcher: fall back to the heuristic view */
|
|
1708
|
+
}
|
|
1709
|
+
const map = {};
|
|
1710
|
+
for (const node of this.#numatopology.nodes) {
|
|
1711
|
+
map[node.id] = [...node.cpus];
|
|
1712
|
+
}
|
|
1713
|
+
return { nodes: this.#numatopology.nodes.length, map };
|
|
1714
|
+
}
|
|
1715
|
+
async #readjsonmaybe(filepath) {
|
|
1716
|
+
try {
|
|
1717
|
+
const raw = await readFile(filepath, 'utf8');
|
|
1718
|
+
return JSON.parse(raw);
|
|
1719
|
+
}
|
|
1720
|
+
catch {
|
|
1721
|
+
return undefined;
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1724
|
+
/* ---------------------------------------------------------------- */
|
|
1725
|
+
/* RBAC: default roles, assert, assign, grant/check cache */
|
|
1726
|
+
/* ---------------------------------------------------------------- */
|
|
1727
|
+
#initdefaultrbac() {
|
|
1728
|
+
const roles = [
|
|
1729
|
+
{ id: 'role-admin', name: 'admin', permissions: ['*'] },
|
|
1730
|
+
{
|
|
1731
|
+
id: 'role-operator',
|
|
1732
|
+
name: 'operator',
|
|
1733
|
+
permissions: [
|
|
1734
|
+
'vm:create',
|
|
1735
|
+
'vm:start',
|
|
1736
|
+
'vm:stop',
|
|
1737
|
+
'vm:delete',
|
|
1738
|
+
'vm:migrate',
|
|
1739
|
+
'gpu:assign',
|
|
1740
|
+
'snapshot:create',
|
|
1741
|
+
'metrics:read',
|
|
1742
|
+
],
|
|
1743
|
+
},
|
|
1744
|
+
{ id: 'role-viewer', name: 'viewer', permissions: ['metrics:read'] },
|
|
1745
|
+
{
|
|
1746
|
+
id: 'role-scheduler',
|
|
1747
|
+
name: 'scheduler',
|
|
1748
|
+
permissions: ['vm:create', 'vm:start', 'vm:stop', 'metrics:read'],
|
|
1749
|
+
},
|
|
1750
|
+
];
|
|
1751
|
+
for (const role of roles) {
|
|
1752
|
+
this.#rbacroles.set(role.id, role);
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
/**
|
|
1756
|
+
* asserts that a user holds a permission through one of its roles; the
|
|
1757
|
+
* system actor bypasses everything and roleless users may still read
|
|
1758
|
+
* metrics (viewer fallback). denials publish rbac:denied and throw an
|
|
1759
|
+
* orchestratorerror with ERR_RBAC_DENIED.
|
|
1760
|
+
*/
|
|
1761
|
+
assertpermission(user, perm) {
|
|
1762
|
+
if (user === SYSTEMUSER) {
|
|
1763
|
+
return;
|
|
1764
|
+
}
|
|
1765
|
+
const roles = this.#userroles.get(user) ?? [];
|
|
1766
|
+
for (const roleid of roles) {
|
|
1767
|
+
const role = this.#rbacroles.get(roleid);
|
|
1768
|
+
if (role === undefined) {
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
if (role.permissions.includes('*') || role.permissions.includes(perm)) {
|
|
1772
|
+
return;
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
if (perm === 'metrics:read' && roles.length === 0) {
|
|
1776
|
+
return;
|
|
1777
|
+
}
|
|
1778
|
+
this.#events.publish('rbac:denied', { userid: user, action: perm });
|
|
1779
|
+
this.#metrics.inc('rbac_denied_total', 1, { user: user, perm });
|
|
1780
|
+
throw new orchestratorerror(`rbac denied user=${user} perm=${perm}`, 'ERR_RBAC_DENIED');
|
|
1781
|
+
}
|
|
1782
|
+
/** assigns an extra role to a user. */
|
|
1783
|
+
assignrole(user, role) {
|
|
1784
|
+
if (!this.#rbacroles.has(role)) {
|
|
1785
|
+
throw new orchestratorerror(`role ${role} not found`, 'ERR_RBAC_ROLE');
|
|
1786
|
+
}
|
|
1787
|
+
const current = this.#userroles.get(user) ?? [];
|
|
1788
|
+
if (!current.includes(role)) {
|
|
1789
|
+
current.push(role);
|
|
1790
|
+
}
|
|
1791
|
+
this.#userroles.set(user, current);
|
|
1792
|
+
this.#log(`role ${role} assigned to ${user}`);
|
|
1793
|
+
}
|
|
1794
|
+
/** grants subject actions over a resource key (v6 cache). */
|
|
1795
|
+
grantrbac(principal, resource, actions) {
|
|
1796
|
+
try {
|
|
1797
|
+
const key = `${principal.tenant}::${resource}`;
|
|
1798
|
+
const set = this.#rbaccache.get(key) ?? new Set();
|
|
1799
|
+
for (const action of actions) {
|
|
1800
|
+
set.add(`${principal.subject}:${action}`);
|
|
1801
|
+
}
|
|
1802
|
+
this.#rbaccache.set(key, set);
|
|
1803
|
+
}
|
|
1804
|
+
catch {
|
|
1805
|
+
/* catcher: cache writes are best effort */
|
|
1806
|
+
}
|
|
1807
|
+
}
|
|
1808
|
+
/** checks a grant through subject, roles and wildcards (v6). */
|
|
1809
|
+
checkrbac(principal, resource, action) {
|
|
1810
|
+
const checkkey = (key) => {
|
|
1811
|
+
const set = this.#rbaccache.get(key);
|
|
1812
|
+
if (set === undefined) {
|
|
1813
|
+
return false;
|
|
1814
|
+
}
|
|
1815
|
+
if (set.has(`${principal.subject}:${action}`)) {
|
|
1816
|
+
return true;
|
|
1817
|
+
}
|
|
1818
|
+
for (const role of principal.roles) {
|
|
1819
|
+
if (set.has(`${role}:${action}`) || set.has(`${role}:*`)) {
|
|
1820
|
+
return true;
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
return false;
|
|
1824
|
+
};
|
|
1825
|
+
const ok = checkkey(`${principal.tenant}::${resource}`) ||
|
|
1826
|
+
checkkey(`${principal.tenant}::*:*`) ||
|
|
1827
|
+
checkkey(`${principal.tenant}::${resource.split(':')[0]}:*`);
|
|
1828
|
+
this.#metrics.inc('rbac_check_total', 1, {
|
|
1829
|
+
tenant: principal.tenant,
|
|
1830
|
+
action,
|
|
1831
|
+
ok: ok ? '1' : '0',
|
|
1832
|
+
});
|
|
1833
|
+
return ok;
|
|
1834
|
+
}
|
|
1835
|
+
/* ---------------------------------------------------------------- */
|
|
1836
|
+
/* reservations with TTL + affinity rules */
|
|
1837
|
+
/* ---------------------------------------------------------------- */
|
|
1838
|
+
/**
|
|
1839
|
+
* reserves cpus/memory/gpus for an owner with a TTL (default 300s);
|
|
1840
|
+
* overlapping cpu sets of equal or higher priority block the request.
|
|
1841
|
+
*/
|
|
1842
|
+
reserveresources(owner, resources, ttlsec = 300, priority = 50) {
|
|
1843
|
+
for (const existing of this.#reservations.values()) {
|
|
1844
|
+
const overlap = existing.resources.cpus.some((cpu) => resources.cpus.includes(cpu));
|
|
1845
|
+
if (overlap && Date.now() < existing.expiresat.getTime() && existing.priority >= priority) {
|
|
1846
|
+
throw new orchestratorerror(`resource conflict with reservation ${existing.id} priority=${existing.priority}`, 'ERR_RESERVATION_CONFLICT');
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
const created = {
|
|
1850
|
+
id: `res-${randomUUID()}`,
|
|
1851
|
+
resources,
|
|
1852
|
+
owner,
|
|
1853
|
+
expiresat: new Date(Date.now() + ttlsec * 1000),
|
|
1854
|
+
priority,
|
|
1855
|
+
};
|
|
1856
|
+
this.#reservations.set(created.id, created);
|
|
1857
|
+
this.#metrics.inc('reservation_created_total', 1, { owner: String(owner) });
|
|
1858
|
+
setTimeout(() => {
|
|
1859
|
+
this.#reservations.delete(created.id);
|
|
1860
|
+
}, ttlsec * 1000).unref();
|
|
1861
|
+
return created;
|
|
1862
|
+
}
|
|
1863
|
+
/** releases a reservation by id; true when it existed. */
|
|
1864
|
+
releasereservation(id) {
|
|
1865
|
+
return this.#reservations.delete(id);
|
|
1866
|
+
}
|
|
1867
|
+
/** registers an affinity or anti-affinity rule. */
|
|
1868
|
+
addaffinityrule(rule) {
|
|
1869
|
+
this.#affinityrules.push(rule);
|
|
1870
|
+
this.#log(`affinity rule ${rule.id} ${rule.type} ${rule.subjects.join(',')}`);
|
|
1871
|
+
}
|
|
1872
|
+
/** removes an affinity rule by id. */
|
|
1873
|
+
removeaffinityrule(ruleid) {
|
|
1874
|
+
const index = this.#affinityrules.findIndex((rule) => rule.id === ruleid);
|
|
1875
|
+
if (index === -1) {
|
|
1876
|
+
return false;
|
|
1877
|
+
}
|
|
1878
|
+
this.#affinityrules.splice(index, 1);
|
|
1879
|
+
return true;
|
|
1880
|
+
}
|
|
1881
|
+
/**
|
|
1882
|
+
* evaluates whether a vm definition may land on a node described by
|
|
1883
|
+
* labels: In/NotIn/Exists selectors must all pass and antiaffinity
|
|
1884
|
+
* targets must not overlap the candidate pin set (coreorchestrator).
|
|
1885
|
+
*/
|
|
1886
|
+
resolveaffinity(record, nodelabels) {
|
|
1887
|
+
for (const selector of record.affinity) {
|
|
1888
|
+
const value = nodelabels[selector.key];
|
|
1889
|
+
if (selector.operator === 'in') {
|
|
1890
|
+
if (selector.values === undefined ||
|
|
1891
|
+
value === undefined ||
|
|
1892
|
+
!selector.values.includes(value)) {
|
|
1893
|
+
return false;
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
else if (selector.operator === 'notin') {
|
|
1897
|
+
if (selector.values !== undefined &&
|
|
1898
|
+
value !== undefined &&
|
|
1899
|
+
selector.values.includes(value)) {
|
|
1900
|
+
return false;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
else if (selector.operator === 'exists') {
|
|
1904
|
+
if (value === undefined) {
|
|
1905
|
+
return false;
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
const ownpins = this.#affinityindex.get(record.id);
|
|
1910
|
+
if (record.antiaffinity.length > 0 && ownpins !== undefined) {
|
|
1911
|
+
for (const otherid of record.antiaffinity) {
|
|
1912
|
+
const other = this.#vms.get(otherid);
|
|
1913
|
+
const otherpins = this.#affinityindex.get(otherid);
|
|
1914
|
+
if (other !== undefined && other.state === 'running' && otherpins !== undefined) {
|
|
1915
|
+
const overlap = [...otherpins].some((cpu) => ownpins.has(cpu));
|
|
1916
|
+
if (overlap) {
|
|
1917
|
+
return false;
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
}
|
|
1922
|
+
return true;
|
|
1923
|
+
}
|
|
1924
|
+
/* ---------------------------------------------------------------- */
|
|
1925
|
+
/* NUMA scheduler: placement, runqueues, tick, rebalance */
|
|
1926
|
+
/* ---------------------------------------------------------------- */
|
|
1927
|
+
/**
|
|
1928
|
+
* scores every node as cpuFree - vcpus*10 + memRatio*50 with the
|
|
1929
|
+
* affinity weight applied for numa scoped rules and returns the best.
|
|
1930
|
+
*/
|
|
1931
|
+
#selectbestnumanode(vcpus, _memmb, exclude = []) {
|
|
1932
|
+
let bestid = 0;
|
|
1933
|
+
let bestscore = Number.NEGATIVE_INFINITY;
|
|
1934
|
+
for (const node of this.#numatopology.nodes) {
|
|
1935
|
+
if (exclude.includes(node.id)) {
|
|
1936
|
+
continue;
|
|
1937
|
+
}
|
|
1938
|
+
const queue = this.#runqueues.get(node.id);
|
|
1939
|
+
if (queue === undefined) {
|
|
1940
|
+
continue;
|
|
1941
|
+
}
|
|
1942
|
+
const cpufree = queue.capacity - queue.load;
|
|
1943
|
+
const memratio = this.#numatopology.totalmemorymb > 0 ? node.memorymb / this.#numatopology.totalmemorymb : 0;
|
|
1944
|
+
let score = cpufree - vcpus * 10 + memratio * 50;
|
|
1945
|
+
for (const rule of this.#affinityrules) {
|
|
1946
|
+
if (rule.scope !== 'numa' || rule.subjects.length === 0) {
|
|
1947
|
+
continue;
|
|
1948
|
+
}
|
|
1949
|
+
const subjectsinnode = [...this.#vms.values()].filter((candidate) => rule.subjects.includes(candidate.id) && candidate.numanode === node.id).length;
|
|
1950
|
+
if (subjectsinnode > 0) {
|
|
1951
|
+
score += rule.type === 'affinity' ? rule.weight : -rule.weight;
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
if (score > bestscore) {
|
|
1955
|
+
bestscore = score;
|
|
1956
|
+
bestid = node.id;
|
|
1957
|
+
}
|
|
1958
|
+
}
|
|
1959
|
+
return bestid;
|
|
1960
|
+
}
|
|
1961
|
+
#allocatevcpupins(numanode, count) {
|
|
1962
|
+
const node = this.#numatopology.nodes.find((candidate) => candidate.id === numanode);
|
|
1963
|
+
if (node === undefined) {
|
|
1964
|
+
throw new orchestratorerror(`numa ${numanode} not found`, 'ERR_NUMA_NODE');
|
|
1965
|
+
}
|
|
1966
|
+
const map = new Map();
|
|
1967
|
+
const available = [...node.cpus];
|
|
1968
|
+
for (let index = 0; index < count; index += 1) {
|
|
1969
|
+
const cpu = available[index % available.length];
|
|
1970
|
+
if (cpu !== undefined) {
|
|
1971
|
+
map.set(index, [cpu]);
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
return map;
|
|
1975
|
+
}
|
|
1976
|
+
/** enqueues a vCPU task onto its numa run queue, spilling to the best
|
|
1977
|
+
* alternative node when the queue is at capacity. */
|
|
1978
|
+
#enqueuevcpu(task) {
|
|
1979
|
+
const queue = this.#runqueues.get(task.numanode);
|
|
1980
|
+
if (queue === undefined) {
|
|
1981
|
+
throw new orchestratorerror(`numa ${task.numanode} not found`, 'ERR_NUMA_NODE');
|
|
1982
|
+
}
|
|
1983
|
+
if (queue.load >= queue.capacity) {
|
|
1984
|
+
const alt = this.#selectbestnumanode(1, 0, [task.numanode]);
|
|
1985
|
+
if (alt !== task.numanode) {
|
|
1986
|
+
task.numanode = alt;
|
|
1987
|
+
const altqueue = this.#runqueues.get(alt);
|
|
1988
|
+
if (altqueue !== undefined) {
|
|
1989
|
+
altqueue.tasks.push(task);
|
|
1990
|
+
this.#metrics.inc('vcpu_migrated_total', 1, {
|
|
1991
|
+
from: String(queue.numanode),
|
|
1992
|
+
to: String(alt),
|
|
1993
|
+
});
|
|
1994
|
+
return;
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
queue.tasks.push(task);
|
|
1999
|
+
queue.load += task.shares / 1024;
|
|
2000
|
+
this.#events.publish('vcpu:scheduled', {
|
|
2001
|
+
vmid: task.vmid,
|
|
2002
|
+
vcpuid: task.id,
|
|
2003
|
+
cpu: task.affinity[0] ?? 0,
|
|
2004
|
+
});
|
|
2005
|
+
}
|
|
2006
|
+
/**
|
|
2007
|
+
* periodic scheduler tick: sorts run queues by priority, recomputes
|
|
2008
|
+
* loads and rebalances when the max/min spread exceeds 20 percent.
|
|
2009
|
+
*/
|
|
2010
|
+
scheduletick() {
|
|
2011
|
+
for (const queue of this.#runqueues.values()) {
|
|
2012
|
+
queue.tasks.sort((a, b) => b.priority - a.priority);
|
|
2013
|
+
queue.load = queue.tasks.reduce((sum, task) => sum + task.shares / 1024, 0);
|
|
2014
|
+
this.#metrics.gauge('runqueue_load', queue.load, { numa: String(queue.numanode) });
|
|
2015
|
+
}
|
|
2016
|
+
const loads = [...this.#runqueues.values()].map((queue) => queue.load);
|
|
2017
|
+
if (loads.length > 1) {
|
|
2018
|
+
const max = Math.max(...loads);
|
|
2019
|
+
const min = Math.min(...loads);
|
|
2020
|
+
if (max > 0 && (max - min) / max > 0.2) {
|
|
2021
|
+
void this.#rebalancenuma();
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
/** moves the lowest priority task from the most loaded queue to the
|
|
2026
|
+
* least loaded one when the delta is at least 10 load units. */
|
|
2027
|
+
async #rebalancenuma() {
|
|
2028
|
+
const sorted = [...this.#runqueues.values()].sort((a, b) => b.load - a.load);
|
|
2029
|
+
const overloaded = sorted[0];
|
|
2030
|
+
const underloaded = sorted[sorted.length - 1];
|
|
2031
|
+
if (overloaded === undefined || underloaded === undefined || overloaded.tasks.length === 0) {
|
|
2032
|
+
return;
|
|
2033
|
+
}
|
|
2034
|
+
if (overloaded.load - underloaded.load < 10) {
|
|
2035
|
+
return;
|
|
2036
|
+
}
|
|
2037
|
+
const victim = [...overloaded.tasks].sort((a, b) => a.priority - b.priority)[0];
|
|
2038
|
+
if (victim === undefined) {
|
|
2039
|
+
return;
|
|
2040
|
+
}
|
|
2041
|
+
overloaded.tasks = overloaded.tasks.filter((task) => task.id !== victim.id);
|
|
2042
|
+
overloaded.load -= victim.shares / 1024;
|
|
2043
|
+
victim.numanode = underloaded.numanode;
|
|
2044
|
+
underloaded.tasks.push(victim);
|
|
2045
|
+
underloaded.load += victim.shares / 1024;
|
|
2046
|
+
this.#events.publish('numa:rebalance', {
|
|
2047
|
+
moves: 1,
|
|
2048
|
+
from: overloaded.numanode,
|
|
2049
|
+
to: underloaded.numanode,
|
|
2050
|
+
});
|
|
2051
|
+
this.#metrics.inc('numa_rebalanced_total', 1, {
|
|
2052
|
+
from: String(overloaded.numanode),
|
|
2053
|
+
to: String(underloaded.numanode),
|
|
2054
|
+
});
|
|
2055
|
+
}
|
|
2056
|
+
/** repins one vCPU of a vm onto an explicit cpu set. */
|
|
2057
|
+
setvcpuaffinity(id, vcpuindex, target) {
|
|
2058
|
+
const vm = this.#vms.get(id);
|
|
2059
|
+
if (vm === undefined) {
|
|
2060
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
2061
|
+
}
|
|
2062
|
+
vm.vcpupins.set(vcpuindex, target);
|
|
2063
|
+
for (const queue of this.#runqueues.values()) {
|
|
2064
|
+
for (const task of queue.tasks) {
|
|
2065
|
+
if (task.vmid === id && task.id === `${id}-vcpu-${vcpuindex}`) {
|
|
2066
|
+
task.affinity = [...target];
|
|
2067
|
+
}
|
|
2068
|
+
}
|
|
2069
|
+
}
|
|
2070
|
+
return { ok: true, value: undefined };
|
|
2071
|
+
}
|
|
2072
|
+
/**
|
|
2073
|
+
* validates the topology of a vm, warns on overcommit above 1.5x the
|
|
2074
|
+
* host cpus, computes the pin set (explicit pins or a random rotation)
|
|
2075
|
+
* and walks the machine through scheduling and binding
|
|
2076
|
+
* (coreorchestrator scheduleVcpu).
|
|
2077
|
+
*/
|
|
2078
|
+
async schedulevcpu(vm) {
|
|
2079
|
+
const hostcpus = cpus().length;
|
|
2080
|
+
const topo = vm.vcputopology ?? { sockets: 1, corespersocket: vm.vcpus, threadspercore: 1 };
|
|
2081
|
+
const totallogical = topo.sockets * topo.corespersocket * topo.threadspercore;
|
|
2082
|
+
if (vm.vcputopology !== undefined && totallogical !== vm.vcpus) {
|
|
2083
|
+
throw new orchestratorerror('vcpu topology mismatch', 'ERR_VCPU_TOPO');
|
|
2084
|
+
}
|
|
2085
|
+
const overcommit = topo.overcommitratio ?? 1.5;
|
|
2086
|
+
if (vm.vcpus > hostcpus * overcommit) {
|
|
2087
|
+
this.#events.publish('sched:overcommitwarn', {
|
|
2088
|
+
vmid: vm.id,
|
|
2089
|
+
requested: vm.vcpus,
|
|
2090
|
+
host: hostcpus,
|
|
2091
|
+
overcommit,
|
|
2092
|
+
});
|
|
2093
|
+
}
|
|
2094
|
+
const pinset = this.#computecpupinning(vm, hostcpus);
|
|
2095
|
+
this.#affinityindex.set(vm.id, new Set(pinset.map(String)));
|
|
2096
|
+
this.#metrics.inc('vcpu_scheduled_total', vm.vcpus, { vmid: vm.id });
|
|
2097
|
+
await this.transitionvm(vm.id, 'scheduling');
|
|
2098
|
+
await this.transitionvm(vm.id, 'binding');
|
|
2099
|
+
}
|
|
2100
|
+
#computecpupinning(vm, hostcpus) {
|
|
2101
|
+
if (vm.vcputopology?.pinnedcpus !== undefined) {
|
|
2102
|
+
return vm.vcputopology.pinnedcpus.slice(0, vm.vcpus);
|
|
2103
|
+
}
|
|
2104
|
+
const start = randomInt(0, Math.max(1, hostcpus - vm.vcpus));
|
|
2105
|
+
return Array.from({ length: vm.vcpus }, (_, index) => (start + index) % hostcpus);
|
|
2106
|
+
}
|
|
2107
|
+
/* ---------------------------------------------------------------- */
|
|
2108
|
+
/* vm lifecycle: create, start, stop, destroy, pause, transition */
|
|
2109
|
+
/* ---------------------------------------------------------------- */
|
|
2110
|
+
/** guarded transition on the single machine; publishes vm:phase. */
|
|
2111
|
+
async transitionvm(id, to) {
|
|
2112
|
+
const vm = this.#vms.get(id);
|
|
2113
|
+
if (vm === undefined) {
|
|
2114
|
+
throw new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND');
|
|
2115
|
+
}
|
|
2116
|
+
if (!cantransition(vm.state, to)) {
|
|
2117
|
+
throw new orchestratorerror(`illegal transition ${vm.state} -> ${to} for ${id}`, 'ERR_PHASE');
|
|
2118
|
+
}
|
|
2119
|
+
const from = vm.state;
|
|
2120
|
+
vm.state = to;
|
|
2121
|
+
this.#events.publish('vm:phase', { vmid: id, to });
|
|
2122
|
+
this.#metrics.inc('vm_phase_transition_total', 1, { from, to });
|
|
2123
|
+
return vm;
|
|
2124
|
+
}
|
|
2125
|
+
/**
|
|
2126
|
+
* creates a vm record: quota enforcement, NUMA placement, vCPU pins,
|
|
2127
|
+
* balloon initialization and the scheduling/binding walkthrough.
|
|
2128
|
+
*/
|
|
2129
|
+
async createvm(definition, actor = SYSTEMUSER) {
|
|
2130
|
+
try {
|
|
2131
|
+
this.assertpermission(actor, 'vm:create');
|
|
2132
|
+
const id = definition.id ?? createvmid();
|
|
2133
|
+
if (this.#vms.has(id)) {
|
|
2134
|
+
return {
|
|
2135
|
+
ok: false,
|
|
2136
|
+
error: new orchestratorerror(`vm ${id} already exists`, 'ERR_VM_EXISTS'),
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
2139
|
+
this.#enforcequota(definition.tenantid ?? 'default', definition.vcpus, definition.vrammib, definition.gpu?.length ?? 0);
|
|
2140
|
+
const numanode = this.#selectbestnumanode(definition.vcpus, definition.vrammib);
|
|
2141
|
+
const record = {
|
|
2142
|
+
id,
|
|
2143
|
+
name: definition.name,
|
|
2144
|
+
tenantid: definition.tenantid ?? 'default',
|
|
2145
|
+
vcpus: definition.vcpus,
|
|
2146
|
+
vrammib: definition.vrammib,
|
|
2147
|
+
diskgb: definition.diskgb ?? 40,
|
|
2148
|
+
ostype: definition.ostype ?? 'linux',
|
|
2149
|
+
arch: definition.arch ?? 'x86_64',
|
|
2150
|
+
image: definition.image ?? 'debian:trixie-slim',
|
|
2151
|
+
qemubinary: definition.qemubinary ?? '/usr/bin/qemu-system-x86_64',
|
|
2152
|
+
machine: definition.machine ?? 'q35',
|
|
2153
|
+
accel: definition.accel ?? (this.#options.kvmenabled === false ? 'tcg' : 'kvm'),
|
|
2154
|
+
extraargs: definition.extraargs ?? [],
|
|
2155
|
+
dockerruntime: definition.dockerruntime ?? 'runc',
|
|
2156
|
+
dockernetwork: definition.dockernetwork ?? 'bridge',
|
|
2157
|
+
vcputopology: definition.vcputopology,
|
|
2158
|
+
vramballoon: {
|
|
2159
|
+
requestedmib: definition.vrammib,
|
|
2160
|
+
minmib: Math.floor(definition.vrammib * 0.5),
|
|
2161
|
+
maxmib: Math.floor(definition.vrammib * 1.5),
|
|
2162
|
+
currentmib: definition.vrammib,
|
|
2163
|
+
balloondriver: 'virtio-balloon',
|
|
2164
|
+
hugepages: '2mib',
|
|
2165
|
+
},
|
|
2166
|
+
gpu: definition.gpu !== undefined ? [...definition.gpu] : [],
|
|
2167
|
+
labels: definition.labels ?? {},
|
|
2168
|
+
affinity: definition.affinity ?? [],
|
|
2169
|
+
antiaffinity: definition.antiaffinity ?? [],
|
|
2170
|
+
checkpointenabled: definition.checkpointenabled ?? true,
|
|
2171
|
+
rbacroles: definition.rbacroles ?? [],
|
|
2172
|
+
state: 'pending',
|
|
2173
|
+
createdat: new Date(),
|
|
2174
|
+
numanode,
|
|
2175
|
+
vcpupins: this.#allocatevcpupins(numanode, definition.vcpus),
|
|
2176
|
+
metrics: { cputimens: 0n, memrssmb: 0 },
|
|
2177
|
+
};
|
|
2178
|
+
this.#vms.set(id, record);
|
|
2179
|
+
this.#balloons.set(id, {
|
|
2180
|
+
vmid: id,
|
|
2181
|
+
targetmib: definition.vrammib,
|
|
2182
|
+
actualmib: definition.vrammib,
|
|
2183
|
+
deflatedmib: 0,
|
|
2184
|
+
lastadjust: new Date(),
|
|
2185
|
+
pressurescore: 0,
|
|
2186
|
+
});
|
|
2187
|
+
this.#metrics.inc('vm_created_total', 1, { vmid: id, arch: record.arch });
|
|
2188
|
+
this.#audit(actor, 'vm:create', id, 'success', { name: definition.name });
|
|
2189
|
+
this.#events.publish('vm:created', { vmid: id, tenantid: record.tenantid });
|
|
2190
|
+
this.#log(`vm ${id} created numa=${numanode} vcpus=${definition.vcpus}`);
|
|
2191
|
+
await this.schedulevcpu(record);
|
|
2192
|
+
return { ok: true, value: record };
|
|
2193
|
+
}
|
|
2194
|
+
catch (error) {
|
|
2195
|
+
this.#audit(actor, 'vm:create', definition.id ?? 'new', 'failure', {
|
|
2196
|
+
error: errormessage(error),
|
|
2197
|
+
});
|
|
2198
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
/**
|
|
2202
|
+
* starts a vm: vCPU tasks enter the run queues, qemu spawns with the
|
|
2203
|
+
* monitor/qmp socket layout and the docker bridge attaches for bridge
|
|
2204
|
+
* or passage network modes.
|
|
2205
|
+
*/
|
|
2206
|
+
async startvm(id, actor = SYSTEMUSER) {
|
|
2207
|
+
this.assertpermission(actor, 'vm:start');
|
|
2208
|
+
const vm = this.#vms.get(id);
|
|
2209
|
+
if (vm === undefined) {
|
|
2210
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
2211
|
+
}
|
|
2212
|
+
if (!['binding', 'stopped', 'failed'].includes(vm.state)) {
|
|
2213
|
+
return {
|
|
2214
|
+
ok: false,
|
|
2215
|
+
error: new orchestratorerror(`vm ${id} not startable from ${vm.state}`, 'ERR_VM_STATE'),
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
try {
|
|
2219
|
+
await this.transitionvm(id, 'provisioning');
|
|
2220
|
+
for (let index = 0; index < vm.vcpus; index += 1) {
|
|
2221
|
+
const nodecpus = this.#numatopology.nodes[vm.numanode]?.cpus ?? [0];
|
|
2222
|
+
this.#enqueuevcpu({
|
|
2223
|
+
id: `${id}-vcpu-${index}`,
|
|
2224
|
+
vmid: id,
|
|
2225
|
+
priority: 20,
|
|
2226
|
+
affinity: vm.vcpupins.get(index) ?? nodecpus.slice(0, 1),
|
|
2227
|
+
numanode: vm.numanode,
|
|
2228
|
+
shares: 1024,
|
|
2229
|
+
burstms: 100,
|
|
2230
|
+
});
|
|
2231
|
+
}
|
|
2232
|
+
const qemustrategy = this.#registry.get('qemu');
|
|
2233
|
+
if (!(qemustrategy instanceof qemuruntime)) {
|
|
2234
|
+
throw new orchestratorerror('qemu runtime strategy unavailable', 'ERR_QEMU_STRATEGY');
|
|
2235
|
+
}
|
|
2236
|
+
const processinfo = await qemustrategy.spawn(vm, [...this.#gpuassignments.values()], (exited, code) => {
|
|
2237
|
+
this.#qemuprocesses.delete(exited);
|
|
2238
|
+
const record = this.#vms.get(exited);
|
|
2239
|
+
if (record !== undefined && record.state === 'running') {
|
|
2240
|
+
record.state = 'stopped';
|
|
2241
|
+
}
|
|
2242
|
+
this.#events.publish('qemu:exit', { vmid: exited, code });
|
|
2243
|
+
this.#log(`qemu exit vm=${exited} code=${code ?? 'signal'}`);
|
|
2244
|
+
});
|
|
2245
|
+
this.#qemuprocesses.set(id, processinfo);
|
|
2246
|
+
const probe = await qemustrategy.probeversion();
|
|
2247
|
+
if (!probe.includes(runtimecatalog.qemu)) {
|
|
2248
|
+
this.#events.publish('qemu:versionmismatch', { expected: runtimecatalog.qemu, got: probe });
|
|
2249
|
+
}
|
|
2250
|
+
if (vm.dockernetwork === 'bridge' || vm.dockernetwork === 'passage') {
|
|
2251
|
+
await this.#attachdockerbridge(vm);
|
|
2252
|
+
}
|
|
2253
|
+
await this.transitionvm(id, 'running');
|
|
2254
|
+
vm.startedat = new Date();
|
|
2255
|
+
vm.qemupid = processinfo.pid;
|
|
2256
|
+
this.#metrics.inc('vm_running_total', 1, { vmid: id });
|
|
2257
|
+
this.#events.publish('vm:started', { vmid: id, pid: processinfo.pid });
|
|
2258
|
+
this.#audit(actor, 'vm:start', id, 'success', {});
|
|
2259
|
+
this.#log(`vm ${id} running pid=${processinfo.pid}`);
|
|
2260
|
+
return { ok: true, value: vm };
|
|
2261
|
+
}
|
|
2262
|
+
catch (error) {
|
|
2263
|
+
vm.state = 'failed';
|
|
2264
|
+
this.#events.publish('vm:error', { vmid: id, error: errormessage(error) });
|
|
2265
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
/** stops a vm through the drain path, kills qemu and unschedules its
|
|
2269
|
+
* vCPU tasks. */
|
|
2270
|
+
async stopvm(id, actor = SYSTEMUSER, reason = 'user') {
|
|
2271
|
+
this.assertpermission(actor, 'vm:stop');
|
|
2272
|
+
const vm = this.#vms.get(id);
|
|
2273
|
+
if (vm === undefined) {
|
|
2274
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
2275
|
+
}
|
|
2276
|
+
await this.drainvm(id, {
|
|
2277
|
+
vmid: id,
|
|
2278
|
+
timeoutsec: 30,
|
|
2279
|
+
signal: 'SIGTERM',
|
|
2280
|
+
fallback: 'SIGKILL',
|
|
2281
|
+
});
|
|
2282
|
+
const processinfo = this.#qemuprocesses.get(id);
|
|
2283
|
+
if (processinfo !== undefined) {
|
|
2284
|
+
try {
|
|
2285
|
+
processinfo.child.kill('SIGTERM');
|
|
2286
|
+
setTimeout(() => {
|
|
2287
|
+
try {
|
|
2288
|
+
if (!processinfo.child.killed) {
|
|
2289
|
+
processinfo.child.kill('SIGKILL');
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
catch {
|
|
2293
|
+
/* catcher: the process may already be gone */
|
|
2294
|
+
}
|
|
2295
|
+
}, 5000).unref();
|
|
2296
|
+
}
|
|
2297
|
+
catch {
|
|
2298
|
+
/* catcher: best effort kill */
|
|
2299
|
+
}
|
|
2300
|
+
this.#qemuprocesses.delete(id);
|
|
2301
|
+
}
|
|
2302
|
+
for (const queue of this.#runqueues.values()) {
|
|
2303
|
+
queue.tasks = queue.tasks.filter((task) => task.vmid !== id);
|
|
2304
|
+
}
|
|
2305
|
+
try {
|
|
2306
|
+
await this.transitionvm(id, 'stopped');
|
|
2307
|
+
}
|
|
2308
|
+
catch {
|
|
2309
|
+
/* catcher: non-standard states keep their value */
|
|
2310
|
+
}
|
|
2311
|
+
vm.qemupid = undefined;
|
|
2312
|
+
this.#events.publish('vm:stopped', { vmid: id, reason });
|
|
2313
|
+
this.#audit(actor, 'vm:stop', id, 'success', { reason });
|
|
2314
|
+
this.#metrics.inc('vm_stopped_total', 1, { vmid: id });
|
|
2315
|
+
return { ok: true, value: undefined };
|
|
2316
|
+
}
|
|
2317
|
+
/** destroys a vm: stops it when running, releases gpus, frees
|
|
2318
|
+
* reservations and quota and forgets every side table. */
|
|
2319
|
+
async destroyvm(id, actor = SYSTEMUSER) {
|
|
2320
|
+
this.assertpermission(actor, 'vm:delete');
|
|
2321
|
+
const vm = this.#vms.get(id);
|
|
2322
|
+
if (vm === undefined) {
|
|
2323
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
2324
|
+
}
|
|
2325
|
+
if (vm.state === 'running') {
|
|
2326
|
+
const stopped = await this.stopvm(id, actor, 'destroy');
|
|
2327
|
+
if (!stopped.ok) {
|
|
2328
|
+
return stopped;
|
|
2329
|
+
}
|
|
2330
|
+
}
|
|
2331
|
+
for (const [key, assign] of this.#gpuassignments.entries()) {
|
|
2332
|
+
if (assign.vmid === id) {
|
|
2333
|
+
this.releasegpu(key, actor);
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
for (const [resid, held] of this.#reservations.entries()) {
|
|
2337
|
+
if (held.owner === id) {
|
|
2338
|
+
this.#reservations.delete(resid);
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
this.#deallocatequota(vm);
|
|
2342
|
+
this.#vms.delete(id);
|
|
2343
|
+
this.#balloons.delete(id);
|
|
2344
|
+
this.#dockerbridges.delete(id);
|
|
2345
|
+
this.#qemuprocesses.delete(id);
|
|
2346
|
+
this.#healthprobes.delete(id);
|
|
2347
|
+
this.#affinityindex.delete(id);
|
|
2348
|
+
vm.state = 'destroyed';
|
|
2349
|
+
this.#audit(actor, 'vm:delete', id, 'success', {});
|
|
2350
|
+
return { ok: true, value: undefined };
|
|
2351
|
+
}
|
|
2352
|
+
/** pauses a running vm (qmp stop semantics). */
|
|
2353
|
+
async pausevm(id, actor = SYSTEMUSER) {
|
|
2354
|
+
this.assertpermission(actor, 'vm:stop');
|
|
2355
|
+
const vm = this.#vms.get(id);
|
|
2356
|
+
if (vm === undefined || vm.state !== 'running') {
|
|
2357
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not running`, 'ERR_VM_STATE') };
|
|
2358
|
+
}
|
|
2359
|
+
await this.transitionvm(id, 'paused');
|
|
2360
|
+
this.#log(`vm ${id} paused via qmp`);
|
|
2361
|
+
return { ok: true, value: undefined };
|
|
2362
|
+
}
|
|
2363
|
+
/** lists vm records with optional state and numa filters. */
|
|
2364
|
+
listvms(filter) {
|
|
2365
|
+
let list = [...this.#vms.values()];
|
|
2366
|
+
if (filter?.state !== undefined) {
|
|
2367
|
+
list = list.filter((vm) => vm.state === filter.state);
|
|
2368
|
+
}
|
|
2369
|
+
if (filter?.numa !== undefined) {
|
|
2370
|
+
list = list.filter((vm) => vm.numanode === filter.numa);
|
|
2371
|
+
}
|
|
2372
|
+
return list;
|
|
2373
|
+
}
|
|
2374
|
+
/* ---------------------------------------------------------------- */
|
|
2375
|
+
/* quotas (coreorchestrator) */
|
|
2376
|
+
/* ---------------------------------------------------------------- */
|
|
2377
|
+
/** sets the vcpu/vram/gpu quota of a tenant and publishes quota:set. */
|
|
2378
|
+
setquota(tenantid, quota) {
|
|
2379
|
+
const existing = this.#quotatable.get(tenantid) ?? {
|
|
2380
|
+
vcpu: quota.vcpu,
|
|
2381
|
+
vram: quota.vram,
|
|
2382
|
+
gpu: quota.gpu,
|
|
2383
|
+
usedvcpu: 0,
|
|
2384
|
+
usedvram: 0,
|
|
2385
|
+
usedgpu: 0,
|
|
2386
|
+
};
|
|
2387
|
+
existing.vcpu = quota.vcpu;
|
|
2388
|
+
existing.vram = quota.vram;
|
|
2389
|
+
existing.gpu = quota.gpu;
|
|
2390
|
+
this.#quotatable.set(tenantid, existing);
|
|
2391
|
+
this.#events.publish('quota:set', {
|
|
2392
|
+
tenantid,
|
|
2393
|
+
vcpu: quota.vcpu,
|
|
2394
|
+
vrammib: quota.vram,
|
|
2395
|
+
gpu: quota.gpu,
|
|
2396
|
+
});
|
|
2397
|
+
}
|
|
2398
|
+
#enforcequota(tenantid, vcpu, vram, gpu) {
|
|
2399
|
+
const quota = this.#quotatable.get(tenantid);
|
|
2400
|
+
if (quota === undefined) {
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
if (quota.usedvcpu + vcpu > quota.vcpu) {
|
|
2404
|
+
throw new orchestratorerror(`quota vcpu exceeded for ${tenantid}`, 'ERR_QUOTA_VCPU');
|
|
2405
|
+
}
|
|
2406
|
+
if (quota.usedvram + vram > quota.vram) {
|
|
2407
|
+
throw new orchestratorerror(`quota vram exceeded for ${tenantid}`, 'ERR_QUOTA_VRAM');
|
|
2408
|
+
}
|
|
2409
|
+
if (quota.usedgpu + gpu > quota.gpu) {
|
|
2410
|
+
throw new orchestratorerror(`quota gpu exceeded for ${tenantid}`, 'ERR_QUOTA_GPU');
|
|
2411
|
+
}
|
|
2412
|
+
quota.usedvcpu += vcpu;
|
|
2413
|
+
quota.usedvram += vram;
|
|
2414
|
+
quota.usedgpu += gpu;
|
|
2415
|
+
this.#quotatable.set(tenantid, quota);
|
|
2416
|
+
}
|
|
2417
|
+
#deallocatequota(vm) {
|
|
2418
|
+
const quota = this.#quotatable.get(vm.tenantid);
|
|
2419
|
+
if (quota === undefined) {
|
|
2420
|
+
return;
|
|
2421
|
+
}
|
|
2422
|
+
quota.usedvcpu = Math.max(0, quota.usedvcpu - vm.vcpus);
|
|
2423
|
+
quota.usedvram = Math.max(0, quota.usedvram - vm.vrammib);
|
|
2424
|
+
quota.usedgpu = Math.max(0, quota.usedgpu - vm.gpu.length);
|
|
2425
|
+
this.#quotatable.set(vm.tenantid, quota);
|
|
2426
|
+
}
|
|
2427
|
+
/* ---------------------------------------------------------------- */
|
|
2428
|
+
/* vRAM ballooning (src_core pressure walk + v6 bounds) */
|
|
2429
|
+
/* ---------------------------------------------------------------- */
|
|
2430
|
+
/**
|
|
2431
|
+
* walks the balloon of a vm towards a target with pressure scoring
|
|
2432
|
+
* and a gradual 100 MB per 500 ms adjustment; bounds come from the
|
|
2433
|
+
* vramballoon descriptor (0.5x .. 1.5x of the requested size).
|
|
2434
|
+
*/
|
|
2435
|
+
async balloonvm(id, targetmb, actor = SYSTEMUSER) {
|
|
2436
|
+
this.assertpermission(actor, 'vm:start');
|
|
2437
|
+
const state = this.#balloons.get(id);
|
|
2438
|
+
const vm = this.#vms.get(id);
|
|
2439
|
+
if (state === undefined || vm === undefined) {
|
|
2440
|
+
return {
|
|
2441
|
+
ok: false,
|
|
2442
|
+
error: new orchestratorerror(`balloon state not found for ${id}`, 'ERR_VM_NOTFOUND'),
|
|
2443
|
+
};
|
|
2444
|
+
}
|
|
2445
|
+
const bounds = vm.vramballoon;
|
|
2446
|
+
const minmb = Math.max(128, bounds?.minmib ?? 128);
|
|
2447
|
+
const maxmb = bounds?.maxmib ?? Math.floor(vm.vrammib * 1.5);
|
|
2448
|
+
if (targetmb < minmb || targetmb > maxmb) {
|
|
2449
|
+
return {
|
|
2450
|
+
ok: false,
|
|
2451
|
+
error: new orchestratorerror(`balloon target ${targetmb} outside [${minmb}, ${maxmb}]`, 'ERR_BALLOON_BOUNDS'),
|
|
2452
|
+
};
|
|
2453
|
+
}
|
|
2454
|
+
const pressure = Math.max(0, Math.min(1, 1 - freemem() / totalmem()));
|
|
2455
|
+
state.pressurescore = pressure;
|
|
2456
|
+
state.targetmib = targetmb;
|
|
2457
|
+
const diff = targetmb - state.actualmib;
|
|
2458
|
+
const step = Math.sign(diff) * Math.min(100, Math.abs(diff));
|
|
2459
|
+
state.actualmib += step;
|
|
2460
|
+
state.deflatedmib = Math.max(0, state.actualmib - targetmb);
|
|
2461
|
+
state.lastadjust = new Date();
|
|
2462
|
+
if (state.actualmib !== targetmb) {
|
|
2463
|
+
setTimeout(() => {
|
|
2464
|
+
void this.balloonvm(id, targetmb, actor).catch(() => { });
|
|
2465
|
+
}, 500).unref();
|
|
2466
|
+
}
|
|
2467
|
+
this.#events.publish('vram:adjusted', {
|
|
2468
|
+
vmid: id,
|
|
2469
|
+
targetmib: targetmb,
|
|
2470
|
+
actualmib: state.actualmib,
|
|
2471
|
+
});
|
|
2472
|
+
this.#metrics.gauge('balloon_actual_mb', state.actualmib, { vmid: id });
|
|
2473
|
+
this.#metrics.gauge('balloon_pressure', pressure, { vmid: id });
|
|
2474
|
+
return { ok: true, value: { ...state } };
|
|
2475
|
+
}
|
|
2476
|
+
/** strict v6 balloon write with explicit bounds errors. */
|
|
2477
|
+
async balloonvram(id, targetmib) {
|
|
2478
|
+
const vm = this.#vms.get(id);
|
|
2479
|
+
if (vm === undefined) {
|
|
2480
|
+
throw new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND');
|
|
2481
|
+
}
|
|
2482
|
+
const balloon = vm.vramballoon ?? {
|
|
2483
|
+
requestedmib: vm.vrammib,
|
|
2484
|
+
minmib: Math.floor(vm.vrammib * 0.5),
|
|
2485
|
+
maxmib: Math.floor(vm.vrammib * 1.5),
|
|
2486
|
+
currentmib: vm.vrammib,
|
|
2487
|
+
balloondriver: 'virtio-balloon',
|
|
2488
|
+
hugepages: '2mib',
|
|
2489
|
+
};
|
|
2490
|
+
if (targetmib < balloon.minmib || targetmib > balloon.maxmib) {
|
|
2491
|
+
throw new orchestratorerror(`balloon target ${targetmib} outside [${balloon.minmib}, ${balloon.maxmib}]`, 'ERR_BALLOON_BOUNDS');
|
|
2492
|
+
}
|
|
2493
|
+
balloon.currentmib = targetmib;
|
|
2494
|
+
vm.vramballoon = balloon;
|
|
2495
|
+
vm.vrammib = targetmib;
|
|
2496
|
+
this.#events.publish('vram:adjusted', {
|
|
2497
|
+
vmid: id,
|
|
2498
|
+
targetmib,
|
|
2499
|
+
actualmib: balloon.currentmib,
|
|
2500
|
+
});
|
|
2501
|
+
this.#metrics.inc('vram_balloon_total', targetmib, { vmid: id });
|
|
2502
|
+
return balloon;
|
|
2503
|
+
}
|
|
2504
|
+
/** returns the balloon state of a vm, when present. */
|
|
2505
|
+
getballoonstate(id) {
|
|
2506
|
+
return this.#balloons.get(id);
|
|
2507
|
+
}
|
|
2508
|
+
/* ---------------------------------------------------------------- */
|
|
2509
|
+
/* GPU passthrough + SR-IOV (src_core) + gpus.json requests (v6) */
|
|
2510
|
+
/* ---------------------------------------------------------------- */
|
|
2511
|
+
/** registers a gpu; sriov enabled gpus get a PF view with 7 VFs
|
|
2512
|
+
* (Blackwell typical). */
|
|
2513
|
+
registergpu(gpu) {
|
|
2514
|
+
if (gpu.sriovenabled && !this.#sriovpfs.has(gpu.pciaddr)) {
|
|
2515
|
+
this.#sriovpfs.set(gpu.pciaddr, {
|
|
2516
|
+
pfpci: gpu.pciaddr,
|
|
2517
|
+
totalvfs: 7,
|
|
2518
|
+
activevfs: 0,
|
|
2519
|
+
vfs: [],
|
|
2520
|
+
driver: 'nvidia',
|
|
2521
|
+
});
|
|
2522
|
+
}
|
|
2523
|
+
this.#metrics.inc('gpu_registered_total', 1, { model: gpu.model, pci: gpu.pciaddr });
|
|
2524
|
+
this.#log(`gpu registered ${gpu.model} ${gpu.pciaddr} mode=${gpu.mode}`);
|
|
2525
|
+
}
|
|
2526
|
+
/**
|
|
2527
|
+
* assigns a gpu (or one of its VFs) to a vm; the iommu group is read
|
|
2528
|
+
* from the last id bytes or drawn randomly when unavailable.
|
|
2529
|
+
*/
|
|
2530
|
+
async assigngpu(id, gpu, actor = SYSTEMUSER, usevf = false) {
|
|
2531
|
+
this.assertpermission(actor, 'gpu:assign');
|
|
2532
|
+
const vm = this.#vms.get(id);
|
|
2533
|
+
if (vm === undefined) {
|
|
2534
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
2535
|
+
}
|
|
2536
|
+
if (this.#gpuassignments.has(gpu)) {
|
|
2537
|
+
return {
|
|
2538
|
+
ok: false,
|
|
2539
|
+
error: new orchestratorerror(`gpu ${gpu} already assigned`, 'ERR_GPU_ASSIGNED'),
|
|
2540
|
+
};
|
|
2541
|
+
}
|
|
2542
|
+
const iommugroup = Number.parseInt(gpu.slice(-2), 16) || randomInt(0, 20);
|
|
2543
|
+
let vfid;
|
|
2544
|
+
let pciaddr = `0000:0${randomInt(0, 8)}:00.0`;
|
|
2545
|
+
if (usevf) {
|
|
2546
|
+
const pf = [...this.#sriovpfs.values()].find((candidate) => candidate.vfs.length < candidate.totalvfs || candidate.activevfs < candidate.totalvfs);
|
|
2547
|
+
if (pf !== undefined) {
|
|
2548
|
+
const newvfid = `${pf.pfpci}-vf-${pf.activevfs}`;
|
|
2549
|
+
const vfpci = `${pf.pfpci.slice(0, -1)}${pf.activevfs + 1}`;
|
|
2550
|
+
pf.vfs.push({ vfid: newvfid, pci: vfpci, assignedto: id });
|
|
2551
|
+
pf.activevfs += 1;
|
|
2552
|
+
vfid = newvfid;
|
|
2553
|
+
pciaddr = vfpci;
|
|
2554
|
+
this.#events.publish('sriov:vfcreated', { pf: pf.pfpci, vfid: newvfid });
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
const assignment = {
|
|
2558
|
+
vmid: id,
|
|
2559
|
+
gpuid: gpu,
|
|
2560
|
+
vfid,
|
|
2561
|
+
pciaddr,
|
|
2562
|
+
iommugroup,
|
|
2563
|
+
bounddriver: 'vfio-pci',
|
|
2564
|
+
};
|
|
2565
|
+
this.#gpuassignments.set(gpu, assignment);
|
|
2566
|
+
this.#events.publish('gpu:assigned', { vmid: id, gpuid: gpu, vfid: vfid });
|
|
2567
|
+
this.#metrics.inc('gpu_assigned_total', 1, { gpuid: gpu, vmid: id });
|
|
2568
|
+
this.#audit(actor, 'gpu:assign', gpu, 'success', { vmid: id, vfid });
|
|
2569
|
+
return { ok: true, value: assignment };
|
|
2570
|
+
}
|
|
2571
|
+
/** releases a gpu assignment and frees its VF, if any. */
|
|
2572
|
+
releasegpu(gpu, actor = SYSTEMUSER) {
|
|
2573
|
+
const assign = this.#gpuassignments.get(gpu);
|
|
2574
|
+
if (assign === undefined) {
|
|
2575
|
+
return {
|
|
2576
|
+
ok: false,
|
|
2577
|
+
error: new orchestratorerror(`gpu ${gpu} not assigned`, 'ERR_GPU_ASSIGNED'),
|
|
2578
|
+
};
|
|
2579
|
+
}
|
|
2580
|
+
if (assign.vfid !== undefined) {
|
|
2581
|
+
for (const pf of this.#sriovpfs.values()) {
|
|
2582
|
+
const index = pf.vfs.findIndex((vf) => vf.vfid === assign.vfid);
|
|
2583
|
+
if (index !== -1) {
|
|
2584
|
+
pf.vfs.splice(index, 1);
|
|
2585
|
+
pf.activevfs -= 1;
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
}
|
|
2589
|
+
this.#gpuassignments.delete(gpu);
|
|
2590
|
+
this.#events.publish('gpu:released', { gpuid: gpu, vmid: assign.vmid });
|
|
2591
|
+
this.#audit(actor, 'gpu:release', gpu, 'success', {});
|
|
2592
|
+
return { ok: true, value: undefined };
|
|
2593
|
+
}
|
|
2594
|
+
/** lists the SR-IOV physical functions known to the engine. */
|
|
2595
|
+
listsriov() {
|
|
2596
|
+
return [...this.#sriovpfs.values()];
|
|
2597
|
+
}
|
|
2598
|
+
/**
|
|
2599
|
+
* provisions VFs on a PF through the sysfs sriov_numvfs knob; when the
|
|
2600
|
+
* sysfs node is absent (containers) the call reports the simulated
|
|
2601
|
+
* outcome instead of failing (coreorchestrator).
|
|
2602
|
+
*/
|
|
2603
|
+
async provisionsriov(pf, vfcount) {
|
|
2604
|
+
if (vfcount < 1 || vfcount > 64) {
|
|
2605
|
+
throw new orchestratorerror('vfcount must be 1-64', 'ERR_SRIOV_BOUNDS');
|
|
2606
|
+
}
|
|
2607
|
+
const sysfs = `/sys/bus/pci/devices/${pf}/sriov_numvfs`;
|
|
2608
|
+
const present = existsSync(sysfs);
|
|
2609
|
+
const vfids = Array.from({ length: vfcount }, (_, index) => `${pf}_vf${index}`);
|
|
2610
|
+
if (!present) {
|
|
2611
|
+
this.#metrics.inc('sriov_vf_provisioned_total', vfcount, { pf, mode: 'simulated' });
|
|
2612
|
+
return vfids;
|
|
2613
|
+
}
|
|
2614
|
+
try {
|
|
2615
|
+
writeFileSync(sysfs, String(vfcount), 'utf8');
|
|
2616
|
+
this.#metrics.inc('sriov_vf_provisioned_total', vfcount, { pf, mode: 'hardware' });
|
|
2617
|
+
}
|
|
2618
|
+
catch {
|
|
2619
|
+
this.#metrics.inc('sriov_vf_provisioned_total', vfcount, { pf, mode: 'failed' });
|
|
2620
|
+
}
|
|
2621
|
+
return vfids;
|
|
2622
|
+
}
|
|
2623
|
+
/**
|
|
2624
|
+
* attaches gpu passthrough requests to a vm by filtering the flat
|
|
2625
|
+
* gpus.json catalog (coreorchestrator); vfio must be requested.
|
|
2626
|
+
*/
|
|
2627
|
+
async attachgpupassthrough(id, requests) {
|
|
2628
|
+
const vm = this.#vms.get(id);
|
|
2629
|
+
if (vm === undefined) {
|
|
2630
|
+
throw new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND');
|
|
2631
|
+
}
|
|
2632
|
+
const catalog = (await this.#readjsonmaybe(path.join(this.#rootdir, 'gpus.json')));
|
|
2633
|
+
const devices = catalog ?? [];
|
|
2634
|
+
const allocated = [];
|
|
2635
|
+
for (const request of requests) {
|
|
2636
|
+
if (!request.vfio) {
|
|
2637
|
+
throw new orchestratorerror('vfio must be true for passthrough', 'ERR_VFIO_REQUIRED');
|
|
2638
|
+
}
|
|
2639
|
+
const candidates = devices.filter((device) => {
|
|
2640
|
+
const vendorok = String(device.vendor ?? '')
|
|
2641
|
+
.toLowerCase()
|
|
2642
|
+
.includes(request.vendor);
|
|
2643
|
+
const modelok = request.model === undefined ||
|
|
2644
|
+
String(device.model ?? '')
|
|
2645
|
+
.toLowerCase()
|
|
2646
|
+
.includes(request.model.toLowerCase());
|
|
2647
|
+
return vendorok && modelok;
|
|
2648
|
+
});
|
|
2649
|
+
if (candidates.length < request.count) {
|
|
2650
|
+
throw new orchestratorerror(`insufficient gpus for ${request.model ?? request.vendor}`, 'ERR_GPU_INSUFFICIENT');
|
|
2651
|
+
}
|
|
2652
|
+
for (let index = 0; index < request.count; index += 1) {
|
|
2653
|
+
allocated.push(String(candidates[index]?.deviceid ?? randomUUID()));
|
|
2654
|
+
}
|
|
2655
|
+
}
|
|
2656
|
+
vm.gpu = [...requests];
|
|
2657
|
+
this.#metrics.inc('gpu_attached_total', allocated.length, { vmid: id });
|
|
2658
|
+
return allocated;
|
|
2659
|
+
}
|
|
2660
|
+
/* ---------------------------------------------------------------- */
|
|
2661
|
+
/* passage routing (merged src_core + coreorchestrator shapes) */
|
|
2662
|
+
/* ---------------------------------------------------------------- */
|
|
2663
|
+
/**
|
|
2664
|
+
* adds a passage route between two vms; the default latency follows
|
|
2665
|
+
* the numa distance (15us same node, 45us remote) plus 20us of vxlan
|
|
2666
|
+
* encapsulation, and ivshmem routes are capped at 50000 Mbps.
|
|
2667
|
+
*/
|
|
2668
|
+
addpassageroute(route) {
|
|
2669
|
+
const stored = { ...route, latencyus: route.latencyus ?? 15 };
|
|
2670
|
+
if (this.#passageroutes.has(stored.id)) {
|
|
2671
|
+
return {
|
|
2672
|
+
ok: false,
|
|
2673
|
+
error: new orchestratorerror(`route ${stored.id} exists`, 'ERR_PASSAGE_EXISTS'),
|
|
2674
|
+
};
|
|
2675
|
+
}
|
|
2676
|
+
const src = this.#vms.get(stored.srcvm);
|
|
2677
|
+
const dst = this.#vms.get(stored.dstvm);
|
|
2678
|
+
if (src === undefined || dst === undefined) {
|
|
2679
|
+
return {
|
|
2680
|
+
ok: false,
|
|
2681
|
+
error: new orchestratorerror('src or dst vm not found', 'ERR_PASSAGE_VM'),
|
|
2682
|
+
};
|
|
2683
|
+
}
|
|
2684
|
+
if (stored.protocol === 'ivshmem' && stored.bandwidthgbps * 1000 > 50000) {
|
|
2685
|
+
return {
|
|
2686
|
+
ok: false,
|
|
2687
|
+
error: new orchestratorerror('ivshmem bandwidth unrealistic', 'ERR_PASSAGE_BW'),
|
|
2688
|
+
};
|
|
2689
|
+
}
|
|
2690
|
+
if (route.latencyus === undefined) {
|
|
2691
|
+
stored.latencyus =
|
|
2692
|
+
(src.numanode === dst.numanode ? 15 : 45) + (stored.protocol === 'vxlan' ? 20 : 0);
|
|
2693
|
+
}
|
|
2694
|
+
this.#passageroutes.set(stored.id, stored);
|
|
2695
|
+
this.#events.publish('passage:routecreated', {
|
|
2696
|
+
id: stored.id,
|
|
2697
|
+
srcvm: stored.srcvm,
|
|
2698
|
+
dstvm: stored.dstvm,
|
|
2699
|
+
protocol: stored.protocol,
|
|
2700
|
+
});
|
|
2701
|
+
this.#metrics.inc('passage_routes_total', 1, {
|
|
2702
|
+
protocol: stored.protocol,
|
|
2703
|
+
passage: stored.passage,
|
|
2704
|
+
});
|
|
2705
|
+
this.#log(`passage route ${stored.id} ${stored.srcvm}->${stored.dstvm} ${stored.protocol} ${stored.latencyus}us`);
|
|
2706
|
+
return { ok: true, value: stored };
|
|
2707
|
+
}
|
|
2708
|
+
/** v6 alias returning the first route between two vms. */
|
|
2709
|
+
resolvepassageroute(src, dst) {
|
|
2710
|
+
for (const route of this.#passageroutes.values()) {
|
|
2711
|
+
if (route.srcvm === src && route.dstvm === dst) {
|
|
2712
|
+
return route;
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
return undefined;
|
|
2716
|
+
}
|
|
2717
|
+
/** every route between two vms (src_core). */
|
|
2718
|
+
resolvepassage(src, dst) {
|
|
2719
|
+
return [...this.#passageroutes.values()].filter((route) => route.srcvm === src && route.dstvm === dst);
|
|
2720
|
+
}
|
|
2721
|
+
/** removes a passage route by id. */
|
|
2722
|
+
removepassageroute(routeid) {
|
|
2723
|
+
return this.#passageroutes.delete(routeid);
|
|
2724
|
+
}
|
|
2725
|
+
/* ---------------------------------------------------------------- */
|
|
2726
|
+
/* MTTG queue (deps, retry, anti-starvation, qos preemption) */
|
|
2727
|
+
/* ---------------------------------------------------------------- */
|
|
2728
|
+
/** enqueues an MTTG job; unresolved dependencies are logged so the
|
|
2729
|
+
* pipeline can trace the wait chain. */
|
|
2730
|
+
enqueuemttg(job) {
|
|
2731
|
+
const full = { ...job, createdat: new Date(), retries: 0 };
|
|
2732
|
+
const unresolved = full.dependencies.filter((dep) => this.#mttgqueue.list().some((queued) => queued.id === dep) || this.#mttgprocessing.has(dep));
|
|
2733
|
+
if (unresolved.length > 0) {
|
|
2734
|
+
this.#log(`mttg job ${full.id} waiting for deps ${unresolved.join(',')}`);
|
|
2735
|
+
}
|
|
2736
|
+
this.#mttgqueue.push(full);
|
|
2737
|
+
this.#metrics.inc('mttg_enqueued_total', 1, {
|
|
2738
|
+
group: full.groupid,
|
|
2739
|
+
priority: String(full.priority),
|
|
2740
|
+
});
|
|
2741
|
+
this.#events.publish('mttg:enqueued', { jobid: full.id, tenant: full.tenant, qos: full.qos });
|
|
2742
|
+
}
|
|
2743
|
+
/**
|
|
2744
|
+
* drains the MTTG queue with a bounded concurrency (default 4):
|
|
2745
|
+
* dependencies requeue at a lower priority (anti-starvation) and
|
|
2746
|
+
* failures retry up to three times with priority decay.
|
|
2747
|
+
*/
|
|
2748
|
+
async processmttgqueue(handler, concurrency = 4) {
|
|
2749
|
+
const active = [];
|
|
2750
|
+
const processone = async (job) => {
|
|
2751
|
+
this.#mttgprocessing.set(job.id, job);
|
|
2752
|
+
const start = Date.now();
|
|
2753
|
+
try {
|
|
2754
|
+
await handler(job);
|
|
2755
|
+
this.#metrics.inc('mttg_completed_total', 1, { group: job.groupid });
|
|
2756
|
+
this.#events.publish('mttg:jobdone', { jobid: job.id, durationms: Date.now() - start });
|
|
2757
|
+
}
|
|
2758
|
+
catch {
|
|
2759
|
+
job.retries += 1;
|
|
2760
|
+
if (job.retries < 3) {
|
|
2761
|
+
this.#mttgqueue.push({ ...job, priority: job.priority - 1 });
|
|
2762
|
+
}
|
|
2763
|
+
else {
|
|
2764
|
+
this.#metrics.inc('mttg_failed_total', 1, { group: job.groupid });
|
|
2765
|
+
}
|
|
2766
|
+
}
|
|
2767
|
+
finally {
|
|
2768
|
+
this.#mttgprocessing.delete(job.id);
|
|
2769
|
+
}
|
|
2770
|
+
};
|
|
2771
|
+
while (this.#mttgqueue.size > 0 || active.length > 0) {
|
|
2772
|
+
while (active.length < concurrency && this.#mttgqueue.size > 0) {
|
|
2773
|
+
const job = this.#mttgqueue.pop();
|
|
2774
|
+
if (job === undefined) {
|
|
2775
|
+
break;
|
|
2776
|
+
}
|
|
2777
|
+
const depspending = job.dependencies.some((dep) => this.#mttgprocessing.has(dep) ||
|
|
2778
|
+
this.#mttgqueue.list().some((queued) => queued.id === dep));
|
|
2779
|
+
if (depspending) {
|
|
2780
|
+
this.#mttgqueue.push({ ...job, priority: job.priority - 1 });
|
|
2781
|
+
continue;
|
|
2782
|
+
}
|
|
2783
|
+
const promise = processone(job);
|
|
2784
|
+
active.push(promise);
|
|
2785
|
+
void promise.finally(() => {
|
|
2786
|
+
const index = active.indexOf(promise);
|
|
2787
|
+
if (index !== -1) {
|
|
2788
|
+
active.splice(index, 1);
|
|
2789
|
+
}
|
|
2790
|
+
});
|
|
2791
|
+
}
|
|
2792
|
+
if (active.length > 0) {
|
|
2793
|
+
await Promise.race(active);
|
|
2794
|
+
}
|
|
2795
|
+
}
|
|
2796
|
+
}
|
|
2797
|
+
/** dequeues the next job by priority and deadline (v6). */
|
|
2798
|
+
async dequeuemttg() {
|
|
2799
|
+
const job = [...this.#mttgqueue.list()].sort((a, b) => b.priority - a.priority || (a.deadlinems ?? Infinity) - (b.deadlinems ?? Infinity))[0];
|
|
2800
|
+
if (job === undefined) {
|
|
2801
|
+
return undefined;
|
|
2802
|
+
}
|
|
2803
|
+
this.#mttgqueue.remove(job.id);
|
|
2804
|
+
this.#metrics.inc('mttg_dequeued_total', 1, { tenant: job.tenant, qos: job.qos });
|
|
2805
|
+
return job;
|
|
2806
|
+
}
|
|
2807
|
+
/**
|
|
2808
|
+
* preempts up to three besteffort jobs of lower priority when a
|
|
2809
|
+
* guaranteed job arrives (coreorchestrator QoS preemption).
|
|
2810
|
+
*/
|
|
2811
|
+
async preemptifneeded(incoming) {
|
|
2812
|
+
if (incoming.qos !== 'guaranteed') {
|
|
2813
|
+
return [];
|
|
2814
|
+
}
|
|
2815
|
+
const preemptible = this.#mttgqueue
|
|
2816
|
+
.list()
|
|
2817
|
+
.filter((job) => job.qos === 'besteffort' && job.priority < incoming.priority)
|
|
2818
|
+
.slice(0, 3);
|
|
2819
|
+
for (const job of preemptible) {
|
|
2820
|
+
this.#mttgqueue.remove(job.id);
|
|
2821
|
+
}
|
|
2822
|
+
if (preemptible.length > 0) {
|
|
2823
|
+
this.#events.publish('mttg:preempted', {
|
|
2824
|
+
incoming: incoming.id,
|
|
2825
|
+
preempted: preemptible.map((job) => job.id),
|
|
2826
|
+
});
|
|
2827
|
+
this.#metrics.inc('preemptions_total', preemptible.length, { tenant: incoming.tenant });
|
|
2828
|
+
}
|
|
2829
|
+
return preemptible;
|
|
2830
|
+
}
|
|
2831
|
+
/* ---------------------------------------------------------------- */
|
|
2832
|
+
/* docker bridge (IPAM 10.88.N.O/24, mac 02:42) */
|
|
2833
|
+
/* ---------------------------------------------------------------- */
|
|
2834
|
+
async #attachdockerbridge(vm) {
|
|
2835
|
+
const bridgename = `br-vhe-${vm.numanode}`;
|
|
2836
|
+
const octet = randomInt(10, 210);
|
|
2837
|
+
const mapping = {
|
|
2838
|
+
vmid: vm.id,
|
|
2839
|
+
containerid: `vhe-${vm.id}-${Date.now()}`,
|
|
2840
|
+
bridge: bridgename,
|
|
2841
|
+
vethhost: `veth-${vm.id.slice(0, 8)}-h`,
|
|
2842
|
+
vethguest: `veth-${vm.id.slice(0, 8)}-g`,
|
|
2843
|
+
ipam: {
|
|
2844
|
+
ipv4: `10.88.${vm.numanode}.${octet}/24`,
|
|
2845
|
+
mac: `02:42:ac:${vm.numanode.toString(16).padStart(2, '0')}:${octet
|
|
2846
|
+
.toString(16)
|
|
2847
|
+
.padStart(2, '0')}:${randomInt(0, 255).toString(16).padStart(2, '0')}`,
|
|
2848
|
+
},
|
|
2849
|
+
};
|
|
2850
|
+
this.#dockerbridges.set(vm.id, mapping);
|
|
2851
|
+
this.#events.publish('docker:attached', {
|
|
2852
|
+
vmid: vm.id,
|
|
2853
|
+
bridge: bridgename,
|
|
2854
|
+
ipv4: mapping.ipam.ipv4,
|
|
2855
|
+
});
|
|
2856
|
+
this.#metrics.inc('docker_attached_total', 1, { bridge: bridgename });
|
|
2857
|
+
this.#log(`docker bridge attached vm=${vm.id} ${mapping.ipam.ipv4} ${bridgename}`);
|
|
2858
|
+
return mapping;
|
|
2859
|
+
}
|
|
2860
|
+
/** returns the docker bridge mapping of a vm, when attached. */
|
|
2861
|
+
getdockerbridge(id) {
|
|
2862
|
+
return this.#dockerbridges.get(id);
|
|
2863
|
+
}
|
|
2864
|
+
/**
|
|
2865
|
+
* builds the docker run sidecar argv of a vm from docker.config
|
|
2866
|
+
* (coreorchestrator); the network name falls back to the numa bridge.
|
|
2867
|
+
*/
|
|
2868
|
+
async builddockerbridgeargs(id) {
|
|
2869
|
+
const vm = this.#vms.get(id);
|
|
2870
|
+
if (vm === undefined) {
|
|
2871
|
+
throw new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND');
|
|
2872
|
+
}
|
|
2873
|
+
const dockerconf = (await this.loadconfig('docker.config'));
|
|
2874
|
+
const network = String(dockerconf?.network ?? `br-vhe-${vm.numanode}`);
|
|
2875
|
+
const args = [
|
|
2876
|
+
'run',
|
|
2877
|
+
'--rm',
|
|
2878
|
+
'--name',
|
|
2879
|
+
`vhe-${id}`,
|
|
2880
|
+
'--network',
|
|
2881
|
+
network,
|
|
2882
|
+
'--cpus',
|
|
2883
|
+
String(vm.vcpus),
|
|
2884
|
+
'--memory',
|
|
2885
|
+
`${vm.vrammib}m`,
|
|
2886
|
+
];
|
|
2887
|
+
if (vm.gpu.length > 0) {
|
|
2888
|
+
args.push('--gpus', 'all');
|
|
2889
|
+
}
|
|
2890
|
+
args.push(vm.image);
|
|
2891
|
+
return args;
|
|
2892
|
+
}
|
|
2893
|
+
/* ---------------------------------------------------------------- */
|
|
2894
|
+
/* health monitor (checks of src_core + probes of v6) */
|
|
2895
|
+
/* ---------------------------------------------------------------- */
|
|
2896
|
+
/** starts the periodic health monitor. */
|
|
2897
|
+
starthealthmonitor(intervalms = 15000) {
|
|
2898
|
+
if (this.#healthinterval !== null) {
|
|
2899
|
+
clearInterval(this.#healthinterval);
|
|
2900
|
+
}
|
|
2901
|
+
this.#healthinterval = setInterval(() => {
|
|
2902
|
+
void this.#runhealthchecks();
|
|
2903
|
+
}, intervalms);
|
|
2904
|
+
this.#healthinterval.unref();
|
|
2905
|
+
this.#log(`health monitor started interval=${intervalms}ms`);
|
|
2906
|
+
}
|
|
2907
|
+
/** stops the periodic health monitor. */
|
|
2908
|
+
stophealthmonitor() {
|
|
2909
|
+
if (this.#healthinterval !== null) {
|
|
2910
|
+
clearInterval(this.#healthinterval);
|
|
2911
|
+
this.#healthinterval = null;
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
/** registers a health check over a vm, the host or a gpu. */
|
|
2915
|
+
registerhealthcheck(check) {
|
|
2916
|
+
this.#healthchecks.set(check.id, {
|
|
2917
|
+
...check,
|
|
2918
|
+
laststatus: 'unknown',
|
|
2919
|
+
lastcheck: new Date(0),
|
|
2920
|
+
failurecount: 0,
|
|
2921
|
+
});
|
|
2922
|
+
}
|
|
2923
|
+
async #runhealthchecks() {
|
|
2924
|
+
for (const check of this.#healthchecks.values()) {
|
|
2925
|
+
const previous = check.laststatus;
|
|
2926
|
+
let next = 'healthy';
|
|
2927
|
+
if (typeof check.target === 'string' && check.target.startsWith('gpu:')) {
|
|
2928
|
+
const gpu = check.target.slice(4);
|
|
2929
|
+
next = this.#gpuassignments.has(gpu) ? 'healthy' : 'unknown';
|
|
2930
|
+
}
|
|
2931
|
+
else if (check.target === 'host') {
|
|
2932
|
+
const freeratio = freemem() / totalmem();
|
|
2933
|
+
if (freeratio < 0.1) {
|
|
2934
|
+
next = 'unhealthy';
|
|
2935
|
+
}
|
|
2936
|
+
else if (freeratio < 0.2) {
|
|
2937
|
+
next = 'degraded';
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2940
|
+
else {
|
|
2941
|
+
const vm = this.#vms.get(check.target);
|
|
2942
|
+
if (vm === undefined || vm.state !== 'running') {
|
|
2943
|
+
next = 'unhealthy';
|
|
2944
|
+
}
|
|
2945
|
+
}
|
|
2946
|
+
check.laststatus = next;
|
|
2947
|
+
check.lastcheck = new Date();
|
|
2948
|
+
check.failurecount = next === 'unhealthy' ? check.failurecount + 1 : 0;
|
|
2949
|
+
if (previous !== next) {
|
|
2950
|
+
this.#events.publish('health:change', {
|
|
2951
|
+
checkid: check.id,
|
|
2952
|
+
old: previous,
|
|
2953
|
+
new: next,
|
|
2954
|
+
});
|
|
2955
|
+
this.#metrics.inc('health_change_total', 1, { check: check.id, status: next });
|
|
2956
|
+
}
|
|
2957
|
+
this.#metrics.gauge('health_status', next === 'healthy' ? 1 : 0, { checkid: check.id });
|
|
2958
|
+
}
|
|
2959
|
+
}
|
|
2960
|
+
/**
|
|
2961
|
+
* records an external health probe (v6); a failing probe moves a
|
|
2962
|
+
* running vm to degraded.
|
|
2963
|
+
*/
|
|
2964
|
+
async recordhealth(probe) {
|
|
2965
|
+
this.#healthprobes.set(probe.vmid, probe);
|
|
2966
|
+
this.#metrics.inc('vm_health_total', probe.ok ? 1 : 0, { vmid: probe.vmid });
|
|
2967
|
+
if (!probe.ok) {
|
|
2968
|
+
const vm = this.#vms.get(probe.vmid);
|
|
2969
|
+
if (vm !== undefined && vm.state === 'running') {
|
|
2970
|
+
await this.transitionvm(vm.id, 'degraded').catch(() => { });
|
|
2971
|
+
}
|
|
2972
|
+
}
|
|
2973
|
+
}
|
|
2974
|
+
/** returns the currently failing health probes. */
|
|
2975
|
+
failingvms() {
|
|
2976
|
+
return [...this.#healthprobes.values()].filter((probe) => !probe.ok);
|
|
2977
|
+
}
|
|
2978
|
+
/* ---------------------------------------------------------------- */
|
|
2979
|
+
/* autoscale (policy engine of src_core + tenant view of v6) */
|
|
2980
|
+
/* ---------------------------------------------------------------- */
|
|
2981
|
+
/** registers an autoscale policy. */
|
|
2982
|
+
registerautoscalepolicy(policy) {
|
|
2983
|
+
this.#scalepolicies.set(policy.id, policy);
|
|
2984
|
+
this.#log(`autoscale policy registered ${policy.id} metric=${policy.metric}`);
|
|
2985
|
+
}
|
|
2986
|
+
/** starts the periodic autoscaler. */
|
|
2987
|
+
startautoscaler(intervalms = 30000) {
|
|
2988
|
+
if (this.#autoscaleinterval !== null) {
|
|
2989
|
+
clearInterval(this.#autoscaleinterval);
|
|
2990
|
+
}
|
|
2991
|
+
this.#autoscaleinterval = setInterval(() => {
|
|
2992
|
+
void this.#evaluateautoscale();
|
|
2993
|
+
}, intervalms);
|
|
2994
|
+
this.#autoscaleinterval.unref();
|
|
2995
|
+
}
|
|
2996
|
+
/** stops the periodic autoscaler. */
|
|
2997
|
+
stopautoscaler() {
|
|
2998
|
+
if (this.#autoscaleinterval !== null) {
|
|
2999
|
+
clearInterval(this.#autoscaleinterval);
|
|
3000
|
+
this.#autoscaleinterval = null;
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
async #evaluateautoscale() {
|
|
3004
|
+
for (const policy of this.#scalepolicies.values()) {
|
|
3005
|
+
const groupvms = [...this.#vms.values()].filter((vm) => vm.name.startsWith(policy.vmgroup));
|
|
3006
|
+
const current = groupvms.length;
|
|
3007
|
+
let metricval = 0;
|
|
3008
|
+
switch (policy.metric) {
|
|
3009
|
+
case 'cpu':
|
|
3010
|
+
metricval =
|
|
3011
|
+
groupvms.reduce((sum, vm) => sum + Number(vm.metrics.cputimens % 1000n), 0) /
|
|
3012
|
+
Math.max(1, groupvms.length);
|
|
3013
|
+
break;
|
|
3014
|
+
case 'mem':
|
|
3015
|
+
metricval =
|
|
3016
|
+
groupvms.reduce((sum, vm) => sum + vm.metrics.memrssmb, 0) /
|
|
3017
|
+
Math.max(1, groupvms.length);
|
|
3018
|
+
break;
|
|
3019
|
+
case 'queue_depth':
|
|
3020
|
+
metricval = this.#mttgqueue.size;
|
|
3021
|
+
break;
|
|
3022
|
+
case 'gpu':
|
|
3023
|
+
metricval = (this.#gpuassignments.size / Math.max(1, this.#sriovpfs.size * 7)) * 100;
|
|
3024
|
+
break;
|
|
3025
|
+
default:
|
|
3026
|
+
metricval = 0;
|
|
3027
|
+
}
|
|
3028
|
+
if (metricval > policy.thresholdhigh && current < policy.maxreplicas) {
|
|
3029
|
+
const need = Math.min(policy.maxreplicas - current, Math.ceil((metricval - policy.thresholdhigh) / 10) || 1);
|
|
3030
|
+
this.#events.publish('autoscale:trigger', {
|
|
3031
|
+
policyid: policy.id,
|
|
3032
|
+
action: 'scaleup',
|
|
3033
|
+
count: need,
|
|
3034
|
+
});
|
|
3035
|
+
this.#log(`autoscale up policy=${policy.id} metric=${metricval.toFixed(1)} need=${need}`);
|
|
3036
|
+
const template = groupvms[0];
|
|
3037
|
+
for (let index = 0; index < need; index += 1) {
|
|
3038
|
+
const newid = createvmid();
|
|
3039
|
+
const created = await this.createvm({
|
|
3040
|
+
id: newid,
|
|
3041
|
+
name: `${policy.vmgroup}-${Date.now()}-${index}`,
|
|
3042
|
+
tenantid: template?.tenantid ?? 'autoscale',
|
|
3043
|
+
vcpus: template?.vcpus ?? 4,
|
|
3044
|
+
vrammib: template?.vrammib ?? 4096,
|
|
3045
|
+
});
|
|
3046
|
+
if (created.ok) {
|
|
3047
|
+
await this.startvm(newid);
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
3050
|
+
}
|
|
3051
|
+
else if (metricval < policy.thresholdlow && current > policy.minreplicas) {
|
|
3052
|
+
const excess = current - policy.minreplicas;
|
|
3053
|
+
const toremove = Math.min(excess, Math.ceil((policy.thresholdlow - metricval) / 10) || 1);
|
|
3054
|
+
this.#events.publish('autoscale:trigger', {
|
|
3055
|
+
policyid: policy.id,
|
|
3056
|
+
action: 'scaledown',
|
|
3057
|
+
count: toremove,
|
|
3058
|
+
});
|
|
3059
|
+
this.#log(`autoscale down policy=${policy.id} metric=${metricval.toFixed(1)} remove=${toremove}`);
|
|
3060
|
+
for (const victim of groupvms.slice(0, toremove)) {
|
|
3061
|
+
await this.stopvm(victim.id, SYSTEMUSER, 'autoscaledown');
|
|
3062
|
+
await this.destroyvm(victim.id, SYSTEMUSER);
|
|
3063
|
+
}
|
|
3064
|
+
}
|
|
3065
|
+
this.#metrics.gauge('autoscale_metric', metricval, {
|
|
3066
|
+
policy: policy.id,
|
|
3067
|
+
metric: policy.metric,
|
|
3068
|
+
});
|
|
3069
|
+
}
|
|
3070
|
+
}
|
|
3071
|
+
/**
|
|
3072
|
+
* tenant view of autoscaling (v6): averages the cpu steal of the
|
|
3073
|
+
* tenant probes and proposes one step up above 75 percent or one step
|
|
3074
|
+
* down below 20 percent.
|
|
3075
|
+
*/
|
|
3076
|
+
async evaluatetenantautoscale(tenantid) {
|
|
3077
|
+
const current = [...this.#vms.values()].filter((vm) => vm.tenantid === tenantid && vm.state === 'running').length;
|
|
3078
|
+
const probes = [...this.#healthprobes.values()].filter((probe) => {
|
|
3079
|
+
const vm = this.#vms.get(probe.vmid);
|
|
3080
|
+
return vm?.tenantid === tenantid;
|
|
3081
|
+
});
|
|
3082
|
+
const avgcpu = probes.length === 0
|
|
3083
|
+
? 0
|
|
3084
|
+
: probes.reduce((sum, probe) => sum + (probe.cpustealpct ?? 0), 0) / probes.length;
|
|
3085
|
+
let desired = current;
|
|
3086
|
+
if (avgcpu > 75 && current < 20) {
|
|
3087
|
+
desired = current + 1;
|
|
3088
|
+
}
|
|
3089
|
+
else if (avgcpu < 20 && current > 1) {
|
|
3090
|
+
desired = current - 1;
|
|
3091
|
+
}
|
|
3092
|
+
const action = desired > current ? 'up' : desired < current ? 'down' : 'none';
|
|
3093
|
+
this.#metrics.gauge('autoscale_desired', desired, { tenant: tenantid });
|
|
3094
|
+
return { desired, current, action };
|
|
3095
|
+
}
|
|
3096
|
+
/* ---------------------------------------------------------------- */
|
|
3097
|
+
/* live migration (stages of src_core + vfio guard of v6) */
|
|
3098
|
+
/* ---------------------------------------------------------------- */
|
|
3099
|
+
/**
|
|
3100
|
+
* migrates a running vm to another host; live migrations walk the
|
|
3101
|
+
* precopy, dirty iteration and stop-copy stages with a 50-250ms
|
|
3102
|
+
* downtime window; vfio attached vms require the vfio-migration label.
|
|
3103
|
+
*/
|
|
3104
|
+
async startmigration(id, desthost, type = 'live', actor = SYSTEMUSER) {
|
|
3105
|
+
this.assertpermission(actor, 'vm:migrate');
|
|
3106
|
+
const vm = this.#vms.get(id);
|
|
3107
|
+
if (vm === undefined) {
|
|
3108
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
3109
|
+
}
|
|
3110
|
+
if (vm.state !== 'running') {
|
|
3111
|
+
return {
|
|
3112
|
+
ok: false,
|
|
3113
|
+
error: new orchestratorerror(`vm ${id} not running, cannot live migrate`, 'ERR_MIGRATE_PHASE'),
|
|
3114
|
+
};
|
|
3115
|
+
}
|
|
3116
|
+
if (vm.gpu.some((request) => request.vfio) && vm.labels['vfio-migration'] !== 'enabled') {
|
|
3117
|
+
return {
|
|
3118
|
+
ok: false,
|
|
3119
|
+
error: new orchestratorerror('vfio migration not enabled for this vm', 'ERR_VFIO_MIGRATION'),
|
|
3120
|
+
};
|
|
3121
|
+
}
|
|
3122
|
+
const job = {
|
|
3123
|
+
id: `mig-${randomUUID()}`,
|
|
3124
|
+
vmid: id,
|
|
3125
|
+
type,
|
|
3126
|
+
sourcehost: this.#nodeid,
|
|
3127
|
+
desthost,
|
|
3128
|
+
state: 'pending',
|
|
3129
|
+
bandwidthmbps: 10000,
|
|
3130
|
+
progress: 0,
|
|
3131
|
+
};
|
|
3132
|
+
await this.transitionvm(id, 'migrating');
|
|
3133
|
+
this.#migrations.set(job.id, job);
|
|
3134
|
+
const stages = type === 'live' || type === 'postcopy'
|
|
3135
|
+
? ['precopy', 'dirtyiter', 'stopcopy', 'completed']
|
|
3136
|
+
: ['pending', 'completed'];
|
|
3137
|
+
for (const stage of stages) {
|
|
3138
|
+
job.state = stage;
|
|
3139
|
+
job.progress = stage === 'completed' ? 100 : job.progress + 25;
|
|
3140
|
+
if (stage === 'stopcopy') {
|
|
3141
|
+
job.downtimems = randomInt(200) + 50;
|
|
3142
|
+
}
|
|
3143
|
+
this.#events.publish('migration:progress', {
|
|
3144
|
+
jobid: job.id,
|
|
3145
|
+
vmid: id,
|
|
3146
|
+
state: stage,
|
|
3147
|
+
progress: job.progress,
|
|
3148
|
+
});
|
|
3149
|
+
this.#metrics.gauge('migration_progress', job.progress, { vmid: id, dest: desthost });
|
|
3150
|
+
await sleep(type === 'live' || type === 'postcopy' ? 800 : 200);
|
|
3151
|
+
if (this.#shuttingdown) {
|
|
3152
|
+
job.state = 'failed';
|
|
3153
|
+
await this.transitionvm(id, 'running');
|
|
3154
|
+
return {
|
|
3155
|
+
ok: false,
|
|
3156
|
+
error: new orchestratorerror('migration aborted: shutting down', 'ERR_MIGRATION_ABORT'),
|
|
3157
|
+
};
|
|
3158
|
+
}
|
|
3159
|
+
}
|
|
3160
|
+
await this.transitionvm(id, 'running');
|
|
3161
|
+
this.#audit(actor, 'vm:migrate', id, 'success', { desthost, type });
|
|
3162
|
+
this.#log(`migration ${job.id} vm=${id} -> ${desthost} downtime=${job.downtimems ?? 0}ms`);
|
|
3163
|
+
return { ok: true, value: job };
|
|
3164
|
+
}
|
|
3165
|
+
/** v6 alias for startmigration with the live type. */
|
|
3166
|
+
async startlivemigration(id, targethost) {
|
|
3167
|
+
const outcome = await this.startmigration(id, targethost, 'live');
|
|
3168
|
+
if (!outcome.ok) {
|
|
3169
|
+
throw outcome.error;
|
|
3170
|
+
}
|
|
3171
|
+
return outcome.value;
|
|
3172
|
+
}
|
|
3173
|
+
/* ---------------------------------------------------------------- */
|
|
3174
|
+
/* snapshots + checkpoints (disk level, qcow2 + meta + checksums) */
|
|
3175
|
+
/* ---------------------------------------------------------------- */
|
|
3176
|
+
/**
|
|
3177
|
+
* creates a disk snapshot: a qcow2 artifact (with a metadata payload
|
|
3178
|
+
* in the simulated layout), an optional ram image and a json sidecar;
|
|
3179
|
+
* the vm walks running -> snapshotted -> running on the single
|
|
3180
|
+
* machine.
|
|
3181
|
+
*/
|
|
3182
|
+
async createsnapshot(id, name, includeram = false, actor = SYSTEMUSER) {
|
|
3183
|
+
this.assertpermission(actor, 'snapshot:create');
|
|
3184
|
+
const vm = this.#vms.get(id);
|
|
3185
|
+
if (vm === undefined) {
|
|
3186
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
3187
|
+
}
|
|
3188
|
+
try {
|
|
3189
|
+
const snapshotid = `snap-${randomUUID()}`;
|
|
3190
|
+
await this.transitionvm(id, 'snapshotted');
|
|
3191
|
+
const basepath = path.join(this.#rootdir, 'snapshots', id);
|
|
3192
|
+
mkdirSync(basepath, { recursive: true });
|
|
3193
|
+
const qcow2path = path.join(basepath, `${name}.qcow2`);
|
|
3194
|
+
const rampath = includeram ? path.join(basepath, `${name}.ram`) : undefined;
|
|
3195
|
+
await sleep(400);
|
|
3196
|
+
writeFileSync(qcow2path, JSON.stringify({ vmid: id, name, created: new Date().toISOString(), vcpus: vm.vcpus }), 'utf8');
|
|
3197
|
+
if (rampath !== undefined) {
|
|
3198
|
+
writeFileSync(rampath, Buffer.alloc(1024 * 1024, 0));
|
|
3199
|
+
}
|
|
3200
|
+
const meta = {
|
|
3201
|
+
snapshotid,
|
|
3202
|
+
vmid: id,
|
|
3203
|
+
createdat: new Date().toISOString(),
|
|
3204
|
+
vcpu: vm.vcpus,
|
|
3205
|
+
vrammib: vm.vrammib,
|
|
3206
|
+
qemu: runtimecatalog.qemu,
|
|
3207
|
+
};
|
|
3208
|
+
writeFileSync(path.join(basepath, `${name}.json`), JSON.stringify(meta, null, 2), 'utf8');
|
|
3209
|
+
const sizemb = existsSync(qcow2path)
|
|
3210
|
+
? Math.ceil((await stat(qcow2path)).size / 1024 / 1024)
|
|
3211
|
+
: 1;
|
|
3212
|
+
const snap = {
|
|
3213
|
+
id: snapshotid,
|
|
3214
|
+
vmid: id,
|
|
3215
|
+
name,
|
|
3216
|
+
createdat: new Date(),
|
|
3217
|
+
sizemb,
|
|
3218
|
+
qcow2path,
|
|
3219
|
+
rampath,
|
|
3220
|
+
};
|
|
3221
|
+
this.#snapshots.set(snapshotid, snap);
|
|
3222
|
+
await this.transitionvm(id, 'running');
|
|
3223
|
+
this.#events.publish('snapshot:created', { snapshotid, vmid: id });
|
|
3224
|
+
this.#metrics.inc('snapshot_created_total', 1, { vmid: id });
|
|
3225
|
+
this.#audit(actor, 'snapshot:create', id, 'success', { name, includeram });
|
|
3226
|
+
return { ok: true, value: snap };
|
|
3227
|
+
}
|
|
3228
|
+
catch (error) {
|
|
3229
|
+
if (vm.state === 'snapshotted') {
|
|
3230
|
+
vm.state = 'running';
|
|
3231
|
+
}
|
|
3232
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
3233
|
+
}
|
|
3234
|
+
}
|
|
3235
|
+
/** lists snapshots, optionally filtered by vm. */
|
|
3236
|
+
listsnapshots(id) {
|
|
3237
|
+
const all = [...this.#snapshots.values()];
|
|
3238
|
+
return id === undefined ? all : all.filter((snap) => snap.vmid === id);
|
|
3239
|
+
}
|
|
3240
|
+
/** restores a snapshot; the vm walks back to stopped. */
|
|
3241
|
+
async restoresnapshot(snapshot, actor = SYSTEMUSER) {
|
|
3242
|
+
this.assertpermission(actor, 'vm:start');
|
|
3243
|
+
const snap = this.#snapshots.get(snapshot);
|
|
3244
|
+
if (snap === undefined) {
|
|
3245
|
+
return {
|
|
3246
|
+
ok: false,
|
|
3247
|
+
error: new orchestratorerror(`snapshot ${snapshot} not found`, 'ERR_SNAPSHOT_NOTFOUND'),
|
|
3248
|
+
};
|
|
3249
|
+
}
|
|
3250
|
+
const vm = this.#vms.get(snap.vmid);
|
|
3251
|
+
if (vm === undefined) {
|
|
3252
|
+
return {
|
|
3253
|
+
ok: false,
|
|
3254
|
+
error: new orchestratorerror(`vm ${snap.vmid} not found`, 'ERR_VM_NOTFOUND'),
|
|
3255
|
+
};
|
|
3256
|
+
}
|
|
3257
|
+
// restore path honors the single state machine: a live vm moves through
|
|
3258
|
+
// restoring back to running, a stopped vm reprovisions; both routes are
|
|
3259
|
+
// legal edges of the guard map (running->restoring->running and
|
|
3260
|
+
// stopped->provisioning->stopped).
|
|
3261
|
+
if (vm.state === 'stopped') {
|
|
3262
|
+
await this.transitionvm(snap.vmid, 'provisioning');
|
|
3263
|
+
await sleep(600);
|
|
3264
|
+
await this.transitionvm(snap.vmid, 'stopped');
|
|
3265
|
+
}
|
|
3266
|
+
else {
|
|
3267
|
+
await this.transitionvm(snap.vmid, 'restoring');
|
|
3268
|
+
await sleep(600);
|
|
3269
|
+
await this.transitionvm(snap.vmid, 'running');
|
|
3270
|
+
}
|
|
3271
|
+
this.#audit(actor, 'snapshot:restore', snap.vmid, 'success', { snapshot: snapshot });
|
|
3272
|
+
return { ok: true, value: undefined };
|
|
3273
|
+
}
|
|
3274
|
+
/**
|
|
3275
|
+
* checkpoints a vm with criu, qemu savevm or docker checkpoint
|
|
3276
|
+
* semantics; the passage routes touching the vm are checksummed so the
|
|
3277
|
+
* restore can verify the network plan (v6).
|
|
3278
|
+
*/
|
|
3279
|
+
async checkpointvm(id, method = 'qemu_savevm', incremental = false, actor = SYSTEMUSER) {
|
|
3280
|
+
const vm = this.#vms.get(id);
|
|
3281
|
+
if (vm === undefined) {
|
|
3282
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
3283
|
+
}
|
|
3284
|
+
if (!vm.checkpointenabled) {
|
|
3285
|
+
return {
|
|
3286
|
+
ok: false,
|
|
3287
|
+
error: new orchestratorerror('checkpoint disabled', 'ERR_CHECKPOINT_DISABLED'),
|
|
3288
|
+
};
|
|
3289
|
+
}
|
|
3290
|
+
try {
|
|
3291
|
+
const checkpointid = `ckpt-${randomUUID()}`;
|
|
3292
|
+
const basepath = path.join(this.#rootdir, 'checkpoints', id, checkpointid);
|
|
3293
|
+
mkdirSync(basepath, { recursive: true });
|
|
3294
|
+
if (method === 'criu' && vm.qemupid !== undefined) {
|
|
3295
|
+
const result = await run('criu', [
|
|
3296
|
+
'dump',
|
|
3297
|
+
'-t',
|
|
3298
|
+
String(vm.qemupid),
|
|
3299
|
+
'--images-dir',
|
|
3300
|
+
basepath,
|
|
3301
|
+
'--leave-running',
|
|
3302
|
+
]);
|
|
3303
|
+
if (result.code !== 0) {
|
|
3304
|
+
throw new orchestratorerror(`criu dump exited ${result.code}: ${result.stderr.trim()}`, 'ERR_CRIU_DUMP');
|
|
3305
|
+
}
|
|
3306
|
+
}
|
|
3307
|
+
else {
|
|
3308
|
+
await sleep(500);
|
|
3309
|
+
}
|
|
3310
|
+
const routes = [...this.#passageroutes.values()].filter((route) => route.srcvm === id || route.dstvm === id);
|
|
3311
|
+
const checksum = createHash('sha256')
|
|
3312
|
+
.update(JSON.stringify(routes))
|
|
3313
|
+
.digest('hex')
|
|
3314
|
+
.slice(0, 16);
|
|
3315
|
+
const checkpoint = {
|
|
3316
|
+
id: checkpointid,
|
|
3317
|
+
vmid: id,
|
|
3318
|
+
method,
|
|
3319
|
+
path: basepath,
|
|
3320
|
+
incremental,
|
|
3321
|
+
createdat: new Date(),
|
|
3322
|
+
passagechecksum: checksum,
|
|
3323
|
+
};
|
|
3324
|
+
this.#checkpoints.set(checkpointid, checkpoint);
|
|
3325
|
+
writeFileSync(path.join(basepath, 'meta.json'), JSON.stringify({ ...checkpoint, createdat: checkpoint.createdat.toISOString() }, null, 2), 'utf8');
|
|
3326
|
+
this.#events.publish('checkpoint:created', { checkpointid, vmid: id, method });
|
|
3327
|
+
this.#metrics.inc('checkpoint_created_total', 1, { vmid: id, method });
|
|
3328
|
+
this.#audit(actor, 'checkpoint:create', id, 'success', { method, incremental });
|
|
3329
|
+
return { ok: true, value: checkpoint };
|
|
3330
|
+
}
|
|
3331
|
+
catch (error) {
|
|
3332
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
3333
|
+
}
|
|
3334
|
+
}
|
|
3335
|
+
/** restores a checkpoint; the vm walks provisioning -> running. */
|
|
3336
|
+
async restorecheckpoint(checkpoint, actor = SYSTEMUSER) {
|
|
3337
|
+
const ckpt = this.#checkpoints.get(checkpoint);
|
|
3338
|
+
if (ckpt === undefined) {
|
|
3339
|
+
return {
|
|
3340
|
+
ok: false,
|
|
3341
|
+
error: new orchestratorerror(`checkpoint ${checkpoint} not found`, 'ERR_CHECKPOINT_NOTFOUND'),
|
|
3342
|
+
};
|
|
3343
|
+
}
|
|
3344
|
+
const vm = this.#vms.get(ckpt.vmid);
|
|
3345
|
+
if (vm === undefined) {
|
|
3346
|
+
return {
|
|
3347
|
+
ok: false,
|
|
3348
|
+
error: new orchestratorerror(`vm ${ckpt.vmid} not found`, 'ERR_VM_NOTFOUND'),
|
|
3349
|
+
};
|
|
3350
|
+
}
|
|
3351
|
+
await this.transitionvm(ckpt.vmid, 'provisioning');
|
|
3352
|
+
await sleep(700);
|
|
3353
|
+
await this.transitionvm(ckpt.vmid, 'running');
|
|
3354
|
+
this.#audit(actor, 'checkpoint:restore', ckpt.vmid, 'success', { checkpoint });
|
|
3355
|
+
return { ok: true, value: undefined };
|
|
3356
|
+
}
|
|
3357
|
+
/* ---------------------------------------------------------------- */
|
|
3358
|
+
/* metrics: snapshots, prometheus and the OTLP HTTP exporter */
|
|
3359
|
+
/* ---------------------------------------------------------------- */
|
|
3360
|
+
/** refreshes the runtime gauges and returns the OTel ring tail. */
|
|
3361
|
+
getmetricssnapshot() {
|
|
3362
|
+
for (const vm of this.#vms.values()) {
|
|
3363
|
+
this.#metrics.gauge('vm_state', vm.state === 'running' ? 1 : 0, {
|
|
3364
|
+
vmid: vm.id,
|
|
3365
|
+
state: vm.state,
|
|
3366
|
+
});
|
|
3367
|
+
}
|
|
3368
|
+
this.#metrics.gauge('host_cpus', cpus().length, { node: this.#nodeid });
|
|
3369
|
+
this.#metrics.gauge('host_mem_free_mb', Math.floor(freemem() / 1024 / 1024), {
|
|
3370
|
+
node: this.#nodeid,
|
|
3371
|
+
});
|
|
3372
|
+
return this.#metrics.otelsnapshot();
|
|
3373
|
+
}
|
|
3374
|
+
/** renders the Prometheus exposition of the single metrics store. */
|
|
3375
|
+
exportprometheus() {
|
|
3376
|
+
return this.#metrics.exportprometheus();
|
|
3377
|
+
}
|
|
3378
|
+
/**
|
|
3379
|
+
* starts the OTLP style HTTP exporter serving /metrics and /health;
|
|
3380
|
+
* the bind host is a user choice (explicit argument or resolvehost)
|
|
3381
|
+
* and the port is random in [30000, 59999] unless pinned — the engine
|
|
3382
|
+
* never binds a loopback default.
|
|
3383
|
+
*/
|
|
3384
|
+
async startotelexporter(host, port) {
|
|
3385
|
+
const bindhost = resolvehost(host);
|
|
3386
|
+
const bindport = allocport(port);
|
|
3387
|
+
const server = createServer((req, res) => {
|
|
3388
|
+
try {
|
|
3389
|
+
if (req.url === '/metrics' && req.method === 'GET') {
|
|
3390
|
+
res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' });
|
|
3391
|
+
res.end(this.exportprometheus());
|
|
3392
|
+
return;
|
|
3393
|
+
}
|
|
3394
|
+
if (req.url === '/health' && req.method === 'GET') {
|
|
3395
|
+
const failing = this.failingvms().length;
|
|
3396
|
+
res.writeHead(200, { 'content-type': 'application/json' });
|
|
3397
|
+
res.end(JSON.stringify({ ok: failing === 0, failing, vmcount: this.#vms.size }, null, 2));
|
|
3398
|
+
return;
|
|
3399
|
+
}
|
|
3400
|
+
res.writeHead(404);
|
|
3401
|
+
res.end('not found');
|
|
3402
|
+
}
|
|
3403
|
+
catch {
|
|
3404
|
+
res.writeHead(500);
|
|
3405
|
+
res.end('exporter error');
|
|
3406
|
+
}
|
|
3407
|
+
});
|
|
3408
|
+
await new Promise((resolve) => {
|
|
3409
|
+
server.listen(bindport, bindhost, () => {
|
|
3410
|
+
resolve();
|
|
3411
|
+
});
|
|
3412
|
+
});
|
|
3413
|
+
this.#events.publish('otel:exporterstarted', { host: bindhost, port: bindport });
|
|
3414
|
+
this.#log(`otel exporter listening on ${bindhost}:${bindport}`);
|
|
3415
|
+
return {
|
|
3416
|
+
host: bindhost,
|
|
3417
|
+
port: bindport,
|
|
3418
|
+
close: () => {
|
|
3419
|
+
server.close();
|
|
3420
|
+
},
|
|
3421
|
+
};
|
|
3422
|
+
}
|
|
3423
|
+
/* ---------------------------------------------------------------- */
|
|
3424
|
+
/* config hot-reload + hardware view bootstrap */
|
|
3425
|
+
/* ---------------------------------------------------------------- */
|
|
3426
|
+
/** the six dotted config names every deployment must carry. */
|
|
3427
|
+
static mandatoryconfigs() {
|
|
3428
|
+
return [
|
|
3429
|
+
'vm.config',
|
|
3430
|
+
'gpu.config',
|
|
3431
|
+
'passage.config',
|
|
3432
|
+
'qemu.config',
|
|
3433
|
+
'mttg.config',
|
|
3434
|
+
'docker.config',
|
|
3435
|
+
];
|
|
3436
|
+
}
|
|
3437
|
+
/**
|
|
3438
|
+
* default contents of the flat config files written by start(); the
|
|
3439
|
+
* version strings carry the v2 anchors (qemu 11.1.0, docker 29.7.2,
|
|
3440
|
+
* node 26.7.0), replacing the stale Meta banners.
|
|
3441
|
+
*/
|
|
3442
|
+
#defaultconfigcontents() {
|
|
3443
|
+
return {
|
|
3444
|
+
'vm.config': [
|
|
3445
|
+
`# saddle v6 vm.config — qemu ${runtimecatalog.qemu} — ${new Date().toISOString()}`,
|
|
3446
|
+
'vm_default_vcpus=4',
|
|
3447
|
+
'vm_default_ram_mb=4096',
|
|
3448
|
+
'vm_default_disk_gb=40',
|
|
3449
|
+
'vm_default_arch=x86_64',
|
|
3450
|
+
'',
|
|
3451
|
+
].join('\n'),
|
|
3452
|
+
'gpu.config': [
|
|
3453
|
+
'# saddle v6 gpu.config — blackwell / ryzen x',
|
|
3454
|
+
'gpu_mode=passthrough',
|
|
3455
|
+
'gpu_default_vram=24576',
|
|
3456
|
+
'sriov_enabled=true',
|
|
3457
|
+
'',
|
|
3458
|
+
].join('\n'),
|
|
3459
|
+
'passage.config': [
|
|
3460
|
+
'# saddle v6 passage.config — ovs 3.7.1 / dpdk 26.07.0',
|
|
3461
|
+
'passage_bridge=br-vhe',
|
|
3462
|
+
'passageMtu=9000',
|
|
3463
|
+
'vxlan_port=4789',
|
|
3464
|
+
'',
|
|
3465
|
+
].join('\n'),
|
|
3466
|
+
'qemu.config': [
|
|
3467
|
+
`# saddle v6 qemu.config — qemu ${runtimecatalog.qemu}`,
|
|
3468
|
+
'qemu_binary=/usr/bin/qemu-system-x86_64',
|
|
3469
|
+
'qemuMachine=q35',
|
|
3470
|
+
'qemu_accel=kvm',
|
|
3471
|
+
`qemuVersion=${runtimecatalog.qemu}`,
|
|
3472
|
+
'',
|
|
3473
|
+
].join('\n'),
|
|
3474
|
+
'mttg.config': [
|
|
3475
|
+
'# saddle v6 mttg.config — cgroups v2 / sched_ext 6.12',
|
|
3476
|
+
'mttg_workers=16',
|
|
3477
|
+
'mttg_priority_levels=8',
|
|
3478
|
+
'mttg_max_retries=3',
|
|
3479
|
+
'',
|
|
3480
|
+
].join('\n'),
|
|
3481
|
+
'docker.config': [
|
|
3482
|
+
`# saddle v6 docker.config — docker ${runtimecatalog.docker}`,
|
|
3483
|
+
'docker_runtime=runc',
|
|
3484
|
+
'docker_network=bridge',
|
|
3485
|
+
'dockerBuildx=vhe-builder',
|
|
3486
|
+
`dockerVersion=${runtimecatalog.docker}`,
|
|
3487
|
+
'',
|
|
3488
|
+
].join('\n'),
|
|
3489
|
+
'boards.json': JSON.stringify({
|
|
3490
|
+
boards: [
|
|
3491
|
+
{ id: 'b550-aorus', chipset: 'B550', socket: 'AM4', version: 'v2' },
|
|
3492
|
+
{ id: 'x670e-hero', chipset: 'X670E', socket: 'AM5', version: 'v2' },
|
|
3493
|
+
],
|
|
3494
|
+
version: this.version,
|
|
3495
|
+
}, null, 2),
|
|
3496
|
+
'cores.json': JSON.stringify({
|
|
3497
|
+
cores: Array.from({ length: 32 }, (_, index) => ({
|
|
3498
|
+
id: index,
|
|
3499
|
+
freqmhz: 4500 + (index % 4) * 100,
|
|
3500
|
+
enabled: true,
|
|
3501
|
+
})),
|
|
3502
|
+
version: this.version,
|
|
3503
|
+
}, null, 2),
|
|
3504
|
+
'processors.json': JSON.stringify({
|
|
3505
|
+
processors: [
|
|
3506
|
+
{
|
|
3507
|
+
id: 'ryzen-9-9950x3d',
|
|
3508
|
+
series: 'ryzen x',
|
|
3509
|
+
cores: 16,
|
|
3510
|
+
threads: 32,
|
|
3511
|
+
baseghz: 4.3,
|
|
3512
|
+
boostghz: 5.7,
|
|
3513
|
+
tdpw: 170,
|
|
3514
|
+
arch: 'zen5',
|
|
3515
|
+
},
|
|
3516
|
+
],
|
|
3517
|
+
version: this.version,
|
|
3518
|
+
}, null, 2),
|
|
3519
|
+
'gpus.json': JSON.stringify({
|
|
3520
|
+
gpus: [
|
|
3521
|
+
{
|
|
3522
|
+
id: 'gpu-blackwell-b200',
|
|
3523
|
+
model: 'nvidia blackwell b200',
|
|
3524
|
+
vendor: 'nvidia',
|
|
3525
|
+
vrammb: 192000,
|
|
3526
|
+
tdpw: 1000,
|
|
3527
|
+
pci: '0000:01:00.0',
|
|
3528
|
+
arch: 'blackwell',
|
|
3529
|
+
},
|
|
3530
|
+
{
|
|
3531
|
+
id: 'gpu-rdna4-9070xt',
|
|
3532
|
+
model: 'amd radeon rx 9070 xt',
|
|
3533
|
+
vendor: 'amd',
|
|
3534
|
+
vrammb: 16384,
|
|
3535
|
+
tdpw: 304,
|
|
3536
|
+
pci: '0000:03:00.0',
|
|
3537
|
+
arch: 'rdna4',
|
|
3538
|
+
},
|
|
3539
|
+
],
|
|
3540
|
+
version: this.version,
|
|
3541
|
+
}, null, 2),
|
|
3542
|
+
'vcpus.json': JSON.stringify({ vcpus: { total: 32, reserved: 2, allocationmap: {} }, version: this.version }, null, 2),
|
|
3543
|
+
'vram.json': JSON.stringify({ vram: { totalmb: 131072, reservedmb: 2048, hugepages2m: 2048 }, version: this.version }, null, 2),
|
|
3544
|
+
'amdryzenxseries.json': JSON.stringify({
|
|
3545
|
+
series: 'amd ryzen x',
|
|
3546
|
+
models: [
|
|
3547
|
+
{ model: '9950X', cores: 16, tdp: 170, ccd: 2, boost: 5.7, x3d: false },
|
|
3548
|
+
{
|
|
3549
|
+
model: '9950X3D',
|
|
3550
|
+
cores: 16,
|
|
3551
|
+
tdp: 170,
|
|
3552
|
+
ccd: 2,
|
|
3553
|
+
boost: 5.7,
|
|
3554
|
+
x3d: true,
|
|
3555
|
+
l3cachemb: 128,
|
|
3556
|
+
},
|
|
3557
|
+
],
|
|
3558
|
+
arch: 'zen5',
|
|
3559
|
+
node: '4nm',
|
|
3560
|
+
version: this.version,
|
|
3561
|
+
}, null, 2),
|
|
3562
|
+
'nvidiablackwell.json': JSON.stringify({
|
|
3563
|
+
series: 'nvidia blackwell',
|
|
3564
|
+
models: [
|
|
3565
|
+
{ model: 'B200', sm: 208, tensorgen: 5, vramgb: 192, bandwidthgbs: 8000 },
|
|
3566
|
+
{ model: 'GB200', sm: 208, issuperchip: true, vramgb: 192, nvlinkbw: 3600 },
|
|
3567
|
+
],
|
|
3568
|
+
interconnect: 'nvlink 8',
|
|
3569
|
+
process: '4np',
|
|
3570
|
+
version: this.version,
|
|
3571
|
+
}, null, 2),
|
|
3572
|
+
};
|
|
3573
|
+
}
|
|
3574
|
+
/**
|
|
3575
|
+
* watches the six dotted configs plus the eight json catalogs; missing
|
|
3576
|
+
* files are created with the default contents before watching starts.
|
|
3577
|
+
*/
|
|
3578
|
+
enablehotreload(files = orchestrator.mandatoryconfigs()) {
|
|
3579
|
+
const allfiles = [
|
|
3580
|
+
...files,
|
|
3581
|
+
'boards.json',
|
|
3582
|
+
'cores.json',
|
|
3583
|
+
'processors.json',
|
|
3584
|
+
'gpus.json',
|
|
3585
|
+
'vcpus.json',
|
|
3586
|
+
'vram.json',
|
|
3587
|
+
'amdryzenxseries.json',
|
|
3588
|
+
'nvidiablackwell.json',
|
|
3589
|
+
];
|
|
3590
|
+
const defaults = this.#defaultconfigcontents();
|
|
3591
|
+
for (const name of allfiles) {
|
|
3592
|
+
try {
|
|
3593
|
+
const fullpath = path.join(this.#rootdir, name);
|
|
3594
|
+
if (!existsSync(fullpath)) {
|
|
3595
|
+
const fallback = defaults[name];
|
|
3596
|
+
writeFileSync(fullpath, fallback ?? (name.endsWith('.json') ? '{}' : '# saddle v6'), 'utf8');
|
|
3597
|
+
}
|
|
3598
|
+
const watcher = watch(fullpath, (event) => {
|
|
3599
|
+
if (event === 'change') {
|
|
3600
|
+
void this.handleconfigreload(name, fullpath);
|
|
3601
|
+
}
|
|
3602
|
+
});
|
|
3603
|
+
this.#watchers.set(name, watcher);
|
|
3604
|
+
}
|
|
3605
|
+
catch (error) {
|
|
3606
|
+
this.#log(`cannot watch ${name}: ${errormessage(error)}`);
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
3609
|
+
this.#log(`hot-reload enabled for ${allfiles.length} files`);
|
|
3610
|
+
}
|
|
3611
|
+
/** closes every config watcher. */
|
|
3612
|
+
disablehotreload() {
|
|
3613
|
+
for (const watcher of this.#watchers.values()) {
|
|
3614
|
+
try {
|
|
3615
|
+
watcher.close();
|
|
3616
|
+
}
|
|
3617
|
+
catch {
|
|
3618
|
+
/* catcher: watchers may already be closed */
|
|
3619
|
+
}
|
|
3620
|
+
}
|
|
3621
|
+
this.#watchers.clear();
|
|
3622
|
+
}
|
|
3623
|
+
/**
|
|
3624
|
+
* loads one config file with caching; json files parse into objects
|
|
3625
|
+
* and key=value files parse into a raw string record.
|
|
3626
|
+
*/
|
|
3627
|
+
async loadconfig(name) {
|
|
3628
|
+
const fullpath = path.join(this.#rootdir, name);
|
|
3629
|
+
try {
|
|
3630
|
+
const raw = await readFile(fullpath, 'utf8');
|
|
3631
|
+
let parsed;
|
|
3632
|
+
try {
|
|
3633
|
+
parsed = JSON.parse(raw);
|
|
3634
|
+
}
|
|
3635
|
+
catch {
|
|
3636
|
+
const lines = raw
|
|
3637
|
+
.split('\n')
|
|
3638
|
+
.filter((line) => line.trim().length > 0 && !line.startsWith('#'));
|
|
3639
|
+
const kv = {};
|
|
3640
|
+
for (const line of lines) {
|
|
3641
|
+
const separator = line.indexOf('=');
|
|
3642
|
+
if (separator > 0) {
|
|
3643
|
+
kv[line.slice(0, separator).trim()] = line.slice(separator + 1).trim();
|
|
3644
|
+
}
|
|
3645
|
+
}
|
|
3646
|
+
parsed = kv;
|
|
3647
|
+
}
|
|
3648
|
+
this.#configcache.set(name, parsed);
|
|
3649
|
+
return parsed;
|
|
3650
|
+
}
|
|
3651
|
+
catch {
|
|
3652
|
+
return this.#configcache.get(name);
|
|
3653
|
+
}
|
|
3654
|
+
}
|
|
3655
|
+
async handleconfigreload(name, fullpath) {
|
|
3656
|
+
try {
|
|
3657
|
+
const content = await readFile(fullpath, 'utf8');
|
|
3658
|
+
if (name.endsWith('.json')) {
|
|
3659
|
+
JSON.parse(content);
|
|
3660
|
+
}
|
|
3661
|
+
else {
|
|
3662
|
+
content.split('\n').filter((line) => line.trim().length > 0 && !line.startsWith('#'));
|
|
3663
|
+
}
|
|
3664
|
+
await this.loadconfig(name);
|
|
3665
|
+
this.#metrics.inc('config_reloaded_total', 1, { file: name });
|
|
3666
|
+
this.#events.publish('config:reloaded', { file: name });
|
|
3667
|
+
this.#log(`config reloaded ${name} ${content.length} bytes`);
|
|
3668
|
+
}
|
|
3669
|
+
catch (error) {
|
|
3670
|
+
this.#log(`config parse error ${name}: ${errormessage(error)}`);
|
|
3671
|
+
}
|
|
3672
|
+
}
|
|
3673
|
+
/**
|
|
3674
|
+
* bootstraps the hardware view from the eight flat json catalogs and
|
|
3675
|
+
* reports the missing ones (coreorchestrator).
|
|
3676
|
+
*/
|
|
3677
|
+
async bootstraphardwareview() {
|
|
3678
|
+
const names = [
|
|
3679
|
+
'boards.json',
|
|
3680
|
+
'cores.json',
|
|
3681
|
+
'processors.json',
|
|
3682
|
+
'gpus.json',
|
|
3683
|
+
'vcpus.json',
|
|
3684
|
+
'vram.json',
|
|
3685
|
+
'amdryzenxseries.json',
|
|
3686
|
+
'nvidiablackwell.json',
|
|
3687
|
+
];
|
|
3688
|
+
const view = {};
|
|
3689
|
+
for (const name of names) {
|
|
3690
|
+
const loaded = await this.#readjsonmaybe(path.join(this.#rootdir, name));
|
|
3691
|
+
view[name.replace('.json', '')] = loaded ?? { note: `missing ${name}` };
|
|
3692
|
+
}
|
|
3693
|
+
this.#log(`hardware view bootstrapped keys=${Object.keys(view).join(',')}`);
|
|
3694
|
+
return view;
|
|
3695
|
+
}
|
|
3696
|
+
/* ---------------------------------------------------------------- */
|
|
3697
|
+
/* plugin system */
|
|
3698
|
+
/* ---------------------------------------------------------------- */
|
|
3699
|
+
/**
|
|
3700
|
+
* activates a plugin with a context that exposes the orchestrator, a
|
|
3701
|
+
* prefixed logger and typed hook registration on the single bus.
|
|
3702
|
+
*/
|
|
3703
|
+
async loadplugin(plugin, actor = SYSTEMUSER) {
|
|
3704
|
+
this.assertpermission(actor, 'plugin:manage');
|
|
3705
|
+
if (this.#plugins.has(plugin.id)) {
|
|
3706
|
+
return {
|
|
3707
|
+
ok: false,
|
|
3708
|
+
error: new orchestratorerror(`plugin ${plugin.id} already loaded`, 'ERR_PLUGIN_EXISTS'),
|
|
3709
|
+
};
|
|
3710
|
+
}
|
|
3711
|
+
const context = {
|
|
3712
|
+
orchestrator: this,
|
|
3713
|
+
log: (message) => {
|
|
3714
|
+
this.#log(`[plugin ${plugin.name}] ${message}`);
|
|
3715
|
+
},
|
|
3716
|
+
registerhook: (hook, handler) => {
|
|
3717
|
+
this.#events.subscribe(hook, handler);
|
|
3718
|
+
},
|
|
3719
|
+
};
|
|
3720
|
+
try {
|
|
3721
|
+
await plugin.activate(context);
|
|
3722
|
+
this.#plugins.set(plugin.id, plugin);
|
|
3723
|
+
this.#events.publish('plugin:loaded', { pluginid: plugin.id });
|
|
3724
|
+
this.#metrics.inc('plugin_loaded_total', 1, { plugin: plugin.name });
|
|
3725
|
+
this.#audit(actor, 'plugin:load', plugin.id, 'success', { version: plugin.version });
|
|
3726
|
+
return { ok: true, value: undefined };
|
|
3727
|
+
}
|
|
3728
|
+
catch (error) {
|
|
3729
|
+
return { ok: false, error: error instanceof Error ? error : new Error(String(error)) };
|
|
3730
|
+
}
|
|
3731
|
+
}
|
|
3732
|
+
/** deactivates a plugin and forgets it. */
|
|
3733
|
+
async unloadplugin(id, actor = SYSTEMUSER) {
|
|
3734
|
+
this.assertpermission(actor, 'plugin:manage');
|
|
3735
|
+
const plugin = this.#plugins.get(id);
|
|
3736
|
+
if (plugin === undefined) {
|
|
3737
|
+
return {
|
|
3738
|
+
ok: false,
|
|
3739
|
+
error: new orchestratorerror(`plugin ${id} not found`, 'ERR_PLUGIN_NOTFOUND'),
|
|
3740
|
+
};
|
|
3741
|
+
}
|
|
3742
|
+
try {
|
|
3743
|
+
await plugin.deactivate();
|
|
3744
|
+
}
|
|
3745
|
+
catch {
|
|
3746
|
+
/* catcher: a broken deactivate must not block unload */
|
|
3747
|
+
}
|
|
3748
|
+
this.#plugins.delete(id);
|
|
3749
|
+
this.#events.publish('plugin:unloaded', { pluginid: id });
|
|
3750
|
+
this.#audit(actor, 'plugin:unload', id, 'success', {});
|
|
3751
|
+
return { ok: true, value: undefined };
|
|
3752
|
+
}
|
|
3753
|
+
/** v6 alias for loadplugin under the system actor. */
|
|
3754
|
+
async registerplugin(plugin) {
|
|
3755
|
+
const outcome = await this.loadplugin(plugin);
|
|
3756
|
+
if (!outcome.ok) {
|
|
3757
|
+
throw outcome.error;
|
|
3758
|
+
}
|
|
3759
|
+
}
|
|
3760
|
+
/** v6 alias for unloadplugin under the system actor. */
|
|
3761
|
+
async unregisterplugin(id) {
|
|
3762
|
+
const outcome = await this.unloadplugin(id);
|
|
3763
|
+
if (!outcome.ok) {
|
|
3764
|
+
throw outcome.error;
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
/** lists loaded plugins. */
|
|
3768
|
+
listplugins() {
|
|
3769
|
+
return [...this.#plugins.values()];
|
|
3770
|
+
}
|
|
3771
|
+
/* ---------------------------------------------------------------- */
|
|
3772
|
+
/* audit: ring + JSONL persistence */
|
|
3773
|
+
/* ---------------------------------------------------------------- */
|
|
3774
|
+
#audit(actor, action, target, out, detail) {
|
|
3775
|
+
const entry = {
|
|
3776
|
+
id: `audit-${randomUUID()}`,
|
|
3777
|
+
timestamp: new Date(),
|
|
3778
|
+
userid: actor,
|
|
3779
|
+
action,
|
|
3780
|
+
target: String(target),
|
|
3781
|
+
result: out,
|
|
3782
|
+
detail,
|
|
3783
|
+
};
|
|
3784
|
+
this.#auditring.push(entry);
|
|
3785
|
+
if (this.#auditring.length > 5000) {
|
|
3786
|
+
this.#auditring.splice(0, this.#auditring.length - 5000);
|
|
3787
|
+
}
|
|
3788
|
+
this.#events.publish('audit:entry', {
|
|
3789
|
+
actor: actor,
|
|
3790
|
+
action,
|
|
3791
|
+
target: entry.target,
|
|
3792
|
+
result: out,
|
|
3793
|
+
});
|
|
3794
|
+
this.#metrics.inc('audit_total', 1, { action, result: out });
|
|
3795
|
+
}
|
|
3796
|
+
/** filters the in-memory audit ring by user, action and since date. */
|
|
3797
|
+
getauditlog(filter) {
|
|
3798
|
+
let list = this.#auditring;
|
|
3799
|
+
if (filter?.userid !== undefined) {
|
|
3800
|
+
list = list.filter((entry) => entry.userid === filter.userid);
|
|
3801
|
+
}
|
|
3802
|
+
if (filter?.action !== undefined) {
|
|
3803
|
+
list = list.filter((entry) => entry.action === filter.action);
|
|
3804
|
+
}
|
|
3805
|
+
if (filter?.since !== undefined) {
|
|
3806
|
+
list = list.filter((entry) => entry.timestamp >= filter.since);
|
|
3807
|
+
}
|
|
3808
|
+
return list.slice(-1000);
|
|
3809
|
+
}
|
|
3810
|
+
/**
|
|
3811
|
+
* appends one audit line to the tenant JSONL file under audit/
|
|
3812
|
+
* (coreorchestrator); the ring mirrors the entry for fast queries.
|
|
3813
|
+
*/
|
|
3814
|
+
async auditlog(entry) {
|
|
3815
|
+
try {
|
|
3816
|
+
const auditdir = path.join(this.#rootdir, 'audit');
|
|
3817
|
+
await mkdir(auditdir, { recursive: true });
|
|
3818
|
+
const line = `${JSON.stringify({ ...entry, at: new Date().toISOString(), id: randomUUID() })}\n`;
|
|
3819
|
+
await appendFile(path.join(auditdir, `${entry.tenant}.jsonl`), line, 'utf8');
|
|
3820
|
+
this.#metrics.inc('audit_logged_total', 1, { tenant: entry.tenant, action: entry.action });
|
|
3821
|
+
}
|
|
3822
|
+
catch {
|
|
3823
|
+
/* catcher: audit persistence must never break the caller */
|
|
3824
|
+
}
|
|
3825
|
+
}
|
|
3826
|
+
/* ---------------------------------------------------------------- */
|
|
3827
|
+
/* drain: vm drain, graceful shutdown, node drain */
|
|
3828
|
+
/* ---------------------------------------------------------------- */
|
|
3829
|
+
/**
|
|
3830
|
+
* drains one vm: signals the guest for a graceful stop and waits up
|
|
3831
|
+
* to the task timeout (capped at two seconds in simulated mode).
|
|
3832
|
+
*/
|
|
3833
|
+
async drainvm(id, task) {
|
|
3834
|
+
const vm = this.#vms.get(id);
|
|
3835
|
+
if (vm === undefined) {
|
|
3836
|
+
return { ok: false, error: new orchestratorerror(`vm ${id} not found`, 'ERR_VM_NOTFOUND') };
|
|
3837
|
+
}
|
|
3838
|
+
const previous = vm.state;
|
|
3839
|
+
if (cantransition(vm.state, 'draining')) {
|
|
3840
|
+
vm.state = 'draining';
|
|
3841
|
+
}
|
|
3842
|
+
this.#log(`draining vm ${id} signal=${task.signal} timeout=${task.timeoutsec}s fallback=${task.fallback}`);
|
|
3843
|
+
await sleep(Math.min(task.timeoutsec * 1000, 2000));
|
|
3844
|
+
vm.state = previous;
|
|
3845
|
+
return { ok: true, value: undefined };
|
|
3846
|
+
}
|
|
3847
|
+
/**
|
|
3848
|
+
* graceful shutdown of the whole orchestrator: monitors stop, hot
|
|
3849
|
+
* reload closes, running vms drain with concurrency four against an
|
|
3850
|
+
* overall timeout and leftover qemu processes get SIGKILL.
|
|
3851
|
+
*/
|
|
3852
|
+
async gracefulshutdown(timeoutsec = 60) {
|
|
3853
|
+
if (this.#shuttingdown) {
|
|
3854
|
+
return;
|
|
3855
|
+
}
|
|
3856
|
+
this.#shuttingdown = true;
|
|
3857
|
+
this.#log(`graceful shutdown initiated timeout=${timeoutsec}s vms=${this.#vms.size}`);
|
|
3858
|
+
this.stophealthmonitor();
|
|
3859
|
+
this.stopautoscaler();
|
|
3860
|
+
this.disablehotreload();
|
|
3861
|
+
const running = [...this.#vms.values()].filter((vm) => vm.state === 'running');
|
|
3862
|
+
let index = 0;
|
|
3863
|
+
const worker = async () => {
|
|
3864
|
+
while (index < running.length) {
|
|
3865
|
+
const vm = running[index];
|
|
3866
|
+
index += 1;
|
|
3867
|
+
if (vm === undefined) {
|
|
3868
|
+
continue;
|
|
3869
|
+
}
|
|
3870
|
+
await this.drainvm(vm.id, {
|
|
3871
|
+
vmid: vm.id,
|
|
3872
|
+
timeoutsec: 15,
|
|
3873
|
+
signal: 'SIGTERM',
|
|
3874
|
+
fallback: 'SIGKILL',
|
|
3875
|
+
});
|
|
3876
|
+
await this.stopvm(vm.id, SYSTEMUSER, 'shutdown');
|
|
3877
|
+
}
|
|
3878
|
+
};
|
|
3879
|
+
const workers = Array.from({ length: 4 }, () => worker());
|
|
3880
|
+
await Promise.race([
|
|
3881
|
+
Promise.all(workers),
|
|
3882
|
+
new Promise((resolve) => {
|
|
3883
|
+
setTimeout(() => {
|
|
3884
|
+
resolve();
|
|
3885
|
+
}, timeoutsec * 1000).unref();
|
|
3886
|
+
}),
|
|
3887
|
+
]);
|
|
3888
|
+
for (const processinfo of this.#qemuprocesses.values()) {
|
|
3889
|
+
try {
|
|
3890
|
+
processinfo.child.kill('SIGKILL');
|
|
3891
|
+
}
|
|
3892
|
+
catch {
|
|
3893
|
+
/* catcher: the process may already be gone */
|
|
3894
|
+
}
|
|
3895
|
+
}
|
|
3896
|
+
this.#qemuprocesses.clear();
|
|
3897
|
+
await this.destroyall();
|
|
3898
|
+
this.#log('graceful shutdown completed');
|
|
3899
|
+
}
|
|
3900
|
+
/**
|
|
3901
|
+
* drains a node identified by a label pair: running vms that carry the
|
|
3902
|
+
* label migrate away (or terminate when migration fails) unless they
|
|
3903
|
+
* are marked drain-protected (coreorchestrator).
|
|
3904
|
+
*/
|
|
3905
|
+
async drainnode(labelkey, labelvalue) {
|
|
3906
|
+
const victims = [...this.#vms.values()].filter((vm) => vm.labels[labelkey] === labelvalue && vm.state === 'running');
|
|
3907
|
+
const drained = [];
|
|
3908
|
+
for (const vm of victims) {
|
|
3909
|
+
if (vm.labels['drain-protected'] === 'true') {
|
|
3910
|
+
continue;
|
|
3911
|
+
}
|
|
3912
|
+
await this.startlivemigration(vm.id, `drain-target-${labelvalue}`).catch(async () => {
|
|
3913
|
+
await this.transitionvm(vm.id, 'draining').catch(() => { });
|
|
3914
|
+
});
|
|
3915
|
+
drained.push(vm.id);
|
|
3916
|
+
}
|
|
3917
|
+
this.#events.publish('node:drained', { labelkey, labelvalue, drained });
|
|
3918
|
+
return drained;
|
|
3919
|
+
}
|
|
3920
|
+
/* ---------------------------------------------------------------- */
|
|
3921
|
+
/* leader election (bully with lease) */
|
|
3922
|
+
/* ---------------------------------------------------------------- */
|
|
3923
|
+
/**
|
|
3924
|
+
* starts the bully leader election over the peers with a lease in
|
|
3925
|
+
* seconds; the highest node id wins whenever the lease expires.
|
|
3926
|
+
*/
|
|
3927
|
+
startleaderelection(peers, leasesec = 15) {
|
|
3928
|
+
const attempt = () => {
|
|
3929
|
+
if (this.#shuttingdown) {
|
|
3930
|
+
return;
|
|
3931
|
+
}
|
|
3932
|
+
const allnodes = [...peers, this.#nodeid].sort();
|
|
3933
|
+
const highest = allnodes[allnodes.length - 1] ?? this.#nodeid;
|
|
3934
|
+
const leaseexpired = this.#leader.leaseexpiry === undefined || Date.now() > this.#leader.leaseexpiry.getTime();
|
|
3935
|
+
if (leaseexpired) {
|
|
3936
|
+
if (this.#nodeid === highest) {
|
|
3937
|
+
const oldleader = this.#leader.leaderid;
|
|
3938
|
+
this.#leader.term += 1;
|
|
3939
|
+
this.#leader.isleader = true;
|
|
3940
|
+
this.#leader.leaderid = this.#nodeid;
|
|
3941
|
+
this.#leader.leaseexpiry = new Date(Date.now() + leasesec * 1000);
|
|
3942
|
+
this.#leader.voters = allnodes;
|
|
3943
|
+
if (oldleader !== this.#leader.leaderid) {
|
|
3944
|
+
this.#events.publish('leader:changed', {
|
|
3945
|
+
leader: this.#leader.leaderid,
|
|
3946
|
+
term: this.#leader.term,
|
|
3947
|
+
});
|
|
3948
|
+
this.#log(`elected leader ${this.#leader.leaderid} term=${this.#leader.term}`);
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
else {
|
|
3952
|
+
this.#leader.isleader = false;
|
|
3953
|
+
this.#leader.leaderid = highest;
|
|
3954
|
+
}
|
|
3955
|
+
}
|
|
3956
|
+
else if (this.#leader.isleader) {
|
|
3957
|
+
this.#leader.leaseexpiry = new Date(Date.now() + leasesec * 1000);
|
|
3958
|
+
}
|
|
3959
|
+
this.#metrics.gauge('leader_is_leader', this.#leader.isleader ? 1 : 0, {
|
|
3960
|
+
node: this.#nodeid,
|
|
3961
|
+
term: String(this.#leader.term),
|
|
3962
|
+
});
|
|
3963
|
+
};
|
|
3964
|
+
const interval = setInterval(attempt, leasesec * 500);
|
|
3965
|
+
interval.unref();
|
|
3966
|
+
attempt();
|
|
3967
|
+
}
|
|
3968
|
+
/** returns a readonly view of the leader state. */
|
|
3969
|
+
getleaderstate() {
|
|
3970
|
+
return this.#leader;
|
|
3971
|
+
}
|
|
3972
|
+
/** true when this node currently holds leadership. */
|
|
3973
|
+
isleader() {
|
|
3974
|
+
return this.#leader.isleader;
|
|
3975
|
+
}
|
|
3976
|
+
/* ---------------------------------------------------------------- */
|
|
3977
|
+
/* sandbox plane api (v5 preserved verbatim) */
|
|
3978
|
+
/* ---------------------------------------------------------------- */
|
|
3979
|
+
/** fresh spec builder (builder pattern entry point). */
|
|
3980
|
+
builder() {
|
|
3981
|
+
return new sandboxbuilder();
|
|
3982
|
+
}
|
|
3983
|
+
/** probes every runtime binary. */
|
|
3984
|
+
async detectruntimes() {
|
|
3985
|
+
return this.#registry.detectall();
|
|
3986
|
+
}
|
|
3987
|
+
/** creates a sandbox through the strategy for its runtime. */
|
|
3988
|
+
async create(spec) {
|
|
3989
|
+
const sandbox = await this.#factory.create(spec);
|
|
3990
|
+
this.#active.add(sandbox);
|
|
3991
|
+
return sandbox;
|
|
3992
|
+
}
|
|
3993
|
+
/** restores a firecracker sandbox from a snapshot file. */
|
|
3994
|
+
async restore(spec, snapshotfile) {
|
|
3995
|
+
const strategy = this.#registry.get(spec.runtime);
|
|
3996
|
+
if (!(strategy instanceof firecrackerruntime)) {
|
|
3997
|
+
throw new Error(`restore requires the firecracker runtime, got ${spec.runtime}`);
|
|
3998
|
+
}
|
|
3999
|
+
try {
|
|
4000
|
+
await mkdir(path.join(rundir(), spec.id), { recursive: true });
|
|
4001
|
+
const argv = strategy.restorecommand(spec.id, snapshotfile);
|
|
4002
|
+
const [bin, ...rest] = argv;
|
|
4003
|
+
const child = spawn(bin, rest, { env: { ...process.env, ...spec.env } });
|
|
4004
|
+
const handle = new sandboxhandle(spec.id, strategy.name, child);
|
|
4005
|
+
await handle.started();
|
|
4006
|
+
this.#metrics.inc(`sandbox.restored.${spec.runtime}`);
|
|
4007
|
+
this.#events.emit('restored', { id: spec.id, snapshotfile });
|
|
4008
|
+
const proxy = new sandboxproxy(spec, handle, strategy, this.#events, this.#metrics);
|
|
4009
|
+
this.#active.add(proxy);
|
|
4010
|
+
return proxy;
|
|
4011
|
+
}
|
|
4012
|
+
catch (error) {
|
|
4013
|
+
this.#events.emit('error', { id: spec.id, phase: 'restore', message: errormessage(error) });
|
|
4014
|
+
throw new Error(`restore failed for ${spec.id}: ${errormessage(error)}`);
|
|
4015
|
+
}
|
|
4016
|
+
}
|
|
4017
|
+
/** enables the warm pool with a template spec. */
|
|
4018
|
+
async enablewarmpool(min, max, template) {
|
|
4019
|
+
await this.disablewarmpool();
|
|
4020
|
+
this.#pool = new warmpool({
|
|
4021
|
+
min,
|
|
4022
|
+
max,
|
|
4023
|
+
events: this.#events,
|
|
4024
|
+
metrics: this.#metrics,
|
|
4025
|
+
template: async () => {
|
|
4026
|
+
const sandbox = await this.#factory.create(template);
|
|
4027
|
+
await sandbox.start();
|
|
4028
|
+
return sandbox;
|
|
4029
|
+
},
|
|
4030
|
+
});
|
|
4031
|
+
await this.#pool.prewarm();
|
|
4032
|
+
this.#pool.startrefill();
|
|
4033
|
+
}
|
|
4034
|
+
/** acquires a warm sandbox. */
|
|
4035
|
+
async createpooled() {
|
|
4036
|
+
if (this.#pool === null) {
|
|
4037
|
+
throw new Error('warm pool is disabled; call enablewarmpool first');
|
|
4038
|
+
}
|
|
4039
|
+
const sandbox = await this.#pool.acquire();
|
|
4040
|
+
this.#active.add(sandbox);
|
|
4041
|
+
return sandbox;
|
|
4042
|
+
}
|
|
4043
|
+
/** disables the warm pool and destroys the warm sandboxes. */
|
|
4044
|
+
async disablewarmpool() {
|
|
4045
|
+
if (this.#pool !== null) {
|
|
4046
|
+
await this.#pool.drain();
|
|
4047
|
+
this.#pool = null;
|
|
4048
|
+
}
|
|
4049
|
+
}
|
|
4050
|
+
/** criu checkpoint of a live sandbox process tree. */
|
|
4051
|
+
async checkpoint(sandbox, dir) {
|
|
4052
|
+
await sandbox.criucheckpoint(this.#checkpointer, dir ?? path.join(rundir(), sandbox.id, 'criu'));
|
|
4053
|
+
}
|
|
4054
|
+
/** criu restore of a checkpointed image directory. */
|
|
4055
|
+
async criurestore(dir) {
|
|
4056
|
+
await this.#checkpointer.restore(dir);
|
|
4057
|
+
}
|
|
4058
|
+
/** destroys every active sandbox. */
|
|
4059
|
+
async destroyall() {
|
|
4060
|
+
for (const sandbox of this.#active) {
|
|
4061
|
+
try {
|
|
4062
|
+
await sandbox.destroy();
|
|
4063
|
+
}
|
|
4064
|
+
catch (error) {
|
|
4065
|
+
this.#events.emit('error', {
|
|
4066
|
+
id: sandbox.id,
|
|
4067
|
+
phase: 'destroyall',
|
|
4068
|
+
message: errormessage(error),
|
|
4069
|
+
});
|
|
4070
|
+
}
|
|
4071
|
+
}
|
|
4072
|
+
this.#active.clear();
|
|
4073
|
+
await this.disablewarmpool();
|
|
4074
|
+
}
|
|
4075
|
+
/** compose 5.5.0 rendering of a spec. */
|
|
4076
|
+
composeyaml(spec) {
|
|
4077
|
+
return composeyaml(spec);
|
|
4078
|
+
}
|
|
4079
|
+
/* ---------------------------------------------------------------- */
|
|
4080
|
+
/* start / stop / dumpstate */
|
|
4081
|
+
/* ---------------------------------------------------------------- */
|
|
4082
|
+
/**
|
|
4083
|
+
* prepares the runtime directories, materializes the default flat
|
|
4084
|
+
* configs, optionally enables hot-reload and starts the health
|
|
4085
|
+
* monitor, the autoscaler and the scheduler tick.
|
|
4086
|
+
*/
|
|
4087
|
+
async start() {
|
|
4088
|
+
this.#log(`starting orchestrator v${this.version} stack=${JSON.stringify(stackcatalog)}`);
|
|
4089
|
+
for (const dir of ['snapshots', 'checkpoints', 'logs', 'run', 'audit']) {
|
|
4090
|
+
try {
|
|
4091
|
+
mkdirSync(path.join(this.#rootdir, dir), { recursive: true });
|
|
4092
|
+
}
|
|
4093
|
+
catch {
|
|
4094
|
+
/* catcher: read-only roots keep the orchestrator memory-only */
|
|
4095
|
+
}
|
|
4096
|
+
}
|
|
4097
|
+
const defaults = this.#defaultconfigcontents();
|
|
4098
|
+
for (const [name, content] of Object.entries(defaults)) {
|
|
4099
|
+
const fullpath = path.join(this.#rootdir, name);
|
|
4100
|
+
try {
|
|
4101
|
+
if (!existsSync(fullpath)) {
|
|
4102
|
+
writeFileSync(fullpath, content, 'utf8');
|
|
4103
|
+
}
|
|
4104
|
+
}
|
|
4105
|
+
catch {
|
|
4106
|
+
/* catcher: config materialization is best effort */
|
|
4107
|
+
}
|
|
4108
|
+
}
|
|
4109
|
+
if (this.#options.enablehotreload === true) {
|
|
4110
|
+
this.enablehotreload();
|
|
4111
|
+
}
|
|
4112
|
+
this.starthealthmonitor();
|
|
4113
|
+
this.startautoscaler();
|
|
4114
|
+
this.#schedulerinterval = setInterval(() => {
|
|
4115
|
+
this.scheduletick();
|
|
4116
|
+
}, 2000);
|
|
4117
|
+
this.#schedulerinterval.unref();
|
|
4118
|
+
this.#log('orchestrator started');
|
|
4119
|
+
}
|
|
4120
|
+
/** graceful shutdown plus scheduler teardown. */
|
|
4121
|
+
async stop() {
|
|
4122
|
+
await this.gracefulshutdown(30);
|
|
4123
|
+
if (this.#schedulerinterval !== null) {
|
|
4124
|
+
clearInterval(this.#schedulerinterval);
|
|
4125
|
+
this.#schedulerinterval = null;
|
|
4126
|
+
}
|
|
4127
|
+
}
|
|
4128
|
+
/** full diagnostics dump of both planes. */
|
|
4129
|
+
dumpstate() {
|
|
4130
|
+
return {
|
|
4131
|
+
version: this.version,
|
|
4132
|
+
nodeid: this.#nodeid,
|
|
4133
|
+
stack: stackcatalog,
|
|
4134
|
+
vms: this.#vms.size,
|
|
4135
|
+
running: [...this.#vms.values()].filter((vm) => vm.state === 'running').length,
|
|
4136
|
+
runqueues: [...this.#runqueues.values()].map((queue) => ({
|
|
4137
|
+
numa: queue.numanode,
|
|
4138
|
+
load: queue.load,
|
|
4139
|
+
tasks: queue.tasks.length,
|
|
4140
|
+
})),
|
|
4141
|
+
balloons: this.#balloons.size,
|
|
4142
|
+
gpus: this.#gpuassignments.size,
|
|
4143
|
+
sriov: this.#sriovpfs.size,
|
|
4144
|
+
passageroutes: this.#passageroutes.size,
|
|
4145
|
+
mttgqueue: this.#mttgqueue.size,
|
|
4146
|
+
dockerbridges: this.#dockerbridges.size,
|
|
4147
|
+
qemuprocesses: this.#qemuprocesses.size,
|
|
4148
|
+
migrations: this.#migrations.size,
|
|
4149
|
+
snapshots: this.#snapshots.size,
|
|
4150
|
+
checkpoints: this.#checkpoints.size,
|
|
4151
|
+
plugins: this.#plugins.size,
|
|
4152
|
+
reservations: this.#reservations.size,
|
|
4153
|
+
quotas: [...this.#quotatable.keys()],
|
|
4154
|
+
leader: this.#leader,
|
|
4155
|
+
numa: this.#numatopology,
|
|
4156
|
+
sandboxactive: this.#active.size,
|
|
4157
|
+
metricscount: this.#metrics.otelsnapshot().length,
|
|
4158
|
+
};
|
|
4159
|
+
}
|
|
4160
|
+
[Symbol.dispose]() {
|
|
4161
|
+
try {
|
|
4162
|
+
void this.stop();
|
|
4163
|
+
}
|
|
4164
|
+
catch {
|
|
4165
|
+
/* catcher: disposal must not throw */
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
4169
|
+
/* ------------------------------------------------------------------ */
|
|
4170
|
+
/* context: singleton + end-to-end demo with `using` disposal */
|
|
4171
|
+
/* ------------------------------------------------------------------ */
|
|
4172
|
+
/** cached default instance of the orchestrator (getOrchestrator merge). */
|
|
4173
|
+
let defaultorchestrator;
|
|
4174
|
+
/**
|
|
4175
|
+
* returns the singleton orchestrator, creating it on the first call with
|
|
4176
|
+
* options; later calls ignore their options and always return the same
|
|
4177
|
+
* instance.
|
|
4178
|
+
*/
|
|
4179
|
+
function getorchestrator(options) {
|
|
4180
|
+
if (defaultorchestrator !== undefined) {
|
|
4181
|
+
return defaultorchestrator;
|
|
4182
|
+
}
|
|
4183
|
+
if (options === undefined) {
|
|
4184
|
+
throw new orchestratorerror('orchestrator not initialized; provide options on the first call', 'ERR_SINGLETON');
|
|
4185
|
+
}
|
|
4186
|
+
defaultorchestrator = new orchestrator(options);
|
|
4187
|
+
return defaultorchestrator;
|
|
4188
|
+
}
|
|
4189
|
+
/**
|
|
4190
|
+
* end-to-end demo: a vm is created and scheduled, an mttg job flows
|
|
4191
|
+
* through the queue, a passage route resolves and the metrics snapshot
|
|
4192
|
+
* is returned; the vm plane is destroyed before returning so the demo
|
|
4193
|
+
* leaves no state behind.
|
|
4194
|
+
*/
|
|
4195
|
+
export async function orchestratordemo() {
|
|
4196
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
4197
|
+
try {
|
|
4198
|
+
const engine = new orchestrator();
|
|
4199
|
+
const created = await engine.createvm({ name: 'demo-vm', vcpus: 2, vrammib: 2048 });
|
|
4200
|
+
if (created.ok) {
|
|
4201
|
+
engine.enqueuemttg({
|
|
4202
|
+
id: 'demo-job',
|
|
4203
|
+
tenant: 'default',
|
|
4204
|
+
qos: 'burstable',
|
|
4205
|
+
priority: 5,
|
|
4206
|
+
groupid: 'demo',
|
|
4207
|
+
payloadref: 'demo:payload',
|
|
4208
|
+
dependencies: [],
|
|
4209
|
+
});
|
|
4210
|
+
await engine.processmttgqueue(async (job) => {
|
|
4211
|
+
engine.events.publish('mttg:jobdone', { jobid: job.id, durationms: 1 });
|
|
4212
|
+
}, 1);
|
|
4213
|
+
await engine.destroyvm(created.value.id);
|
|
4214
|
+
}
|
|
4215
|
+
const sandbox = __addDisposableResource(env_1, await engine.create(engine
|
|
4216
|
+
.builder()
|
|
4217
|
+
.withimage('debian:trixie-slim')
|
|
4218
|
+
.withruntime('docker')
|
|
4219
|
+
.withcpus(2)
|
|
4220
|
+
.withmemory(2048)
|
|
4221
|
+
.withgpuspoof('rtx5090')
|
|
4222
|
+
.withport(8080)
|
|
4223
|
+
.build()), false);
|
|
4224
|
+
void sandbox;
|
|
4225
|
+
return engine.metrics.snapshot();
|
|
4226
|
+
}
|
|
4227
|
+
catch (e_1) {
|
|
4228
|
+
env_1.error = e_1;
|
|
4229
|
+
env_1.hasError = true;
|
|
4230
|
+
}
|
|
4231
|
+
finally {
|
|
4232
|
+
__disposeResources(env_1);
|
|
4233
|
+
}
|
|
4234
|
+
}
|
|
4235
|
+
/**
|
|
4236
|
+
* plans the Flux GitOps repository layout for SADDLE CRDs (ledger
|
|
4237
|
+
* F-064): branch model (main promoted through staging), one
|
|
4238
|
+
* flux-system bootstrap per cluster, an infrastructure layer holding
|
|
4239
|
+
* the VirtualMachine CRD (buildvmcrdmanifest in compute.ts) plus OPA
|
|
4240
|
+
* gatekeeper policies, and an apps layer holding the per-cluster
|
|
4241
|
+
* VirtualMachine custom resources. the planner emits the full
|
|
4242
|
+
* kustomization.yaml content for each overlay and the exact flux
|
|
4243
|
+
* bootstrap argv (with the GHCR image repository and sync intervals),
|
|
4244
|
+
* so `flux bootstrap` plus a git push is everything the operator runs.
|
|
4245
|
+
*/
|
|
4246
|
+
export function planfluxgitops(opts = {}) {
|
|
4247
|
+
try {
|
|
4248
|
+
const repo = opts.repo ?? 'github.com/wenathlan/saddle-gitops';
|
|
4249
|
+
if (!/^[a-z0-9./:-]+$/.test(repo))
|
|
4250
|
+
throw new Error(`repo ${repo} must be a lowercase git url`);
|
|
4251
|
+
const clusters = opts.clusters ?? ['staging', 'prod'];
|
|
4252
|
+
if (clusters.length === 0)
|
|
4253
|
+
throw new Error('at least one cluster is required');
|
|
4254
|
+
if (clusters.length > 16)
|
|
4255
|
+
throw new Error('at most 16 clusters per repository');
|
|
4256
|
+
const seen = new Set();
|
|
4257
|
+
for (const cluster of clusters) {
|
|
4258
|
+
if (!/^[a-z][a-z0-9-]*$/.test(cluster)) {
|
|
4259
|
+
throw new Error(`cluster ${cluster} must be lowercase alnum/dash`);
|
|
4260
|
+
}
|
|
4261
|
+
if (seen.has(cluster))
|
|
4262
|
+
throw new Error(`duplicate cluster ${cluster}`);
|
|
4263
|
+
seen.add(cluster);
|
|
4264
|
+
}
|
|
4265
|
+
const interval = opts.interval ?? '1m';
|
|
4266
|
+
if (!/^[0-9]+(s|m|h)$/.test(interval))
|
|
4267
|
+
throw new Error('interval must look like 30s, 1m, 2h');
|
|
4268
|
+
const kustomization = (path, resources) => ({
|
|
4269
|
+
path,
|
|
4270
|
+
resources,
|
|
4271
|
+
yaml: [
|
|
4272
|
+
'apiVersion: kustomize.config.k8s.io/v1beta1',
|
|
4273
|
+
'kind: Kustomization',
|
|
4274
|
+
`resources:${resources.map((resource) => `\n - ${resource}`).join('')}`,
|
|
4275
|
+
'',
|
|
4276
|
+
].join('\n'),
|
|
4277
|
+
});
|
|
4278
|
+
const kustomizations = [
|
|
4279
|
+
kustomization('infrastructure/crd', ['virtualmachine-crd.yaml', 'opa-policies.yaml']),
|
|
4280
|
+
kustomization('apps/base', ['virtualmachines.yaml']),
|
|
4281
|
+
...clusters.map((cluster) => kustomization(`apps/overlays/${cluster}`, ['../../base'])),
|
|
4282
|
+
];
|
|
4283
|
+
const bootstrapargv = [
|
|
4284
|
+
'flux bootstrap github',
|
|
4285
|
+
` --owner=${repo.split('/')[0]}`,
|
|
4286
|
+
` --repository=${repo.split('/').pop()}`,
|
|
4287
|
+
' --branch=main',
|
|
4288
|
+
' --path=clusters/${cluster}/flux-system',
|
|
4289
|
+
` --interval=${interval}`,
|
|
4290
|
+
opts.prune === false ? ' --no-prune' : ' --prune=true',
|
|
4291
|
+
opts.selfheal === false ? ' --no-self-heal' : ' --self-heal=true',
|
|
4292
|
+
' --components=source-controller,kustomize-controller,helm-controller,notification-controller',
|
|
4293
|
+
];
|
|
4294
|
+
return {
|
|
4295
|
+
repo,
|
|
4296
|
+
branches: ['main', 'staging'],
|
|
4297
|
+
clusters,
|
|
4298
|
+
kustomizations,
|
|
4299
|
+
bootstrapargv,
|
|
4300
|
+
syncpolicy: { interval, prune: opts.prune !== false, selfheal: opts.selfheal !== false },
|
|
4301
|
+
};
|
|
4302
|
+
}
|
|
4303
|
+
catch (error) {
|
|
4304
|
+
throw new Error(`planfluxgitops failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
4307
|
+
export { allocport, cantransition, clhruntime, composeyaml, createvmid, cricheckpointer, dockerruntime, fcrequest, firecrackerruntime, getorchestrator, gvisorruntime, kataruntime, kvmavailable, lifecyclefromphase, orchestrator, orchestratorerror, performetrics, priorityqueue, qemuruntime, resolvehost, runtimecatalog, runtimeregistry, sandboxbuilder, sandboxevents, sandboxfactory, sandboxhandle, sandboxmetrics, sandboxproxy, stackcatalog, transitions, vmimage, warmpool, };
|
|
4308
|
+
//# sourceMappingURL=orchestrator.js.map
|