@hotcell/daemon 0.1.22 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/server.d.ts +2 -0
- package/dist/api/server.js +145 -14
- package/dist/api/server.js.map +1 -1
- package/dist/audit-check.d.ts +1 -0
- package/dist/audit-check.js +113 -0
- package/dist/audit-check.js.map +1 -0
- package/dist/audit.d.ts +66 -0
- package/dist/audit.js +260 -0
- package/dist/audit.js.map +1 -0
- package/dist/boot-scheduler.d.ts +25 -0
- package/dist/boot-scheduler.js +81 -0
- package/dist/boot-scheduler.js.map +1 -0
- package/dist/config.d.ts +15 -3
- package/dist/config.js +22 -0
- package/dist/config.js.map +1 -1
- package/dist/driver/container.d.ts +15 -0
- package/dist/driver/container.js +70 -13
- package/dist/driver/container.js.map +1 -1
- package/dist/driver/router.d.ts +5 -0
- package/dist/driver/router.js +6 -0
- package/dist/driver/router.js.map +1 -1
- package/dist/driver/types.d.ts +2 -0
- package/dist/e2e-harness.js +39 -26
- package/dist/e2e-harness.js.map +1 -1
- package/dist/egress-check.js +7 -0
- package/dist/egress-check.js.map +1 -1
- package/dist/index.js +16 -4
- package/dist/index.js.map +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/lifecycle.js +24 -2
- package/dist/lifecycle.js.map +1 -1
- package/dist/net/firewall.d.ts +26 -1
- package/dist/net/firewall.js +175 -35
- package/dist/net/firewall.js.map +1 -1
- package/dist/proxy/egress.js +25 -0
- package/dist/proxy/egress.js.map +1 -1
- package/dist/smoke.js +8 -0
- package/dist/smoke.js.map +1 -1
- package/dist/store.d.ts +8 -0
- package/dist/store.js +25 -0
- package/dist/store.js.map +1 -1
- package/dist/tracing.d.ts +1 -0
- package/dist/tracing.js +17 -3
- package/dist/tracing.js.map +1 -1
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { get as httpGet } from "node:http";
|
|
3
3
|
import { BackupRegistry } from "./backups.js";
|
|
4
|
+
import { configureAudit, stopAudit } from "./audit.js";
|
|
4
5
|
import { Capacity } from "./capacity.js";
|
|
6
|
+
import { BootScheduler } from "./boot-scheduler.js";
|
|
5
7
|
import { loadConfig, loadProviderConfigs } from "./config.js";
|
|
6
8
|
import { DriverRouter } from "./driver/router.js";
|
|
7
9
|
import { startReaper } from "./lifecycle.js";
|
|
@@ -14,7 +16,7 @@ import { createProxyServer } from "./proxy/server.js";
|
|
|
14
16
|
import { buildProviders, createEgressProxy } from "./proxy/egress.js";
|
|
15
17
|
import { loadProviderKeyMap } from "./keystore.js";
|
|
16
18
|
import { loadModelPrices } from "./pricing.js";
|
|
17
|
-
import { ensureEgressFirewall } from "./net/firewall.js";
|
|
19
|
+
import { ensureEgressFirewall, startEgressDropMonitor } from "./net/firewall.js";
|
|
18
20
|
import { errorMessage } from "./util.js";
|
|
19
21
|
/** A loopback bind host (unreachable from sandboxes on native-Linux dockerd). */
|
|
20
22
|
function isLoopback(host) {
|
|
@@ -54,9 +56,11 @@ function reportPortInUse(host, port) {
|
|
|
54
56
|
async function main() {
|
|
55
57
|
const config = loadConfig();
|
|
56
58
|
configureLogger({ level: config.logLevel, format: config.logFormat });
|
|
59
|
+
configureAudit({ path: config.auditPath, endpoint: config.auditOtlpEndpoint, headers: config.auditOtlpHeaders });
|
|
57
60
|
configureTracing({
|
|
58
|
-
serviceName: "
|
|
61
|
+
serviceName: "hotcell",
|
|
59
62
|
otlpEndpoint: config.otlpEndpoint,
|
|
63
|
+
headers: config.otlpHeaders,
|
|
60
64
|
ringSize: config.traceRing,
|
|
61
65
|
});
|
|
62
66
|
const store = new SandboxStore(config.dbPath);
|
|
@@ -94,6 +98,7 @@ async function main() {
|
|
|
94
98
|
// Host capacity for the meter + admission control (best-effort detection).
|
|
95
99
|
const host = await driver.hostInfo().catch(() => null);
|
|
96
100
|
const capacity = new Capacity(store, config, host, history, () => driver.poolStats());
|
|
101
|
+
const bootScheduler = new BootScheduler(config.microvmBootConcurrency, config.microvmBootQueueTimeoutMs);
|
|
97
102
|
if (host) {
|
|
98
103
|
log.info("host capacity", {
|
|
99
104
|
memoryMb: host.memoryMb,
|
|
@@ -115,7 +120,7 @@ async function main() {
|
|
|
115
120
|
log.info("provider keys reloaded", { providers: Object.keys(rebuilt).length });
|
|
116
121
|
return { providers: Object.keys(rebuilt).length };
|
|
117
122
|
};
|
|
118
|
-
const server = createApiServer({ config, driver, store, backups, history, capacity, reloadKeys });
|
|
123
|
+
const server = createApiServer({ config, driver, store, backups, history, capacity, bootScheduler, reloadKeys });
|
|
119
124
|
server.on("error", (err) => {
|
|
120
125
|
if (err.code === "EADDRINUSE")
|
|
121
126
|
return reportPortInUse(config.host, config.port);
|
|
@@ -129,6 +134,7 @@ async function main() {
|
|
|
129
134
|
image: config.defaultImage,
|
|
130
135
|
auth: config.apiKey ? "on" : "off",
|
|
131
136
|
otlp: config.otlpEndpoint || "off",
|
|
137
|
+
auditOtlp: config.auditOtlpEndpoint || "off",
|
|
132
138
|
});
|
|
133
139
|
});
|
|
134
140
|
const proxy = createProxyServer({ config, driver, store, capacity });
|
|
@@ -149,13 +155,17 @@ async function main() {
|
|
|
149
155
|
// provider keys (SBX_PROVIDER_KEY_*) are set.
|
|
150
156
|
// Install the host default-deny egress firewall (Linux + SBX_EGRESS_ENFORCE only;
|
|
151
157
|
// advisory no-op otherwise). Best-effort: logs and continues if it can't.
|
|
152
|
-
|
|
158
|
+
let stopEgressDropMonitor = () => { };
|
|
159
|
+
const dropLoggingInstalled = await ensureEgressFirewall(config);
|
|
153
160
|
// Reconcile the egress bridge network at boot so a subnet collision (e.g. a
|
|
154
161
|
// stale network from a pre-rename daemon) fails fast with an actionable message
|
|
155
162
|
// instead of a 422 on the first create.
|
|
156
163
|
if (config.egressEnforce && typeof driver.initEgress === "function") {
|
|
157
164
|
await driver.initEgress();
|
|
158
165
|
}
|
|
166
|
+
if (dropLoggingInstalled) {
|
|
167
|
+
stopEgressDropMonitor = startEgressDropMonitor((sourceIp) => driver.resolveEgressSourceIp(sourceIp), config);
|
|
168
|
+
}
|
|
159
169
|
const providers = buildProviders(config);
|
|
160
170
|
const prices = loadModelPrices(config.modelPricesPath);
|
|
161
171
|
// Native-Linux enforcement footgun: sandboxes reach the daemon via the bridge
|
|
@@ -206,6 +216,7 @@ async function main() {
|
|
|
206
216
|
clearInterval(reaper);
|
|
207
217
|
if (sampler)
|
|
208
218
|
clearInterval(sampler);
|
|
219
|
+
stopEgressDropMonitor();
|
|
209
220
|
// Failsafe: `server.close()`'s callback waits for every connection to drain,
|
|
210
221
|
// and an attached terminal WebSocket / SSE exec stream / keep-alive client
|
|
211
222
|
// NEVER drains — without this the process lingers forever as a listener-less
|
|
@@ -222,6 +233,7 @@ async function main() {
|
|
|
222
233
|
failsafe.unref();
|
|
223
234
|
await driver.shutdown().catch(() => { }); // tear down the VZ warm pool, if any
|
|
224
235
|
stopTracing();
|
|
236
|
+
await stopAudit();
|
|
225
237
|
egress?.close();
|
|
226
238
|
egress?.closeAllConnections();
|
|
227
239
|
proxy.close();
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,GAAG,IAAI,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAqB,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,GAAG,IAAI,OAAO,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAqB,MAAM,mBAAmB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEzC,iFAAiF;AACjF,SAAS,UAAU,CAAC,IAAY;IAC9B,OAAO,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,KAAK,KAAK,CAAC;AACxE,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,IAAI,GAAG,CAAC,KAAe,EAAE,EAAE;QAC/B,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;QAC1E,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QACnC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;YACjB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC;gBAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,IAAI,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YACzD,IAAI,CAAC,OAAO;gBACV,CAAC,CAAC,CAAC,0CAA0C,IAAI,IAAI,IAAI,EAAE;oBACxD,iGAAiG,CAAC;gBACrG,CAAC,CAAC,CAAC,QAAQ,IAAI,uCAAuC;oBACnD,kEAAkE,CAAC,CAAC,CAAC;QAC5E,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IACvC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,oBAAoB;QAC1D,kEAAkE,CAAC,CAAC,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;IAC5B,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACtE,cAAc,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,iBAAiB,EAAE,OAAO,EAAE,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACjH,gBAAgB,CAAC;QACf,WAAW,EAAE,SAAS;QACtB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,OAAO,EAAE,MAAM,CAAC,WAAW;QAC3B,QAAQ,EAAE,MAAM,CAAC,SAAS;KAC3B,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9C,0EAA0E;IAC1E,8EAA8E;IAC9E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAE1D,2EAA2E;IAC3E,4EAA4E;IAC5E,4EAA4E;IAC5E,4EAA4E;IAC5E,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CAAC,uBAAuB,MAAM,CAAC,IAAI,UAAU,EAAE,EAAE,KAAK,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yEAAyE;IACzE,yEAAyE;IACzE,uEAAuE;IACvE,8CAA8C;IAC9C,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,UAAU;YAAE,SAAS;QAC3C,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;QACxB,MAAM,CAAC,YAAY,GAAG,gCAAgC,CAAC;QACvD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClB,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACpF,KAAK,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC/C,GAAG,CAAC,IAAI,CAAC,0DAA0D,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,2EAA2E;IAC3E,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IACtF,MAAM,aAAa,GAAG,IAAI,aAAa,CACrC,MAAM,CAAC,sBAAsB,EAC7B,MAAM,CAAC,yBAAyB,CACjC,CAAC;IACF,IAAI,IAAI,EAAE,CAAC;QACT,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE;YACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK;SACjD,CAAC,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,uEAAuE;IACvE,iEAAiE;IACjE,IAAI,MAAgC,CAAC;IACrC,MAAM,UAAU,GAAG,GAAG,EAAE;QACtB,MAAM,CAAC,YAAY,GAAG,kBAAkB,EAAE,CAAC;QAC3C,yEAAyE;QACzE,6DAA6D;QAC7D,MAAM,CAAC,eAAe,GAAG,mBAAmB,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACvC,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,CAAC;QACjC,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/E,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;IACpD,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,eAAe,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC;IACjH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;QAChD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAChF,GAAG,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QAC3C,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE;YAC3B,GAAG,EAAE,UAAU,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE;YAC3C,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,MAAM,CAAC,YAAY;YAC1B,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK;YAClC,IAAI,EAAE,MAAM,CAAC,YAAY,IAAI,KAAK;YAClC,SAAS,EAAE,MAAM,CAAC,iBAAiB,IAAI,KAAK;SAC7C,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrE,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;QAC/C,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,eAAe,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC1F,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE;QACpD,GAAG,CAAC,IAAI,CAAC,yBAAyB,EAAE;YAClC,GAAG,EAAE,UAAU,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,EAAE;YACrD,QAAQ,EAAE,gCAAgC,MAAM,CAAC,SAAS,GAAG;SAC9D,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,6EAA6E;IAC7E,8EAA8E;IAC9E,8CAA8C;IAC9C,kFAAkF;IAClF,0EAA0E;IAC1E,IAAI,qBAAqB,GAAG,GAAG,EAAE,GAAE,CAAC,CAAC;IACrC,MAAM,oBAAoB,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;IAEhE,4EAA4E;IAC5E,gFAAgF;IAChF,wCAAwC;IACxC,IAAI,MAAM,CAAC,aAAa,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;QACpE,MAAM,MAAM,CAAC,UAAU,EAAE,CAAC;IAC5B,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,qBAAqB,GAAG,sBAAsB,CAC5C,CAAC,QAAQ,EAAE,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EACpD,MAAM,CACP,CAAC;IACJ,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACvD,8EAA8E;IAC9E,+EAA+E;IAC/E,gFAAgF;IAChF,4EAA4E;IAC5E,iFAAiF;IACjF,8EAA8E;IAC9E,kBAAkB;IAClB,MAAM,aAAa,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACzF,MAAM,cAAc,GAClB,MAAM,CAAC,aAAa,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,UAAU,CAAC;QACrG,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;IACxB,IAAI,cAAc,KAAK,MAAM,CAAC,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,IAAI,CAAC,4FAA4F,EAAE;YACrG,IAAI,EAAE,MAAM,CAAC,UAAU;SACxB,CAAC,CAAC;IACL,CAAC;IACD,MAAM;QACJ,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9F,MAAM,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,GAA0B,EAAE,EAAE;QACjD,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,eAAe,CAAC,cAAc,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;QACzF,GAAG,CAAC,KAAK,CAAC,8BAA8B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IACH,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,cAAc,EAAE,GAAG,EAAE;QACrD,GAAG,CAAC,IAAI,CAAC,wBAAwB,EAAE;YACjC,GAAG,EAAE,UAAU,cAAc,IAAI,MAAM,CAAC,UAAU,EAAE;YACpD,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB;SAChG,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,uEAAuE;IACvE,MAAM,MAAM,GACV,MAAM,CAAC,cAAc,GAAG,CAAC;QACvB,CAAC,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;QACnE,CAAC,CAAC,SAAS,CAAC;IAEhB,2EAA2E;IAC3E,MAAM,OAAO,GACX,MAAM,CAAC,iBAAiB,GAAG,CAAC;QAC1B,CAAC,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC;QAChF,CAAC,CAAC,SAAS,CAAC;IAEhB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IAAI,YAAY;YAAE,OAAO;QACzB,YAAY,GAAG,IAAI,CAAC;QACpB,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAC1B,IAAI,MAAM;YAAE,aAAa,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,OAAO;YAAE,aAAa,CAAC,OAAO,CAAC,CAAC;QACpC,qBAAqB,EAAE,CAAC;QACxB,6EAA6E;QAC7E,2EAA2E;QAC3E,6EAA6E;QAC7E,2EAA2E;QAC3E,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE;YAC/B,IAAI,CAAC;gBACH,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;YAAC,MAAM,CAAC;gBACP,oBAAoB;YACtB,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,EAAE,IAAI,CAAC,CAAC;QACT,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,qCAAqC;QAC9E,WAAW,EAAE,CAAC;QACd,MAAM,SAAS,EAAE,CAAC;QAClB,MAAM,EAAE,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,mBAAmB,EAAE,CAAC;QAC9B,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,yEAAyE;QACzE,uDAAuD;QACtD,KAA8C,CAAC,mBAAmB,EAAE,EAAE,CAAC;QACxE,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;YAChB,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,8EAA8E;QAC9E,MAAM,CAAC,mBAAmB,EAAE,CAAC;IAC/B,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;IAC5C,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/lifecycle.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ import type { SandboxRecord } from "./types.js";
|
|
|
28
28
|
* processes, exposed ports) is only dropped on a cold pause — after a snapshot
|
|
29
29
|
* pause it stays valid, because resume restores those processes live.
|
|
30
30
|
*/
|
|
31
|
-
export declare function pauseSandbox(driver: Driver, store: SandboxStore, record: SandboxRecord): Promise<void>;
|
|
31
|
+
export declare function pauseSandbox(driver: Driver, store: SandboxStore, record: SandboxRecord, automatic?: boolean): Promise<void>;
|
|
32
32
|
/**
|
|
33
33
|
* Resume a paused (or stopped) sandbox: recreate its container, reattaching the
|
|
34
34
|
* persistent volume, and mark it running + active. When a `capacity` meter is
|
package/dist/lifecycle.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { CapacityError } from "./capacity.js";
|
|
2
|
+
import { audit } from "./audit.js";
|
|
2
3
|
import { log } from "./logger.js";
|
|
3
4
|
/**
|
|
4
5
|
* Lifecycle FSM transitions and the idle reaper.
|
|
@@ -26,7 +27,7 @@ import { log } from "./logger.js";
|
|
|
26
27
|
* processes, exposed ports) is only dropped on a cold pause — after a snapshot
|
|
27
28
|
* pause it stays valid, because resume restores those processes live.
|
|
28
29
|
*/
|
|
29
|
-
export async function pauseSandbox(driver, store, record) {
|
|
30
|
+
export async function pauseSandbox(driver, store, record, automatic = false) {
|
|
30
31
|
let hibernated = false;
|
|
31
32
|
if (typeof driver.snapshot === "function" && (driver.canSnapshot?.(record.id) ?? true)) {
|
|
32
33
|
try {
|
|
@@ -46,6 +47,7 @@ export async function pauseSandbox(driver, store, record) {
|
|
|
46
47
|
}
|
|
47
48
|
record.status = "paused";
|
|
48
49
|
store.add(record);
|
|
50
|
+
audit({ event: "sandbox.pause", sandboxId: record.id, driver: record.driver ?? driver.name, attributes: { automatic } });
|
|
49
51
|
}
|
|
50
52
|
/**
|
|
51
53
|
* Resume a paused (or stopped) sandbox: recreate its container, reattaching the
|
|
@@ -54,6 +56,7 @@ export async function pauseSandbox(driver, store, record) {
|
|
|
54
56
|
* `CapacityError` instead of over-subscribing the host.
|
|
55
57
|
*/
|
|
56
58
|
export async function resumeSandbox(driver, store, record, capacity) {
|
|
59
|
+
const previousStatus = record.status;
|
|
57
60
|
// Reserve the sandbox's memory for the duration of the start: once the record
|
|
58
61
|
// is written back as `running` it's counted by the meter itself, so the
|
|
59
62
|
// pending reservation is released synchronously after the status write.
|
|
@@ -81,6 +84,7 @@ export async function resumeSandbox(driver, store, record, capacity) {
|
|
|
81
84
|
record.status = "running";
|
|
82
85
|
record.lastActivityAt = new Date().toISOString();
|
|
83
86
|
store.add(record);
|
|
87
|
+
audit({ event: previousStatus === "stopped" ? "sandbox.start" : "sandbox.resume", sandboxId: record.id, driver: record.driver ?? driver.name });
|
|
84
88
|
admitted.release();
|
|
85
89
|
}
|
|
86
90
|
/**
|
|
@@ -99,6 +103,11 @@ export async function reapIdle(driver, store, now = Date.now()) {
|
|
|
99
103
|
continue;
|
|
100
104
|
if (!record.sleepAfterMs || record.sleepAfterMs <= 0)
|
|
101
105
|
continue;
|
|
106
|
+
// An open exec stream is live work, whatever the timestamps say: `touch`
|
|
107
|
+
// marks request ARRIVAL, so a long agent run looks idle while it is still
|
|
108
|
+
// streaming — and cold-pausing it means SIGKILLing that work mid-flight.
|
|
109
|
+
if (store.openExecs(record.id) > 0)
|
|
110
|
+
continue;
|
|
102
111
|
const hasService = store.listExposed(record.id).length > 0;
|
|
103
112
|
const hasLiveProcs = store.listProcesses(record.id).some((p) => p.status === "running");
|
|
104
113
|
if ((hasService || hasLiveProcs) && !driver.canSnapshot?.(record.id))
|
|
@@ -106,7 +115,7 @@ export async function reapIdle(driver, store, now = Date.now()) {
|
|
|
106
115
|
if (now - Date.parse(record.lastActivityAt) < record.sleepAfterMs)
|
|
107
116
|
continue;
|
|
108
117
|
try {
|
|
109
|
-
await pauseSandbox(driver, store, record);
|
|
118
|
+
await pauseSandbox(driver, store, record, true);
|
|
110
119
|
paused.push(record.id);
|
|
111
120
|
}
|
|
112
121
|
catch (err) {
|
|
@@ -121,7 +130,20 @@ export async function reapIdle(driver, store, now = Date.now()) {
|
|
|
121
130
|
*/
|
|
122
131
|
export function startReaper(opts) {
|
|
123
132
|
const { driver, store, intervalMs } = opts;
|
|
133
|
+
let lastTickAt = Date.now();
|
|
124
134
|
const timer = setInterval(() => {
|
|
135
|
+
const gap = Date.now() - lastTickAt;
|
|
136
|
+
lastTickAt = Date.now();
|
|
137
|
+
// A tick arriving far later than scheduled means the host slept — every
|
|
138
|
+
// sandbox's lastActivityAt is stale through no fault of its owner, whose
|
|
139
|
+
// own clocks froze with ours. Reaping on that evidence once SIGKILLed a
|
|
140
|
+
// healthy agent seconds after wake. Sit this cycle out; owners that are
|
|
141
|
+
// really alive re-beat within one interval, and true orphans are still
|
|
142
|
+
// reaped on the next tick.
|
|
143
|
+
if (gap > intervalMs * 2) {
|
|
144
|
+
log.info("host slept; skipping one reap cycle", { gapMs: gap });
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
125
147
|
reapIdle(driver, store)
|
|
126
148
|
.then((ids) => {
|
|
127
149
|
if (ids.length)
|
package/dist/lifecycle.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../src/lifecycle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"lifecycle.js","sourceRoot":"","sources":["../src/lifecycle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAiB,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAEnC,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAIlC;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAc,EACd,KAAmB,EACnB,MAAqB,EACrB,SAAS,GAAG,KAAK;IAEjB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,UAAU,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QACvF,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACjC,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,oDAAoD,EAAE;gBAC7D,OAAO,EAAE,MAAM,CAAC,EAAE;gBAClB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC7B,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,wCAAwC;IAC9E,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;IACzB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;AAC3H,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAc,EACd,KAAmB,EACnB,MAAqB,EACrB,QAAmB;IAEnB,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC;IACrC,8EAA8E;IAC9E,wEAAwE;IACxE,wEAAwE;IACxE,MAAM,QAAQ,GAAG,QAAQ,EAAE,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI;QACpE,EAAE,EAAE,IAAa;QACjB,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC;KAClB,CAAC;IACF,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,KAAK,CAAC;YACjB,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,QAAQ,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,GAAG,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,CAAC,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACjD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,EAAE,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAChJ,QAAQ,CAAC,OAAO,EAAE,CAAC;AACrB,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAc,EACd,KAAmB,EACnB,MAAc,IAAI,CAAC,GAAG,EAAE;IAExB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;YAAE,SAAS;QAC1C,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,IAAI,CAAC;YAAE,SAAS;QAC/D,yEAAyE;QACzE,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC;YAAE,SAAS;QAC7C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3D,MAAM,YAAY,GAAG,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QACxF,IAAI,CAAC,UAAU,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YAAE,SAAS;QAC/E,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG,MAAM,CAAC,YAAY;YAAE,SAAS;QAC5E,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;YAChD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,gCAAgC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,IAI3B;IACC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IAC3C,IAAI,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;QACpC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,wEAAwE;QACxE,yEAAyE;QACzE,wEAAwE;QACxE,wEAAwE;QACxE,uEAAuE;QACvE,2BAA2B;QAC3B,IAAI,GAAG,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,IAAI,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;YAChE,OAAO;QACT,CAAC;QACD,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC;aACpB,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YACZ,IAAI,GAAG,CAAC,MAAM;gBAAE,GAAG,CAAC,IAAI,CAAC,4BAA4B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAClE,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,CAAC,EAAE,UAAU,CAAC,CAAC;IACf,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/net/firewall.d.ts
CHANGED
|
@@ -1,4 +1,29 @@
|
|
|
1
1
|
import type { Config } from "../config.js";
|
|
2
|
+
export interface FirewallDrop {
|
|
3
|
+
sourceIp: string;
|
|
4
|
+
destinationIp: string;
|
|
5
|
+
protocol: string;
|
|
6
|
+
port: number;
|
|
7
|
+
}
|
|
8
|
+
export interface EgressSandboxRef {
|
|
9
|
+
sandboxId: string;
|
|
10
|
+
driver: string;
|
|
11
|
+
}
|
|
12
|
+
export type EgressSourceResolver = (sourceIp: string) => Promise<EgressSandboxRef | undefined>;
|
|
2
13
|
/** Derive the bridge gateway IP (`.1`) from a `/24`-style subnet base. */
|
|
3
14
|
export declare function gatewayOf(subnet: string): string;
|
|
4
|
-
|
|
15
|
+
/** Parse one Linux kernel LOG record emitted by the Hotcell default-deny rule. */
|
|
16
|
+
export declare function parseEgressFirewallDrop(line: string): FirewallDrop | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* Host-side default-deny egress firewall for sandboxes (Linux only).
|
|
19
|
+
* Traffic that misses the gateway/DNS allow rules is logged (rate-limited) and
|
|
20
|
+
* then dropped. The daemon consumes those kernel records and resolves the source
|
|
21
|
+
* container IP through Docker's `sbx.id` label before emitting an audit event.
|
|
22
|
+
*/
|
|
23
|
+
export declare function ensureEgressFirewall(config: Config): Promise<boolean>;
|
|
24
|
+
/**
|
|
25
|
+
* Consume Linux kernel firewall logs. `/dev/kmsg` is preferred; if its permissions
|
|
26
|
+
* prevent access, fall back to `journalctl -k -f`. The rate-limited firewall rule
|
|
27
|
+
* protects the kernel log from a sandbox generating a denial storm.
|
|
28
|
+
*/
|
|
29
|
+
export declare function startEgressDropMonitor(resolveSource: EgressSourceResolver, config: Config): () => void;
|
package/dist/net/firewall.js
CHANGED
|
@@ -1,29 +1,13 @@
|
|
|
1
|
-
import { execFile } from "node:child_process";
|
|
1
|
+
import { execFile, spawn } from "node:child_process";
|
|
2
|
+
import { createReadStream, readFileSync } from "node:fs";
|
|
2
3
|
import { platform } from "node:os";
|
|
3
4
|
import { promisify } from "node:util";
|
|
5
|
+
import { audit } from "../audit.js";
|
|
4
6
|
import { log } from "../logger.js";
|
|
5
7
|
const exec = promisify(execFile);
|
|
6
|
-
/**
|
|
7
|
-
* Host-side default-deny egress firewall for sandboxes (Linux only).
|
|
8
|
-
*
|
|
9
|
-
* When `SBX_EGRESS_ENFORCE=true`, sandboxes run on a dedicated bridge network
|
|
10
|
-
* (`config.egressSubnet`). This installs `iptables` rules in the `DOCKER-USER`
|
|
11
|
-
* chain — the supported, operator-owned hook Docker consults before its own rules
|
|
12
|
-
* — so traffic from that subnet may reach ONLY the egress gateway (the bridge
|
|
13
|
-
* gateway `.1` on `egressPort`) and the pinned DNS resolver; everything else is
|
|
14
|
-
* dropped. The sandbox holds no `CAP_NET_ADMIN`, so it cannot edit these rules.
|
|
15
|
-
*
|
|
16
|
-
* Rules live in a dedicated `SBX-EGRESS` chain that is flushed + rebuilt on every
|
|
17
|
-
* startup (idempotent), with a single jump from `DOCKER-USER`.
|
|
18
|
-
*
|
|
19
|
-
* NOT a hard isolation boundary: a kernel-shared container is weaker than a VM,
|
|
20
|
-
* and on Docker Desktop (macOS) `DOCKER-USER` isn't host-editable — there
|
|
21
|
-
* enforcement degrades to advisory (proxy env is still set, but direct egress is
|
|
22
|
-
* not blocked). Transparent, in-guest-enforced default-deny is the microVM driver's
|
|
23
|
-
* job; this is the best that plain Docker on Linux can do. Requires the daemon to
|
|
24
|
-
* run with iptables privileges (root or `CAP_NET_ADMIN`).
|
|
25
|
-
*/
|
|
26
8
|
const CHAIN = "SBX-EGRESS";
|
|
9
|
+
const DROP_LOG_PREFIX = "HOTCELL_EGRESS_DROP ";
|
|
10
|
+
const DROP_RULE = "hotcell-default-deny";
|
|
27
11
|
/** Derive the bridge gateway IP (`.1`) from a `/24`-style subnet base. */
|
|
28
12
|
export function gatewayOf(subnet) {
|
|
29
13
|
const base = subnet.split("/")[0] ?? subnet;
|
|
@@ -33,45 +17,201 @@ export function gatewayOf(subnet) {
|
|
|
33
17
|
octets[3] = "1";
|
|
34
18
|
return octets.join(".");
|
|
35
19
|
}
|
|
20
|
+
/** Parse one Linux kernel LOG record emitted by the Hotcell default-deny rule. */
|
|
21
|
+
export function parseEgressFirewallDrop(line) {
|
|
22
|
+
const marker = line.indexOf(DROP_LOG_PREFIX);
|
|
23
|
+
if (marker < 0)
|
|
24
|
+
return undefined;
|
|
25
|
+
const message = line.slice(marker + DROP_LOG_PREFIX.length);
|
|
26
|
+
const fields = {};
|
|
27
|
+
for (const match of message.matchAll(/(?:^|\s)([A-Z][A-Z0-9_]*)=([^\s]*)/g)) {
|
|
28
|
+
fields[match[1]] = match[2];
|
|
29
|
+
}
|
|
30
|
+
if (!fields.SRC || !fields.DST)
|
|
31
|
+
return undefined;
|
|
32
|
+
const parsedPort = Number(fields.DPT);
|
|
33
|
+
return {
|
|
34
|
+
sourceIp: fields.SRC,
|
|
35
|
+
destinationIp: fields.DST,
|
|
36
|
+
protocol: (fields.PROTO ?? "unknown").toLowerCase(),
|
|
37
|
+
port: Number.isInteger(parsedPort) && parsedPort >= 0 ? parsedPort : 0,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/** Kernel log timestamps are monotonic microseconds since boot (`/dev/kmsg`). */
|
|
41
|
+
function kernelUptimeUs(line) {
|
|
42
|
+
const semicolon = line.indexOf(";");
|
|
43
|
+
if (semicolon < 0)
|
|
44
|
+
return undefined; // journalctl output has no kmsg metadata
|
|
45
|
+
const metadata = line.slice(0, semicolon).replace(/^<\d+>/, "").split(",");
|
|
46
|
+
const value = Number(metadata[2]);
|
|
47
|
+
return Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Host-side default-deny egress firewall for sandboxes (Linux only).
|
|
51
|
+
* Traffic that misses the gateway/DNS allow rules is logged (rate-limited) and
|
|
52
|
+
* then dropped. The daemon consumes those kernel records and resolves the source
|
|
53
|
+
* container IP through Docker's `sbx.id` label before emitting an audit event.
|
|
54
|
+
*/
|
|
36
55
|
export async function ensureEgressFirewall(config) {
|
|
37
56
|
if (!config.egressEnforce)
|
|
38
|
-
return;
|
|
57
|
+
return false;
|
|
39
58
|
if (platform() !== "linux") {
|
|
40
59
|
log.warn("egress enforcement requested but host is not Linux — fail-closed firewall is ADVISORY only " +
|
|
41
60
|
"(proxy env is injected, but direct egress is NOT blocked). Use the microVM driver for hard isolation.", { platform: platform() });
|
|
42
|
-
return;
|
|
61
|
+
return false;
|
|
43
62
|
}
|
|
44
63
|
const subnet = config.egressSubnet;
|
|
45
64
|
const gateway = gatewayOf(subnet);
|
|
46
65
|
const port = String(config.egressPort);
|
|
47
66
|
const dns = config.egressDnsResolver;
|
|
48
|
-
// The ordered rule set for the SBX-EGRESS chain. Allows return traffic, the
|
|
49
|
-
// gateway, and DNS (to the pinned resolver if set, else the bridge gateway, which
|
|
50
|
-
// Docker's embedded resolver forwards through); drops the rest.
|
|
51
|
-
const rules = [
|
|
52
|
-
["-m", "state", "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT"],
|
|
53
|
-
["-p", "tcp", "-d", gateway, "--dport", port, "-j", "ACCEPT"],
|
|
54
|
-
["-p", "udp", "-d", dns || gateway, "--dport", "53", "-j", "ACCEPT"],
|
|
55
|
-
["-p", "tcp", "-d", dns || gateway, "--dport", "53", "-j", "ACCEPT"],
|
|
56
|
-
["-j", "DROP"],
|
|
57
|
-
];
|
|
58
67
|
try {
|
|
59
68
|
// Create (ignore "exists"), then flush so we rebuild cleanly each boot.
|
|
60
69
|
await ipt(["-N", CHAIN]).catch(() => { });
|
|
61
70
|
await ipt(["-F", CHAIN]);
|
|
62
|
-
|
|
71
|
+
const allowRules = [
|
|
72
|
+
["-m", "state", "--state", "ESTABLISHED,RELATED", "-j", "ACCEPT"],
|
|
73
|
+
["-p", "tcp", "-d", gateway, "--dport", port, "-j", "ACCEPT"],
|
|
74
|
+
["-p", "udp", "-d", dns || gateway, "--dport", "53", "-j", "ACCEPT"],
|
|
75
|
+
["-p", "tcp", "-d", dns || gateway, "--dport", "53", "-j", "ACCEPT"],
|
|
76
|
+
];
|
|
77
|
+
for (const rule of allowRules)
|
|
63
78
|
await ipt(["-A", CHAIN, ...rule]);
|
|
79
|
+
// Logging is optional; install the DROP regardless if LOG/limit is unsupported.
|
|
80
|
+
let logInstalled = false;
|
|
81
|
+
try {
|
|
82
|
+
await ipt([
|
|
83
|
+
"-A", CHAIN,
|
|
84
|
+
"-m", "conntrack", "--ctstate", "NEW",
|
|
85
|
+
"-m", "limit", "--limit", "20/second", "--limit-burst", "50",
|
|
86
|
+
"-j", "LOG", "--log-prefix", DROP_LOG_PREFIX, "--log-level", "4",
|
|
87
|
+
]);
|
|
88
|
+
logInstalled = true;
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
log.warn("egress firewall drop logging unavailable; DROP enforcement remains active", {
|
|
92
|
+
error: String(err?.message ?? err),
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
await ipt(["-A", CHAIN, "-j", "DROP"]);
|
|
64
96
|
// Jump from DOCKER-USER for our subnet, exactly once.
|
|
65
97
|
const jump = ["-s", subnet, "-j", CHAIN];
|
|
66
98
|
const present = await ipt(["-C", "DOCKER-USER", ...jump]).then(() => true, () => false);
|
|
67
99
|
if (!present)
|
|
68
100
|
await ipt(["-I", "DOCKER-USER", ...jump]);
|
|
69
|
-
log.info("egress firewall installed (default-deny)", {
|
|
101
|
+
log.info("egress firewall installed (default-deny)", {
|
|
102
|
+
subnet, gateway, port, dns: dns || "embedded", dropAudit: logInstalled,
|
|
103
|
+
});
|
|
104
|
+
return logInstalled;
|
|
70
105
|
}
|
|
71
106
|
catch (err) {
|
|
72
107
|
log.error("egress firewall: failed to install iptables rules — egress is NOT locked down. " +
|
|
73
108
|
"Run the daemon with root / CAP_NET_ADMIN, or disable SBX_EGRESS_ENFORCE.", { error: String(err?.message ?? err) });
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Consume Linux kernel firewall logs. `/dev/kmsg` is preferred; if its permissions
|
|
114
|
+
* prevent access, fall back to `journalctl -k -f`. The rate-limited firewall rule
|
|
115
|
+
* protects the kernel log from a sandbox generating a denial storm.
|
|
116
|
+
*/
|
|
117
|
+
export function startEgressDropMonitor(resolveSource, config) {
|
|
118
|
+
if (platform() !== "linux")
|
|
119
|
+
return () => { };
|
|
120
|
+
let stopped = false;
|
|
121
|
+
let reader;
|
|
122
|
+
let journal;
|
|
123
|
+
let pending = "";
|
|
124
|
+
let processing = Promise.resolve();
|
|
125
|
+
let startUptimeUs = 0;
|
|
126
|
+
try {
|
|
127
|
+
startUptimeUs = Number(readFileSync("/proc/uptime", "utf8").trim().split(/\s+/)[0]) * 1_000_000;
|
|
128
|
+
if (!Number.isFinite(startUptimeUs))
|
|
129
|
+
startUptimeUs = 0;
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
// Without /proc uptime, the parser still works; it may replay buffered records.
|
|
74
133
|
}
|
|
134
|
+
const handleLine = (line) => {
|
|
135
|
+
const drop = parseEgressFirewallDrop(line);
|
|
136
|
+
if (!drop)
|
|
137
|
+
return;
|
|
138
|
+
const eventTime = kernelUptimeUs(line);
|
|
139
|
+
if (eventTime !== undefined && eventTime < startUptimeUs)
|
|
140
|
+
return;
|
|
141
|
+
processing = processing.then(async () => {
|
|
142
|
+
if (stopped)
|
|
143
|
+
return;
|
|
144
|
+
let source;
|
|
145
|
+
try {
|
|
146
|
+
source = await resolveSource(drop.sourceIp);
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
log.warn("could not map a firewall drop to a sandbox", {
|
|
150
|
+
sourceIp: drop.sourceIp,
|
|
151
|
+
error: String(err?.message ?? err),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
audit({
|
|
155
|
+
event: "egress.connection.denied",
|
|
156
|
+
sandboxId: source?.sandboxId,
|
|
157
|
+
driver: source?.driver ?? "container",
|
|
158
|
+
attributes: {
|
|
159
|
+
host: drop.destinationIp,
|
|
160
|
+
port: drop.port,
|
|
161
|
+
protocol: drop.protocol,
|
|
162
|
+
rule: DROP_RULE,
|
|
163
|
+
direction: "outbound",
|
|
164
|
+
"source.ip": drop.sourceIp,
|
|
165
|
+
network: config.egressNetwork,
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
}).catch((err) => log.warn("failed to process an egress firewall record", {
|
|
169
|
+
error: String(err?.message ?? err),
|
|
170
|
+
}));
|
|
171
|
+
};
|
|
172
|
+
const handleChunk = (chunk) => {
|
|
173
|
+
pending += chunk.toString();
|
|
174
|
+
let newline;
|
|
175
|
+
while ((newline = pending.indexOf("\n")) >= 0) {
|
|
176
|
+
const line = pending.slice(0, newline);
|
|
177
|
+
pending = pending.slice(newline + 1);
|
|
178
|
+
handleLine(line);
|
|
179
|
+
}
|
|
180
|
+
if (pending.length > 16_384)
|
|
181
|
+
pending = "";
|
|
182
|
+
};
|
|
183
|
+
const startJournalFallback = () => {
|
|
184
|
+
if (stopped || journal)
|
|
185
|
+
return;
|
|
186
|
+
const child = spawn("journalctl", ["-k", "-f", "-o", "cat", "--since=now"], {
|
|
187
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
188
|
+
});
|
|
189
|
+
journal = child;
|
|
190
|
+
child.stdout?.on("data", handleChunk);
|
|
191
|
+
child.once("error", (err) => log.warn("egress drop audit unavailable (no kernel log reader)", {
|
|
192
|
+
error: String(err),
|
|
193
|
+
}));
|
|
194
|
+
child.once("exit", (code) => {
|
|
195
|
+
if (!stopped)
|
|
196
|
+
log.warn("kernel journal reader exited", { code });
|
|
197
|
+
});
|
|
198
|
+
};
|
|
199
|
+
const stream = createReadStream("/dev/kmsg", { encoding: "utf8" });
|
|
200
|
+
reader = stream;
|
|
201
|
+
stream.on("data", handleChunk);
|
|
202
|
+
stream.once("error", (err) => {
|
|
203
|
+
if (stopped)
|
|
204
|
+
return;
|
|
205
|
+
log.warn("cannot read /dev/kmsg; trying journalctl for egress drop audit", {
|
|
206
|
+
error: String(err),
|
|
207
|
+
});
|
|
208
|
+
startJournalFallback();
|
|
209
|
+
});
|
|
210
|
+
return () => {
|
|
211
|
+
stopped = true;
|
|
212
|
+
reader?.destroy();
|
|
213
|
+
journal?.kill("SIGTERM");
|
|
214
|
+
};
|
|
75
215
|
}
|
|
76
216
|
/** Run an `iptables` command, throwing with stderr on failure. */
|
|
77
217
|
async function ipt(args) {
|
package/dist/net/firewall.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"firewall.js","sourceRoot":"","sources":["../../src/net/firewall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"firewall.js","sourceRoot":"","sources":["../../src/net/firewall.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEzD,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEpC,OAAO,EAAE,GAAG,EAAE,MAAM,cAAc,CAAC;AAEnC,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AACjC,MAAM,KAAK,GAAG,YAAY,CAAC;AAC3B,MAAM,eAAe,GAAG,sBAAsB,CAAC;AAC/C,MAAM,SAAS,GAAG,sBAAsB,CAAC;AAgBzC,0EAA0E;AAC1E,MAAM,UAAU,SAAS,CAAC,MAAc;IACtC,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;IAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC;IAChB,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7C,IAAI,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACjC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,qCAAqC,CAAC,EAAE,CAAC;QAC5E,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IACjD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO;QACL,QAAQ,EAAE,MAAM,CAAC,GAAG;QACpB,aAAa,EAAE,MAAM,CAAC,GAAG;QACzB,QAAQ,EAAE,CAAC,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC,CAAC,WAAW,EAAE;QACnD,IAAI,EAAE,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;KACvE,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,SAAS,cAAc,CAAC,IAAY;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,yCAAyC;IAC9E,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,MAAc;IACvD,IAAI,CAAC,MAAM,CAAC,aAAa;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,QAAQ,EAAE,KAAK,OAAO,EAAE,CAAC;QAC3B,GAAG,CAAC,IAAI,CACN,6FAA6F;YAC3F,uGAAuG,EACzG,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CACzB,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC;IACnC,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAErC,IAAI,CAAC;QACH,wEAAwE;QACxE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzB,MAAM,UAAU,GAAe;YAC7B,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,qBAAqB,EAAE,IAAI,EAAE,QAAQ,CAAC;YACjE,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;YAC7D,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;YACpE,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC;SACrE,CAAC;QACF,KAAK,MAAM,IAAI,IAAI,UAAU;YAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QAEjE,gFAAgF;QAChF,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,GAAG,CAAC;gBACR,IAAI,EAAE,KAAK;gBACX,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,KAAK;gBACrC,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,eAAe,EAAE,IAAI;gBAC5D,IAAI,EAAE,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,GAAG;aACjE,CAAC,CAAC;YACH,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,2EAA2E,EAAE;gBACpF,KAAK,EAAE,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC;aAC9C,CAAC,CAAC;QACL,CAAC;QACD,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;QAEvC,sDAAsD;QACtD,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAC5D,GAAG,EAAE,CAAC,IAAI,EACV,GAAG,EAAE,CAAC,KAAK,CACZ,CAAC;QACF,IAAI,CAAC,OAAO;YAAE,MAAM,GAAG,CAAC,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;QACxD,GAAG,CAAC,IAAI,CAAC,0CAA0C,EAAE;YACnD,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,UAAU,EAAE,SAAS,EAAE,YAAY;SACvE,CAAC,CAAC;QACH,OAAO,YAAY,CAAC;IACtB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,KAAK,CACP,iFAAiF;YAC/E,0EAA0E,EAC5E,EAAE,KAAK,EAAE,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC,EAAE,CAClD,CAAC;QACF,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,sBAAsB,CACpC,aAAmC,EACnC,MAAc;IAEd,IAAI,QAAQ,EAAE,KAAK,OAAO;QAAE,OAAO,GAAG,EAAE,GAAE,CAAC,CAAC;IAE5C,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,MAA4B,CAAC;IACjC,IAAI,OAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,UAAU,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IACnC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC;QACH,aAAa,GAAG,MAAM,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QAChG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;YAAE,aAAa,GAAG,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,gFAAgF;IAClF,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,EAAE;QAClC,MAAM,IAAI,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,aAAa;YAAE,OAAO;QACjE,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,OAAO;gBAAE,OAAO;YACpB,IAAI,MAAoC,CAAC;YACzC,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,4CAA4C,EAAE;oBACrD,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,KAAK,EAAE,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC;iBAC9C,CAAC,CAAC;YACL,CAAC;YACD,KAAK,CAAC;gBACJ,KAAK,EAAE,0BAA0B;gBACjC,SAAS,EAAE,MAAM,EAAE,SAAS;gBAC5B,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,WAAW;gBACrC,UAAU,EAAE;oBACV,IAAI,EAAE,IAAI,CAAC,aAAa;oBACxB,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,IAAI,EAAE,SAAS;oBACf,SAAS,EAAE,UAAU;oBACrB,WAAW,EAAE,IAAI,CAAC,QAAQ;oBAC1B,OAAO,EAAE,MAAM,CAAC,aAAa;iBAC9B;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,6CAA6C,EAAE;YACxE,KAAK,EAAE,MAAM,CAAE,GAAa,EAAE,OAAO,IAAI,GAAG,CAAC;SAC9C,CAAC,CAAC,CAAC;IACN,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,KAAsB,EAAE,EAAE;QAC7C,OAAO,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC5B,IAAI,OAAe,CAAC;QACpB,OAAO,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACvC,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,UAAU,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,MAAM;YAAE,OAAO,GAAG,EAAE,CAAC;IAC5C,CAAC,CAAC;IAEF,MAAM,oBAAoB,GAAG,GAAG,EAAE;QAChC,IAAI,OAAO,IAAI,OAAO;YAAE,OAAO;QAC/B,MAAM,KAAK,GAAG,KAAK,CAAC,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,EAAE;YAC1E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC;QACH,OAAO,GAAG,KAAK,CAAC;QAChB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,sDAAsD,EAAE;YAC5F,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;SACnB,CAAC,CAAC,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC1B,IAAI,CAAC,OAAO;gBAAE,GAAG,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACnE,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,gBAAgB,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,MAAM,GAAG,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAC/B,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;QAC3B,IAAI,OAAO;YAAE,OAAO;QACpB,GAAG,CAAC,IAAI,CAAC,gEAAgE,EAAE;YACzE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;SACnB,CAAC,CAAC;QACH,oBAAoB,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,EAAE;QACV,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,EAAE,OAAO,EAAE,CAAC;QAClB,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,GAAG,CAAC,IAAc;IAC/B,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC;AAC1C,CAAC"}
|