@nylorun/runtime 0.25.0-beta → 0.27.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +169 -0
- package/README.md +26 -16
- package/dist/adapters/execution/restate.d.ts +15 -18
- package/dist/adapters/execution/restate.js +31 -27
- package/dist/adapters/replication/pgoutput.d.ts +1 -1
- package/dist/adapters/replication/pgoutput.js +3 -2
- package/dist/adapters/sandbox/local.js +8 -5
- package/dist/adapters/sandbox/virtual.js +2 -2
- package/dist/adapters/streams/s2.d.ts +1 -1
- package/dist/adapters/streams/s2.js +18 -14
- package/dist/api/a2a/card.js +1 -1
- package/dist/api/a2a/endpoint.js +3 -4
- package/dist/api/a2a/protocol.js +4 -0
- package/dist/api/a2a/routes.d.ts +2 -6
- package/dist/api/a2a/routes.js +12 -18
- package/dist/api/ag-ui/endpoint.js +2 -3
- package/dist/api/ag-ui/routes.js +4 -4
- package/dist/api/components.d.ts +326 -45
- package/dist/api/components.js +5 -1
- package/dist/api/gate/keys.js +7 -4
- package/dist/api/gate/routes.d.ts +6 -2
- package/dist/api/gate/routes.js +15 -11
- package/dist/api/http/app.d.ts +2 -2
- package/dist/api/http/app.js +2 -2
- package/dist/api/http/body.d.ts +15 -3
- package/dist/api/http/body.js +15 -7
- package/dist/api/http/define.d.ts +1 -1
- package/dist/api/http/respond.js +7 -3
- package/dist/api/http/routes/access.js +5 -4
- package/dist/api/http/routes/artifacts.js +19 -7
- package/dist/api/http/routes/coverage.js +1 -1
- package/dist/api/http/routes/environment-manifests.js +1 -1
- package/dist/api/http/routes/environments.js +6 -6
- package/dist/api/http/routes/files.d.ts +2 -1
- package/dist/api/http/routes/files.js +31 -5
- package/dist/api/http/routes/mcp.js +1 -1
- package/dist/api/http/routes/reads.d.ts +35 -0
- package/dist/api/http/routes/reads.js +50 -22
- package/dist/api/http/routes/sessions.js +12 -11
- package/dist/api/http/routes/tenant.d.ts +2 -2
- package/dist/api/http/routes/tenant.js +69 -16
- package/dist/api/inbound-files.d.ts +3 -9
- package/dist/api/inbound-files.js +18 -24
- package/dist/api/openapi.js +6 -2
- package/dist/artifacts/disk.d.ts +0 -4
- package/dist/artifacts/disk.js +77 -73
- package/dist/artifacts/files.d.ts +13 -4
- package/dist/artifacts/files.js +27 -9
- package/dist/artifacts/folder-upload.d.ts +75 -0
- package/dist/artifacts/folder-upload.js +263 -0
- package/dist/artifacts/folders.d.ts +14 -8
- package/dist/artifacts/folders.js +19 -11
- package/dist/artifacts/links.js +7 -6
- package/dist/artifacts/parts.d.ts +9 -14
- package/dist/artifacts/parts.js +11 -9
- package/dist/artifacts/pipe.d.ts +15 -0
- package/dist/artifacts/pipe.js +71 -0
- package/dist/artifacts/quota.d.ts +36 -0
- package/dist/artifacts/quota.js +44 -0
- package/dist/artifacts/service.d.ts +25 -1
- package/dist/artifacts/service.js +70 -25
- package/dist/blob/fs.js +2 -1
- package/dist/blob/s3.d.ts +6 -2
- package/dist/blob/s3.js +9 -7
- package/dist/blob/types.d.ts +1 -1
- package/dist/blob/types.js +1 -1
- package/dist/core/limits.d.ts +11 -35
- package/dist/core/limits.js +5 -73
- package/dist/core/sandbox-routes.js +0 -1
- package/dist/core/turn-manifest.d.ts +1 -2
- package/dist/core/turn-manifest.js +4 -28
- package/dist/execution/memory.d.ts +8 -5
- package/dist/execution/memory.js +8 -6
- package/dist/execution/types.d.ts +1 -6
- package/dist/execution/types.js +0 -5
- package/dist/gates/contract.d.ts +3 -2
- package/dist/gates/contract.js +1 -1
- package/dist/gates/egress-credentials.js +4 -3
- package/dist/gates/egress.d.ts +9 -9
- package/dist/gates/egress.js +19 -19
- package/dist/gates/fixture-model.js +13 -2
- package/dist/gates/handler.d.ts +1 -3
- package/dist/gates/handler.js +4 -3
- package/dist/gates/http-client.d.ts +4 -3
- package/dist/gates/http-client.js +5 -7
- package/dist/gates/http-tool.d.ts +24 -4
- package/dist/gates/http-tool.js +53 -17
- package/dist/gates/in-process.d.ts +2 -1
- package/dist/gates/in-process.js +17 -9
- package/dist/gates/inflight.d.ts +2 -2
- package/dist/gates/inflight.js +3 -2
- package/dist/gates/journal.js +5 -7
- package/dist/gates/mcp-handler.d.ts +4 -3
- package/dist/gates/mcp-handler.js +15 -5
- package/dist/gates/model-gate.d.ts +11 -2
- package/dist/gates/policy.d.ts +28 -5
- package/dist/gates/policy.js +60 -17
- package/dist/gates/provider-gate.d.ts +22 -0
- package/dist/gates/provider-gate.js +30 -0
- package/dist/gates/tool-calls.d.ts +1 -3
- package/dist/gates/tool-calls.js +2 -3
- package/dist/gates/tool-client.d.ts +1 -1
- package/dist/gates/tool-client.js +5 -6
- package/dist/gates/tool-contract.d.ts +8 -5
- package/dist/gates/tool-contract.js +1 -1
- package/dist/gates/tool-gate.d.ts +18 -7
- package/dist/gates/tool-gate.js +49 -7
- package/dist/harness/calls.d.ts +27 -25
- package/dist/harness/calls.js +40 -46
- package/dist/harness/executors.d.ts +10 -3
- package/dist/harness/executors.js +53 -23
- package/dist/harness/main.js +2 -2
- package/dist/harness/pod.d.ts +0 -2
- package/dist/harness/pod.js +7 -6
- package/dist/harness/run-tokens.d.ts +4 -1
- package/dist/harness/run-tokens.js +6 -2
- package/dist/harness/service.d.ts +17 -14
- package/dist/harness/service.js +21 -35
- package/dist/harness/setup.d.ts +0 -2
- package/dist/harness/setup.js +4 -9
- package/dist/harness/ws-client.d.ts +2 -2
- package/dist/harness/ws-client.js +7 -5
- package/dist/harness/ws-port.d.ts +3 -4
- package/dist/harness/ws-port.js +3 -4
- package/dist/harness-api/artifacts.d.ts +20 -0
- package/dist/harness-api/artifacts.js +478 -0
- package/dist/harness-api/in-process.d.ts +2 -2
- package/dist/harness-api/in-process.js +18 -16
- package/dist/harness-api/record.d.ts +10 -9
- package/dist/harness-api/record.js +102 -157
- package/dist/harness-api/renew.d.ts +1 -1
- package/dist/harness-api/renew.js +2 -2
- package/dist/harness-api/server.d.ts +1 -16
- package/dist/harness-api/server.js +82 -73
- package/dist/harness-api/start.d.ts +20 -2
- package/dist/harness-api/start.js +57 -18
- package/dist/harness-api/tool-journal.d.ts +15 -10
- package/dist/harness-api/tool-journal.js +26 -24
- package/dist/harness-api/workspace.d.ts +0 -4
- package/dist/harness-api/workspace.js +2 -45
- package/dist/harness-api/ws-server.js +10 -10
- package/dist/host/create-host.js +2 -3
- package/dist/host/execution.js +2 -3
- package/dist/host/gates.d.ts +2 -9
- package/dist/host/gates.js +15 -12
- package/dist/host/http.d.ts +0 -2
- package/dist/host/http.js +0 -27
- package/dist/host/main.js +10 -3
- package/dist/host/operate.js +12 -3
- package/dist/host/stack-config.d.ts +11 -0
- package/dist/host/stack-config.js +21 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/infra/index.js +2 -1
- package/dist/infra/readiness.d.ts +0 -2
- package/dist/infra/readiness.js +4 -3
- package/dist/keys/client.d.ts +1 -0
- package/dist/keys/client.js +2 -3
- package/dist/keys/contract.d.ts +1 -3
- package/dist/keys/contract.js +1 -3
- package/dist/keys/egress-ca.d.ts +0 -2
- package/dist/keys/egress-ca.js +2 -3
- package/dist/keys/keys.js +6 -1
- package/dist/keys/x509.d.ts +8 -4
- package/dist/keys/x509.js +14 -5
- package/dist/management-openapi.json +1961 -994
- package/dist/mcp/connect.d.ts +35 -26
- package/dist/mcp/connect.js +112 -141
- package/dist/mcp/pool.d.ts +27 -22
- package/dist/mcp/pool.js +34 -48
- package/dist/mcp/preview.d.ts +13 -6
- package/dist/mcp/preview.js +37 -20
- package/dist/mcp/snapshot.d.ts +35 -86
- package/dist/mcp/snapshot.js +93 -135
- package/dist/model/auth-store.js +5 -3
- package/dist/model/capabilities.d.ts +1 -1
- package/dist/model/catalog.d.ts +9 -36
- package/dist/model/catalog.js +6 -33
- package/dist/model/descriptor.d.ts +51 -0
- package/dist/model/descriptor.js +83 -0
- package/dist/model/models.d.ts +0 -3
- package/dist/model/models.js +3 -5
- package/dist/model/native-blocks.js +1 -1
- package/dist/model/pi-model.d.ts +16 -5
- package/dist/model/pi-model.js +86 -105
- package/dist/openapi.json +866 -1218
- package/dist/reads/cursor.js +3 -1
- package/dist/record/append.d.ts +34 -2
- package/dist/record/append.js +50 -1
- package/dist/record/environment.d.ts +2 -0
- package/dist/record/environment.js +2 -0
- package/dist/record/index.d.ts +1 -1
- package/dist/record/index.js +1 -1
- package/dist/redact.d.ts +1 -1
- package/dist/redact.js +3 -2
- package/dist/sandbox/egress-token.d.ts +3 -3
- package/dist/sandbox/egress-token.js +3 -3
- package/dist/sandbox/join.js +3 -2
- package/dist/sandbox/manager.d.ts +41 -41
- package/dist/sandbox/manager.js +112 -44
- package/dist/sandbox/placement.d.ts +1 -1
- package/dist/sandbox/placement.js +5 -8
- package/dist/sandbox/pods/client.d.ts +1 -10
- package/dist/sandbox/pods/client.js +8 -4
- package/dist/sandbox/pods/lifecycle.d.ts +19 -11
- package/dist/sandbox/pods/lifecycle.js +45 -23
- package/dist/sandbox/pods/network-gate.d.ts +4 -5
- package/dist/sandbox/pods/network-gate.js +7 -7
- package/dist/sandbox/pods/reconcile.js +4 -1
- package/dist/sandbox/pods/spec.js +8 -5
- package/dist/sandbox/policy.d.ts +7 -1
- package/dist/sandbox/policy.js +10 -3
- package/dist/sandbox/resolve.d.ts +8 -3
- package/dist/sandbox/resolve.js +12 -2
- package/dist/sandbox/setup.d.ts +3 -1
- package/dist/sandbox/setup.js +34 -11
- package/dist/sandbox/share.d.ts +5 -14
- package/dist/sandbox/share.js +5 -34
- package/dist/sandbox/skills.d.ts +7 -13
- package/dist/sandbox/skills.js +16 -31
- package/dist/sandbox/tenant-config.d.ts +2 -2
- package/dist/sandbox/tenant-config.js +4 -3
- package/dist/sandbox/tools.d.ts +7 -1
- package/dist/sandbox/tools.js +26 -26
- package/dist/sandbox/types.d.ts +10 -3
- package/dist/sandbox/types.js +9 -0
- package/dist/store/postgres/connect.d.ts +6 -5
- package/dist/store/postgres/connect.js +14 -11
- package/dist/store/postgres/control.js +13 -13
- package/dist/store/postgres/drizzle/0028_artifact_write_keys.sql +2 -0
- package/dist/store/postgres/drizzle/0029_protocol_15.sql +11 -0
- package/dist/store/postgres/drizzle/0030_protocol_15_runtime_block.sql +11 -0
- package/dist/store/postgres/drizzle/0031_protocol_15_mcp_snapshot.sql +8 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
- package/dist/store/postgres/migrate.js +1 -1
- package/dist/store/postgres/reads.js +10 -2
- package/dist/store/postgres/schema.d.ts +19 -0
- package/dist/store/postgres/schema.js +7 -0
- package/dist/store/postgres/store.js +55 -26
- package/dist/store/types.d.ts +17 -7
- package/dist/streams/relay/core.d.ts +3 -3
- package/dist/streams/relay/core.js +9 -10
- package/dist/tenant/advance.js +21 -31
- package/dist/tenant/commands.js +9 -16
- package/dist/tenant/context.d.ts +14 -17
- package/dist/tenant/definition-files.d.ts +25 -4
- package/dist/tenant/definition-files.js +85 -47
- package/dist/tenant/effects.js +9 -36
- package/dist/tenant/environment-manifests.js +9 -1
- package/dist/tenant/environments.d.ts +0 -2
- package/dist/tenant/environments.js +54 -30
- package/dist/tenant/ephemeral.d.ts +1 -1
- package/dist/tenant/ephemeral.js +2 -1
- package/dist/tenant/history.d.ts +1 -1
- package/dist/tenant/history.js +1 -1
- package/dist/tenant/host-token.d.ts +3 -3
- package/dist/tenant/host-token.js +12 -11
- package/dist/tenant/http.d.ts +5 -8
- package/dist/tenant/http.js +9 -0
- package/dist/tenant/identity-file.d.ts +24 -1
- package/dist/tenant/identity-file.js +35 -7
- package/dist/tenant/issuers.d.ts +5 -5
- package/dist/tenant/issuers.js +31 -23
- package/dist/tenant/jwt.d.ts +8 -2
- package/dist/tenant/jwt.js +10 -2
- package/dist/tenant/limits.d.ts +90 -0
- package/dist/tenant/limits.js +175 -0
- package/dist/tenant/module.d.ts +4 -1
- package/dist/tenant/module.js +9 -8
- package/dist/tenant/outbound.d.ts +17 -19
- package/dist/tenant/outbound.js +18 -21
- package/dist/tenant/run-grants.js +3 -2
- package/dist/tenant/run-token.d.ts +0 -4
- package/dist/tenant/run-token.js +6 -11
- package/dist/tenant/runtime-limits.d.ts +24 -0
- package/dist/tenant/runtime-limits.js +27 -0
- package/dist/tenant/runtime.js +30 -43
- package/dist/tenant/session-streams.js +7 -6
- package/dist/tenant/session.d.ts +1 -1
- package/dist/tenant/session.js +1 -1
- package/dist/tenant/sessions.js +32 -29
- package/dist/tenant/signing-keys.d.ts +1 -1
- package/dist/tenant/signing-keys.js +5 -6
- package/dist/tenant/status.js +4 -3
- package/dist/tenant/streams.js +18 -20
- package/dist/tenant/sweep.d.ts +1 -21
- package/dist/tenant/sweep.js +39 -48
- package/dist/tenant/transcript.d.ts +70 -23
- package/dist/tenant/transcript.js +28 -22
- package/dist/tenant/types.d.ts +11 -16
- package/dist/tenant/worker.d.ts +2 -10
- package/dist/tenant/worker.js +5 -14
- package/dist/traces/ids.js +5 -1
- package/dist/traces/map.d.ts +9 -6
- package/dist/traces/map.js +9 -11
- package/dist/traces/otlp.d.ts +3 -3
- package/dist/traces/otlp.js +5 -3
- package/dist/traces/projector.d.ts +9 -4
- package/dist/traces/projector.js +13 -7
- package/dist/traces/read.js +6 -1
- package/dist/tuning.d.ts +383 -0
- package/dist/tuning.js +658 -0
- package/dist/vault/headers.js +4 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -3
- package/dist/artifacts/export.d.ts +0 -8
- package/dist/artifacts/export.js +0 -204
- package/dist/artifacts/media-types.d.ts +0 -30
- package/dist/artifacts/media-types.js +0 -137
- package/dist/artifacts/workspace.d.ts +0 -60
- package/dist/artifacts/workspace.js +0 -45
- package/dist/media.d.ts +0 -10
- package/dist/media.js +0 -58
- package/dist/sandbox/runtime-tools.d.ts +0 -9
- package/dist/sandbox/runtime-tools.js +0 -16
- package/dist/sandbox/session-sandbox.d.ts +0 -16
- package/dist/sandbox/session-sandbox.js +0 -77
- package/dist/tenant/artifact-tool.d.ts +0 -13
- package/dist/tenant/artifact-tool.js +0 -243
- package/dist/tenant/skill-tool.d.ts +0 -20
- package/dist/tenant/skill-tool.js +0 -67
- package/dist/tenant/tool-results.d.ts +0 -57
- package/dist/tenant/tool-results.js +0 -298
- package/dist/tenant/tool-search.d.ts +0 -39
- package/dist/tenant/tool-search.js +0 -103
package/dist/tenant/runtime.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { mkdirSync } from "node:fs";
|
|
2
|
-
import { resolveFlowLimits } from "../core/limits.js";
|
|
3
2
|
import { scriptedModel, toolFixtureModel, } from "../core/provider.js";
|
|
4
3
|
import { tenantModelGate } from "../gates/in-process.js";
|
|
5
4
|
import { createKekFile, readVaultKek } from "../vault/kek.js";
|
|
6
5
|
import { SigningKeys } from "./signing-keys.js";
|
|
6
|
+
import { readRuntimeLimits } from "./runtime-limits.js";
|
|
7
7
|
import { VaultService } from "../vault/service.js";
|
|
8
8
|
import { McpPool } from "../mcp/pool.js";
|
|
9
9
|
import { SandboxManager } from "../sandbox/manager.js";
|
|
@@ -20,7 +20,8 @@ import { advance } from "./advance.js";
|
|
|
20
20
|
import { sweep } from "./sweep.js";
|
|
21
21
|
import { TenantWorkers, WORKER_ID, } from "./worker.js";
|
|
22
22
|
import { authorize } from "./effects.js";
|
|
23
|
-
import { inProcessToolGate } from "../gates/tool-gate.js";
|
|
23
|
+
import { inProcessToolGate, openMcpThrough } from "../gates/tool-gate.js";
|
|
24
|
+
import { providerModelGate } from "../gates/provider-gate.js";
|
|
24
25
|
import { inProcessKeys } from "../keys/keys.js";
|
|
25
26
|
import { tenantApi } from "../api/http/app.js";
|
|
26
27
|
import { createFsBlobStore } from "../blob/index.js";
|
|
@@ -31,24 +32,16 @@ import { localWorkspace, remoteWorkspace, withPodWorkspaces, } from "../harness-
|
|
|
31
32
|
import { hostAuthority } from "../sandbox/join.js";
|
|
32
33
|
import { harnessOfToken } from "./harness-tokens.js";
|
|
33
34
|
import { reconcileSandbox } from "../sandbox/pods/reconcile.js";
|
|
34
|
-
import {
|
|
35
|
-
import { readDefinitionFile } from "./definition-files.js";
|
|
36
|
-
/** Default ownership lease of an advance; the heartbeat renews it every third. */
|
|
37
|
-
const DEFAULT_OWNER_LEASE_MS = 30_000;
|
|
35
|
+
import { hostTuning } from "../tuning.js";
|
|
38
36
|
function validateConfig(config) {
|
|
39
37
|
for (const key of ["leaseMs", "ownerLeaseMs", "sweepIntervalMs"]) {
|
|
40
38
|
const value = config[key];
|
|
41
39
|
if (value !== undefined && (!Number.isFinite(value) || value <= 0))
|
|
42
40
|
throw new Error(`${key} must be finite and positive`);
|
|
43
41
|
}
|
|
44
|
-
const flowLimits = resolveFlowLimits({
|
|
45
|
-
flow: config.flow,
|
|
46
|
-
env: config.flowEnv,
|
|
47
|
-
});
|
|
48
42
|
if ((config.model.kind === "fixture" || config.model.kind === "scripted") &&
|
|
49
43
|
config.mode === "shared")
|
|
50
44
|
throw new Error("fixture/scripted models require ephemeral or test mode");
|
|
51
|
-
return flowLimits;
|
|
52
45
|
}
|
|
53
46
|
export class TenantRuntime {
|
|
54
47
|
ctx;
|
|
@@ -81,7 +74,7 @@ export class TenantRuntime {
|
|
|
81
74
|
}
|
|
82
75
|
static async open(config, hooks) {
|
|
83
76
|
const envelope = hooks.envelope;
|
|
84
|
-
|
|
77
|
+
validateConfig(config);
|
|
85
78
|
const paths = config.paths;
|
|
86
79
|
mkdirSync(paths.root, { recursive: true });
|
|
87
80
|
mkdirSync(paths.home, { recursive: true });
|
|
@@ -128,6 +121,7 @@ export class TenantRuntime {
|
|
|
128
121
|
inProcessToolGate(config.delivery ?? {}, {
|
|
129
122
|
session: (sessionId) => opened.tx((t) => t.get("sessions", sessionId)),
|
|
130
123
|
authorize: (sessionId, request) => authorize(ctx, sessionId, request),
|
|
124
|
+
limits: () => opened.tx((t) => readRuntimeLimits(t)),
|
|
131
125
|
logger: config.logger,
|
|
132
126
|
});
|
|
133
127
|
const signingKeys = new SigningKeys({ tenantId: config.tenantId, kek: ensureKek });
|
|
@@ -144,13 +138,9 @@ export class TenantRuntime {
|
|
|
144
138
|
const preference = seededBackend ?? config.sandbox.backend;
|
|
145
139
|
// With a harness in this process, the Tenant's MCP pool and SandboxManager are its
|
|
146
140
|
// executors; otherwise the harness holds both (F6.2).
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
policy: config.delivery ?? {},
|
|
151
|
-
...(toolGate.openMcp ? { openRemote: (server) => toolGate.openMcp(server) } : {}),
|
|
152
|
-
})
|
|
153
|
-
: undefined;
|
|
141
|
+
// The pool opens every server through the Tool Gate: core never authorizes an MCP
|
|
142
|
+
// connection outside a gate (protocol 15).
|
|
143
|
+
const mcp = inProcess ? new McpPool({ openRemote: openMcpThrough(toolGate) }) : undefined;
|
|
154
144
|
const ephemeral = config.mode === "ephemeral";
|
|
155
145
|
const manager = inProcess
|
|
156
146
|
? new SandboxManager({
|
|
@@ -160,12 +150,6 @@ export class TenantRuntime {
|
|
|
160
150
|
defaultSandboxBackends({ root: paths.sandboxes }),
|
|
161
151
|
preference,
|
|
162
152
|
ephemeral,
|
|
163
|
-
definitionFile: async (sha256) => {
|
|
164
|
-
const bytes = await readDefinitionFile(blobs, sha256);
|
|
165
|
-
if (!bytes)
|
|
166
|
-
throw new Error(`The Object store has no bytes for ${sha256}`);
|
|
167
|
-
return bytes;
|
|
168
|
-
},
|
|
169
153
|
emit: async (sessionId, turnId, type, payload) => {
|
|
170
154
|
if (ctx.closed)
|
|
171
155
|
return;
|
|
@@ -201,9 +185,13 @@ export class TenantRuntime {
|
|
|
201
185
|
isPod: async (sandboxId) => (await opened.tx((t) => t.environment(sandboxId)))?.kind === "pod",
|
|
202
186
|
})
|
|
203
187
|
: workspaces;
|
|
204
|
-
|
|
188
|
+
// A model that is not the vault's (scripted, the fixture, an embedder's provider) is
|
|
189
|
+
// served by a gate over it, metered like any (protocol 15): no harness calls a provider.
|
|
190
|
+
const scripted = config.model.kind !== "vault";
|
|
205
191
|
let modelProvider;
|
|
206
|
-
if (
|
|
192
|
+
if (!scripted)
|
|
193
|
+
modelProvider = undefined;
|
|
194
|
+
else if (hooks.modelProvider)
|
|
207
195
|
modelProvider = hooks.modelProvider;
|
|
208
196
|
else if (config.model.kind === "scripted")
|
|
209
197
|
modelProvider = scriptedModel(config.model.output);
|
|
@@ -214,21 +202,23 @@ export class TenantRuntime {
|
|
|
214
202
|
const blobs = hooks.blobs ?? createFsBlobStore({ root: paths.blobs });
|
|
215
203
|
// With the gates service the loop never reads a model credential: only a Runtime
|
|
216
204
|
// without one (embedding, the ephemeral Runtime, tests) reads it here.
|
|
217
|
-
const modelGate =
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
205
|
+
const modelGate = modelProvider
|
|
206
|
+
? providerModelGate(modelProvider, { store: opened, logger: config.logger })
|
|
207
|
+
: (hooks.modelGate ??
|
|
208
|
+
tenantModelGate({
|
|
209
|
+
store: opened,
|
|
210
|
+
kek: ensureKek,
|
|
211
|
+
root: paths.home,
|
|
212
|
+
logger: config.logger,
|
|
213
|
+
blobs,
|
|
214
|
+
}));
|
|
226
215
|
const sessionStreams = createSessionStreams();
|
|
227
216
|
const workers = hooks.execution?.workers ?? new TenantWorkers();
|
|
228
217
|
const local = hooks.execution
|
|
229
218
|
? undefined
|
|
230
219
|
: new MemoryExecution({
|
|
231
|
-
sweepIntervalMs: config.sweepIntervalMs ??
|
|
220
|
+
sweepIntervalMs: config.sweepIntervalMs ??
|
|
221
|
+
Math.min(config.leaseMs ?? Infinity, hostTuning().sweep.intervalMs),
|
|
232
222
|
// Close's `detach` step waits no longer for running work than for its advances.
|
|
233
223
|
stopGraceMs: workers.graceMs,
|
|
234
224
|
onError: (error) => config.logger.error("tenant execution failed", {
|
|
@@ -245,9 +235,7 @@ export class TenantRuntime {
|
|
|
245
235
|
vault,
|
|
246
236
|
...(mcp ? { mcp } : {}),
|
|
247
237
|
sandbox,
|
|
248
|
-
|
|
249
|
-
modelProvider,
|
|
250
|
-
useVaultModel,
|
|
238
|
+
scriptedModel: scripted,
|
|
251
239
|
modelGate,
|
|
252
240
|
toolGate,
|
|
253
241
|
get harness() {
|
|
@@ -260,11 +248,10 @@ export class TenantRuntime {
|
|
|
260
248
|
signingKeys,
|
|
261
249
|
keys,
|
|
262
250
|
blobs,
|
|
263
|
-
workspaces: sandboxWorkspaceReader(sandbox),
|
|
264
251
|
...(hooks.traces ? { traces: hooks.traces } : {}),
|
|
265
252
|
...(hooks.runGrants ? { runGrants: hooks.runGrants } : {}),
|
|
266
253
|
workerId: hooks.workerId ?? WORKER_ID,
|
|
267
|
-
ownerLeaseMs: config.ownerLeaseMs ??
|
|
254
|
+
ownerLeaseMs: config.ownerLeaseMs ?? hostTuning().tenant.ownerLeaseMs,
|
|
268
255
|
wake: async (sessionId, wake) => {
|
|
269
256
|
if (ctx.closing || ctx.closed)
|
|
270
257
|
return false;
|
|
@@ -372,7 +359,7 @@ export class TenantRuntime {
|
|
|
372
359
|
harness: harnessStatusOf(this.ctx),
|
|
373
360
|
};
|
|
374
361
|
}
|
|
375
|
-
drain(activeWork, timeoutMs =
|
|
362
|
+
drain(activeWork, timeoutMs = hostTuning().tenant.drainTimeoutMs) {
|
|
376
363
|
return drain(this.ctx, activeWork, timeoutMs);
|
|
377
364
|
}
|
|
378
365
|
/**
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
import { historyCursor, historyResume } from "../reads/cursor.js";
|
|
22
22
|
import { decodeCursor, encodeCursor } from "../record/index.js";
|
|
23
23
|
import { sleep } from "../retry.js";
|
|
24
|
+
import { hostTuning } from "../tuning.js";
|
|
24
25
|
import { basinOf } from "../streams/basin.js";
|
|
25
26
|
import { sessionStream, } from "../streams/types.js";
|
|
26
27
|
import { fail } from "./http.js";
|
|
@@ -33,8 +34,6 @@ export function streamHolderOf(scope) {
|
|
|
33
34
|
export function createSessionStreams() {
|
|
34
35
|
return { wiring: undefined, generation: 0, sessions: new Map() };
|
|
35
36
|
}
|
|
36
|
-
const RETRY_MIN_MS = 100;
|
|
37
|
-
const RETRY_MAX_MS = 2000;
|
|
38
37
|
function frame(event) {
|
|
39
38
|
return `id: ${event.cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`;
|
|
40
39
|
}
|
|
@@ -66,7 +65,7 @@ export function openSse(request, response, onClose) {
|
|
|
66
65
|
"x-accel-buffering": "no",
|
|
67
66
|
});
|
|
68
67
|
response.flushHeaders();
|
|
69
|
-
const timer = setInterval(() => send(response, ": keepalive\n\n"),
|
|
68
|
+
const timer = setInterval(() => send(response, ": keepalive\n\n"), hostTuning().api.eventStreamKeepaliveMs);
|
|
70
69
|
timer.unref();
|
|
71
70
|
request.on("close", () => {
|
|
72
71
|
clearInterval(timer);
|
|
@@ -313,7 +312,9 @@ function runFeed(ctx, feed) {
|
|
|
313
312
|
const read = feed.read;
|
|
314
313
|
const signal = read.signal;
|
|
315
314
|
void (async () => {
|
|
316
|
-
|
|
315
|
+
// A failed read is retried after `streams.retry` (Host tuning), doubling.
|
|
316
|
+
const { minMs: retryMinMs, maxMs: retryMaxMs } = hostTuning().streams.retry;
|
|
317
|
+
let delay = retryMinMs;
|
|
317
318
|
while (!signal.aborted) {
|
|
318
319
|
const streams = ctx.sessionStreams.wiring?.streams;
|
|
319
320
|
if (!streams)
|
|
@@ -326,7 +327,7 @@ function runFeed(ctx, feed) {
|
|
|
326
327
|
if (record.seq < feed.next)
|
|
327
328
|
continue;
|
|
328
329
|
feed.next = record.seq + 1;
|
|
329
|
-
delay =
|
|
330
|
+
delay = retryMinMs;
|
|
330
331
|
for (const observer of feed.observers)
|
|
331
332
|
deliver(observer, record);
|
|
332
333
|
}
|
|
@@ -345,7 +346,7 @@ function runFeed(ctx, feed) {
|
|
|
345
346
|
if (!(await stillCurrent(ctx, feed)))
|
|
346
347
|
break;
|
|
347
348
|
await sleep(delay, signal);
|
|
348
|
-
delay = Math.min(delay * 2,
|
|
349
|
+
delay = Math.min(delay * 2, retryMaxMs);
|
|
349
350
|
}
|
|
350
351
|
if (signal.aborted || feed.read !== read)
|
|
351
352
|
return;
|
package/dist/tenant/session.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
import type { AgentManifest } from "@nylorun/core/define";
|
|
10
10
|
import type { TurnManifestStore } from "../core/turn-manifest.js";
|
|
11
11
|
import type { Session } from "./context.js";
|
|
12
|
-
export { manifestFor
|
|
12
|
+
export { manifestFor } from "../mcp/snapshot.js";
|
|
13
13
|
/** Turn-manifest store backed by the session's `variants` map (no schema change). */
|
|
14
14
|
export declare function variantStore(session: Session): TurnManifestStore;
|
|
15
15
|
/** Manifest this turn's checkpoint is pinned to (session pin or a stored variant). */
|
package/dist/tenant/session.js
CHANGED
package/dist/tenant/sessions.js
CHANGED
|
@@ -9,14 +9,17 @@ import { approvalRefusal, codeToolRefusal } from "@nylorun/core/define";
|
|
|
9
9
|
import { aggregateWaits, isWorkflowManifest } from "../core/flow-host.js";
|
|
10
10
|
import { canonical } from "../store/canonical.js";
|
|
11
11
|
import { sandboxSpecOf } from "../sandbox/resolve.js";
|
|
12
|
-
import {
|
|
13
|
-
import { authoredRuntimeBlocks, declaredSandboxes } from "../sandbox/share.js";
|
|
12
|
+
import { declaredSandboxes } from "../sandbox/share.js";
|
|
14
13
|
import { effectiveSandboxConfig, readSandboxConfig } from "../sandbox/tenant-config.js";
|
|
15
14
|
import { HARNESS_ID, checkPlacement, requirePods } from "../sandbox/placement.js";
|
|
16
15
|
import { openEnvironment, recordAttachment } from "./environments.js";
|
|
17
16
|
import { harnessOf, } from "./context.js";
|
|
18
17
|
import { fail } from "./http.js";
|
|
19
|
-
import {
|
|
18
|
+
import { definitionFileIssues } from "./definition-files.js";
|
|
19
|
+
import { limitsInForce, readSkillLimits, readTenantLimits } from "./limits.js";
|
|
20
|
+
import { readRuntimeLimits } from "./runtime-limits.js";
|
|
21
|
+
import { httpToolTimeoutIssues } from "../gates/http-tool.js";
|
|
22
|
+
import { requiresIssues } from "./limits.js";
|
|
20
23
|
/** A session's creation identity: everything but the request id. */
|
|
21
24
|
const sessionIdentity = (value) => {
|
|
22
25
|
const { requestId: _requestId, ...body } = value ?? {};
|
|
@@ -66,18 +69,25 @@ export async function putDefinition(ctx, agentId, body, options = {}) {
|
|
|
66
69
|
const declared = declaredSandboxes(body.manifest);
|
|
67
70
|
if (declared.length > 0)
|
|
68
71
|
fail(400, `${declared.join(", ")} declares a sandbox. Agents no longer declare one: remove .sandbox(), create an Environment with the sandbox (PUT /v1/environments/{id}) and open the session on it, createSession({ environment }). See MIGRATION.md.`);
|
|
69
|
-
const runtime = authoredRuntimeBlocks(body.manifest);
|
|
70
|
-
if (runtime.length > 0)
|
|
71
|
-
fail(400, `${runtime.join(", ")} sets runtime, which only the Runtime writes when it pins a session's agent. Remove it and register the manifest your builder makes.`);
|
|
72
72
|
const definition = {
|
|
73
73
|
...body,
|
|
74
74
|
manifestHash: hashManifest(body.manifest),
|
|
75
75
|
};
|
|
76
76
|
const files = [...definitionFilesOf(body.manifest)];
|
|
77
77
|
await ctx.store.tx(async (t) => {
|
|
78
|
-
const
|
|
78
|
+
const limits = await readRuntimeLimits(t);
|
|
79
|
+
const issues = [
|
|
80
|
+
...httpToolTimeoutIssues(body.manifest, limits.httpToolTimeoutMs),
|
|
81
|
+
...requiresIssues(body.manifest.requires, limits),
|
|
82
|
+
];
|
|
83
|
+
if (issues.length > 0)
|
|
84
|
+
fail(400, issues.join(" "));
|
|
85
|
+
const { totalBytes } = await readSkillLimits(t);
|
|
86
|
+
const { missing, oversized } = await definitionFileIssues(t, body.manifest, totalBytes);
|
|
79
87
|
if (missing.length > 0)
|
|
80
88
|
fail(400, `The definition names ${missing.length} file(s) the Runtime does not hold; upload each with PUT /v1/files/<sha256> first: ${missing.join(", ")}`, { code: "definition_files_missing", details: { missing } });
|
|
89
|
+
if (oversized.length > 0)
|
|
90
|
+
fail(413, `This Tenant allows a skill's files at most ${totalBytes} bytes together: ${oversized.map((skill) => `'${skill.name}' holds ${skill.bytes}`).join(", ")}`, { code: "limit_exceeded", details: { limitBytes: totalBytes, skills: oversized } });
|
|
81
91
|
if (options.dryRun)
|
|
82
92
|
return;
|
|
83
93
|
await t.put("definitions", agentId, definition);
|
|
@@ -158,6 +168,8 @@ export async function putSession(ctx, id, body, access, options = {}) {
|
|
|
158
168
|
});
|
|
159
169
|
if (!isWorkflowManifest(definition.manifest))
|
|
160
170
|
await checkRequirements(t, definition.manifest, environment);
|
|
171
|
+
if (environment !== undefined)
|
|
172
|
+
await checkSkillLimit(t, definition.manifest, environment);
|
|
161
173
|
const config = await readSandboxConfig(t);
|
|
162
174
|
// The harness the session runs for its life (R4.1): its Environment's, else ours. A flow
|
|
163
175
|
// runs in core and our engine, whatever its Environment names.
|
|
@@ -171,13 +183,14 @@ export async function putSession(ctx, id, body, access, options = {}) {
|
|
|
171
183
|
const spec = environment === undefined || environment.kind === "none"
|
|
172
184
|
? undefined
|
|
173
185
|
: sandboxSpecOf(environment.applied, effectiveSandboxConfig(config).limits.idle);
|
|
174
|
-
|
|
186
|
+
// The session pins its definition as registered (protocol 15): the sandbox's spec stays on
|
|
187
|
+
// the session, and the harness words the facts and defines the tools.
|
|
175
188
|
const created = {
|
|
176
189
|
id,
|
|
177
190
|
agentId: body.agentId,
|
|
178
191
|
ownerUserId: body.ownerUserId,
|
|
179
|
-
manifest:
|
|
180
|
-
manifestHash:
|
|
192
|
+
manifest: definition.manifest,
|
|
193
|
+
manifestHash: definition.manifestHash,
|
|
181
194
|
definitionHash: definition.manifestHash,
|
|
182
195
|
implementationVersion: definition.implementationVersion,
|
|
183
196
|
info: body.info,
|
|
@@ -224,26 +237,16 @@ async function checkRequirements(t, manifest, environment) {
|
|
|
224
237
|
fail(409, `Agent ${manifest.id} requires ${missing.map((gap) => (gap.startsWith("secret:") ? `the secret ${gap.slice(7)}` : gap === "pod" ? "a pod Environment" : "a sandbox")).join(", ")}, and ${where}.`, { code: "environment_unsuitable", details: { missing } });
|
|
225
238
|
}
|
|
226
239
|
/**
|
|
227
|
-
*
|
|
228
|
-
* `
|
|
240
|
+
* Refuses a session whose agent carries a skill larger than its Environment allows: its
|
|
241
|
+
* manifest's `limits.skills.totalBytes`, else the Tenant's, never above the Tenant's.
|
|
242
|
+
* `409 environment_unsuitable`, with each skill and its size in `details.skills`.
|
|
229
243
|
*/
|
|
230
|
-
function
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
return { manifest: pinned.manifest, manifestHash: pinned.manifestHash };
|
|
237
|
-
}
|
|
238
|
-
/** The manifest a session with a sandbox pins: the definition with the sandbox in its `runtime` block. */
|
|
239
|
-
function pin(definition, spec) {
|
|
240
|
-
// A workflow's manifest stays as registered: its agents inherit the sandbox when they start.
|
|
241
|
-
if (isWorkflowManifest(definition.manifest))
|
|
242
|
-
return {};
|
|
243
|
-
const sandboxed = withSandbox(definition.manifest, spec);
|
|
244
|
-
if (!sandboxed.ok)
|
|
245
|
-
return fail(400, sandboxed.message);
|
|
246
|
-
return { manifest: sandboxed.manifest, manifestHash: sandboxed.manifestHash };
|
|
244
|
+
async function checkSkillLimit(t, manifest, environment) {
|
|
245
|
+
const limitBytes = limitsInForce(await readTenantLimits(t), environment.applied.limits).skills.totalBytes;
|
|
246
|
+
const { oversized } = await definitionFileIssues(t, manifest, limitBytes);
|
|
247
|
+
if (oversized.length === 0)
|
|
248
|
+
return;
|
|
249
|
+
fail(409, `Environment ${environment.id} allows a skill's files at most ${limitBytes} bytes together, and agent ${manifest.id} carries ${oversized.map((skill) => `'${skill.name}' (${skill.bytes})`).join(", ")}.`, { code: "environment_unsuitable", details: { limitBytes, skills: oversized } });
|
|
247
250
|
}
|
|
248
251
|
/** The session resource returned by `GET`/`PUT /v1/sessions/:id`. */
|
|
249
252
|
export async function sessionView(t, s) {
|
|
@@ -48,7 +48,7 @@ export declare class SigningKeys {
|
|
|
48
48
|
/**
|
|
49
49
|
* Rotates: previous → revoked, current → previous, standby → current, new standby. Refused
|
|
50
50
|
* with 409 while the previous key may still verify live tokens (retired less than the
|
|
51
|
-
* longest token lifetime, `maxTtlSeconds` or `
|
|
51
|
+
* longest token lifetime, `maxTtlSeconds` or `keys.tokenTtlMaxSeconds`, plus a minute ago), unless
|
|
52
52
|
* `force`.
|
|
53
53
|
*/
|
|
54
54
|
rotate(t: Tx, kek: Buffer, maxTtlSeconds: number, force: boolean): Promise<SigningKeyRow[]>;
|
|
@@ -15,12 +15,10 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { exportJWK, exportPKCS8, generateKeyPair, importJWK, importPKCS8, } from "jose";
|
|
17
17
|
import { newSigningKeyId, SIGNING_KEY_ID_PATTERN, } from "@nylorun/core/compatibility";
|
|
18
|
-
import { TOKEN_TTL_MAX_SECONDS } from "@nylorun/core/contracts";
|
|
19
18
|
import { decryptSecret, encryptSecret } from "../vault/crypto.js";
|
|
20
19
|
import { fail } from "./http.js";
|
|
20
|
+
import { hostTuning } from "../tuning.js";
|
|
21
21
|
const ALG = "ES256";
|
|
22
|
-
/** Grace added to the longest token lifetime before a previous key may be revoked. */
|
|
23
|
-
const ROTATION_GRACE_MS = 60_000;
|
|
24
22
|
export class SigningKeys {
|
|
25
23
|
options;
|
|
26
24
|
publicKeys = new Map();
|
|
@@ -107,7 +105,7 @@ export class SigningKeys {
|
|
|
107
105
|
/**
|
|
108
106
|
* Rotates: previous → revoked, current → previous, standby → current, new standby. Refused
|
|
109
107
|
* with 409 while the previous key may still verify live tokens (retired less than the
|
|
110
|
-
* longest token lifetime, `maxTtlSeconds` or `
|
|
108
|
+
* longest token lifetime, `maxTtlSeconds` or `keys.tokenTtlMaxSeconds`, plus a minute ago), unless
|
|
111
109
|
* `force`.
|
|
112
110
|
*/
|
|
113
111
|
async rotate(t, kek, maxTtlSeconds, force) {
|
|
@@ -118,8 +116,9 @@ export class SigningKeys {
|
|
|
118
116
|
const [previous] = await t.signingKeys(["previous"]);
|
|
119
117
|
if (previous && !force) {
|
|
120
118
|
const retired = Date.parse(previous.retiredAt ?? previous.createdAt);
|
|
121
|
-
const longest = Math.max(maxTtlSeconds,
|
|
122
|
-
|
|
119
|
+
const longest = Math.max(maxTtlSeconds, hostTuning().keys.tokenTtlMaxSeconds);
|
|
120
|
+
// The grace added to the longest token lifetime before a previous key may be revoked.
|
|
121
|
+
const until = retired + longest * 1000 + hostTuning().tenant.signingKeyRotationGraceMs;
|
|
123
122
|
if (until > now.getTime())
|
|
124
123
|
fail(409, "The previous key may still verify live tokens", {
|
|
125
124
|
code: "request_rejected",
|
package/dist/tenant/status.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { FIXTURE_MODEL_SETTING, seedFixtureModel } from "./model-setting.js";
|
|
2
2
|
import { ENVIRONMENTS_CONFIG_SETTING, writeSandboxConfig } from "../sandbox/tenant-config.js";
|
|
3
|
+
import { hostTuning } from "../tuning.js";
|
|
3
4
|
/** The Tenant's harnesses, for its status and the Host's. */
|
|
4
5
|
export function harnessStatusOf(ctx) {
|
|
5
6
|
return { mode: ctx.mcp ? "in-process" : "remote", ...ctx.harness.status() };
|
|
6
7
|
}
|
|
7
|
-
/** How long status waits for the execution to list stuck invocations. */
|
|
8
|
-
const STUCK_TIMEOUT_MS = 5000;
|
|
9
8
|
/** `TenantStatus.execution`: paused and backing-off invocations, or why they are unknown. */
|
|
10
9
|
async function executionStatus(stuckInvocations) {
|
|
11
10
|
let timer;
|
|
@@ -13,7 +12,9 @@ async function executionStatus(stuckInvocations) {
|
|
|
13
12
|
const stuck = await Promise.race([
|
|
14
13
|
stuckInvocations(),
|
|
15
14
|
new Promise((_, reject) => {
|
|
16
|
-
timer = setTimeout(() => reject(new Error("Timed out listing stuck invocations")),
|
|
15
|
+
timer = setTimeout(() => reject(new Error("Timed out listing stuck invocations")),
|
|
16
|
+
// How long status waits for the execution to list stuck invocations.
|
|
17
|
+
hostTuning().tenant.statusTimeoutMs);
|
|
17
18
|
}),
|
|
18
19
|
]);
|
|
19
20
|
return {
|
package/dist/tenant/streams.js
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { basinOf } from "../streams/basin.js";
|
|
2
2
|
import { createStreamRelay, } from "../streams/relay/core.js";
|
|
3
3
|
import { retry } from "../retry.js";
|
|
4
|
+
import { hostTuning } from "../tuning.js";
|
|
4
5
|
import { checkSessionStreams, currentBasin } from "./session-streams.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
/** How long a retired basin stays for readers that have not moved yet. */
|
|
14
|
-
const RETIRE_GRACE_MS = 60_000;
|
|
6
|
+
/*
|
|
7
|
+
* Timings are the Host tuning's `streams` group (`tuning.ts`): `retry` for deletions and
|
|
8
|
+
* repairs, `basinRetryMaxMs` (the longest wait between two `ensureTenant` attempts while the
|
|
9
|
+
* basin is being repaired), `basinRepairCooldownMs` (a repair started within this long after the
|
|
10
|
+
* last one succeeded waits for the next failure), `feedCheckMs` (how often every session stream is
|
|
11
|
+
* checked against its session, the backstop for lost signals), `retireGraceMs` (how long a retired
|
|
12
|
+
* basin stays for readers that have not moved yet).
|
|
13
|
+
*/
|
|
15
14
|
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
16
15
|
/**
|
|
17
16
|
* Wires the Tenant to Durable Streams: reads its basin generation, checks the basin, starts
|
|
@@ -25,7 +24,7 @@ export async function wireStreams(ctx, options) {
|
|
|
25
24
|
const { store, streams, tenantId } = options;
|
|
26
25
|
const logger = ctx.config.logger;
|
|
27
26
|
const stop = new AbortController();
|
|
28
|
-
const graceMs = options.retireGraceMs ??
|
|
27
|
+
const graceMs = options.retireGraceMs ?? hostTuning().streams.retireGraceMs;
|
|
29
28
|
// Nothing is reported once the wiring stops: the Tenant (and its log directory) may be gone,
|
|
30
29
|
// and a throwing report in a detached loop would be an unhandled rejection.
|
|
31
30
|
const report = (what) => (error) => {
|
|
@@ -53,7 +52,7 @@ export async function wireStreams(ctx, options) {
|
|
|
53
52
|
});
|
|
54
53
|
relay?.start();
|
|
55
54
|
// Streams whose session was reset end on the `sessions.reset` signal; this catches lost ones.
|
|
56
|
-
const feedCheck = setInterval(() => void checkSessionStreams(ctx).catch(report("session stream check failed")),
|
|
55
|
+
const feedCheck = setInterval(() => void checkSessionStreams(ctx).catch(report("session stream check failed")), hostTuning().streams.feedCheckMs);
|
|
57
56
|
feedCheck.unref();
|
|
58
57
|
const retiring = new Set();
|
|
59
58
|
const timers = new Set();
|
|
@@ -94,9 +93,8 @@ export async function wireStreams(ctx, options) {
|
|
|
94
93
|
async function deleteRetired(generation) {
|
|
95
94
|
try {
|
|
96
95
|
await retry(() => streams.deleteTenant(basinOf(tenantId, generation)), {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
attempts: 5,
|
|
96
|
+
...hostTuning().streams.retry,
|
|
97
|
+
attempts: hostTuning().streams.retireAttempts,
|
|
100
98
|
});
|
|
101
99
|
await store.tx((t) => t.forgetRetiredGeneration(generation));
|
|
102
100
|
logger.info("retired stream basin deleted", { generation });
|
|
@@ -160,7 +158,7 @@ export async function streamsStatus(ctx, options = {}) {
|
|
|
160
158
|
const wiring = ctx.sessionStreams.wiring;
|
|
161
159
|
return {
|
|
162
160
|
reachable: wiring
|
|
163
|
-
? await probe(wiring.streams, options.probeTimeoutMs ??
|
|
161
|
+
? await probe(wiring.streams, options.probeTimeoutMs ?? hostTuning().streams.probeTimeoutMs)
|
|
164
162
|
: false,
|
|
165
163
|
basin: wiring?.basin() ?? { ready: false, failures: 0, lastError: null },
|
|
166
164
|
generation: ctx.sessionStreams.generation,
|
|
@@ -215,17 +213,17 @@ function basinKeeper(streams, basinKey, signal, onError) {
|
|
|
215
213
|
repair() {
|
|
216
214
|
if (repairing || signal.aborted)
|
|
217
215
|
return;
|
|
218
|
-
if (ready && readyFor === basinKey() && Date.now() - repairedAt <
|
|
216
|
+
if (ready && readyFor === basinKey() && Date.now() - repairedAt < hostTuning().streams.basinRepairCooldownMs)
|
|
219
217
|
return;
|
|
220
218
|
repairing = true;
|
|
221
219
|
ready = false;
|
|
222
220
|
void (async () => {
|
|
223
221
|
await retry(ensure, {
|
|
224
|
-
minMs:
|
|
225
|
-
maxMs:
|
|
222
|
+
minMs: hostTuning().streams.retry.minMs,
|
|
223
|
+
maxMs: hostTuning().streams.basinRetryMaxMs,
|
|
226
224
|
signal,
|
|
227
225
|
onError: (error) => {
|
|
228
|
-
if (failures === 1 || failures %
|
|
226
|
+
if (failures === 1 || failures % hostTuning().streams.basinLogEveryFailures === 0)
|
|
229
227
|
onError(error);
|
|
230
228
|
},
|
|
231
229
|
}).then(() => (repairedAt = Date.now()), () => undefined);
|
package/dist/tenant/sweep.d.ts
CHANGED
|
@@ -1,32 +1,12 @@
|
|
|
1
1
|
import type { PendingRequest } from "../store/types.js";
|
|
2
2
|
import type { TenantContext } from "./context.js";
|
|
3
|
-
/**
|
|
4
|
-
* How old an outbox row is before the sweep delivers it, unless its delivery after commit
|
|
5
|
-
* already failed (then it is due at once): longer than that delivery can take with its
|
|
6
|
-
* retries, so the sweep does not send again, and double the load on, an execution that is
|
|
7
|
-
* only slow. Sending one twice is harmless either way (the idempotency key).
|
|
8
|
-
*/
|
|
9
|
-
export declare const OUTBOX_GRACE_MS = 30000;
|
|
10
|
-
/**
|
|
11
|
-
* How often the orphan scan, and the stale pod sandbox check, run after the first pass of a
|
|
12
|
-
* Tenant opened on this process.
|
|
13
|
-
*/
|
|
14
|
-
export declare const ORPHAN_SCAN_MS = 60000;
|
|
15
|
-
/**
|
|
16
|
-
* How long one pass may spend draining the outbox, and paging through orphaned sessions.
|
|
17
|
-
* A backlog is worked off over several passes rather than holding one (and with it the
|
|
18
|
-
* Tenant's sweep object in the execution) for long.
|
|
19
|
-
*/
|
|
20
|
-
export declare const DRAIN_BUDGET_MS = 2000;
|
|
21
3
|
export declare function sweep(ctx: TenantContext): Promise<void>;
|
|
22
|
-
/** Failed deliveries after which the sweep parks an outbox row (`Tx.failOutbox`). */
|
|
23
|
-
export declare const MAX_OUTBOX_ATTEMPTS = 10;
|
|
24
4
|
/**
|
|
25
5
|
* Deliver the outbox rows that are due (`Tx.pendingOutbox`: older than `graceMs`, or whose
|
|
26
6
|
* delivery failed), oldest first, batch after batch until none is left or `budgetMs` is
|
|
27
7
|
* spent, and delete the delivered ones. Each row is delivered on its own: one that fails
|
|
28
8
|
* (Restate refuses it, or a Runtime that does not know a wake's reason) is tried again later,
|
|
29
|
-
* with backoff, so it never holds back the rows behind it, and after `
|
|
9
|
+
* with backoff, so it never holds back the rows behind it, and after `sweep.outboxAttempts`
|
|
30
10
|
* failures it is parked: logged once, at error, and kept in the outbox for an operator. A
|
|
31
11
|
* parked wake's session is still re-woken by the orphan scan while it is runnable, and a
|
|
32
12
|
* parked sandbox signal's pod by the stale pod check. A row the Tenant declines (it is
|