@nylorun/runtime 0.26.0-beta → 0.27.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -0
- package/README.md +18 -14
- package/dist/adapters/execution/restate.d.ts +15 -18
- package/dist/adapters/execution/restate.js +31 -27
- package/dist/adapters/replication/pgoutput.d.ts +1 -1
- package/dist/adapters/replication/pgoutput.js +3 -2
- package/dist/adapters/sandbox/local.js +8 -5
- package/dist/adapters/sandbox/virtual.js +2 -2
- package/dist/adapters/streams/s2.d.ts +1 -1
- package/dist/adapters/streams/s2.js +18 -14
- package/dist/api/a2a/card.js +1 -1
- package/dist/api/a2a/endpoint.js +3 -4
- package/dist/api/a2a/protocol.js +4 -0
- package/dist/api/a2a/routes.d.ts +2 -6
- package/dist/api/a2a/routes.js +12 -18
- package/dist/api/ag-ui/endpoint.js +2 -3
- package/dist/api/ag-ui/routes.js +4 -4
- package/dist/api/components.d.ts +326 -45
- package/dist/api/components.js +5 -1
- package/dist/api/gate/keys.js +7 -4
- package/dist/api/gate/routes.d.ts +6 -2
- package/dist/api/gate/routes.js +15 -11
- package/dist/api/http/body.d.ts +15 -3
- package/dist/api/http/body.js +15 -7
- package/dist/api/http/respond.js +6 -2
- package/dist/api/http/routes/access.js +5 -4
- package/dist/api/http/routes/artifacts.js +19 -7
- package/dist/api/http/routes/coverage.js +1 -1
- package/dist/api/http/routes/environment-manifests.js +1 -1
- package/dist/api/http/routes/environments.js +6 -6
- package/dist/api/http/routes/files.d.ts +2 -1
- package/dist/api/http/routes/files.js +31 -5
- package/dist/api/http/routes/mcp.js +1 -1
- package/dist/api/http/routes/reads.d.ts +35 -0
- package/dist/api/http/routes/reads.js +50 -22
- package/dist/api/http/routes/sessions.js +11 -10
- package/dist/api/http/routes/tenant.d.ts +2 -2
- package/dist/api/http/routes/tenant.js +69 -16
- package/dist/api/inbound-files.d.ts +3 -9
- package/dist/api/inbound-files.js +18 -24
- package/dist/api/openapi.js +6 -2
- package/dist/artifacts/disk.d.ts +0 -4
- package/dist/artifacts/disk.js +77 -73
- package/dist/artifacts/files.d.ts +13 -4
- package/dist/artifacts/files.js +27 -9
- package/dist/artifacts/folder-upload.d.ts +75 -0
- package/dist/artifacts/folder-upload.js +263 -0
- package/dist/artifacts/folders.d.ts +14 -8
- package/dist/artifacts/folders.js +19 -11
- package/dist/artifacts/links.js +7 -6
- package/dist/artifacts/parts.d.ts +9 -14
- package/dist/artifacts/parts.js +11 -9
- package/dist/artifacts/pipe.d.ts +15 -0
- package/dist/artifacts/pipe.js +71 -0
- package/dist/artifacts/quota.d.ts +36 -0
- package/dist/artifacts/quota.js +44 -0
- package/dist/artifacts/service.d.ts +25 -1
- package/dist/artifacts/service.js +70 -25
- package/dist/blob/fs.js +2 -1
- package/dist/blob/s3.d.ts +6 -2
- package/dist/blob/s3.js +9 -7
- package/dist/blob/types.d.ts +1 -1
- package/dist/blob/types.js +1 -1
- package/dist/core/limits.d.ts +11 -35
- package/dist/core/limits.js +5 -73
- package/dist/core/sandbox-routes.js +0 -1
- package/dist/core/turn-manifest.d.ts +1 -2
- package/dist/core/turn-manifest.js +4 -28
- package/dist/execution/memory.d.ts +8 -5
- package/dist/execution/memory.js +8 -6
- package/dist/execution/types.d.ts +1 -6
- package/dist/execution/types.js +0 -5
- package/dist/gates/contract.d.ts +3 -2
- package/dist/gates/contract.js +1 -1
- package/dist/gates/egress-credentials.js +4 -3
- package/dist/gates/egress.d.ts +9 -9
- package/dist/gates/egress.js +14 -16
- package/dist/gates/fixture-model.js +13 -2
- package/dist/gates/handler.d.ts +1 -3
- package/dist/gates/handler.js +4 -3
- package/dist/gates/http-client.d.ts +4 -3
- package/dist/gates/http-client.js +5 -7
- package/dist/gates/http-tool.d.ts +24 -4
- package/dist/gates/http-tool.js +40 -12
- package/dist/gates/in-process.d.ts +2 -1
- package/dist/gates/in-process.js +17 -9
- package/dist/gates/inflight.d.ts +2 -2
- package/dist/gates/inflight.js +3 -2
- package/dist/gates/journal.js +5 -7
- package/dist/gates/mcp-handler.d.ts +4 -3
- package/dist/gates/mcp-handler.js +15 -5
- package/dist/gates/model-gate.d.ts +11 -2
- package/dist/gates/policy.d.ts +28 -5
- package/dist/gates/policy.js +60 -17
- package/dist/gates/provider-gate.d.ts +22 -0
- package/dist/gates/provider-gate.js +30 -0
- package/dist/gates/tool-calls.d.ts +1 -3
- package/dist/gates/tool-calls.js +2 -3
- package/dist/gates/tool-client.d.ts +1 -1
- package/dist/gates/tool-client.js +5 -6
- package/dist/gates/tool-contract.d.ts +8 -5
- package/dist/gates/tool-contract.js +1 -1
- package/dist/gates/tool-gate.d.ts +18 -7
- package/dist/gates/tool-gate.js +49 -7
- package/dist/harness/calls.d.ts +27 -25
- package/dist/harness/calls.js +40 -46
- package/dist/harness/executors.d.ts +10 -3
- package/dist/harness/executors.js +53 -23
- package/dist/harness/main.js +2 -2
- package/dist/harness/pod.d.ts +0 -2
- package/dist/harness/pod.js +7 -6
- package/dist/harness/run-tokens.d.ts +4 -1
- package/dist/harness/run-tokens.js +6 -2
- package/dist/harness/service.d.ts +17 -14
- package/dist/harness/service.js +21 -35
- package/dist/harness/setup.d.ts +0 -2
- package/dist/harness/setup.js +4 -9
- package/dist/harness/ws-client.d.ts +2 -2
- package/dist/harness/ws-client.js +7 -5
- package/dist/harness/ws-port.d.ts +3 -4
- package/dist/harness/ws-port.js +3 -4
- package/dist/harness-api/artifacts.d.ts +20 -0
- package/dist/harness-api/artifacts.js +478 -0
- package/dist/harness-api/in-process.d.ts +2 -2
- package/dist/harness-api/in-process.js +18 -16
- package/dist/harness-api/record.d.ts +10 -9
- package/dist/harness-api/record.js +102 -157
- package/dist/harness-api/renew.d.ts +1 -1
- package/dist/harness-api/renew.js +2 -2
- package/dist/harness-api/server.d.ts +1 -16
- package/dist/harness-api/server.js +82 -73
- package/dist/harness-api/start.d.ts +20 -2
- package/dist/harness-api/start.js +57 -18
- package/dist/harness-api/tool-journal.d.ts +15 -10
- package/dist/harness-api/tool-journal.js +26 -24
- package/dist/harness-api/workspace.d.ts +0 -4
- package/dist/harness-api/workspace.js +2 -45
- package/dist/harness-api/ws-server.js +10 -10
- package/dist/host/create-host.js +2 -3
- package/dist/host/execution.js +2 -3
- package/dist/host/gates.d.ts +2 -9
- package/dist/host/gates.js +15 -12
- package/dist/host/http.d.ts +0 -2
- package/dist/host/http.js +0 -27
- package/dist/host/main.js +10 -3
- package/dist/host/operate.js +12 -3
- package/dist/host/stack-config.d.ts +11 -0
- package/dist/host/stack-config.js +21 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +0 -1
- package/dist/infra/index.js +2 -1
- package/dist/infra/readiness.d.ts +0 -2
- package/dist/infra/readiness.js +4 -3
- package/dist/keys/client.d.ts +1 -0
- package/dist/keys/client.js +2 -3
- package/dist/keys/contract.d.ts +0 -2
- package/dist/keys/contract.js +0 -2
- package/dist/keys/egress-ca.d.ts +0 -2
- package/dist/keys/egress-ca.js +2 -3
- package/dist/keys/keys.js +6 -1
- package/dist/keys/x509.d.ts +8 -4
- package/dist/keys/x509.js +14 -5
- package/dist/management-openapi.json +1961 -994
- package/dist/mcp/connect.d.ts +35 -26
- package/dist/mcp/connect.js +112 -141
- package/dist/mcp/pool.d.ts +27 -22
- package/dist/mcp/pool.js +34 -48
- package/dist/mcp/preview.d.ts +12 -5
- package/dist/mcp/preview.js +36 -19
- package/dist/mcp/snapshot.d.ts +35 -86
- package/dist/mcp/snapshot.js +93 -135
- package/dist/model/auth-store.js +5 -3
- package/dist/model/capabilities.d.ts +1 -1
- package/dist/model/catalog.d.ts +9 -36
- package/dist/model/catalog.js +6 -33
- package/dist/model/descriptor.d.ts +51 -0
- package/dist/model/descriptor.js +83 -0
- package/dist/model/models.d.ts +0 -3
- package/dist/model/models.js +3 -5
- package/dist/model/native-blocks.js +1 -1
- package/dist/model/pi-model.d.ts +16 -5
- package/dist/model/pi-model.js +86 -105
- package/dist/openapi.json +838 -1218
- package/dist/reads/cursor.js +3 -1
- package/dist/record/append.d.ts +34 -2
- package/dist/record/append.js +50 -1
- package/dist/record/environment.d.ts +2 -0
- package/dist/record/environment.js +2 -0
- package/dist/record/index.d.ts +1 -1
- package/dist/record/index.js +1 -1
- package/dist/redact.d.ts +1 -1
- package/dist/redact.js +3 -2
- package/dist/sandbox/egress-token.d.ts +3 -3
- package/dist/sandbox/egress-token.js +3 -3
- package/dist/sandbox/join.js +3 -2
- package/dist/sandbox/manager.d.ts +41 -41
- package/dist/sandbox/manager.js +112 -44
- package/dist/sandbox/placement.d.ts +1 -1
- package/dist/sandbox/placement.js +5 -8
- package/dist/sandbox/pods/client.d.ts +1 -10
- package/dist/sandbox/pods/client.js +8 -4
- package/dist/sandbox/pods/lifecycle.d.ts +19 -11
- package/dist/sandbox/pods/lifecycle.js +45 -23
- package/dist/sandbox/pods/network-gate.d.ts +4 -5
- package/dist/sandbox/pods/network-gate.js +7 -7
- package/dist/sandbox/pods/reconcile.js +4 -1
- package/dist/sandbox/pods/spec.js +8 -5
- package/dist/sandbox/policy.d.ts +7 -1
- package/dist/sandbox/policy.js +10 -3
- package/dist/sandbox/resolve.d.ts +8 -3
- package/dist/sandbox/resolve.js +12 -2
- package/dist/sandbox/setup.d.ts +3 -1
- package/dist/sandbox/setup.js +34 -11
- package/dist/sandbox/share.d.ts +5 -14
- package/dist/sandbox/share.js +5 -34
- package/dist/sandbox/skills.d.ts +7 -13
- package/dist/sandbox/skills.js +16 -31
- package/dist/sandbox/tenant-config.d.ts +2 -2
- package/dist/sandbox/tenant-config.js +4 -3
- package/dist/sandbox/tools.d.ts +7 -1
- package/dist/sandbox/tools.js +26 -26
- package/dist/sandbox/types.d.ts +10 -3
- package/dist/sandbox/types.js +9 -0
- package/dist/store/postgres/connect.d.ts +6 -5
- package/dist/store/postgres/connect.js +14 -11
- package/dist/store/postgres/control.js +13 -13
- package/dist/store/postgres/drizzle/0028_artifact_write_keys.sql +2 -0
- package/dist/store/postgres/drizzle/0029_protocol_15.sql +11 -0
- package/dist/store/postgres/drizzle/0030_protocol_15_runtime_block.sql +11 -0
- package/dist/store/postgres/drizzle/0031_protocol_15_mcp_snapshot.sql +8 -0
- package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
- package/dist/store/postgres/reads.js +10 -2
- package/dist/store/postgres/schema.d.ts +19 -0
- package/dist/store/postgres/schema.js +7 -0
- package/dist/store/postgres/store.js +55 -26
- package/dist/store/types.d.ts +17 -7
- package/dist/streams/relay/core.d.ts +3 -3
- package/dist/streams/relay/core.js +9 -10
- package/dist/tenant/advance.js +21 -31
- package/dist/tenant/commands.js +9 -16
- package/dist/tenant/context.d.ts +14 -17
- package/dist/tenant/definition-files.d.ts +25 -4
- package/dist/tenant/definition-files.js +85 -47
- package/dist/tenant/effects.js +9 -36
- package/dist/tenant/environment-manifests.js +9 -1
- package/dist/tenant/environments.d.ts +0 -2
- package/dist/tenant/environments.js +49 -28
- package/dist/tenant/ephemeral.js +2 -1
- package/dist/tenant/history.d.ts +1 -1
- package/dist/tenant/history.js +1 -1
- package/dist/tenant/host-token.d.ts +3 -3
- package/dist/tenant/host-token.js +11 -10
- package/dist/tenant/http.d.ts +5 -8
- package/dist/tenant/http.js +9 -0
- package/dist/tenant/identity-file.d.ts +23 -0
- package/dist/tenant/identity-file.js +34 -6
- package/dist/tenant/issuers.d.ts +5 -5
- package/dist/tenant/issuers.js +31 -23
- package/dist/tenant/jwt.d.ts +8 -2
- package/dist/tenant/jwt.js +10 -2
- package/dist/tenant/limits.d.ts +90 -0
- package/dist/tenant/limits.js +175 -0
- package/dist/tenant/module.d.ts +4 -1
- package/dist/tenant/module.js +9 -8
- package/dist/tenant/outbound.d.ts +11 -14
- package/dist/tenant/outbound.js +10 -15
- package/dist/tenant/run-grants.js +3 -2
- package/dist/tenant/run-token.d.ts +0 -4
- package/dist/tenant/run-token.js +6 -11
- package/dist/tenant/runtime-limits.d.ts +24 -0
- package/dist/tenant/runtime-limits.js +27 -0
- package/dist/tenant/runtime.js +30 -43
- package/dist/tenant/session-streams.js +7 -6
- package/dist/tenant/session.d.ts +1 -1
- package/dist/tenant/session.js +1 -1
- package/dist/tenant/sessions.js +32 -29
- package/dist/tenant/signing-keys.d.ts +1 -1
- package/dist/tenant/signing-keys.js +5 -6
- package/dist/tenant/status.js +4 -3
- package/dist/tenant/streams.js +18 -20
- package/dist/tenant/sweep.d.ts +1 -21
- package/dist/tenant/sweep.js +39 -48
- package/dist/tenant/transcript.d.ts +70 -23
- package/dist/tenant/transcript.js +28 -22
- package/dist/tenant/types.d.ts +9 -14
- package/dist/tenant/worker.d.ts +2 -10
- package/dist/tenant/worker.js +5 -14
- package/dist/traces/ids.js +5 -1
- package/dist/traces/map.d.ts +9 -6
- package/dist/traces/map.js +9 -11
- package/dist/traces/otlp.d.ts +3 -3
- package/dist/traces/otlp.js +5 -3
- package/dist/traces/projector.d.ts +9 -4
- package/dist/traces/projector.js +13 -7
- package/dist/traces/read.js +6 -1
- package/dist/tuning.d.ts +383 -0
- package/dist/tuning.js +658 -0
- package/dist/vault/headers.js +4 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +3 -3
- package/dist/artifacts/export.d.ts +0 -8
- package/dist/artifacts/export.js +0 -204
- package/dist/artifacts/media-types.d.ts +0 -30
- package/dist/artifacts/media-types.js +0 -137
- package/dist/artifacts/workspace.d.ts +0 -60
- package/dist/artifacts/workspace.js +0 -45
- package/dist/media.d.ts +0 -10
- package/dist/media.js +0 -58
- package/dist/sandbox/runtime-tools.d.ts +0 -9
- package/dist/sandbox/runtime-tools.js +0 -16
- package/dist/sandbox/session-sandbox.d.ts +0 -16
- package/dist/sandbox/session-sandbox.js +0 -77
- package/dist/tenant/artifact-tool.d.ts +0 -13
- package/dist/tenant/artifact-tool.js +0 -243
- package/dist/tenant/skill-tool.d.ts +0 -20
- package/dist/tenant/skill-tool.js +0 -67
- package/dist/tenant/tool-results.d.ts +0 -57
- package/dist/tenant/tool-results.js +0 -298
- package/dist/tenant/tool-search.d.ts +0 -39
- package/dist/tenant/tool-search.js +0 -103
package/dist/tenant/sessions.js
CHANGED
|
@@ -9,14 +9,17 @@ import { approvalRefusal, codeToolRefusal } from "@nylorun/core/define";
|
|
|
9
9
|
import { aggregateWaits, isWorkflowManifest } from "../core/flow-host.js";
|
|
10
10
|
import { canonical } from "../store/canonical.js";
|
|
11
11
|
import { sandboxSpecOf } from "../sandbox/resolve.js";
|
|
12
|
-
import {
|
|
13
|
-
import { authoredRuntimeBlocks, declaredSandboxes } from "../sandbox/share.js";
|
|
12
|
+
import { declaredSandboxes } from "../sandbox/share.js";
|
|
14
13
|
import { effectiveSandboxConfig, readSandboxConfig } from "../sandbox/tenant-config.js";
|
|
15
14
|
import { HARNESS_ID, checkPlacement, requirePods } from "../sandbox/placement.js";
|
|
16
15
|
import { openEnvironment, recordAttachment } from "./environments.js";
|
|
17
16
|
import { harnessOf, } from "./context.js";
|
|
18
17
|
import { fail } from "./http.js";
|
|
19
|
-
import {
|
|
18
|
+
import { definitionFileIssues } from "./definition-files.js";
|
|
19
|
+
import { limitsInForce, readSkillLimits, readTenantLimits } from "./limits.js";
|
|
20
|
+
import { readRuntimeLimits } from "./runtime-limits.js";
|
|
21
|
+
import { httpToolTimeoutIssues } from "../gates/http-tool.js";
|
|
22
|
+
import { requiresIssues } from "./limits.js";
|
|
20
23
|
/** A session's creation identity: everything but the request id. */
|
|
21
24
|
const sessionIdentity = (value) => {
|
|
22
25
|
const { requestId: _requestId, ...body } = value ?? {};
|
|
@@ -66,18 +69,25 @@ export async function putDefinition(ctx, agentId, body, options = {}) {
|
|
|
66
69
|
const declared = declaredSandboxes(body.manifest);
|
|
67
70
|
if (declared.length > 0)
|
|
68
71
|
fail(400, `${declared.join(", ")} declares a sandbox. Agents no longer declare one: remove .sandbox(), create an Environment with the sandbox (PUT /v1/environments/{id}) and open the session on it, createSession({ environment }). See MIGRATION.md.`);
|
|
69
|
-
const runtime = authoredRuntimeBlocks(body.manifest);
|
|
70
|
-
if (runtime.length > 0)
|
|
71
|
-
fail(400, `${runtime.join(", ")} sets runtime, which only the Runtime writes when it pins a session's agent. Remove it and register the manifest your builder makes.`);
|
|
72
72
|
const definition = {
|
|
73
73
|
...body,
|
|
74
74
|
manifestHash: hashManifest(body.manifest),
|
|
75
75
|
};
|
|
76
76
|
const files = [...definitionFilesOf(body.manifest)];
|
|
77
77
|
await ctx.store.tx(async (t) => {
|
|
78
|
-
const
|
|
78
|
+
const limits = await readRuntimeLimits(t);
|
|
79
|
+
const issues = [
|
|
80
|
+
...httpToolTimeoutIssues(body.manifest, limits.httpToolTimeoutMs),
|
|
81
|
+
...requiresIssues(body.manifest.requires, limits),
|
|
82
|
+
];
|
|
83
|
+
if (issues.length > 0)
|
|
84
|
+
fail(400, issues.join(" "));
|
|
85
|
+
const { totalBytes } = await readSkillLimits(t);
|
|
86
|
+
const { missing, oversized } = await definitionFileIssues(t, body.manifest, totalBytes);
|
|
79
87
|
if (missing.length > 0)
|
|
80
88
|
fail(400, `The definition names ${missing.length} file(s) the Runtime does not hold; upload each with PUT /v1/files/<sha256> first: ${missing.join(", ")}`, { code: "definition_files_missing", details: { missing } });
|
|
89
|
+
if (oversized.length > 0)
|
|
90
|
+
fail(413, `This Tenant allows a skill's files at most ${totalBytes} bytes together: ${oversized.map((skill) => `'${skill.name}' holds ${skill.bytes}`).join(", ")}`, { code: "limit_exceeded", details: { limitBytes: totalBytes, skills: oversized } });
|
|
81
91
|
if (options.dryRun)
|
|
82
92
|
return;
|
|
83
93
|
await t.put("definitions", agentId, definition);
|
|
@@ -158,6 +168,8 @@ export async function putSession(ctx, id, body, access, options = {}) {
|
|
|
158
168
|
});
|
|
159
169
|
if (!isWorkflowManifest(definition.manifest))
|
|
160
170
|
await checkRequirements(t, definition.manifest, environment);
|
|
171
|
+
if (environment !== undefined)
|
|
172
|
+
await checkSkillLimit(t, definition.manifest, environment);
|
|
161
173
|
const config = await readSandboxConfig(t);
|
|
162
174
|
// The harness the session runs for its life (R4.1): its Environment's, else ours. A flow
|
|
163
175
|
// runs in core and our engine, whatever its Environment names.
|
|
@@ -171,13 +183,14 @@ export async function putSession(ctx, id, body, access, options = {}) {
|
|
|
171
183
|
const spec = environment === undefined || environment.kind === "none"
|
|
172
184
|
? undefined
|
|
173
185
|
: sandboxSpecOf(environment.applied, effectiveSandboxConfig(config).limits.idle);
|
|
174
|
-
|
|
186
|
+
// The session pins its definition as registered (protocol 15): the sandbox's spec stays on
|
|
187
|
+
// the session, and the harness words the facts and defines the tools.
|
|
175
188
|
const created = {
|
|
176
189
|
id,
|
|
177
190
|
agentId: body.agentId,
|
|
178
191
|
ownerUserId: body.ownerUserId,
|
|
179
|
-
manifest:
|
|
180
|
-
manifestHash:
|
|
192
|
+
manifest: definition.manifest,
|
|
193
|
+
manifestHash: definition.manifestHash,
|
|
181
194
|
definitionHash: definition.manifestHash,
|
|
182
195
|
implementationVersion: definition.implementationVersion,
|
|
183
196
|
info: body.info,
|
|
@@ -224,26 +237,16 @@ async function checkRequirements(t, manifest, environment) {
|
|
|
224
237
|
fail(409, `Agent ${manifest.id} requires ${missing.map((gap) => (gap.startsWith("secret:") ? `the secret ${gap.slice(7)}` : gap === "pod" ? "a pod Environment" : "a sandbox")).join(", ")}, and ${where}.`, { code: "environment_unsuitable", details: { missing } });
|
|
225
238
|
}
|
|
226
239
|
/**
|
|
227
|
-
*
|
|
228
|
-
* `
|
|
240
|
+
* Refuses a session whose agent carries a skill larger than its Environment allows: its
|
|
241
|
+
* manifest's `limits.skills.totalBytes`, else the Tenant's, never above the Tenant's.
|
|
242
|
+
* `409 environment_unsuitable`, with each skill and its size in `details.skills`.
|
|
229
243
|
*/
|
|
230
|
-
function
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
return { manifest: pinned.manifest, manifestHash: pinned.manifestHash };
|
|
237
|
-
}
|
|
238
|
-
/** The manifest a session with a sandbox pins: the definition with the sandbox in its `runtime` block. */
|
|
239
|
-
function pin(definition, spec) {
|
|
240
|
-
// A workflow's manifest stays as registered: its agents inherit the sandbox when they start.
|
|
241
|
-
if (isWorkflowManifest(definition.manifest))
|
|
242
|
-
return {};
|
|
243
|
-
const sandboxed = withSandbox(definition.manifest, spec);
|
|
244
|
-
if (!sandboxed.ok)
|
|
245
|
-
return fail(400, sandboxed.message);
|
|
246
|
-
return { manifest: sandboxed.manifest, manifestHash: sandboxed.manifestHash };
|
|
244
|
+
async function checkSkillLimit(t, manifest, environment) {
|
|
245
|
+
const limitBytes = limitsInForce(await readTenantLimits(t), environment.applied.limits).skills.totalBytes;
|
|
246
|
+
const { oversized } = await definitionFileIssues(t, manifest, limitBytes);
|
|
247
|
+
if (oversized.length === 0)
|
|
248
|
+
return;
|
|
249
|
+
fail(409, `Environment ${environment.id} allows a skill's files at most ${limitBytes} bytes together, and agent ${manifest.id} carries ${oversized.map((skill) => `'${skill.name}' (${skill.bytes})`).join(", ")}.`, { code: "environment_unsuitable", details: { limitBytes, skills: oversized } });
|
|
247
250
|
}
|
|
248
251
|
/** The session resource returned by `GET`/`PUT /v1/sessions/:id`. */
|
|
249
252
|
export async function sessionView(t, s) {
|
|
@@ -48,7 +48,7 @@ export declare class SigningKeys {
|
|
|
48
48
|
/**
|
|
49
49
|
* Rotates: previous → revoked, current → previous, standby → current, new standby. Refused
|
|
50
50
|
* with 409 while the previous key may still verify live tokens (retired less than the
|
|
51
|
-
* longest token lifetime, `maxTtlSeconds` or `
|
|
51
|
+
* longest token lifetime, `maxTtlSeconds` or `keys.tokenTtlMaxSeconds`, plus a minute ago), unless
|
|
52
52
|
* `force`.
|
|
53
53
|
*/
|
|
54
54
|
rotate(t: Tx, kek: Buffer, maxTtlSeconds: number, force: boolean): Promise<SigningKeyRow[]>;
|
|
@@ -15,12 +15,10 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { exportJWK, exportPKCS8, generateKeyPair, importJWK, importPKCS8, } from "jose";
|
|
17
17
|
import { newSigningKeyId, SIGNING_KEY_ID_PATTERN, } from "@nylorun/core/compatibility";
|
|
18
|
-
import { TOKEN_TTL_MAX_SECONDS } from "@nylorun/core/contracts";
|
|
19
18
|
import { decryptSecret, encryptSecret } from "../vault/crypto.js";
|
|
20
19
|
import { fail } from "./http.js";
|
|
20
|
+
import { hostTuning } from "../tuning.js";
|
|
21
21
|
const ALG = "ES256";
|
|
22
|
-
/** Grace added to the longest token lifetime before a previous key may be revoked. */
|
|
23
|
-
const ROTATION_GRACE_MS = 60_000;
|
|
24
22
|
export class SigningKeys {
|
|
25
23
|
options;
|
|
26
24
|
publicKeys = new Map();
|
|
@@ -107,7 +105,7 @@ export class SigningKeys {
|
|
|
107
105
|
/**
|
|
108
106
|
* Rotates: previous → revoked, current → previous, standby → current, new standby. Refused
|
|
109
107
|
* with 409 while the previous key may still verify live tokens (retired less than the
|
|
110
|
-
* longest token lifetime, `maxTtlSeconds` or `
|
|
108
|
+
* longest token lifetime, `maxTtlSeconds` or `keys.tokenTtlMaxSeconds`, plus a minute ago), unless
|
|
111
109
|
* `force`.
|
|
112
110
|
*/
|
|
113
111
|
async rotate(t, kek, maxTtlSeconds, force) {
|
|
@@ -118,8 +116,9 @@ export class SigningKeys {
|
|
|
118
116
|
const [previous] = await t.signingKeys(["previous"]);
|
|
119
117
|
if (previous && !force) {
|
|
120
118
|
const retired = Date.parse(previous.retiredAt ?? previous.createdAt);
|
|
121
|
-
const longest = Math.max(maxTtlSeconds,
|
|
122
|
-
|
|
119
|
+
const longest = Math.max(maxTtlSeconds, hostTuning().keys.tokenTtlMaxSeconds);
|
|
120
|
+
// The grace added to the longest token lifetime before a previous key may be revoked.
|
|
121
|
+
const until = retired + longest * 1000 + hostTuning().tenant.signingKeyRotationGraceMs;
|
|
123
122
|
if (until > now.getTime())
|
|
124
123
|
fail(409, "The previous key may still verify live tokens", {
|
|
125
124
|
code: "request_rejected",
|
package/dist/tenant/status.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { FIXTURE_MODEL_SETTING, seedFixtureModel } from "./model-setting.js";
|
|
2
2
|
import { ENVIRONMENTS_CONFIG_SETTING, writeSandboxConfig } from "../sandbox/tenant-config.js";
|
|
3
|
+
import { hostTuning } from "../tuning.js";
|
|
3
4
|
/** The Tenant's harnesses, for its status and the Host's. */
|
|
4
5
|
export function harnessStatusOf(ctx) {
|
|
5
6
|
return { mode: ctx.mcp ? "in-process" : "remote", ...ctx.harness.status() };
|
|
6
7
|
}
|
|
7
|
-
/** How long status waits for the execution to list stuck invocations. */
|
|
8
|
-
const STUCK_TIMEOUT_MS = 5000;
|
|
9
8
|
/** `TenantStatus.execution`: paused and backing-off invocations, or why they are unknown. */
|
|
10
9
|
async function executionStatus(stuckInvocations) {
|
|
11
10
|
let timer;
|
|
@@ -13,7 +12,9 @@ async function executionStatus(stuckInvocations) {
|
|
|
13
12
|
const stuck = await Promise.race([
|
|
14
13
|
stuckInvocations(),
|
|
15
14
|
new Promise((_, reject) => {
|
|
16
|
-
timer = setTimeout(() => reject(new Error("Timed out listing stuck invocations")),
|
|
15
|
+
timer = setTimeout(() => reject(new Error("Timed out listing stuck invocations")),
|
|
16
|
+
// How long status waits for the execution to list stuck invocations.
|
|
17
|
+
hostTuning().tenant.statusTimeoutMs);
|
|
17
18
|
}),
|
|
18
19
|
]);
|
|
19
20
|
return {
|
package/dist/tenant/streams.js
CHANGED
|
@@ -1,17 +1,16 @@
|
|
|
1
1
|
import { basinOf } from "../streams/basin.js";
|
|
2
2
|
import { createStreamRelay, } from "../streams/relay/core.js";
|
|
3
3
|
import { retry } from "../retry.js";
|
|
4
|
+
import { hostTuning } from "../tuning.js";
|
|
4
5
|
import { checkSessionStreams, currentBasin } from "./session-streams.js";
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
/** How long a retired basin stays for readers that have not moved yet. */
|
|
14
|
-
const RETIRE_GRACE_MS = 60_000;
|
|
6
|
+
/*
|
|
7
|
+
* Timings are the Host tuning's `streams` group (`tuning.ts`): `retry` for deletions and
|
|
8
|
+
* repairs, `basinRetryMaxMs` (the longest wait between two `ensureTenant` attempts while the
|
|
9
|
+
* basin is being repaired), `basinRepairCooldownMs` (a repair started within this long after the
|
|
10
|
+
* last one succeeded waits for the next failure), `feedCheckMs` (how often every session stream is
|
|
11
|
+
* checked against its session, the backstop for lost signals), `retireGraceMs` (how long a retired
|
|
12
|
+
* basin stays for readers that have not moved yet).
|
|
13
|
+
*/
|
|
15
14
|
const messageOf = (error) => error instanceof Error ? error.message : String(error);
|
|
16
15
|
/**
|
|
17
16
|
* Wires the Tenant to Durable Streams: reads its basin generation, checks the basin, starts
|
|
@@ -25,7 +24,7 @@ export async function wireStreams(ctx, options) {
|
|
|
25
24
|
const { store, streams, tenantId } = options;
|
|
26
25
|
const logger = ctx.config.logger;
|
|
27
26
|
const stop = new AbortController();
|
|
28
|
-
const graceMs = options.retireGraceMs ??
|
|
27
|
+
const graceMs = options.retireGraceMs ?? hostTuning().streams.retireGraceMs;
|
|
29
28
|
// Nothing is reported once the wiring stops: the Tenant (and its log directory) may be gone,
|
|
30
29
|
// and a throwing report in a detached loop would be an unhandled rejection.
|
|
31
30
|
const report = (what) => (error) => {
|
|
@@ -53,7 +52,7 @@ export async function wireStreams(ctx, options) {
|
|
|
53
52
|
});
|
|
54
53
|
relay?.start();
|
|
55
54
|
// Streams whose session was reset end on the `sessions.reset` signal; this catches lost ones.
|
|
56
|
-
const feedCheck = setInterval(() => void checkSessionStreams(ctx).catch(report("session stream check failed")),
|
|
55
|
+
const feedCheck = setInterval(() => void checkSessionStreams(ctx).catch(report("session stream check failed")), hostTuning().streams.feedCheckMs);
|
|
57
56
|
feedCheck.unref();
|
|
58
57
|
const retiring = new Set();
|
|
59
58
|
const timers = new Set();
|
|
@@ -94,9 +93,8 @@ export async function wireStreams(ctx, options) {
|
|
|
94
93
|
async function deleteRetired(generation) {
|
|
95
94
|
try {
|
|
96
95
|
await retry(() => streams.deleteTenant(basinOf(tenantId, generation)), {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
attempts: 5,
|
|
96
|
+
...hostTuning().streams.retry,
|
|
97
|
+
attempts: hostTuning().streams.retireAttempts,
|
|
100
98
|
});
|
|
101
99
|
await store.tx((t) => t.forgetRetiredGeneration(generation));
|
|
102
100
|
logger.info("retired stream basin deleted", { generation });
|
|
@@ -160,7 +158,7 @@ export async function streamsStatus(ctx, options = {}) {
|
|
|
160
158
|
const wiring = ctx.sessionStreams.wiring;
|
|
161
159
|
return {
|
|
162
160
|
reachable: wiring
|
|
163
|
-
? await probe(wiring.streams, options.probeTimeoutMs ??
|
|
161
|
+
? await probe(wiring.streams, options.probeTimeoutMs ?? hostTuning().streams.probeTimeoutMs)
|
|
164
162
|
: false,
|
|
165
163
|
basin: wiring?.basin() ?? { ready: false, failures: 0, lastError: null },
|
|
166
164
|
generation: ctx.sessionStreams.generation,
|
|
@@ -215,17 +213,17 @@ function basinKeeper(streams, basinKey, signal, onError) {
|
|
|
215
213
|
repair() {
|
|
216
214
|
if (repairing || signal.aborted)
|
|
217
215
|
return;
|
|
218
|
-
if (ready && readyFor === basinKey() && Date.now() - repairedAt <
|
|
216
|
+
if (ready && readyFor === basinKey() && Date.now() - repairedAt < hostTuning().streams.basinRepairCooldownMs)
|
|
219
217
|
return;
|
|
220
218
|
repairing = true;
|
|
221
219
|
ready = false;
|
|
222
220
|
void (async () => {
|
|
223
221
|
await retry(ensure, {
|
|
224
|
-
minMs:
|
|
225
|
-
maxMs:
|
|
222
|
+
minMs: hostTuning().streams.retry.minMs,
|
|
223
|
+
maxMs: hostTuning().streams.basinRetryMaxMs,
|
|
226
224
|
signal,
|
|
227
225
|
onError: (error) => {
|
|
228
|
-
if (failures === 1 || failures %
|
|
226
|
+
if (failures === 1 || failures % hostTuning().streams.basinLogEveryFailures === 0)
|
|
229
227
|
onError(error);
|
|
230
228
|
},
|
|
231
229
|
}).then(() => (repairedAt = Date.now()), () => undefined);
|
package/dist/tenant/sweep.d.ts
CHANGED
|
@@ -1,32 +1,12 @@
|
|
|
1
1
|
import type { PendingRequest } from "../store/types.js";
|
|
2
2
|
import type { TenantContext } from "./context.js";
|
|
3
|
-
/**
|
|
4
|
-
* How old an outbox row is before the sweep delivers it, unless its delivery after commit
|
|
5
|
-
* already failed (then it is due at once): longer than that delivery can take with its
|
|
6
|
-
* retries, so the sweep does not send again, and double the load on, an execution that is
|
|
7
|
-
* only slow. Sending one twice is harmless either way (the idempotency key).
|
|
8
|
-
*/
|
|
9
|
-
export declare const OUTBOX_GRACE_MS = 30000;
|
|
10
|
-
/**
|
|
11
|
-
* How often the orphan scan, and the stale pod sandbox check, run after the first pass of a
|
|
12
|
-
* Tenant opened on this process.
|
|
13
|
-
*/
|
|
14
|
-
export declare const ORPHAN_SCAN_MS = 60000;
|
|
15
|
-
/**
|
|
16
|
-
* How long one pass may spend draining the outbox, and paging through orphaned sessions.
|
|
17
|
-
* A backlog is worked off over several passes rather than holding one (and with it the
|
|
18
|
-
* Tenant's sweep object in the execution) for long.
|
|
19
|
-
*/
|
|
20
|
-
export declare const DRAIN_BUDGET_MS = 2000;
|
|
21
3
|
export declare function sweep(ctx: TenantContext): Promise<void>;
|
|
22
|
-
/** Failed deliveries after which the sweep parks an outbox row (`Tx.failOutbox`). */
|
|
23
|
-
export declare const MAX_OUTBOX_ATTEMPTS = 10;
|
|
24
4
|
/**
|
|
25
5
|
* Deliver the outbox rows that are due (`Tx.pendingOutbox`: older than `graceMs`, or whose
|
|
26
6
|
* delivery failed), oldest first, batch after batch until none is left or `budgetMs` is
|
|
27
7
|
* spent, and delete the delivered ones. Each row is delivered on its own: one that fails
|
|
28
8
|
* (Restate refuses it, or a Runtime that does not know a wake's reason) is tried again later,
|
|
29
|
-
* with backoff, so it never holds back the rows behind it, and after `
|
|
9
|
+
* with backoff, so it never holds back the rows behind it, and after `sweep.outboxAttempts`
|
|
30
10
|
* failures it is parked: logged once, at error, and kept in the outbox for an operator. A
|
|
31
11
|
* parked wake's session is still re-woken by the orphan scan while it is runnable, and a
|
|
32
12
|
* parked sandbox signal's pod by the stale pod check. A row the Tenant declines (it is
|
package/dist/tenant/sweep.js
CHANGED
|
@@ -8,19 +8,19 @@
|
|
|
8
8
|
* a delivery after commit that failed, or that a crash between the commit and the send cut
|
|
9
9
|
* off. The first pass on this process takes every row, so a previous process's are not
|
|
10
10
|
* left waiting; later passes take those whose delivery after commit failed and those older
|
|
11
|
-
* than `
|
|
12
|
-
* `
|
|
11
|
+
* than `sweep.outboxGraceMs` (Host tuning) (the database's clock), batch after batch within
|
|
12
|
+
* `sweep.drainBudgetMs`. Each row carries the idempotency key it was first sent with, so a
|
|
13
13
|
* request the execution did accept before the crash, or one still being sent after its
|
|
14
14
|
* commit, runs once. Rows are delivered one by one: a row that fails is retried with
|
|
15
|
-
* backoff and parked after `
|
|
15
|
+
* backoff and parked after `sweep.outboxAttempts` (`deliverOutbox`), and no delivery fails
|
|
16
16
|
* the step.
|
|
17
17
|
* 2. **Linked agents.** Pending workflow `agent` effects whose linked turn already settled
|
|
18
18
|
* are completed and the workflow is woken. A linked turn's settle does both in its own
|
|
19
19
|
* transaction; this is the backstop for one that settle left pending, such as an effect
|
|
20
20
|
* an older Runtime wrote.
|
|
21
21
|
* 3. **Orphaned sessions.** On this process's first pass for the Tenant, then at most every
|
|
22
|
-
* `
|
|
23
|
-
* expired, are woken with reason `recover`, page by page within `
|
|
22
|
+
* `sweep.orphanScanMs`: `running` or `runnable` sessions with no owner, or an owner whose lease
|
|
23
|
+
* expired, are woken with reason `recover`, page by page within `sweep.drainBudgetMs`; a scan
|
|
24
24
|
* the budget cut short goes on at the next pass. Neither a lost wake (step 1) nor a Worker
|
|
25
25
|
* that died mid-advance (the execution retries that advance, which takes over, §11.4)
|
|
26
26
|
* needs it: it is the backstop for what the execution itself lost or ended without an
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
* orphan scan, pod Environments whose change stalled get a reconcile (`reconcileStalePods`).
|
|
35
35
|
* Environments unused for their `lifecycle.deleteAfter` are deleted
|
|
36
36
|
* (`sweepUnusedEnvironments`).
|
|
37
|
-
* 5. **Control signals.** Signals on the control bus older than `
|
|
37
|
+
* 5. **Control signals.** Signals on the control bus older than `sweep.signalRetentionMs` are
|
|
38
38
|
* deleted: followers read back only the last two minutes (`store/postgres/control.ts`).
|
|
39
39
|
* 6. **Hooks.** Callbacks registered with `ctx.onSweep`.
|
|
40
40
|
*
|
|
@@ -46,27 +46,20 @@
|
|
|
46
46
|
import { sweepUnusedEnvironments } from "./environments.js";
|
|
47
47
|
import { sweepPendingSeeds } from "../artifacts/disk.js";
|
|
48
48
|
import { pendingAgentEffects, reconcilePendingAgentEffect, } from "../core/flow-host.js";
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*/
|
|
63
|
-
export const ORPHAN_SCAN_MS = 60_000;
|
|
64
|
-
/**
|
|
65
|
-
* How long one pass may spend draining the outbox, and paging through orphaned sessions.
|
|
66
|
-
* A backlog is worked off over several passes rather than holding one (and with it the
|
|
67
|
-
* Tenant's sweep object in the execution) for long.
|
|
49
|
+
import { hostTuning } from "../tuning.js";
|
|
50
|
+
/*
|
|
51
|
+
* The sweep's timings are the Host tuning's `sweep` group (`tuning.ts`):
|
|
52
|
+
* - `outboxGraceMs`: how old an outbox row is before the sweep delivers it, unless its delivery
|
|
53
|
+
* after commit already failed (then it is due at once): longer than that delivery can take with
|
|
54
|
+
* its retries, so the sweep does not send again, and double the load on, an execution that is
|
|
55
|
+
* only slow. Sending one twice is harmless either way (the idempotency key).
|
|
56
|
+
* - `orphanScanMs`: how often the orphan scan, and the stale pod sandbox check, run after the
|
|
57
|
+
* first pass of a Tenant opened on this process.
|
|
58
|
+
* - `drainBudgetMs`: how long one pass may spend draining the outbox, and paging through
|
|
59
|
+
* orphaned sessions. A backlog is worked off over several passes rather than holding one (and
|
|
60
|
+
* with it the Tenant's sweep object in the execution) for long.
|
|
61
|
+
* - `batchSize`, `signalRetentionMs`, `outboxAttempts`, `outboxRetry`, `stalePodMs`.
|
|
68
62
|
*/
|
|
69
|
-
export const DRAIN_BUDGET_MS = 2000;
|
|
70
63
|
/**
|
|
71
64
|
* Each open Tenant's sweep state on this process: when its last orphan scan finished, the
|
|
72
65
|
* last session id of one that ran out of budget, which the next pass goes on from, and when
|
|
@@ -82,10 +75,10 @@ export async function sweep(ctx) {
|
|
|
82
75
|
passes.set(ctx, pass);
|
|
83
76
|
const scanOrphans = pass.orphansAt === undefined ||
|
|
84
77
|
pass.orphansAfter !== undefined ||
|
|
85
|
-
now.getTime() - pass.orphansAt >=
|
|
86
|
-
const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >=
|
|
78
|
+
now.getTime() - pass.orphansAt >= hostTuning().sweep.orphanScanMs;
|
|
79
|
+
const checkPods = pass.podsAt === undefined || now.getTime() - pass.podsAt >= hostTuning().sweep.orphanScanMs;
|
|
87
80
|
const steps = [
|
|
88
|
-
["outbox", () => deliverOutbox(ctx, first ? 0 :
|
|
81
|
+
["outbox", () => deliverOutbox(ctx, first ? 0 : hostTuning().sweep.outboxGraceMs)],
|
|
89
82
|
["linked", () => reconcileLinkedAgents(ctx)],
|
|
90
83
|
...(scanOrphans
|
|
91
84
|
? [
|
|
@@ -127,7 +120,7 @@ export async function sweep(ctx) {
|
|
|
127
120
|
["mcp", async () => ctx.mcp?.sweep()],
|
|
128
121
|
[
|
|
129
122
|
"signals",
|
|
130
|
-
() => ctx.store.tx((t) => t.pruneSignals(new Date(now.getTime() -
|
|
123
|
+
() => ctx.store.tx((t) => t.pruneSignals(new Date(now.getTime() - hostTuning().sweep.signalRetentionMs))),
|
|
131
124
|
],
|
|
132
125
|
...[...ctx.sweepHooks].map((hook) => ["hook", hook]),
|
|
133
126
|
];
|
|
@@ -149,30 +142,26 @@ export async function sweep(ctx) {
|
|
|
149
142
|
if (failure)
|
|
150
143
|
throw failure.error;
|
|
151
144
|
}
|
|
152
|
-
/** Failed deliveries after which the sweep parks an outbox row (`Tx.failOutbox`). */
|
|
153
|
-
export const MAX_OUTBOX_ATTEMPTS = 10;
|
|
154
|
-
/** How long after its first failed delivery the sweep tries a row again; doubled each time. */
|
|
155
|
-
const RETRY_MS = 5000;
|
|
156
|
-
const RETRY_MAX_MS = 10 * 60_000;
|
|
157
145
|
/**
|
|
158
146
|
* Deliver the outbox rows that are due (`Tx.pendingOutbox`: older than `graceMs`, or whose
|
|
159
147
|
* delivery failed), oldest first, batch after batch until none is left or `budgetMs` is
|
|
160
148
|
* spent, and delete the delivered ones. Each row is delivered on its own: one that fails
|
|
161
149
|
* (Restate refuses it, or a Runtime that does not know a wake's reason) is tried again later,
|
|
162
|
-
* with backoff, so it never holds back the rows behind it, and after `
|
|
150
|
+
* with backoff, so it never holds back the rows behind it, and after `sweep.outboxAttempts`
|
|
163
151
|
* failures it is parked: logged once, at error, and kept in the outbox for an operator. A
|
|
164
152
|
* parked wake's session is still re-woken by the orphan scan while it is runnable, and a
|
|
165
153
|
* parked sandbox signal's pod by the stale pod check. A row the Tenant declines (it is
|
|
166
154
|
* closing) stays as it is, and ends the drain. Never throws for a delivery; returns the rows
|
|
167
155
|
* delivered.
|
|
168
156
|
*/
|
|
169
|
-
export async function deliverOutbox(ctx, graceMs = 0, budgetMs =
|
|
157
|
+
export async function deliverOutbox(ctx, graceMs = 0, budgetMs = hostTuning().sweep.drainBudgetMs) {
|
|
170
158
|
const deadline = Date.now() + budgetMs;
|
|
171
159
|
const delivered = [];
|
|
160
|
+
const batchSize = hostTuning().sweep.batchSize;
|
|
172
161
|
for (;;) {
|
|
173
|
-
const pending = await ctx.store.tx((t) => t.pendingOutbox(graceMs,
|
|
162
|
+
const pending = await ctx.store.tx((t) => t.pendingOutbox(graceMs, batchSize));
|
|
174
163
|
const batch = [];
|
|
175
|
-
let done = pending.length <
|
|
164
|
+
let done = pending.length < batchSize;
|
|
176
165
|
try {
|
|
177
166
|
for (const row of pending) {
|
|
178
167
|
if (Date.now() >= deadline) {
|
|
@@ -206,8 +195,11 @@ export async function deliverOutbox(ctx, graceMs = 0, budgetMs = DRAIN_BUDGET_MS
|
|
|
206
195
|
/** Records a failed delivery of `row`: retried with backoff, parked after the last attempt. */
|
|
207
196
|
async function failed(ctx, row, error) {
|
|
208
197
|
const attempts = row.attempts + 1;
|
|
209
|
-
|
|
210
|
-
|
|
198
|
+
// Parked after `sweep.outboxAttempts` failed deliveries (`Tx.failOutbox`); tried again after
|
|
199
|
+
// `outboxRetry.minMs`, doubled each time.
|
|
200
|
+
const { outboxAttempts, outboxRetry } = hostTuning().sweep;
|
|
201
|
+
const park = attempts >= outboxAttempts;
|
|
202
|
+
const retryInMs = Math.min(outboxRetry.minMs * 2 ** (attempts - 1), outboxRetry.maxMs);
|
|
211
203
|
await ctx.store.tx((t) => t.failOutbox(row, { retryInMs, park }));
|
|
212
204
|
const { request } = row;
|
|
213
205
|
const fields = {
|
|
@@ -235,29 +227,28 @@ export async function reconcileLinkedAgents(ctx) {
|
|
|
235
227
|
* after `after`, until none is left or `budgetMs` is spent. Returns the ids woken, and the last
|
|
236
228
|
* id when the budget ran out first (`next`, where the next scan goes on).
|
|
237
229
|
*/
|
|
238
|
-
export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetMs =
|
|
230
|
+
export async function wakeOrphanedSessions(ctx, now = new Date(), after, budgetMs = hostTuning().sweep.drainBudgetMs) {
|
|
239
231
|
const deadline = Date.now() + budgetMs;
|
|
240
232
|
const woken = [];
|
|
233
|
+
const batchSize = hostTuning().sweep.batchSize;
|
|
241
234
|
let cursor = after;
|
|
242
235
|
for (;;) {
|
|
243
|
-
const page = await ctx.store.tx((t) => t.orphanedSessions(now,
|
|
236
|
+
const page = await ctx.store.tx((t) => t.orphanedSessions(now, batchSize, cursor));
|
|
244
237
|
for (const session of page) {
|
|
245
238
|
await ctx.wake(session.id, { reason: "recover" });
|
|
246
239
|
woken.push(session.id);
|
|
247
240
|
cursor = session.id;
|
|
248
241
|
}
|
|
249
|
-
if (page.length <
|
|
242
|
+
if (page.length < batchSize)
|
|
250
243
|
return { woken };
|
|
251
244
|
if (Date.now() >= deadline)
|
|
252
245
|
return { woken, next: cursor };
|
|
253
246
|
}
|
|
254
247
|
}
|
|
255
|
-
/** A pod sandbox in a change this long without a reconcile gets one from the sweep. */
|
|
256
|
-
const STALE_POD_MS = 60_000;
|
|
257
248
|
/**
|
|
258
249
|
* Pod sandboxes (F7.2) whose change has not finished (creating, deleting, an old volume still
|
|
259
250
|
* to delete, an expiry not yet recorded), or that should run but whose engine is not connected,
|
|
260
|
-
* and whose row has not moved for a minute, get a reconcile: a pod or volume deleted outside
|
|
251
|
+
* and whose row has not moved for `sweep.stalePodMs` (a minute), get a reconcile: a pod or volume deleted outside
|
|
261
252
|
* the Runtime is found, and a change goes on whose reconcile the execution itself lost (its
|
|
262
253
|
* retry timer, an invocation an operator killed, Restate's state wiped) or whose outbox row
|
|
263
254
|
* was parked. A reconcile a commit asked for is never lost: it is in the outbox.
|
|
@@ -267,7 +258,7 @@ async function reconcileStalePods(ctx, now) {
|
|
|
267
258
|
return;
|
|
268
259
|
const stale = await ctx.store.tx(async (t) => (await t.listEnvironments()).filter((row) => {
|
|
269
260
|
const pod = row.pod;
|
|
270
|
-
if (!pod || now.getTime() - Date.parse(row.updatedAt) <
|
|
261
|
+
if (!pod || now.getTime() - Date.parse(row.updatedAt) < hostTuning().sweep.stalePodMs)
|
|
271
262
|
return false;
|
|
272
263
|
return (pod.desired === "deleted" ||
|
|
273
264
|
pod.retiring !== undefined ||
|