@desplega.ai/agent-swarm 1.131.0 → 1.133.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{actions-w53mbns9.js → actions-cskpayd7.js} +11 -9
- package/dist/{anthropic-messages-rjmd2e7n.js → anthropic-messages-tpxtbx7b.js} +28 -12
- package/dist/{app-m9gwqsqv.js → app-ang77nbb.js} +8 -6
- package/dist/{assistant-bp65m6f3.js → assistant-b4exebdw.js} +16 -13
- package/dist/{azure-openai-responses-c0yjmbfd.js → azure-openai-responses-m3921t4y.js} +8 -8
- package/dist/{boot-reembed-62p0at2k.js → boot-reembed-ps2aftpb.js} +9 -7
- package/dist/{boot-reembed-xmfp6wdh.js → boot-reembed-znyk3nxb.js} +10 -8
- package/dist/{boot-scrub-logs-rr8q2dms.js → boot-scrub-logs-2ax3xgz1.js} +7 -5
- package/dist/{claude-adapter-2y47mrkd.js → claude-adapter-3ct8kgxw.js} +13 -9
- package/dist/{claude-managed-adapter-gv19tdew.js → claude-managed-adapter-akx4ntf2.js} +2 -2
- package/dist/{cli-t1xze5qh.js → cli-13w2hwkh.js} +1 -1
- package/dist/{cli-efxfnq6q.js → cli-1532szb1.js} +8 -6
- package/dist/{cli-d2tfkcga.js → cli-23d3rseb.js} +33 -17
- package/dist/{cli-nm5gtp1x.js → cli-2a75z9c5.js} +196 -7
- package/dist/{cli-fywms69x.js → cli-2t30rj0d.js} +1 -1
- package/dist/{cli-cvgjwnh1.js → cli-3c3tdgab.js} +10 -10
- package/dist/{cli-pm6xbh84.js → cli-3zzxa9jz.js} +46 -16
- package/dist/{cli-cdeg08wx.js → cli-4k6233gc.js} +1 -1
- package/dist/{cli-t55yw3a1.js → cli-5p2tz05r.js} +2 -2
- package/dist/{cli-0rcyq6d9.js → cli-5z2yhzvd.js} +125 -47
- package/dist/{cli-3kf51j15.js → cli-6zfq31nj.js} +9 -3
- package/dist/{cli-er3a4bb0.js → cli-8m9de10a.js} +6 -1
- package/dist/{cli-gm9cbhbq.js → cli-8t14vg9w.js} +27 -8
- package/dist/{cli-72q8m8gk.js → cli-9nfc2es8.js} +6 -6
- package/dist/{cli-5ze9rv97.js → cli-9ta3pwms.js} +11 -4
- package/dist/{cli-78xwvkhj.js → cli-avr0ns7z.js} +2 -2
- package/dist/{cli-n6h8qt8n.js → cli-c897wyhw.js} +12 -7
- package/dist/{cli-2q72qz2s.js → cli-dgk20ft7.js} +15 -6
- package/dist/cli-et1bbwqf.js +13 -0
- package/dist/{cli-bqw0mt1n.js → cli-f5w27pj3.js} +3 -3
- package/dist/{cli-2wadx7p2.js → cli-f8dre4bz.js} +2 -2
- package/dist/{cli-e7ckcg4v.js → cli-ftwpv451.js} +170 -387
- package/dist/{cli-g6b8x1h8.js → cli-g854s2dw.js} +1 -1
- package/dist/{cli-5kx36evn.js → cli-ggmdxeqa.js} +13 -8
- package/dist/cli-hkwfe3qj.js +530 -0
- package/dist/{cli-5dsbcjjw.js → cli-jqwnqx9m.js} +1 -1
- package/dist/{cli-9dr5qzc0.js → cli-ny122vh4.js} +21 -10
- package/dist/{cli-5v4rw6yd.js → cli-pvzqhbs9.js} +2 -2
- package/dist/cli-pzq5rjym.js +15 -0
- package/dist/{cli-xpxde7rm.js → cli-qes8ef67.js} +48 -30
- package/dist/{cli-sajgpkqg.js → cli-qs9hhsvb.js} +1723 -325
- package/dist/cli-qys3rhhz.js +27 -0
- package/dist/{cli-rzbpmxdr.js → cli-r6tpm37h.js} +2 -2
- package/dist/{cli-qv61hp37.js → cli-rch80c76.js} +2 -2
- package/dist/{cli-0258203w.js → cli-rpp0x4zy.js} +137 -17
- package/dist/{cli-5n6e5yfn.js → cli-s1k0wxhe.js} +12 -7
- package/dist/{cli-psh3c4va.js → cli-sfd9v7sj.js} +1 -1
- package/dist/{cli-70xqj0z3.js → cli-sh3ytsyj.js} +49 -39
- package/dist/{cli-zsspvvm2.js → cli-w1mp9jy7.js} +1 -1
- package/dist/{cli-d3x3x1c9.js → cli-xdb4mpnp.js} +2 -2
- package/dist/{cli-57khth04.js → cli-xmyqv00z.js} +120 -4
- package/dist/{cli-yw8vghx2.js → cli-xv8t8p3c.js} +65 -32
- package/dist/{cli-f1am9szv.js → cli-xxv5pn4f.js} +1 -1
- package/dist/{cli-t76vw5db.js → cli-y08jvxzx.js} +601 -140
- package/dist/{cli-a4jzdpa3.js → cli-zfbf3srj.js} +650 -245
- package/dist/{cli-a40rnf71.js → cli-zmq89k34.js} +6 -6
- package/dist/{cli-qmdnhpej.js → cli-zr9q011v.js} +23 -13
- package/dist/cli.js +22 -22
- package/dist/{codex-adapter-d24q75wk.js → codex-adapter-bef93pq4.js} +9 -7
- package/dist/{codex-hook-hq54t9ft.js → codex-hook-e8dba65p.js} +3 -2
- package/dist/{codex-session-runner-g50vcsex.js → codex-session-runner-5pkxys7x.js} +9 -7
- package/dist/{commands-c3nrc5hg.js → commands-e23s1bff.js} +9 -7
- package/dist/{db-nt7fg3tw.js → db-m3nabatn.js} +17 -5
- package/dist/{google-generative-ai-hysrd8qx.js → google-generative-ai-8maw55xh.js} +11 -8
- package/dist/{google-vertex-p5arrga0.js → google-vertex-8vjqmx8g.js} +11 -8
- package/dist/{handlers-fkr6swat.js → handlers-akazedvr.js} +16 -13
- package/dist/{hook-77wpvnjc.js → hook-ze4cpf0v.js} +55 -41
- package/dist/{hook-g019yehy.js → hook-zfqyyenw.js} +10 -7
- package/dist/{http-jewj0syk.js → http-sja0n6kw.js} +730 -150
- package/dist/index-0gr8vd6k.js +39 -0
- package/dist/{index-7n2xpbd4.js → index-354vkhdv.js} +154 -74
- package/dist/{index-070v4eyb.js → index-5qthek5t.js} +2 -3
- package/dist/{index-hfvggd0z.js → index-95t3cqq5.js} +16 -14
- package/dist/{index-9znhar3x.js → index-sea66619.js} +3 -4
- package/dist/{index-6gm5he8h.js → index-vpn36mma.js} +15 -13
- package/dist/{keepalive-j0fjvqfd.js → keepalive-3sp3meqj.js} +16 -10
- package/dist/{lead-1n8x30xe.js → lead-mefeewkf.js} +39 -36
- package/dist/{maintenance-gvev11sa.js → maintenance-8tbrbjfg.js} +10 -8
- package/dist/mistral-conversations-00zrp7g4.js +717 -0
- package/dist/{oauth-refresh-sweep-s9ecsvh0.js → oauth-refresh-sweep-srjwjz6m.js} +8 -6
- package/dist/{onboard-pvm5b38w.js → onboard-mt2tgyjg.js} +7 -7
- package/dist/{openai-codex-responses-5g27pd6f.js → openai-codex-responses-map32h2s.js} +18 -18
- package/dist/{openai-completions-3vkmfz2a.js → openai-completions-w871b1cq.js} +12 -11
- package/dist/{openai-responses-dg4t7j7a.js → openai-responses-b0j7xnw8.js} +15 -12
- package/dist/{opencode-adapter-1gm38aba.js → opencode-adapter-b39jefcj.js} +5 -3
- package/dist/{openrouter-images-g3gm8931.js → openrouter-images-srfkywy0.js} +4 -4
- package/dist/{otel-impl-p6jvgd9c.js → otel-impl-z6pxt3j9.js} +224 -9
- package/dist/{pi-mono-adapter-t091s27g.js → pi-mono-adapter-btvqmj58.js} +1138 -408
- package/dist/{pricing-refresh-k49rcyed.js → pricing-refresh-kyq8cezh.js} +9 -7
- package/dist/{rbac-roles-cr80g96t.js → rbac-roles-3ryfrfna.js} +8 -6
- package/dist/{rbac-roles-30pspvy7.js → rbac-roles-6qfvbee1.js} +9 -7
- package/dist/{render-v2-xm457ng1.js → render-v2-p61ppjey.js} +8 -6
- package/dist/{seed-pricing-yhpv35fr.js → seed-pricing-j56bbj4y.js} +8 -6
- package/dist/{setup-nytnmwwr.js → setup-tpr6c1kn.js} +2 -2
- package/dist/{worker-msytr2cw.js → worker-9w1v0mbb.js} +39 -36
- package/openapi.json +542 -16
- package/package.json +6 -6
- package/src/be/budget-refusal-notify.ts +14 -7
- package/src/be/db-queries/task-artifacts.ts +182 -0
- package/src/be/db.ts +783 -169
- package/src/be/events.ts +5 -0
- package/src/be/migrations/130_scratch_script_retention_grace.sql +6 -0
- package/src/be/migrations/131_session_logs_created_at_index.sql +2 -0
- package/src/be/migrations/132_multi_runtime_instances.sql +44 -0
- package/src/be/migrations/133_workflow_run_trigger_type.sql +12 -0
- package/src/be/migrations/135_backfill_task_pull_request_attachments.sql +228 -0
- package/src/be/migrations/136_task_requester_provenance.sql +37 -0
- package/src/be/migrations/137_memory_retrieval_composite_index.sql +1 -0
- package/src/be/multi-runtime.ts +430 -0
- package/src/be/scripts/db.ts +43 -1
- package/src/be/scripts/extract-schema.ts +3 -1
- package/src/be/scripts/retention.ts +164 -0
- package/src/be/scripts/run-saved.ts +16 -1
- package/src/be/scripts/typecheck.ts +3 -2
- package/src/be/seed-skills/index.ts +7 -0
- package/src/be/steering.ts +1 -1
- package/src/be/swarm-config-guard.ts +18 -0
- package/src/commands/credential-wait.ts +54 -0
- package/src/commands/onboard/steps/core-credentials.tsx +1 -1
- package/src/commands/onboard/steps/custom-templates.tsx +1 -1
- package/src/commands/onboard/steps/harness-credentials.tsx +1 -1
- package/src/commands/onboard/steps/health-check.tsx +1 -1
- package/src/commands/onboard/steps/start.tsx +1 -1
- package/src/commands/profile-sync-audit.ts +56 -0
- package/src/commands/profile-sync.ts +287 -15
- package/src/commands/provider-credentials.ts +46 -19
- package/src/commands/runner.ts +196 -33
- package/src/commands/templates.ts +22 -0
- package/src/github/handlers.ts +55 -9
- package/src/heartbeat/heartbeat.ts +71 -12
- package/src/hooks/hook.ts +49 -33
- package/src/http/active-sessions.ts +11 -1
- package/src/http/agents.ts +261 -30
- package/src/http/all-routes.ts +1 -0
- package/src/http/config.ts +13 -2
- package/src/http/core.ts +102 -4
- package/src/http/db-query-bounded.ts +313 -0
- package/src/http/db-query-shared.ts +249 -0
- package/src/http/db-query.ts +54 -42
- package/src/http/events.ts +2 -0
- package/src/http/index.ts +27 -2
- package/src/http/mcp-bridge.ts +37 -3
- package/src/http/mcp.ts +12 -0
- package/src/http/memory.ts +13 -2
- package/src/http/openapi.ts +1 -0
- package/src/http/poll.ts +70 -27
- package/src/http/route-def.ts +43 -9
- package/src/http/schedules.ts +2 -2
- package/src/http/scripts.ts +31 -4
- package/src/http/session-data.ts +44 -0
- package/src/http/tasks.ts +2 -2
- package/src/http/users.ts +74 -52
- package/src/http/webhooks.ts +5 -2
- package/src/http/workflows.ts +1 -4
- package/src/http/x.ts +14 -0
- package/src/integrations/kapso/inbound.ts +1 -1
- package/src/jira/outbound.ts +39 -8
- package/src/jira/webhook-lifecycle.ts +13 -2
- package/src/linear/outbound.ts +43 -10
- package/src/oauth/keepalive.ts +13 -2
- package/src/prompts/base-prompt.ts +1 -1
- package/src/prompts/session-templates.ts +45 -7
- package/src/providers/claude-adapter.ts +6 -0
- package/src/providers/claude-managed-models.ts +34 -10
- package/src/providers/codex-adapter.ts +6 -0
- package/src/providers/opencode-adapter.ts +6 -0
- package/src/providers/pi-mono-adapter.ts +8 -0
- package/src/providers/pricing-sources.md +24 -0
- package/src/queue-stall-alarm.ts +216 -0
- package/src/scheduler/scheduler.ts +7 -0
- package/src/script-workflows/executor.ts +2 -1
- package/src/scripts-runtime/executors/native.ts +2 -1
- package/src/scripts-runtime/executors/types.ts +5 -0
- package/src/scripts-runtime/loader.ts +5 -0
- package/src/scripts-runtime/swarm-config.ts +8 -0
- package/src/scripts-runtime/swarm-sdk.ts +7 -0
- package/src/scripts-runtime/types/stdlib.d.ts +1 -0
- package/src/scripts-runtime/types/swarm-sdk.d.ts +1 -0
- package/src/tests/budget-claim-gate.test.ts +20 -0
- package/src/tests/claude-adapter.test.ts +32 -0
- package/src/tests/claude-managed-adapter.test.ts +38 -7
- package/src/tests/codex-adapter.test.ts +30 -0
- package/src/tests/create-task-validation.test.ts +128 -0
- package/src/tests/credential-wait.test.ts +67 -1
- package/src/tests/db-query.test.ts +711 -2
- package/src/tests/http-api-integration.test.ts +81 -0
- package/src/tests/http-users.test.ts +166 -1
- package/src/tests/identity-field-budget.test.ts +176 -0
- package/src/tests/mcp-tools-user.test.ts +34 -0
- package/src/tests/mcp-user-route.test.ts +17 -0
- package/src/tests/metrics-http.test.ts +44 -0
- package/src/tests/multi-runtime-registration.test.ts +2703 -0
- package/src/tests/openapi-response-contract.test.ts +24 -0
- package/src/tests/openapi-route-headers.test.ts +83 -0
- package/src/tests/opencode-adapter.test.ts +40 -0
- package/src/tests/outbound-listener-rejection.test.ts +138 -0
- package/src/tests/pi-mono-adapter.test.ts +98 -0
- package/src/tests/profile-sync-warning.test.ts +277 -0
- package/src/tests/profile-sync.test.ts +232 -6
- package/src/tests/prompt-template-session.test.ts +32 -3
- package/src/tests/queue-stall-alarm.test.ts +226 -0
- package/src/tests/requester-comms.test.ts +32 -0
- package/src/tests/runner-requester-profile.test.ts +38 -0
- package/src/tests/sandboxed-process.test.ts +45 -0
- package/src/tests/schedule-target-type.test.ts +38 -0
- package/src/tests/script-connections.test.ts +16 -3
- package/src/tests/scripts-external-api.test.ts +8 -3
- package/src/tests/scripts-http.test.ts +12 -0
- package/src/tests/scripts-retention.test.ts +384 -0
- package/src/tests/scripts-runtime-identity.test.ts +375 -0
- package/src/tests/scripts-runtime.test.ts +4 -3
- package/src/tests/sdk-allowlist.test.ts +47 -0
- package/src/tests/send-task-requested-by.test.ts +9 -0
- package/src/tests/session-costs.test.ts +506 -1
- package/src/tests/shutdown-error-scrubbing.test.ts +92 -0
- package/src/tests/steering-core.test.ts +28 -0
- package/src/tests/task-artifacts.test.ts +386 -0
- package/src/tests/task-attachments-schema.test.ts +17 -1
- package/src/tests/task-completion-idempotency.test.ts +57 -0
- package/src/tests/task-pull-request-backfill-migration.test.ts +321 -0
- package/src/tests/update-profile-api.test.ts +67 -39
- package/src/tests/update-profile-auth.test.ts +48 -4
- package/src/tests/user-attribution-migration.test.ts +54 -0
- package/src/tests/user-identity.test.ts +72 -0
- package/src/tests/workflow-async-v2.test.ts +39 -1
- package/src/tests/workflow-http-v2.test.ts +3 -2
- package/src/tests/workflow-mcp-trigger-schema.test.ts +52 -3
- package/src/tests/workflow-retry-v2.test.ts +45 -1
- package/src/tests/workflow-swarm-script.test.ts +14 -5
- package/src/tools/db-query.ts +18 -8
- package/src/tools/get-task-details.ts +24 -3
- package/src/tools/manage-user.ts +36 -3
- package/src/tools/poll-task.ts +93 -14
- package/src/tools/script-common.ts +5 -0
- package/src/tools/send-task.ts +1 -4
- package/src/tools/store-progress.ts +13 -2
- package/src/tools/swarm-config/delete-config.ts +9 -0
- package/src/tools/swarm-config/set-config.ts +3 -2
- package/src/tools/task-action.ts +30 -0
- package/src/tools/task-tool-ctx.ts +8 -1
- package/src/tools/update-profile.ts +16 -21
- package/src/tools/utils.ts +5 -0
- package/src/tools/workflows/trigger-workflow.ts +7 -1
- package/src/types.ts +165 -1
- package/src/utils/github-pull-request.ts +76 -0
- package/src/utils/identity-field-budget.ts +73 -0
- package/src/utils/multi-runtime.ts +20 -0
- package/src/utils/requester-comms.ts +24 -0
- package/src/utils/sandboxed-process.ts +40 -14
- package/src/workflows/engine.ts +22 -2
- package/src/workflows/executors/swarm-script.ts +24 -1
- package/src/workflows/resume.ts +19 -11
- package/src/workflows/retry-poller.ts +1 -0
- package/src/workflows/triggers.ts +4 -0
- package/templates/schedules/daily-compounding-reflection/content.md +12 -3
- package/templates/skills/db-query-guidance/config.json +14 -0
- package/templates/skills/db-query-guidance/content.md +57 -0
- package/dist/cli-3te4jdkm.js +0 -225
- package/dist/cli-bmze986d.js +0 -7
- package/dist/cli-kh3ba2zf.js +0 -607
- package/dist/index-hp1kdbjj.js +0 -36
- package/dist/mistral-conversations-81fwav0j.js +0 -30578
- package/dist/otel-nqa12wf3.js +0 -3149
- package/dist/{pi-messages-tqj8q6b9.js → pi-messages-sssy32x4.js} +3 -3
|
@@ -0,0 +1,2703 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-runtime registration and lifecycle, exercised through the real HTTP
|
|
3
|
+
* handlers on both sides of MULTI_RUNTIME_ENABLED.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
7
|
+
import { unlink } from "node:fs/promises";
|
|
8
|
+
import { createServer as createHttpServer, type Server } from "node:http";
|
|
9
|
+
import {
|
|
10
|
+
closeDb,
|
|
11
|
+
completeTask,
|
|
12
|
+
createAgent,
|
|
13
|
+
createTaskExtended,
|
|
14
|
+
getActiveSessionForTask,
|
|
15
|
+
getActiveTaskCount,
|
|
16
|
+
getAgentById,
|
|
17
|
+
getDb,
|
|
18
|
+
getSwarmConfigs,
|
|
19
|
+
getTaskById,
|
|
20
|
+
hasCapacity,
|
|
21
|
+
initDb,
|
|
22
|
+
startTask,
|
|
23
|
+
upsertSwarmConfig,
|
|
24
|
+
} from "../be/db";
|
|
25
|
+
import {
|
|
26
|
+
AGENT_MAX_TASKS_CONFIG_KEY,
|
|
27
|
+
countActiveRuntimeInstancesForAgent,
|
|
28
|
+
expireStaleRuntimeInstances,
|
|
29
|
+
getAgentMaxTasksConfig,
|
|
30
|
+
getRuntimeInstanceById,
|
|
31
|
+
hasReadyLiveRuntime,
|
|
32
|
+
} from "../be/multi-runtime";
|
|
33
|
+
import { retryBootStep } from "../commands/credential-wait";
|
|
34
|
+
import { sendCredStatusReport } from "../commands/provider-credentials";
|
|
35
|
+
import { runHeartbeatSweep } from "../heartbeat/heartbeat";
|
|
36
|
+
import { handleActiveSessions } from "../http/active-sessions";
|
|
37
|
+
import { handleAgentRegister, handleAgentsRest } from "../http/agents";
|
|
38
|
+
import { handleConfig } from "../http/config";
|
|
39
|
+
import { handleCore } from "../http/core";
|
|
40
|
+
import { handlePoll } from "../http/poll";
|
|
41
|
+
import { getPathSegments, parseQueryParams } from "../http/utils";
|
|
42
|
+
import { registerPollTaskTool } from "../tools/poll-task";
|
|
43
|
+
import { registerSetConfigTool } from "../tools/swarm-config/set-config";
|
|
44
|
+
import { registerTaskActionTool } from "../tools/task-action";
|
|
45
|
+
import { setRequestAuth } from "../utils/request-auth-context";
|
|
46
|
+
|
|
47
|
+
const TEST_DB_PATH = "./test-multi-runtime-registration.sqlite";
|
|
48
|
+
const TEST_PORT = 13113 + (process.pid % 1000);
|
|
49
|
+
const baseUrl = `http://localhost:${TEST_PORT}`;
|
|
50
|
+
const API_KEY = "test-multi-runtime-key";
|
|
51
|
+
|
|
52
|
+
const LEAD_ID = "44444444-4444-4444-4444-444444444444";
|
|
53
|
+
|
|
54
|
+
async function removeDbFiles(path: string): Promise<void> {
|
|
55
|
+
for (const suffix of ["", "-wal", "-shm"]) {
|
|
56
|
+
await unlink(path + suffix).catch(() => {});
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function createTestServer(): Server {
|
|
61
|
+
return createHttpServer(async (req, res) => {
|
|
62
|
+
const myAgentId = (req.headers["x-agent-id"] as string | undefined) ?? undefined;
|
|
63
|
+
|
|
64
|
+
// /ping and /close run through the real handleCore (its bearer auth gate
|
|
65
|
+
// included) — the runtime-aware semantics under test live there.
|
|
66
|
+
if (req.url === "/ping" || req.url === "/close") {
|
|
67
|
+
const handled = await handleCore(req, res, myAgentId, API_KEY);
|
|
68
|
+
if (handled) return;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (req.url?.includes("/credential-status") || req.url?.includes("/runtime-instances")) {
|
|
72
|
+
const handled = await handleAgentsRest(
|
|
73
|
+
req,
|
|
74
|
+
res,
|
|
75
|
+
getPathSegments(req.url),
|
|
76
|
+
parseQueryParams(req.url),
|
|
77
|
+
myAgentId,
|
|
78
|
+
);
|
|
79
|
+
if (handled) return;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (req.url?.startsWith("/api/active-sessions")) {
|
|
83
|
+
const handled = await handleActiveSessions(
|
|
84
|
+
req,
|
|
85
|
+
res,
|
|
86
|
+
getPathSegments(req.url),
|
|
87
|
+
parseQueryParams(req.url),
|
|
88
|
+
myAgentId,
|
|
89
|
+
);
|
|
90
|
+
if (handled) return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (req.url?.startsWith("/api/poll")) {
|
|
94
|
+
const handled = await handlePoll(
|
|
95
|
+
req,
|
|
96
|
+
res,
|
|
97
|
+
getPathSegments(req.url),
|
|
98
|
+
parseQueryParams(req.url),
|
|
99
|
+
myAgentId,
|
|
100
|
+
);
|
|
101
|
+
if (handled) return;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Production requests pass handleCore's auth first; this mock bypasses it
|
|
105
|
+
// for the register/config routes, so simulate the operator principal the
|
|
106
|
+
// config RBAC gate short-circuits on.
|
|
107
|
+
setRequestAuth(req, { kind: "operator", fingerprint: "test" });
|
|
108
|
+
const pathSegments = getPathSegments(req.url || "");
|
|
109
|
+
const queryParams = parseQueryParams(req.url || "");
|
|
110
|
+
try {
|
|
111
|
+
if (await handleAgentRegister(req, res, pathSegments, myAgentId)) return;
|
|
112
|
+
if (await handleConfig(req, res, pathSegments, queryParams)) return;
|
|
113
|
+
} catch (err) {
|
|
114
|
+
res.writeHead(500, { "Content-Type": "application/json" });
|
|
115
|
+
res.end(JSON.stringify({ error: (err as Error).message }));
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
res.writeHead(404, { "Content-Type": "application/json" });
|
|
119
|
+
res.end(JSON.stringify({ error: "Not found" }));
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async function register(
|
|
124
|
+
agentId: string,
|
|
125
|
+
body: Record<string, unknown> = {},
|
|
126
|
+
): Promise<{ status: number; agent: { maxTasks?: number } }> {
|
|
127
|
+
const res = await fetch(`${baseUrl}/api/agents`, {
|
|
128
|
+
method: "POST",
|
|
129
|
+
headers: { "Content-Type": "application/json", "X-Agent-ID": agentId },
|
|
130
|
+
body: JSON.stringify({ name: "mr-test-agent", ...body }),
|
|
131
|
+
});
|
|
132
|
+
return { status: res.status, agent: (await res.json()) as { maxTasks?: number } };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function putAgentMaxTasks(agentId: string, value: string): Promise<number> {
|
|
136
|
+
const res = await fetch(`${baseUrl}/api/config`, {
|
|
137
|
+
method: "PUT",
|
|
138
|
+
headers: { "Content-Type": "application/json" },
|
|
139
|
+
body: JSON.stringify({
|
|
140
|
+
scope: "agent",
|
|
141
|
+
scopeId: agentId,
|
|
142
|
+
key: AGENT_MAX_TASKS_CONFIG_KEY,
|
|
143
|
+
value,
|
|
144
|
+
}),
|
|
145
|
+
});
|
|
146
|
+
// Drain the body so the socket is released between sequential requests.
|
|
147
|
+
await res.text();
|
|
148
|
+
return res.status;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function pingAgent(agentId: string, runtimeInstanceId?: string): Promise<number> {
|
|
152
|
+
const headers: Record<string, string> = {
|
|
153
|
+
"X-Agent-ID": agentId,
|
|
154
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
155
|
+
};
|
|
156
|
+
if (runtimeInstanceId) headers["X-Runtime-Instance-ID"] = runtimeInstanceId;
|
|
157
|
+
const res = await fetch(`${baseUrl}/ping`, { method: "POST", headers });
|
|
158
|
+
await res.text();
|
|
159
|
+
return res.status;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
async function closeRuntime(agentId: string, runtimeInstanceId?: string): Promise<number> {
|
|
163
|
+
const headers: Record<string, string> = {
|
|
164
|
+
"X-Agent-ID": agentId,
|
|
165
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
166
|
+
};
|
|
167
|
+
if (runtimeInstanceId) headers["X-Runtime-Instance-ID"] = runtimeInstanceId;
|
|
168
|
+
const res = await fetch(`${baseUrl}/close`, { method: "POST", headers });
|
|
169
|
+
await res.text();
|
|
170
|
+
return res.status;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function pollAgent(
|
|
174
|
+
agentId: string,
|
|
175
|
+
runtimeInstanceId?: string,
|
|
176
|
+
): Promise<{ trigger?: { type?: string } } | null> {
|
|
177
|
+
const headers: Record<string, string> = {
|
|
178
|
+
"X-Agent-ID": agentId,
|
|
179
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
180
|
+
};
|
|
181
|
+
if (runtimeInstanceId) headers["X-Runtime-Instance-ID"] = runtimeInstanceId;
|
|
182
|
+
const res = await fetch(`${baseUrl}/api/poll`, { headers });
|
|
183
|
+
const text = await res.text();
|
|
184
|
+
try {
|
|
185
|
+
return JSON.parse(text) as { trigger?: { type?: string } };
|
|
186
|
+
} catch {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function makeAgent(maxTasks: number): string {
|
|
192
|
+
const id = crypto.randomUUID();
|
|
193
|
+
createAgent({
|
|
194
|
+
id,
|
|
195
|
+
name: "mr-existing",
|
|
196
|
+
isLead: false,
|
|
197
|
+
status: "idle",
|
|
198
|
+
capabilities: [],
|
|
199
|
+
maxTasks,
|
|
200
|
+
});
|
|
201
|
+
return id;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** Runtime rows for an agent, ordered as created (test-local: no production reader). */
|
|
205
|
+
function runtimeInstancesFor(agentId: string) {
|
|
206
|
+
return getDb()
|
|
207
|
+
.prepare<{ id: string; status: string; reported_slots: number }, [string]>(
|
|
208
|
+
"SELECT id, status, reported_slots FROM runtime_instances WHERE agent_id = ? ORDER BY created_at ASC, id ASC",
|
|
209
|
+
)
|
|
210
|
+
.all(agentId)
|
|
211
|
+
.map((r) => ({ id: r.id, status: r.status, reportedSlots: r.reported_slots }));
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
async function startSessionFor(agentId: string, taskId: string, runtimeInstanceId: string) {
|
|
215
|
+
const res = await fetch(`${baseUrl}/api/active-sessions`, {
|
|
216
|
+
method: "POST",
|
|
217
|
+
headers: { "Content-Type": "application/json", "X-Agent-ID": agentId },
|
|
218
|
+
body: JSON.stringify({ agentId, taskId, triggerType: "task_assigned", runtimeInstanceId }),
|
|
219
|
+
});
|
|
220
|
+
await res.text();
|
|
221
|
+
return res.status;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async function startupCleanupFor(agentId: string) {
|
|
225
|
+
const res = await fetch(`${baseUrl}/api/active-sessions/cleanup`, {
|
|
226
|
+
method: "POST",
|
|
227
|
+
headers: { "Content-Type": "application/json", "X-Agent-ID": agentId },
|
|
228
|
+
body: JSON.stringify({ agentId }),
|
|
229
|
+
});
|
|
230
|
+
await res.text();
|
|
231
|
+
return res.status;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Backdate a session's heartbeat, as a dead process's session would be. */
|
|
235
|
+
function makeSessionStale(taskId: string, minutesAgo = 30): void {
|
|
236
|
+
const when = new Date(Date.now() - minutesAgo * 60 * 1000).toISOString();
|
|
237
|
+
getDb()
|
|
238
|
+
.prepare("UPDATE active_sessions SET lastHeartbeatAt = ? WHERE taskId = ?")
|
|
239
|
+
.run(when, taskId);
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Backdate a runtime's last ping so liveness checks treat it as stale. */
|
|
243
|
+
function makeRuntimeStale(runtimeInstanceId: string, minutesAgo = 30): void {
|
|
244
|
+
const when = new Date(Date.now() - minutesAgo * 60 * 1000).toISOString();
|
|
245
|
+
getDb()
|
|
246
|
+
.prepare("UPDATE runtime_instances SET last_seen_at = ? WHERE id = ?")
|
|
247
|
+
.run(when, runtimeInstanceId);
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ─── Minimal MCP server mock (same shape as swarm-config-reserved-keys) ─────
|
|
251
|
+
type ToolHandler = (args: unknown, meta: unknown) => Promise<unknown> | unknown;
|
|
252
|
+
|
|
253
|
+
class MockMcpServer {
|
|
254
|
+
handlers = new Map<string, ToolHandler>();
|
|
255
|
+
|
|
256
|
+
registerTool(name: string, _config: unknown, handler: ToolHandler) {
|
|
257
|
+
this.handlers.set(name, handler);
|
|
258
|
+
return { name };
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function makeRequestInfo(agentId = LEAD_ID) {
|
|
263
|
+
return {
|
|
264
|
+
sessionId: "test-session",
|
|
265
|
+
requestInfo: { headers: { "x-agent-id": agentId } },
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
let server: Server;
|
|
270
|
+
const mcpServer = new MockMcpServer();
|
|
271
|
+
const originalFlag = process.env.MULTI_RUNTIME_ENABLED;
|
|
272
|
+
|
|
273
|
+
beforeAll(async () => {
|
|
274
|
+
await removeDbFiles(TEST_DB_PATH);
|
|
275
|
+
initDb(TEST_DB_PATH);
|
|
276
|
+
registerSetConfigTool(mcpServer as unknown as Parameters<typeof registerSetConfigTool>[0]);
|
|
277
|
+
registerPollTaskTool(mcpServer as unknown as Parameters<typeof registerPollTaskTool>[0]);
|
|
278
|
+
registerTaskActionTool(mcpServer as unknown as Parameters<typeof registerTaskActionTool>[0]);
|
|
279
|
+
server = createTestServer();
|
|
280
|
+
await new Promise<void>((resolve) => {
|
|
281
|
+
server.listen(TEST_PORT, () => resolve());
|
|
282
|
+
});
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
afterAll(async () => {
|
|
286
|
+
await new Promise<void>((resolve) => {
|
|
287
|
+
server.close(() => resolve());
|
|
288
|
+
});
|
|
289
|
+
closeDb();
|
|
290
|
+
await removeDbFiles(TEST_DB_PATH);
|
|
291
|
+
if (originalFlag === undefined) delete process.env.MULTI_RUNTIME_ENABLED;
|
|
292
|
+
else process.env.MULTI_RUNTIME_ENABLED = originalFlag;
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
beforeEach(() => {
|
|
296
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
297
|
+
getDb().prepare("DELETE FROM runtime_instances").run();
|
|
298
|
+
getDb().prepare("DELETE FROM active_sessions").run();
|
|
299
|
+
// Pool sweeps cap how many tasks they assign per tick, so leftovers from an
|
|
300
|
+
// earlier test would starve the one under test.
|
|
301
|
+
getDb().prepare("DELETE FROM agent_tasks").run();
|
|
302
|
+
getDb().prepare("DELETE FROM agents").run();
|
|
303
|
+
getDb().prepare("DELETE FROM swarm_config").run();
|
|
304
|
+
// set-config is lead-gated; the MCP mirror tests call it as this lead.
|
|
305
|
+
createAgent({ id: LEAD_ID, name: "mr-lead", isLead: true, status: "idle", capabilities: [] });
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
// ─── Migration 131 ──────────────────────────────────────────────────────────
|
|
309
|
+
|
|
310
|
+
describe("migration 132_multi_runtime_instances", () => {
|
|
311
|
+
test("runtime_instances table exists with the expected columns", () => {
|
|
312
|
+
const cols = getDb()
|
|
313
|
+
.prepare<{ name: string }, []>("PRAGMA table_info(runtime_instances)")
|
|
314
|
+
.all()
|
|
315
|
+
.map((r) => r.name);
|
|
316
|
+
for (const col of [
|
|
317
|
+
"id",
|
|
318
|
+
"agent_id",
|
|
319
|
+
"status",
|
|
320
|
+
"reported_slots",
|
|
321
|
+
"metadata",
|
|
322
|
+
"last_seen_at",
|
|
323
|
+
"created_at",
|
|
324
|
+
"updated_at",
|
|
325
|
+
"created_by",
|
|
326
|
+
"updated_by",
|
|
327
|
+
]) {
|
|
328
|
+
expect(cols).toContain(col);
|
|
329
|
+
}
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
test("active_sessions carries runtime ownership", () => {
|
|
333
|
+
const cols = getDb()
|
|
334
|
+
.prepare<{ name: string }, []>("PRAGMA table_info(active_sessions)")
|
|
335
|
+
.all()
|
|
336
|
+
.map((r) => r.name);
|
|
337
|
+
expect(cols).toContain("runtimeInstanceId");
|
|
338
|
+
});
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
describe("legacy mode (flag off)", () => {
|
|
342
|
+
test("registration creates the agent with the reported maxTasks", async () => {
|
|
343
|
+
const id = crypto.randomUUID();
|
|
344
|
+
const { status, agent } = await register(id, { maxTasks: 2 });
|
|
345
|
+
expect(status).toBe(201);
|
|
346
|
+
expect(agent.maxTasks).toBe(2);
|
|
347
|
+
expect(getAgentById(id)?.maxTasks).toBe(2);
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
test("re-registration writes the reported value through to agents.maxTasks", async () => {
|
|
351
|
+
const id = makeAgent(3);
|
|
352
|
+
const { status } = await register(id, { maxTasks: 5 });
|
|
353
|
+
expect(status).toBe(200);
|
|
354
|
+
expect(getAgentById(id)?.maxTasks).toBe(5);
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
test("no runtime-instance rows and no policy config rows are written, even when the worker sends a runtimeInstanceId", async () => {
|
|
358
|
+
const id = crypto.randomUUID();
|
|
359
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: crypto.randomUUID() });
|
|
360
|
+
await register(id, { maxTasks: 4, runtimeInstanceId: crypto.randomUUID() });
|
|
361
|
+
expect(runtimeInstancesFor(id)).toHaveLength(0);
|
|
362
|
+
expect(getAgentMaxTasksConfig(id)).toBeNull();
|
|
363
|
+
expect(getAgentById(id)?.maxTasks).toBe(4);
|
|
364
|
+
});
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
describe("multi-runtime mode: registration vs logical policy", () => {
|
|
368
|
+
test("two runtimes with different capacities never touch the persisted policy", async () => {
|
|
369
|
+
const id = makeAgent(3);
|
|
370
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
371
|
+
|
|
372
|
+
const r1 = crypto.randomUUID();
|
|
373
|
+
const r2 = crypto.randomUUID();
|
|
374
|
+
await register(id, { maxTasks: 5, runtimeInstanceId: r1 });
|
|
375
|
+
await register(id, { maxTasks: 7, runtimeInstanceId: r2 });
|
|
376
|
+
|
|
377
|
+
expect(getAgentById(id)?.maxTasks).toBe(3);
|
|
378
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("3");
|
|
379
|
+
expect(getRuntimeInstanceById(r1)?.reportedSlots).toBe(5);
|
|
380
|
+
expect(getRuntimeInstanceById(r2)?.reportedSlots).toBe(7);
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
test("a brand-new agent's first registration establishes the logical policy", async () => {
|
|
384
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
385
|
+
const id = crypto.randomUUID();
|
|
386
|
+
const rt = crypto.randomUUID();
|
|
387
|
+
const { status, agent } = await register(id, { maxTasks: 7, runtimeInstanceId: rt });
|
|
388
|
+
expect(status).toBe(201);
|
|
389
|
+
expect(agent.maxTasks).toBe(7);
|
|
390
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("7");
|
|
391
|
+
expect(getRuntimeInstanceById(rt)?.reportedSlots).toBe(7);
|
|
392
|
+
});
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
describe("multi-runtime mode: operator value survives runtime registrations", () => {
|
|
396
|
+
test("registrations after an operator update leave the policy untouched", async () => {
|
|
397
|
+
const id = makeAgent(3);
|
|
398
|
+
expect(await putAgentMaxTasks(id, "4")).toBe(200);
|
|
399
|
+
expect(getAgentById(id)?.maxTasks).toBe(4);
|
|
400
|
+
|
|
401
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
402
|
+
await register(id, { maxTasks: 10, runtimeInstanceId: crypto.randomUUID() });
|
|
403
|
+
await register(id, { maxTasks: 20, runtimeInstanceId: crypto.randomUUID() });
|
|
404
|
+
|
|
405
|
+
expect(getAgentById(id)?.maxTasks).toBe(4);
|
|
406
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("4");
|
|
407
|
+
const rows = getSwarmConfigs({ scope: "agent", scopeId: id, key: AGENT_MAX_TASKS_CONFIG_KEY });
|
|
408
|
+
expect(rows).toHaveLength(1);
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
|
|
412
|
+
describe("multi-runtime mode: enablement seeding", () => {
|
|
413
|
+
test("first registration seeds AGENT_MAX_TASKS from the persisted agents.maxTasks", async () => {
|
|
414
|
+
const id = makeAgent(3);
|
|
415
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
416
|
+
|
|
417
|
+
await register(id, { maxTasks: 9, runtimeInstanceId: crypto.randomUUID() });
|
|
418
|
+
let rows = getSwarmConfigs({ scope: "agent", scopeId: id, key: AGENT_MAX_TASKS_CONFIG_KEY });
|
|
419
|
+
expect(rows).toHaveLength(1);
|
|
420
|
+
expect(rows[0]?.value).toBe("3");
|
|
421
|
+
|
|
422
|
+
await register(id, { maxTasks: 12, runtimeInstanceId: crypto.randomUUID() });
|
|
423
|
+
rows = getSwarmConfigs({ scope: "agent", scopeId: id, key: AGENT_MAX_TASKS_CONFIG_KEY });
|
|
424
|
+
expect(rows).toHaveLength(1);
|
|
425
|
+
expect(rows[0]?.value).toBe("3");
|
|
426
|
+
});
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
describe("multi-runtime mode: existing config preserved", () => {
|
|
430
|
+
test("registration keeps the pre-existing row and repairs the mirror from it", async () => {
|
|
431
|
+
const id = makeAgent(3);
|
|
432
|
+
upsertSwarmConfig({
|
|
433
|
+
scope: "agent",
|
|
434
|
+
scopeId: id,
|
|
435
|
+
key: AGENT_MAX_TASKS_CONFIG_KEY,
|
|
436
|
+
value: "9",
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
440
|
+
await register(id, { maxTasks: 5, runtimeInstanceId: crypto.randomUUID() });
|
|
441
|
+
|
|
442
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("9");
|
|
443
|
+
// The column converges to the config row, not the runtime's report.
|
|
444
|
+
expect(getAgentById(id)?.maxTasks).toBe(9);
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
test("a config row written before the agent exists is applied when the agent registers", async () => {
|
|
448
|
+
const id = crypto.randomUUID();
|
|
449
|
+
upsertSwarmConfig({
|
|
450
|
+
scope: "agent",
|
|
451
|
+
scopeId: id,
|
|
452
|
+
key: AGENT_MAX_TASKS_CONFIG_KEY,
|
|
453
|
+
value: "6",
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
457
|
+
await register(id, { maxTasks: 5, runtimeInstanceId: crypto.randomUUID() });
|
|
458
|
+
|
|
459
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("6");
|
|
460
|
+
expect(getAgentById(id)?.maxTasks).toBe(6);
|
|
461
|
+
});
|
|
462
|
+
});
|
|
463
|
+
|
|
464
|
+
describe("operator updates mirror into agents.maxTasks", () => {
|
|
465
|
+
test("PUT /api/config updates the enforcement mirror atomically", async () => {
|
|
466
|
+
const id = makeAgent(2);
|
|
467
|
+
expect(await putAgentMaxTasks(id, "6")).toBe(200);
|
|
468
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("6");
|
|
469
|
+
expect(getAgentById(id)?.maxTasks).toBe(6);
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
test("invalid values are rejected and leave both sides untouched", async () => {
|
|
473
|
+
const id = makeAgent(2);
|
|
474
|
+
for (const bad of ["0", "-1", "abc", "101", "2.5"]) {
|
|
475
|
+
expect(await putAgentMaxTasks(id, bad)).toBe(400);
|
|
476
|
+
}
|
|
477
|
+
expect(getAgentMaxTasksConfig(id)).toBeNull();
|
|
478
|
+
expect(getAgentById(id)?.maxTasks).toBe(2);
|
|
479
|
+
});
|
|
480
|
+
|
|
481
|
+
test("the MCP set-config tool mirrors the same way", async () => {
|
|
482
|
+
const id = makeAgent(2);
|
|
483
|
+
const handler = mcpServer.handlers.get("set-config");
|
|
484
|
+
expect(handler).toBeDefined();
|
|
485
|
+
const result = (await handler?.(
|
|
486
|
+
{
|
|
487
|
+
scope: "agent",
|
|
488
|
+
scopeId: id,
|
|
489
|
+
key: AGENT_MAX_TASKS_CONFIG_KEY,
|
|
490
|
+
value: "8",
|
|
491
|
+
},
|
|
492
|
+
makeRequestInfo(),
|
|
493
|
+
)) as { isError?: boolean };
|
|
494
|
+
expect(result?.isError).toBeFalsy();
|
|
495
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("8");
|
|
496
|
+
expect(getAgentById(id)?.maxTasks).toBe(8);
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
test("global-scope writes of other keys are untouched by the mirror wrapper", async () => {
|
|
500
|
+
const res = await fetch(`${baseUrl}/api/config`, {
|
|
501
|
+
method: "PUT",
|
|
502
|
+
headers: { "Content-Type": "application/json" },
|
|
503
|
+
body: JSON.stringify({ scope: "global", key: "STEERING_ENABLED", value: "true" }),
|
|
504
|
+
});
|
|
505
|
+
expect(res.status).toBe(200);
|
|
506
|
+
await res.text();
|
|
507
|
+
expect(getSwarmConfigs({ scope: "global", key: "STEERING_ENABLED" })[0]?.value).toBe("true");
|
|
508
|
+
});
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
describe("disable / legacy restoration", () => {
|
|
512
|
+
test("after the flag is turned off, write-through resumes and the config row is left alone", async () => {
|
|
513
|
+
const id = makeAgent(3);
|
|
514
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
515
|
+
await register(id, { maxTasks: 5, runtimeInstanceId: crypto.randomUUID() });
|
|
516
|
+
expect(getAgentById(id)?.maxTasks).toBe(3);
|
|
517
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("3");
|
|
518
|
+
|
|
519
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
520
|
+
const { status } = await register(id, { maxTasks: 8 });
|
|
521
|
+
expect(status).toBe(200);
|
|
522
|
+
expect(getAgentById(id)?.maxTasks).toBe(8);
|
|
523
|
+
// The seeded row persists but no longer gates registration.
|
|
524
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("3");
|
|
525
|
+
});
|
|
526
|
+
});
|
|
527
|
+
|
|
528
|
+
describe("runtime instances", () => {
|
|
529
|
+
test("two instances serving one agent keep independent capacities", async () => {
|
|
530
|
+
const id = makeAgent(3);
|
|
531
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
532
|
+
const r1 = crypto.randomUUID();
|
|
533
|
+
const r2 = crypto.randomUUID();
|
|
534
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: r1 });
|
|
535
|
+
await register(id, { maxTasks: 6, runtimeInstanceId: r2 });
|
|
536
|
+
|
|
537
|
+
const instances = runtimeInstancesFor(id);
|
|
538
|
+
expect(instances).toHaveLength(2);
|
|
539
|
+
const byId = new Map(instances.map((i) => [i.id, i]));
|
|
540
|
+
expect(byId.get(r1)?.reportedSlots).toBe(2);
|
|
541
|
+
expect(byId.get(r2)?.reportedSlots).toBe(6);
|
|
542
|
+
for (const instance of instances) {
|
|
543
|
+
expect(instance.status).toBe("active");
|
|
544
|
+
}
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
test("a re-registration from the same runtime refreshes its row in place", async () => {
|
|
548
|
+
const id = makeAgent(3);
|
|
549
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
550
|
+
const rt = crypto.randomUUID();
|
|
551
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rt });
|
|
552
|
+
await register(id, { maxTasks: 4, runtimeInstanceId: rt });
|
|
553
|
+
|
|
554
|
+
const instances = runtimeInstancesFor(id);
|
|
555
|
+
expect(instances).toHaveLength(1);
|
|
556
|
+
expect(instances[0]?.id).toBe(rt);
|
|
557
|
+
expect(instances[0]?.reportedSlots).toBe(4);
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
test("re-registering a closed runtime makes it active again", async () => {
|
|
561
|
+
const id = makeAgent(3);
|
|
562
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
563
|
+
const rt = crypto.randomUUID();
|
|
564
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rt });
|
|
565
|
+
await closeRuntime(id, rt);
|
|
566
|
+
expect(getRuntimeInstanceById(rt)?.status).toBe("offline");
|
|
567
|
+
|
|
568
|
+
// Registration is the only path that may restore `active`.
|
|
569
|
+
await register(id, { maxTasks: 3, runtimeInstanceId: rt });
|
|
570
|
+
expect(getRuntimeInstanceById(rt)?.status).toBe("active");
|
|
571
|
+
expect(getRuntimeInstanceById(rt)?.reportedSlots).toBe(3);
|
|
572
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
573
|
+
});
|
|
574
|
+
|
|
575
|
+
test("a runtime id belonging to another agent is rejected and never reassigned", async () => {
|
|
576
|
+
const owner = makeAgent(3);
|
|
577
|
+
const other = makeAgent(3);
|
|
578
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
579
|
+
const rt = crypto.randomUUID();
|
|
580
|
+
await register(owner, { maxTasks: 2, runtimeInstanceId: rt });
|
|
581
|
+
|
|
582
|
+
const { status } = await register(other, { maxTasks: 9, runtimeInstanceId: rt });
|
|
583
|
+
expect(status).toBe(400);
|
|
584
|
+
|
|
585
|
+
const instance = getRuntimeInstanceById(rt);
|
|
586
|
+
expect(instance?.agentId).toBe(owner);
|
|
587
|
+
expect(instance?.reportedSlots).toBe(2);
|
|
588
|
+
expect(runtimeInstancesFor(other)).toHaveLength(0);
|
|
589
|
+
});
|
|
590
|
+
|
|
591
|
+
test("registration without a runtimeInstanceId is rejected with 400 when the flag is on", async () => {
|
|
592
|
+
const id = makeAgent(3);
|
|
593
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
594
|
+
const { status } = await register(id, { maxTasks: 5 });
|
|
595
|
+
expect(status).toBe(400);
|
|
596
|
+
expect(getAgentById(id)?.maxTasks).toBe(3);
|
|
597
|
+
expect(getAgentMaxTasksConfig(id)).toBeNull();
|
|
598
|
+
expect(runtimeInstancesFor(id)).toHaveLength(0);
|
|
599
|
+
});
|
|
600
|
+
|
|
601
|
+
test("an unknown agent registering without a runtimeInstanceId is rejected and never created", async () => {
|
|
602
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
603
|
+
const id = crypto.randomUUID();
|
|
604
|
+
const { status } = await register(id, { maxTasks: 5 });
|
|
605
|
+
expect(status).toBe(400);
|
|
606
|
+
expect(getAgentById(id)).toBeNull();
|
|
607
|
+
});
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
// ─── Runtime-instance read API ──────────────────────────────────────────────
|
|
611
|
+
|
|
612
|
+
describe("GET /api/agents/{id}/runtime-instances", () => {
|
|
613
|
+
interface RuntimeInstanceEntry {
|
|
614
|
+
id: string;
|
|
615
|
+
agentId: string;
|
|
616
|
+
status: string;
|
|
617
|
+
isLive: boolean;
|
|
618
|
+
reportedSlots: number;
|
|
619
|
+
credentialReady?: boolean | null;
|
|
620
|
+
lastSeenAt: string;
|
|
621
|
+
createdAt: string;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
async function listRuntimes(agentId: string): Promise<{
|
|
625
|
+
status: number;
|
|
626
|
+
body: {
|
|
627
|
+
runtimeInstances?: RuntimeInstanceEntry[];
|
|
628
|
+
staleThresholdMinutes?: number;
|
|
629
|
+
error?: string;
|
|
630
|
+
};
|
|
631
|
+
}> {
|
|
632
|
+
const res = await fetch(`${baseUrl}/api/agents/${agentId}/runtime-instances`);
|
|
633
|
+
return { status: res.status, body: (await res.json()) as never };
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
async function reportRuntimeReady(
|
|
637
|
+
agentId: string,
|
|
638
|
+
runtimeInstanceId: string,
|
|
639
|
+
ready: boolean,
|
|
640
|
+
): Promise<number> {
|
|
641
|
+
const res = await fetch(`${baseUrl}/api/agents/${agentId}/credential-status`, {
|
|
642
|
+
method: "PUT",
|
|
643
|
+
headers: {
|
|
644
|
+
"Content-Type": "application/json",
|
|
645
|
+
"X-Agent-ID": agentId,
|
|
646
|
+
"X-Runtime-Instance-ID": runtimeInstanceId,
|
|
647
|
+
},
|
|
648
|
+
body: JSON.stringify({ ready }),
|
|
649
|
+
});
|
|
650
|
+
await res.text();
|
|
651
|
+
return res.status;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
test("an agent with no runtimes returns an empty list", async () => {
|
|
655
|
+
const id = makeAgent(2);
|
|
656
|
+
const { status, body } = await listRuntimes(id);
|
|
657
|
+
expect(status).toBe(200);
|
|
658
|
+
expect(body.runtimeInstances).toEqual([]);
|
|
659
|
+
expect(body.staleThresholdMinutes).toBe(5);
|
|
660
|
+
});
|
|
661
|
+
|
|
662
|
+
test("a nonexistent agent returns 404", async () => {
|
|
663
|
+
const { status, body } = await listRuntimes(crypto.randomUUID());
|
|
664
|
+
expect(status).toBe(404);
|
|
665
|
+
expect(body.error).toBe("Agent not found");
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
test("a live runtime is returned with its capacity, liveness, and identity", async () => {
|
|
669
|
+
const id = makeAgent(3);
|
|
670
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
671
|
+
const rt = crypto.randomUUID();
|
|
672
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rt });
|
|
673
|
+
|
|
674
|
+
const { status, body } = await listRuntimes(id);
|
|
675
|
+
expect(status).toBe(200);
|
|
676
|
+
expect(body.runtimeInstances).toHaveLength(1);
|
|
677
|
+
const entry = body.runtimeInstances?.[0];
|
|
678
|
+
expect(entry?.id).toBe(rt);
|
|
679
|
+
expect(entry?.agentId).toBe(id);
|
|
680
|
+
expect(entry?.status).toBe("active");
|
|
681
|
+
expect(entry?.isLive).toBe(true);
|
|
682
|
+
expect(entry?.reportedSlots).toBe(2);
|
|
683
|
+
expect(entry?.credentialReady).toBeNull();
|
|
684
|
+
expect(Date.parse(entry?.lastSeenAt ?? "")).not.toBeNaN();
|
|
685
|
+
// metadata is server-internal and must not leak through the read API.
|
|
686
|
+
expect(entry).not.toContainKey("metadata");
|
|
687
|
+
});
|
|
688
|
+
|
|
689
|
+
test("every runtime serving the agent is returned independently", async () => {
|
|
690
|
+
const id = makeAgent(4);
|
|
691
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
692
|
+
const rA = crypto.randomUUID();
|
|
693
|
+
const rB = crypto.randomUUID();
|
|
694
|
+
const rC = crypto.randomUUID();
|
|
695
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
696
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rB });
|
|
697
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rC });
|
|
698
|
+
|
|
699
|
+
const { body } = await listRuntimes(id);
|
|
700
|
+
expect(body.runtimeInstances).toHaveLength(3);
|
|
701
|
+
const byId = new Map(body.runtimeInstances?.map((r) => [r.id, r]));
|
|
702
|
+
expect(byId.get(rA)?.reportedSlots).toBe(1);
|
|
703
|
+
expect(byId.get(rB)?.reportedSlots).toBe(2);
|
|
704
|
+
expect(byId.get(rC)?.reportedSlots).toBe(1);
|
|
705
|
+
});
|
|
706
|
+
|
|
707
|
+
test("runtimes of another agent never appear", async () => {
|
|
708
|
+
const mine = makeAgent(2);
|
|
709
|
+
const other = makeAgent(2);
|
|
710
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
711
|
+
const rMine = crypto.randomUUID();
|
|
712
|
+
const rOther = crypto.randomUUID();
|
|
713
|
+
await register(mine, { maxTasks: 1, runtimeInstanceId: rMine });
|
|
714
|
+
await register(other, { maxTasks: 1, runtimeInstanceId: rOther });
|
|
715
|
+
|
|
716
|
+
const { body } = await listRuntimes(mine);
|
|
717
|
+
expect(body.runtimeInstances?.map((r) => r.id)).toEqual([rMine]);
|
|
718
|
+
});
|
|
719
|
+
|
|
720
|
+
test("a stale unswept runtime keeps status active but is not presented as live", async () => {
|
|
721
|
+
const id = makeAgent(2);
|
|
722
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
723
|
+
const fresh = crypto.randomUUID();
|
|
724
|
+
const stale = crypto.randomUUID();
|
|
725
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: fresh });
|
|
726
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: stale });
|
|
727
|
+
makeRuntimeStale(stale);
|
|
728
|
+
|
|
729
|
+
const { body } = await listRuntimes(id);
|
|
730
|
+
const byId = new Map(body.runtimeInstances?.map((r) => [r.id, r]));
|
|
731
|
+
expect(byId.get(fresh)?.isLive).toBe(true);
|
|
732
|
+
expect(byId.get(stale)?.status).toBe("active");
|
|
733
|
+
expect(byId.get(stale)?.isLive).toBe(false);
|
|
734
|
+
});
|
|
735
|
+
|
|
736
|
+
test("a closed runtime is reported offline and not live", async () => {
|
|
737
|
+
const id = makeAgent(2);
|
|
738
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
739
|
+
const rt = crypto.randomUUID();
|
|
740
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
741
|
+
await closeRuntime(id, rt);
|
|
742
|
+
|
|
743
|
+
const { body } = await listRuntimes(id);
|
|
744
|
+
expect(body.runtimeInstances?.[0]?.status).toBe("offline");
|
|
745
|
+
expect(body.runtimeInstances?.[0]?.isLive).toBe(false);
|
|
746
|
+
});
|
|
747
|
+
|
|
748
|
+
test("credential readiness surfaces all three states", async () => {
|
|
749
|
+
const id = makeAgent(3);
|
|
750
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
751
|
+
const ready = crypto.randomUUID();
|
|
752
|
+
const waiting = crypto.randomUUID();
|
|
753
|
+
const silent = crypto.randomUUID();
|
|
754
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: ready });
|
|
755
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: waiting });
|
|
756
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: silent });
|
|
757
|
+
expect(await reportRuntimeReady(id, ready, true)).toBe(200);
|
|
758
|
+
expect(await reportRuntimeReady(id, waiting, false)).toBe(200);
|
|
759
|
+
|
|
760
|
+
const { body } = await listRuntimes(id);
|
|
761
|
+
const byId = new Map(body.runtimeInstances?.map((r) => [r.id, r]));
|
|
762
|
+
expect(byId.get(ready)?.credentialReady).toBe(true);
|
|
763
|
+
expect(byId.get(waiting)?.credentialReady).toBe(false);
|
|
764
|
+
expect(byId.get(silent)?.credentialReady).toBeNull();
|
|
765
|
+
});
|
|
766
|
+
|
|
767
|
+
test("listing never mutates runtime state, with the flag on or off", async () => {
|
|
768
|
+
const id = makeAgent(2);
|
|
769
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
770
|
+
const rt = crypto.randomUUID();
|
|
771
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
772
|
+
makeRuntimeStale(rt);
|
|
773
|
+
const before = getDb()
|
|
774
|
+
.prepare<{ last_seen_at: string; updated_at: string; status: string }, [string]>(
|
|
775
|
+
"SELECT last_seen_at, updated_at, status FROM runtime_instances WHERE id = ?",
|
|
776
|
+
)
|
|
777
|
+
.get(rt);
|
|
778
|
+
|
|
779
|
+
await listRuntimes(id);
|
|
780
|
+
// Rows written while the flag was on stay readable after a rollback.
|
|
781
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
782
|
+
const { status, body } = await listRuntimes(id);
|
|
783
|
+
expect(status).toBe(200);
|
|
784
|
+
expect(body.runtimeInstances).toHaveLength(1);
|
|
785
|
+
|
|
786
|
+
const after = getDb()
|
|
787
|
+
.prepare<{ last_seen_at: string; updated_at: string; status: string }, [string]>(
|
|
788
|
+
"SELECT last_seen_at, updated_at, status FROM runtime_instances WHERE id = ?",
|
|
789
|
+
)
|
|
790
|
+
.get(rt);
|
|
791
|
+
expect(after).toEqual(before);
|
|
792
|
+
});
|
|
793
|
+
});
|
|
794
|
+
|
|
795
|
+
// ─── Runtime-aware close ────────────────────────────────────────────────────
|
|
796
|
+
|
|
797
|
+
describe("runtime-aware close", () => {
|
|
798
|
+
test("one of two active runtimes closing leaves the agent available and the sibling untouched", async () => {
|
|
799
|
+
const id = makeAgent(3);
|
|
800
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
801
|
+
const rA = crypto.randomUUID();
|
|
802
|
+
const rB = crypto.randomUUID();
|
|
803
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
804
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rB });
|
|
805
|
+
const bBefore = getRuntimeInstanceById(rB);
|
|
806
|
+
|
|
807
|
+
expect(await closeRuntime(id, rA)).toBe(204);
|
|
808
|
+
|
|
809
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("offline");
|
|
810
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
811
|
+
const bAfter = getRuntimeInstanceById(rB);
|
|
812
|
+
expect(bAfter?.status).toBe("active");
|
|
813
|
+
expect(bAfter?.reportedSlots).toBe(2);
|
|
814
|
+
expect(bAfter?.lastSeenAt).toBe(bBefore?.lastSeenAt ?? "");
|
|
815
|
+
expect(bAfter?.updatedAt).toBe(bBefore?.updatedAt ?? "");
|
|
816
|
+
});
|
|
817
|
+
|
|
818
|
+
test("the last active runtime closing takes the agent offline", async () => {
|
|
819
|
+
const id = makeAgent(3);
|
|
820
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
821
|
+
const rA = crypto.randomUUID();
|
|
822
|
+
const rB = crypto.randomUUID();
|
|
823
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
824
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rB });
|
|
825
|
+
|
|
826
|
+
await closeRuntime(id, rA);
|
|
827
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
828
|
+
|
|
829
|
+
expect(await closeRuntime(id, rB)).toBe(204);
|
|
830
|
+
expect(getRuntimeInstanceById(rB)?.status).toBe("offline");
|
|
831
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
832
|
+
});
|
|
833
|
+
|
|
834
|
+
test("legacy close (flag off) preserves legacy semantics", async () => {
|
|
835
|
+
const id = makeAgent(3);
|
|
836
|
+
expect(await closeRuntime(id)).toBe(204);
|
|
837
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
838
|
+
});
|
|
839
|
+
|
|
840
|
+
test("with the flag off, a runtime-identified close still takes the legacy path", async () => {
|
|
841
|
+
const id = makeAgent(3);
|
|
842
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
843
|
+
const rA = crypto.randomUUID();
|
|
844
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
845
|
+
|
|
846
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
847
|
+
expect(await closeRuntime(id, rA)).toBe(204);
|
|
848
|
+
// Runtime rows stay inert while the flag is off.
|
|
849
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
850
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("active");
|
|
851
|
+
});
|
|
852
|
+
|
|
853
|
+
test("a multi-runtime close without a runtime id fails closed with 400 and mutates nothing", async () => {
|
|
854
|
+
const id = makeAgent(3);
|
|
855
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
856
|
+
const rA = crypto.randomUUID();
|
|
857
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
858
|
+
const before = getRuntimeInstanceById(rA);
|
|
859
|
+
|
|
860
|
+
// Unlike ping, an anonymous close must not fall back to legacy semantics.
|
|
861
|
+
expect(await closeRuntime(id)).toBe(400);
|
|
862
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
863
|
+
const after = getRuntimeInstanceById(rA);
|
|
864
|
+
expect(after?.status).toBe("active");
|
|
865
|
+
expect(after?.updatedAt).toBe(before?.updatedAt ?? "");
|
|
866
|
+
expect(after?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
867
|
+
});
|
|
868
|
+
|
|
869
|
+
test("registration and close share the same identity requirement while the mode is on", async () => {
|
|
870
|
+
const id = makeAgent(3);
|
|
871
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
872
|
+
const { status: registerStatus } = await register(id, { maxTasks: 5 });
|
|
873
|
+
expect(registerStatus).toBe(400);
|
|
874
|
+
expect(await closeRuntime(id)).toBe(400);
|
|
875
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
876
|
+
expect(getAgentById(id)?.maxTasks).toBe(3);
|
|
877
|
+
expect(runtimeInstancesFor(id)).toHaveLength(0);
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
test("transition: a pre-enablement worker's id-less close cannot take the agent offline", async () => {
|
|
881
|
+
const id = makeAgent(3);
|
|
882
|
+
const { status: legacyStatus } = await register(id, { maxTasks: 2 });
|
|
883
|
+
expect(legacyStatus).toBe(200);
|
|
884
|
+
expect(getAgentById(id)?.maxTasks).toBe(2);
|
|
885
|
+
|
|
886
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
887
|
+
const rY = crypto.randomUUID();
|
|
888
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rY });
|
|
889
|
+
expect(getRuntimeInstanceById(rY)?.status).toBe("active");
|
|
890
|
+
|
|
891
|
+
// The old worker's shutdown must not retire the agent Y still serves.
|
|
892
|
+
expect(await closeRuntime(id)).toBe(400);
|
|
893
|
+
expect(getRuntimeInstanceById(rY)?.status).toBe("active");
|
|
894
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
test("a close cannot mark another agent's runtime offline", async () => {
|
|
898
|
+
const id1 = makeAgent(3);
|
|
899
|
+
const id2 = makeAgent(3);
|
|
900
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
901
|
+
const r1 = crypto.randomUUID();
|
|
902
|
+
const r2 = crypto.randomUUID();
|
|
903
|
+
await register(id1, { maxTasks: 1, runtimeInstanceId: r1 });
|
|
904
|
+
await register(id2, { maxTasks: 1, runtimeInstanceId: r2 });
|
|
905
|
+
|
|
906
|
+
// Agent 2 presents agent 1's runtime id.
|
|
907
|
+
expect(await closeRuntime(id2, r1)).toBe(204);
|
|
908
|
+
expect(getRuntimeInstanceById(r1)?.status).toBe("active");
|
|
909
|
+
expect(getRuntimeInstanceById(r2)?.status).toBe("active");
|
|
910
|
+
expect(getAgentById(id2)?.status).toBe("idle");
|
|
911
|
+
});
|
|
912
|
+
});
|
|
913
|
+
|
|
914
|
+
// ─── Runtime liveness via the worker ping ───────────────────────────────────
|
|
915
|
+
|
|
916
|
+
describe("runtime liveness via ping", () => {
|
|
917
|
+
test("a ping refreshes only the pinging runtime's last_seen_at", async () => {
|
|
918
|
+
const id = makeAgent(3);
|
|
919
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
920
|
+
const rA = crypto.randomUUID();
|
|
921
|
+
const rB = crypto.randomUUID();
|
|
922
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
923
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rB });
|
|
924
|
+
const aBefore = getRuntimeInstanceById(rA);
|
|
925
|
+
const bBefore = getRuntimeInstanceById(rB);
|
|
926
|
+
|
|
927
|
+
await Bun.sleep(10);
|
|
928
|
+
expect(await pingAgent(id, rB)).toBe(204);
|
|
929
|
+
|
|
930
|
+
const aAfter = getRuntimeInstanceById(rA);
|
|
931
|
+
const bAfter = getRuntimeInstanceById(rB);
|
|
932
|
+
expect(bAfter && bBefore && bAfter.lastSeenAt > bBefore.lastSeenAt).toBe(true);
|
|
933
|
+
expect(aAfter?.lastSeenAt).toBe(aBefore?.lastSeenAt ?? "");
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
test("a ping without the runtime header is an accepted no-op", async () => {
|
|
937
|
+
const id = makeAgent(3);
|
|
938
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
939
|
+
const rA = crypto.randomUUID();
|
|
940
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
941
|
+
const before = getRuntimeInstanceById(rA);
|
|
942
|
+
|
|
943
|
+
await Bun.sleep(10);
|
|
944
|
+
// Accepted so pre-flag workers keep running, but mutates nothing.
|
|
945
|
+
expect(await pingAgent(id)).toBe(204);
|
|
946
|
+
expect(getRuntimeInstanceById(rA)?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
947
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
948
|
+
});
|
|
949
|
+
|
|
950
|
+
test("an id-less ping does not stomp a busy agent's status", async () => {
|
|
951
|
+
const id = makeAgent(3);
|
|
952
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
953
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
954
|
+
getDb().prepare("UPDATE agents SET status = 'busy' WHERE id = ?").run(id);
|
|
955
|
+
|
|
956
|
+
expect(await pingAgent(id)).toBe(204);
|
|
957
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
958
|
+
});
|
|
959
|
+
|
|
960
|
+
test("with the flag off, a runtime-identified ping is inert", async () => {
|
|
961
|
+
const id = makeAgent(3);
|
|
962
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
963
|
+
const rA = crypto.randomUUID();
|
|
964
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
965
|
+
const before = getRuntimeInstanceById(rA);
|
|
966
|
+
|
|
967
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
968
|
+
await Bun.sleep(10);
|
|
969
|
+
expect(await pingAgent(id, rA)).toBe(204);
|
|
970
|
+
expect(getRuntimeInstanceById(rA)?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
971
|
+
});
|
|
972
|
+
|
|
973
|
+
test("an id-less ping cannot revive an agent that has zero active runtimes", async () => {
|
|
974
|
+
const id = makeAgent(3);
|
|
975
|
+
await register(id, { maxTasks: 2 });
|
|
976
|
+
|
|
977
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
978
|
+
const rY = crypto.randomUUID();
|
|
979
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: rY });
|
|
980
|
+
await closeRuntime(id, rY);
|
|
981
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
982
|
+
|
|
983
|
+
// An anonymous ping must not make the agent look available again.
|
|
984
|
+
expect(await pingAgent(id)).toBe(204);
|
|
985
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
986
|
+
});
|
|
987
|
+
|
|
988
|
+
test("a closed runtime's ping neither reactivates it nor revives the agent", async () => {
|
|
989
|
+
const id = makeAgent(3);
|
|
990
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
991
|
+
const rA = crypto.randomUUID();
|
|
992
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
993
|
+
await closeRuntime(id, rA);
|
|
994
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
995
|
+
const closed = getRuntimeInstanceById(rA);
|
|
996
|
+
|
|
997
|
+
await Bun.sleep(10);
|
|
998
|
+
expect(await pingAgent(id, rA)).toBe(204);
|
|
999
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("offline");
|
|
1000
|
+
expect(getRuntimeInstanceById(rA)?.lastSeenAt).toBe(closed?.lastSeenAt ?? "");
|
|
1001
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1002
|
+
});
|
|
1003
|
+
|
|
1004
|
+
test("an unknown runtime id leaves agent status untouched", async () => {
|
|
1005
|
+
const id = makeAgent(3);
|
|
1006
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1007
|
+
const rA = crypto.randomUUID();
|
|
1008
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1009
|
+
await closeRuntime(id, rA);
|
|
1010
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1011
|
+
|
|
1012
|
+
expect(await pingAgent(id, crypto.randomUUID())).toBe(204);
|
|
1013
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1014
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1015
|
+
});
|
|
1016
|
+
|
|
1017
|
+
test("a foreign runtime id cannot drive the presenting agent's status", async () => {
|
|
1018
|
+
const id1 = makeAgent(3);
|
|
1019
|
+
const id2 = makeAgent(3);
|
|
1020
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1021
|
+
const r1 = crypto.randomUUID();
|
|
1022
|
+
const r2 = crypto.randomUUID();
|
|
1023
|
+
await register(id1, { maxTasks: 1, runtimeInstanceId: r1 });
|
|
1024
|
+
await register(id2, { maxTasks: 1, runtimeInstanceId: r2 });
|
|
1025
|
+
await closeRuntime(id2, r2);
|
|
1026
|
+
expect(getAgentById(id2)?.status).toBe("offline");
|
|
1027
|
+
const before = getRuntimeInstanceById(r1);
|
|
1028
|
+
|
|
1029
|
+
await Bun.sleep(10);
|
|
1030
|
+
expect(await pingAgent(id2, r1)).toBe(204);
|
|
1031
|
+
expect(getAgentById(id2)?.status).toBe("offline");
|
|
1032
|
+
expect(getRuntimeInstanceById(r1)?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
1033
|
+
});
|
|
1034
|
+
|
|
1035
|
+
test("a ping cannot touch another agent's runtime row", async () => {
|
|
1036
|
+
const id1 = makeAgent(3);
|
|
1037
|
+
const id2 = makeAgent(3);
|
|
1038
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1039
|
+
const r1 = crypto.randomUUID();
|
|
1040
|
+
await register(id1, { maxTasks: 1, runtimeInstanceId: r1 });
|
|
1041
|
+
await register(id2, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
1042
|
+
const before = getRuntimeInstanceById(r1);
|
|
1043
|
+
|
|
1044
|
+
await Bun.sleep(10);
|
|
1045
|
+
expect(await pingAgent(id2, r1)).toBe(204);
|
|
1046
|
+
expect(getRuntimeInstanceById(r1)?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
1047
|
+
});
|
|
1048
|
+
});
|
|
1049
|
+
|
|
1050
|
+
// ─── End-to-end multi-runtime lifecycle ─────────────────────────────────────
|
|
1051
|
+
|
|
1052
|
+
describe("end-to-end multi-runtime lifecycle", () => {
|
|
1053
|
+
test("policy, capacity, close, and liveness compose across two runtimes", async () => {
|
|
1054
|
+
const coder = makeAgent(2);
|
|
1055
|
+
expect(await putAgentMaxTasks(coder, "4")).toBe(200);
|
|
1056
|
+
expect(getAgentById(coder)?.maxTasks).toBe(4);
|
|
1057
|
+
|
|
1058
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1059
|
+
const rA = crypto.randomUUID();
|
|
1060
|
+
const rB = crypto.randomUUID();
|
|
1061
|
+
await register(coder, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1062
|
+
await register(coder, { maxTasks: 2, runtimeInstanceId: rB });
|
|
1063
|
+
|
|
1064
|
+
expect(getRuntimeInstanceById(rA)?.reportedSlots).toBe(1);
|
|
1065
|
+
expect(getRuntimeInstanceById(rB)?.reportedSlots).toBe(2);
|
|
1066
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("active");
|
|
1067
|
+
expect(getRuntimeInstanceById(rB)?.status).toBe("active");
|
|
1068
|
+
expect(getAgentById(coder)?.maxTasks).toBe(4);
|
|
1069
|
+
expect(getAgentMaxTasksConfig(coder)?.value).toBe("4");
|
|
1070
|
+
|
|
1071
|
+
await closeRuntime(coder, rA);
|
|
1072
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("offline");
|
|
1073
|
+
expect(getRuntimeInstanceById(rB)?.status).toBe("active");
|
|
1074
|
+
expect(getAgentById(coder)?.status).toBe("idle");
|
|
1075
|
+
expect(getAgentById(coder)?.maxTasks).toBe(4);
|
|
1076
|
+
|
|
1077
|
+
const aSeen = getRuntimeInstanceById(rA)?.lastSeenAt;
|
|
1078
|
+
const bSeen = getRuntimeInstanceById(rB)?.lastSeenAt;
|
|
1079
|
+
await Bun.sleep(10);
|
|
1080
|
+
await pingAgent(coder, rB);
|
|
1081
|
+
expect(getRuntimeInstanceById(rA)?.lastSeenAt).toBe(aSeen ?? "");
|
|
1082
|
+
const bSeenAfter = getRuntimeInstanceById(rB)?.lastSeenAt;
|
|
1083
|
+
expect(bSeenAfter && bSeen && bSeenAfter > bSeen).toBe(true);
|
|
1084
|
+
|
|
1085
|
+
await closeRuntime(coder, rB);
|
|
1086
|
+
expect(getRuntimeInstanceById(rB)?.status).toBe("offline");
|
|
1087
|
+
expect(getAgentById(coder)?.status).toBe("offline");
|
|
1088
|
+
expect(getAgentById(coder)?.maxTasks).toBe(4);
|
|
1089
|
+
expect(getAgentMaxTasksConfig(coder)?.value).toBe("4");
|
|
1090
|
+
});
|
|
1091
|
+
});
|
|
1092
|
+
|
|
1093
|
+
describe("stale runtime liveness", () => {
|
|
1094
|
+
test("a crashed runtime stops counting once its ping goes stale", async () => {
|
|
1095
|
+
const id = makeAgent(3);
|
|
1096
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1097
|
+
const rA = crypto.randomUUID();
|
|
1098
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1099
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(1);
|
|
1100
|
+
|
|
1101
|
+
makeRuntimeStale(rA);
|
|
1102
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(0);
|
|
1103
|
+
});
|
|
1104
|
+
|
|
1105
|
+
test("a stale runtime does not keep the agent available when its sibling closes", async () => {
|
|
1106
|
+
const id = makeAgent(3);
|
|
1107
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1108
|
+
const crashed = crypto.randomUUID();
|
|
1109
|
+
const live = crypto.randomUUID();
|
|
1110
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crashed });
|
|
1111
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
1112
|
+
|
|
1113
|
+
// The crashed process never reaches /close.
|
|
1114
|
+
makeRuntimeStale(crashed);
|
|
1115
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1116
|
+
|
|
1117
|
+
await closeRuntime(id, live);
|
|
1118
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1119
|
+
});
|
|
1120
|
+
|
|
1121
|
+
test("a live sibling keeps the agent available while another is stale", async () => {
|
|
1122
|
+
const id = makeAgent(3);
|
|
1123
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1124
|
+
const crashed = crypto.randomUUID();
|
|
1125
|
+
const live = crypto.randomUUID();
|
|
1126
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crashed });
|
|
1127
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
1128
|
+
makeRuntimeStale(crashed);
|
|
1129
|
+
|
|
1130
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(1);
|
|
1131
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1132
|
+
});
|
|
1133
|
+
|
|
1134
|
+
test("the sweep retires stale runtimes and offlines agents with none left", async () => {
|
|
1135
|
+
const id = makeAgent(3);
|
|
1136
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1137
|
+
const rA = crypto.randomUUID();
|
|
1138
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1139
|
+
makeRuntimeStale(rA);
|
|
1140
|
+
|
|
1141
|
+
const result = expireStaleRuntimeInstances();
|
|
1142
|
+
expect(result.expired).toBe(1);
|
|
1143
|
+
expect(result.agentsOffline).toBe(1);
|
|
1144
|
+
// Retired runtimes are pruned rather than accumulating one row per boot.
|
|
1145
|
+
expect(getRuntimeInstanceById(rA)).toBeNull();
|
|
1146
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1147
|
+
});
|
|
1148
|
+
|
|
1149
|
+
test("the sweep leaves healthy runtimes and their agents alone", async () => {
|
|
1150
|
+
const id = makeAgent(3);
|
|
1151
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1152
|
+
const rA = crypto.randomUUID();
|
|
1153
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1154
|
+
|
|
1155
|
+
const result = expireStaleRuntimeInstances();
|
|
1156
|
+
expect(result.expired).toBe(0);
|
|
1157
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("active");
|
|
1158
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1159
|
+
});
|
|
1160
|
+
|
|
1161
|
+
test("a row left active by a flag-off close does not revive the agent when the flag returns", async () => {
|
|
1162
|
+
const id = makeAgent(3);
|
|
1163
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1164
|
+
const rA = crypto.randomUUID();
|
|
1165
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1166
|
+
|
|
1167
|
+
// Flag off: close takes the legacy path and leaves the row active.
|
|
1168
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1169
|
+
await closeRuntime(id, rA);
|
|
1170
|
+
expect(getRuntimeInstanceById(rA)?.status).toBe("active");
|
|
1171
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1172
|
+
|
|
1173
|
+
// Re-enabled later, that stale row must not count as a live runtime.
|
|
1174
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1175
|
+
makeRuntimeStale(rA);
|
|
1176
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(0);
|
|
1177
|
+
});
|
|
1178
|
+
});
|
|
1179
|
+
|
|
1180
|
+
describe("logical capacity across runtimes", () => {
|
|
1181
|
+
test("reviewing tasks consume a slot", () => {
|
|
1182
|
+
const id = makeAgent(1);
|
|
1183
|
+
const task = createTaskExtended("t", { offeredTo: id });
|
|
1184
|
+
getDb()
|
|
1185
|
+
.prepare("UPDATE agent_tasks SET agentId = ?, status = 'reviewing' WHERE id = ?")
|
|
1186
|
+
.run(id, task.id);
|
|
1187
|
+
expect(getActiveTaskCount(id)).toBe(1);
|
|
1188
|
+
expect(hasCapacity(id)).toBe(false);
|
|
1189
|
+
});
|
|
1190
|
+
|
|
1191
|
+
test("maxTasks=1 admits only one of two concurrent polls", async () => {
|
|
1192
|
+
const id = makeAgent(1);
|
|
1193
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1194
|
+
const rtA = crypto.randomUUID();
|
|
1195
|
+
const rtB = crypto.randomUUID();
|
|
1196
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtA });
|
|
1197
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtB });
|
|
1198
|
+
createTaskExtended("offered-1", { offeredTo: id });
|
|
1199
|
+
createTaskExtended("offered-2", { offeredTo: id });
|
|
1200
|
+
|
|
1201
|
+
// Two runtimes of the same agent polling at once.
|
|
1202
|
+
const [a, b] = await Promise.all([pollAgent(id, rtA), pollAgent(id, rtB)]);
|
|
1203
|
+
const admitted = [a, b].filter((t) => t?.trigger?.type === "task_offered");
|
|
1204
|
+
expect(admitted).toHaveLength(1);
|
|
1205
|
+
expect(getActiveTaskCount(id)).toBe(1);
|
|
1206
|
+
});
|
|
1207
|
+
|
|
1208
|
+
test("maxTasks=2 admits two concurrent polls", async () => {
|
|
1209
|
+
const id = makeAgent(2);
|
|
1210
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1211
|
+
const rtA = crypto.randomUUID();
|
|
1212
|
+
const rtB = crypto.randomUUID();
|
|
1213
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtA });
|
|
1214
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtB });
|
|
1215
|
+
createTaskExtended("offered-1", { offeredTo: id });
|
|
1216
|
+
createTaskExtended("offered-2", { offeredTo: id });
|
|
1217
|
+
|
|
1218
|
+
const [a, b] = await Promise.all([pollAgent(id, rtA), pollAgent(id, rtB)]);
|
|
1219
|
+
const admitted = [a, b].filter((t) => t?.trigger?.type === "task_offered");
|
|
1220
|
+
expect(admitted).toHaveLength(2);
|
|
1221
|
+
});
|
|
1222
|
+
|
|
1223
|
+
test("finishing a task restores capacity", async () => {
|
|
1224
|
+
const id = makeAgent(1);
|
|
1225
|
+
const task = createTaskExtended("t", { offeredTo: id });
|
|
1226
|
+
const first = await pollAgent(id);
|
|
1227
|
+
expect(first?.trigger?.type).toBe("task_offered");
|
|
1228
|
+
expect(hasCapacity(id)).toBe(false);
|
|
1229
|
+
|
|
1230
|
+
completeTask(task.id, "done");
|
|
1231
|
+
expect(hasCapacity(id)).toBe(true);
|
|
1232
|
+
});
|
|
1233
|
+
|
|
1234
|
+
test("capacity is enforced the same way with the flag off", async () => {
|
|
1235
|
+
const id = makeAgent(1);
|
|
1236
|
+
createTaskExtended("offered-1", { offeredTo: id });
|
|
1237
|
+
createTaskExtended("offered-2", { offeredTo: id });
|
|
1238
|
+
|
|
1239
|
+
const [a, b] = await Promise.all([pollAgent(id), pollAgent(id)]);
|
|
1240
|
+
expect([a, b].filter((t) => t?.trigger?.type === "task_offered")).toHaveLength(1);
|
|
1241
|
+
});
|
|
1242
|
+
});
|
|
1243
|
+
|
|
1244
|
+
describe("logical policy seeding", () => {
|
|
1245
|
+
test("a new lead keeps its reported default instead of being forced to one", async () => {
|
|
1246
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1247
|
+
const id = crypto.randomUUID();
|
|
1248
|
+
await register(id, { isLead: true, maxTasks: 2, runtimeInstanceId: crypto.randomUUID() });
|
|
1249
|
+
expect(getAgentById(id)?.maxTasks).toBe(2);
|
|
1250
|
+
expect(getAgentMaxTasksConfig(id)?.value).toBe("2");
|
|
1251
|
+
});
|
|
1252
|
+
|
|
1253
|
+
test("deleting the policy row clears the enforcement mirror", async () => {
|
|
1254
|
+
const id = makeAgent(2);
|
|
1255
|
+
expect(await putAgentMaxTasks(id, "7")).toBe(200);
|
|
1256
|
+
expect(getAgentById(id)?.maxTasks).toBe(7);
|
|
1257
|
+
|
|
1258
|
+
const row = getAgentMaxTasksConfig(id);
|
|
1259
|
+
expect(row).not.toBeNull();
|
|
1260
|
+
const res = await fetch(`${baseUrl}/api/config/${row?.id}`, { method: "DELETE" });
|
|
1261
|
+
await res.text();
|
|
1262
|
+
expect(res.status).toBe(200);
|
|
1263
|
+
|
|
1264
|
+
expect(getAgentMaxTasksConfig(id)).toBeNull();
|
|
1265
|
+
expect(getAgentById(id)?.maxTasks).toBe(1);
|
|
1266
|
+
});
|
|
1267
|
+
});
|
|
1268
|
+
|
|
1269
|
+
describe("sibling runtime session safety", () => {
|
|
1270
|
+
const startSession = startSessionFor;
|
|
1271
|
+
const startupCleanup = startupCleanupFor;
|
|
1272
|
+
|
|
1273
|
+
test("starting a second runtime leaves the first runtime's live session and task alone", async () => {
|
|
1274
|
+
const id = makeAgent(3);
|
|
1275
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1276
|
+
const rA = crypto.randomUUID();
|
|
1277
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1278
|
+
|
|
1279
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
1280
|
+
startTask(task.id);
|
|
1281
|
+
expect(await startSession(id, task.id, rA)).toBe(201);
|
|
1282
|
+
|
|
1283
|
+
// Runtime B boots and runs its startup cleanup.
|
|
1284
|
+
const rB = crypto.randomUUID();
|
|
1285
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rB });
|
|
1286
|
+
expect(await startupCleanup(id)).toBe(200);
|
|
1287
|
+
|
|
1288
|
+
// A's session survives, so its task is not orphaned back to the pool.
|
|
1289
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1290
|
+
expect(getTaskById(task.id)?.status).toBe("in_progress");
|
|
1291
|
+
expect(getRuntimeInstanceById(rB)?.status).toBe("active");
|
|
1292
|
+
});
|
|
1293
|
+
|
|
1294
|
+
test("startup cleanup preserves an activation-window session it cannot prove dead", async () => {
|
|
1295
|
+
// A flag-off worker keeps executing with no runtime row at all; its
|
|
1296
|
+
// session may be quiet past the runtime cutoff (tool-activity heartbeats
|
|
1297
|
+
// only). A sibling booting right after activation must not kill it.
|
|
1298
|
+
const id = makeAgent(3);
|
|
1299
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
1300
|
+
startTask(task.id);
|
|
1301
|
+
await startSession(id, task.id, crypto.randomUUID());
|
|
1302
|
+
makeSessionStale(task.id, 7);
|
|
1303
|
+
|
|
1304
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1305
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
1306
|
+
expect(await startupCleanup(id)).toBe(200);
|
|
1307
|
+
|
|
1308
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1309
|
+
expect(getTaskById(task.id)?.status).toBe("in_progress");
|
|
1310
|
+
});
|
|
1311
|
+
|
|
1312
|
+
test("a crashed predecessor's work is reclaimed by the classifier, not boot cleanup", async () => {
|
|
1313
|
+
const id = makeAgent(3);
|
|
1314
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1315
|
+
const crashed = crypto.randomUUID();
|
|
1316
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crashed });
|
|
1317
|
+
|
|
1318
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
1319
|
+
startTask(task.id);
|
|
1320
|
+
await startSession(id, task.id, crashed);
|
|
1321
|
+
// The crashed process stopped its runtime ping, its session heartbeat,
|
|
1322
|
+
// and its task progress.
|
|
1323
|
+
makeRuntimeStale(crashed);
|
|
1324
|
+
makeSessionStale(task.id, 30);
|
|
1325
|
+
getDb()
|
|
1326
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
1327
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), task.id);
|
|
1328
|
+
|
|
1329
|
+
const replacement = crypto.randomUUID();
|
|
1330
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: replacement });
|
|
1331
|
+
// Boot cleanup deletes nothing in multi-runtime mode — it has no evidence
|
|
1332
|
+
// stronger than the classifier's.
|
|
1333
|
+
expect(await startupCleanup(id)).toBe(200);
|
|
1334
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1335
|
+
|
|
1336
|
+
// The sweep's classifier reclaims the work and its session.
|
|
1337
|
+
await runHeartbeatSweep();
|
|
1338
|
+
expect(getTaskById(task.id)?.status).not.toBe("in_progress");
|
|
1339
|
+
expect(getActiveSessionForTask(task.id)).toBeNull();
|
|
1340
|
+
});
|
|
1341
|
+
|
|
1342
|
+
test("with the flag off, startup cleanup still clears every session for the agent", async () => {
|
|
1343
|
+
const id = makeAgent(3);
|
|
1344
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
1345
|
+
startTask(task.id);
|
|
1346
|
+
await startSession(id, task.id, crypto.randomUUID());
|
|
1347
|
+
|
|
1348
|
+
expect(await startupCleanup(id)).toBe(200);
|
|
1349
|
+
expect(getActiveSessionForTask(task.id)).toBeNull();
|
|
1350
|
+
});
|
|
1351
|
+
});
|
|
1352
|
+
|
|
1353
|
+
describe("rollback to legacy mode", () => {
|
|
1354
|
+
test("stale runtime state is inert while the flag is off", async () => {
|
|
1355
|
+
const id = makeAgent(3);
|
|
1356
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1357
|
+
const rA = crypto.randomUUID();
|
|
1358
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1359
|
+
|
|
1360
|
+
// Operator rolls back; workers stop refreshing their rows.
|
|
1361
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1362
|
+
makeRuntimeStale(rA);
|
|
1363
|
+
|
|
1364
|
+
const result = expireStaleRuntimeInstances();
|
|
1365
|
+
expect(result.expired).toBe(0);
|
|
1366
|
+
expect(getRuntimeInstanceById(rA)).not.toBeNull();
|
|
1367
|
+
// The agent keeps running under legacy semantics.
|
|
1368
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1369
|
+
});
|
|
1370
|
+
|
|
1371
|
+
test("a legacy agent with leftover runtime rows stays assignable", async () => {
|
|
1372
|
+
const id = makeAgent(3);
|
|
1373
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1374
|
+
const rA = crypto.randomUUID();
|
|
1375
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rA });
|
|
1376
|
+
|
|
1377
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1378
|
+
makeRuntimeStale(rA);
|
|
1379
|
+
expireStaleRuntimeInstances();
|
|
1380
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1381
|
+
});
|
|
1382
|
+
});
|
|
1383
|
+
|
|
1384
|
+
describe("runtime expiry retires the runtime, the classifier reclaims its work", () => {
|
|
1385
|
+
test("a crashed runtime's task is recovered by stall remediation while a sibling's survives", async () => {
|
|
1386
|
+
const id = makeAgent(3);
|
|
1387
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1388
|
+
const crashed = crypto.randomUUID();
|
|
1389
|
+
const live = crypto.randomUUID();
|
|
1390
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crashed });
|
|
1391
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
1392
|
+
|
|
1393
|
+
const crashedTask = createTaskExtended("crashed-work", { agentId: id });
|
|
1394
|
+
startTask(crashedTask.id);
|
|
1395
|
+
await startSessionFor(id, crashedTask.id, crashed);
|
|
1396
|
+
|
|
1397
|
+
const liveTask = createTaskExtended("live-work", { agentId: id });
|
|
1398
|
+
startTask(liveTask.id);
|
|
1399
|
+
await startSessionFor(id, liveTask.id, live);
|
|
1400
|
+
|
|
1401
|
+
// A replacement booting before the crash is noticed must not drop either.
|
|
1402
|
+
expect(await startupCleanupFor(id)).toBe(200);
|
|
1403
|
+
expect(getActiveSessionForTask(crashedTask.id)).not.toBeNull();
|
|
1404
|
+
expect(getActiveSessionForTask(liveTask.id)).not.toBeNull();
|
|
1405
|
+
|
|
1406
|
+
// A genuinely crashed process stops both its runtime ping and its
|
|
1407
|
+
// session heartbeat — and its task stops progressing.
|
|
1408
|
+
makeRuntimeStale(crashed);
|
|
1409
|
+
makeSessionStale(crashedTask.id, 30);
|
|
1410
|
+
getDb()
|
|
1411
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
1412
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), crashedTask.id);
|
|
1413
|
+
|
|
1414
|
+
// Expiry only retires the runtime — the session is evidence the stall
|
|
1415
|
+
// classifier owns, and it removes it when it remediates in the same sweep.
|
|
1416
|
+
const result = expireStaleRuntimeInstances();
|
|
1417
|
+
expect(result.expired).toBe(1);
|
|
1418
|
+
expect(getActiveSessionForTask(crashedTask.id)).not.toBeNull();
|
|
1419
|
+
|
|
1420
|
+
await runHeartbeatSweep();
|
|
1421
|
+
|
|
1422
|
+
expect(getTaskById(crashedTask.id)?.status).not.toBe("in_progress");
|
|
1423
|
+
expect(getActiveSessionForTask(crashedTask.id)).toBeNull();
|
|
1424
|
+
// The sibling's work is untouched throughout.
|
|
1425
|
+
expect(getActiveSessionForTask(liveTask.id)).not.toBeNull();
|
|
1426
|
+
expect(getTaskById(liveTask.id)?.status).toBe("in_progress");
|
|
1427
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
1428
|
+
});
|
|
1429
|
+
});
|
|
1430
|
+
|
|
1431
|
+
describe("flag re-enable cycle preserves live sessions", () => {
|
|
1432
|
+
test("a frozen runtime row does not kill a still-heartbeating session on re-enable", async () => {
|
|
1433
|
+
const id = makeAgent(3);
|
|
1434
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1435
|
+
const rt = crypto.randomUUID();
|
|
1436
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1437
|
+
const task = createTaskExtended("in-flight-work", { agentId: id });
|
|
1438
|
+
startTask(task.id);
|
|
1439
|
+
await startSessionFor(id, task.id, rt);
|
|
1440
|
+
|
|
1441
|
+
// Operator disables the flag: nothing refreshes runtime rows, but the
|
|
1442
|
+
// worker keeps executing and heartbeating its session.
|
|
1443
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1444
|
+
makeRuntimeStale(rt);
|
|
1445
|
+
|
|
1446
|
+
// Re-enable past the stale window; the first sweep runs expiry.
|
|
1447
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1448
|
+
await runHeartbeatSweep();
|
|
1449
|
+
|
|
1450
|
+
// The frozen row is retired, but the live session and its task survive —
|
|
1451
|
+
// no supersede, no requeue, no duplicate execution.
|
|
1452
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1453
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1454
|
+
const after = getTaskById(task.id);
|
|
1455
|
+
expect(after?.status).toBe("in_progress");
|
|
1456
|
+
expect(after?.agentId).toBe(id);
|
|
1457
|
+
// The agent needs its next re-registration before it gets NEW work —
|
|
1458
|
+
// safety over availability; registration is the runtime revival path.
|
|
1459
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1460
|
+
});
|
|
1461
|
+
|
|
1462
|
+
test("a quiet-but-healthy session survives the cycle even past the runtime cutoff", async () => {
|
|
1463
|
+
const id = makeAgent(3);
|
|
1464
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1465
|
+
const rt = crypto.randomUUID();
|
|
1466
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1467
|
+
const task = createTaskExtended("quiet-work", { agentId: id });
|
|
1468
|
+
startTask(task.id);
|
|
1469
|
+
await startSessionFor(id, task.id, rt);
|
|
1470
|
+
|
|
1471
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1472
|
+
makeRuntimeStale(rt);
|
|
1473
|
+
// Sessions heartbeat on tool activity only: a long model call or shell
|
|
1474
|
+
// command can be quiet past the runtime cutoff while the worker is fine.
|
|
1475
|
+
makeSessionStale(task.id, 7);
|
|
1476
|
+
|
|
1477
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1478
|
+
await runHeartbeatSweep();
|
|
1479
|
+
|
|
1480
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1481
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1482
|
+
expect(getTaskById(task.id)?.status).toBe("in_progress");
|
|
1483
|
+
// No resume/supersede sibling was minted for the in-flight task.
|
|
1484
|
+
expect(getDb().prepare<{ c: number }, []>("SELECT COUNT(*) c FROM agent_tasks").get()?.c).toBe(
|
|
1485
|
+
1,
|
|
1486
|
+
);
|
|
1487
|
+
});
|
|
1488
|
+
|
|
1489
|
+
test("a stale session heartbeat alone does not fail a task that is still progressing", async () => {
|
|
1490
|
+
// Guards against collapsing the classifier into "heartbeat older than N":
|
|
1491
|
+
// Case B requires the TASK to be stale too before remediation.
|
|
1492
|
+
const id = makeAgent(3);
|
|
1493
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1494
|
+
const rt = crypto.randomUUID();
|
|
1495
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1496
|
+
const task = createTaskExtended("progressing-work", { agentId: id });
|
|
1497
|
+
startTask(task.id);
|
|
1498
|
+
await startSessionFor(id, task.id, rt);
|
|
1499
|
+
|
|
1500
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1501
|
+
makeRuntimeStale(rt);
|
|
1502
|
+
makeSessionStale(task.id, 20);
|
|
1503
|
+
// The task itself progressed recently (store-progress traffic).
|
|
1504
|
+
getDb()
|
|
1505
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
1506
|
+
.run(new Date(Date.now() - 10 * 60000).toISOString(), task.id);
|
|
1507
|
+
|
|
1508
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1509
|
+
await runHeartbeatSweep();
|
|
1510
|
+
|
|
1511
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1512
|
+
expect(getTaskById(task.id)?.status).toBe("in_progress");
|
|
1513
|
+
});
|
|
1514
|
+
|
|
1515
|
+
test("a genuinely dead worker is still recovered across the same cycle", async () => {
|
|
1516
|
+
const id = makeAgent(3);
|
|
1517
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1518
|
+
const rt = crypto.randomUUID();
|
|
1519
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1520
|
+
const task = createTaskExtended("dead-work", { agentId: id });
|
|
1521
|
+
startTask(task.id);
|
|
1522
|
+
await startSessionFor(id, task.id, rt);
|
|
1523
|
+
|
|
1524
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1525
|
+
makeRuntimeStale(rt);
|
|
1526
|
+
// The process died: session heartbeat AND task progress both went stale.
|
|
1527
|
+
makeSessionStale(task.id, 30);
|
|
1528
|
+
getDb()
|
|
1529
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
1530
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), task.id);
|
|
1531
|
+
|
|
1532
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1533
|
+
await runHeartbeatSweep();
|
|
1534
|
+
|
|
1535
|
+
// The classifier (not runtime expiry) reclaimed the work and its session.
|
|
1536
|
+
expect(getTaskById(task.id)?.status).not.toBe("in_progress");
|
|
1537
|
+
expect(getActiveSessionForTask(task.id)).toBeNull();
|
|
1538
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1539
|
+
});
|
|
1540
|
+
|
|
1541
|
+
test("while the flag stays off, expiry touches neither row nor session", async () => {
|
|
1542
|
+
const id = makeAgent(3);
|
|
1543
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1544
|
+
const rt = crypto.randomUUID();
|
|
1545
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1546
|
+
const task = createTaskExtended("legacy-work", { agentId: id });
|
|
1547
|
+
startTask(task.id);
|
|
1548
|
+
await startSessionFor(id, task.id, rt);
|
|
1549
|
+
|
|
1550
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1551
|
+
makeRuntimeStale(rt);
|
|
1552
|
+
|
|
1553
|
+
const result = expireStaleRuntimeInstances();
|
|
1554
|
+
expect(result.expired).toBe(0);
|
|
1555
|
+
expect(getRuntimeInstanceById(rt)).not.toBeNull();
|
|
1556
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1557
|
+
});
|
|
1558
|
+
});
|
|
1559
|
+
|
|
1560
|
+
describe("runtime row retention", () => {
|
|
1561
|
+
test("repeated boot/retire cycles do not accumulate rows", async () => {
|
|
1562
|
+
const id = makeAgent(3);
|
|
1563
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1564
|
+
|
|
1565
|
+
for (let boot = 0; boot < 5; boot++) {
|
|
1566
|
+
const rt = crypto.randomUUID();
|
|
1567
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1568
|
+
await closeRuntime(id, rt);
|
|
1569
|
+
makeRuntimeStale(rt);
|
|
1570
|
+
expireStaleRuntimeInstances();
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
expect(runtimeInstancesFor(id)).toHaveLength(0);
|
|
1574
|
+
});
|
|
1575
|
+
|
|
1576
|
+
test("a gracefully closed runtime is pruned once it goes stale", async () => {
|
|
1577
|
+
const id = makeAgent(3);
|
|
1578
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1579
|
+
const rt = crypto.randomUUID();
|
|
1580
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1581
|
+
await closeRuntime(id, rt);
|
|
1582
|
+
// Closed rows linger only until they age out of the liveness window.
|
|
1583
|
+
expect(getRuntimeInstanceById(rt)).not.toBeNull();
|
|
1584
|
+
|
|
1585
|
+
makeRuntimeStale(rt);
|
|
1586
|
+
expireStaleRuntimeInstances();
|
|
1587
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1588
|
+
});
|
|
1589
|
+
});
|
|
1590
|
+
|
|
1591
|
+
describe("heartbeat ordering vs auto-assignment", () => {
|
|
1592
|
+
test("a dead runtime's agent is retired before pool assignment, leaving the task queued", async () => {
|
|
1593
|
+
const id = makeAgent(3);
|
|
1594
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1595
|
+
const rt = crypto.randomUUID();
|
|
1596
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1597
|
+
|
|
1598
|
+
// The worker dies without /close, so the agent still looks idle.
|
|
1599
|
+
makeRuntimeStale(rt);
|
|
1600
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1601
|
+
|
|
1602
|
+
const task = createTaskExtended("pool-work");
|
|
1603
|
+
expect(getTaskById(task.id)?.status).toBe("unassigned");
|
|
1604
|
+
|
|
1605
|
+
await runHeartbeatSweep();
|
|
1606
|
+
|
|
1607
|
+
// Expiry ran first, so the sweep never handed the task to a dead agent.
|
|
1608
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1609
|
+
expect(getTaskById(task.id)?.status).toBe("unassigned");
|
|
1610
|
+
expect(getTaskById(task.id)?.agentId ?? null).toBeNull();
|
|
1611
|
+
});
|
|
1612
|
+
});
|
|
1613
|
+
|
|
1614
|
+
describe("sweep coverage and rollback inertness", () => {
|
|
1615
|
+
test("a cleanup-only tick still retires a dead runtime", async () => {
|
|
1616
|
+
const id = makeAgent(3);
|
|
1617
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1618
|
+
const rt = crypto.randomUUID();
|
|
1619
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1620
|
+
makeRuntimeStale(rt);
|
|
1621
|
+
|
|
1622
|
+
// No in-progress work and no queued pool task: the preflight gate treats
|
|
1623
|
+
// this tick as "nothing actionable", but expiry must still run.
|
|
1624
|
+
await runHeartbeatSweep();
|
|
1625
|
+
|
|
1626
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1627
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
1628
|
+
});
|
|
1629
|
+
|
|
1630
|
+
test("with the flag off, leftover runtime rows do not block pool assignment", async () => {
|
|
1631
|
+
const id = makeAgent(3);
|
|
1632
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1633
|
+
const rt = crypto.randomUUID();
|
|
1634
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1635
|
+
|
|
1636
|
+
// Rolled back; the legacy worker stops refreshing its retained row.
|
|
1637
|
+
delete process.env.MULTI_RUNTIME_ENABLED;
|
|
1638
|
+
makeRuntimeStale(rt);
|
|
1639
|
+
const task = createTaskExtended("pool-work-legacy");
|
|
1640
|
+
|
|
1641
|
+
await runHeartbeatSweep();
|
|
1642
|
+
|
|
1643
|
+
// The healthy legacy worker still receives the task.
|
|
1644
|
+
expect(getAgentById(id)?.status).not.toBe("offline");
|
|
1645
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
1646
|
+
});
|
|
1647
|
+
});
|
|
1648
|
+
|
|
1649
|
+
describe("enabling the flag while workers are running", () => {
|
|
1650
|
+
test("a live session survives cleanup before its runtime has re-registered", async () => {
|
|
1651
|
+
const id = makeAgent(3);
|
|
1652
|
+
// Worker registered while the flag was off, so it has no runtime row, but
|
|
1653
|
+
// its sessions already carry the id it generated at boot.
|
|
1654
|
+
await register(id, { maxTasks: 2 });
|
|
1655
|
+
const runningRuntime = crypto.randomUUID();
|
|
1656
|
+
const task = createTaskExtended("in-flight-work", { agentId: id });
|
|
1657
|
+
startTask(task.id);
|
|
1658
|
+
|
|
1659
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1660
|
+
await startSessionFor(id, task.id, runningRuntime);
|
|
1661
|
+
|
|
1662
|
+
// A sibling boots and runs startup cleanup before the original worker's
|
|
1663
|
+
// next re-registration materializes its runtime row.
|
|
1664
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: crypto.randomUUID() });
|
|
1665
|
+
expect(await startupCleanupFor(id)).toBe(200);
|
|
1666
|
+
|
|
1667
|
+
expect(getActiveSessionForTask(task.id)).not.toBeNull();
|
|
1668
|
+
expect(getTaskById(task.id)?.status).toBe("in_progress");
|
|
1669
|
+
});
|
|
1670
|
+
});
|
|
1671
|
+
|
|
1672
|
+
describe("dispatch requires a live runtime", () => {
|
|
1673
|
+
test("a retired runtime is given no work while its replacement is", async () => {
|
|
1674
|
+
const id = makeAgent(2);
|
|
1675
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1676
|
+
const retired = crypto.randomUUID();
|
|
1677
|
+
const live = crypto.randomUUID();
|
|
1678
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: retired });
|
|
1679
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
1680
|
+
createTaskExtended("dispatch-work", { offeredTo: id });
|
|
1681
|
+
|
|
1682
|
+
// The retired process reconnects after its row aged out.
|
|
1683
|
+
makeRuntimeStale(retired);
|
|
1684
|
+
const stalePoll = await pollAgent(id, retired);
|
|
1685
|
+
expect(stalePoll?.trigger ?? null).toBeNull();
|
|
1686
|
+
|
|
1687
|
+
// Its replacement still gets the work.
|
|
1688
|
+
const livePoll = await pollAgent(id, live);
|
|
1689
|
+
expect(livePoll?.trigger?.type).toBe("task_offered");
|
|
1690
|
+
});
|
|
1691
|
+
|
|
1692
|
+
test("polling without a runtime identity yields no work while the flag is on", async () => {
|
|
1693
|
+
const id = makeAgent(2);
|
|
1694
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1695
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
1696
|
+
createTaskExtended("dispatch-work", { offeredTo: id });
|
|
1697
|
+
|
|
1698
|
+
expect((await pollAgent(id))?.trigger ?? null).toBeNull();
|
|
1699
|
+
});
|
|
1700
|
+
});
|
|
1701
|
+
|
|
1702
|
+
describe("capacity attribution for offers", () => {
|
|
1703
|
+
test("a lead's own capacity is not consumed by offers it created for a worker", () => {
|
|
1704
|
+
const lead = makeAgent(2);
|
|
1705
|
+
getDb().prepare("UPDATE agents SET isLead = 1 WHERE id = ?").run(lead);
|
|
1706
|
+
const worker = makeAgent(1);
|
|
1707
|
+
|
|
1708
|
+
// POST /api/tasks records the creating lead in agentId while offering to
|
|
1709
|
+
// the worker; the review must count against the worker only.
|
|
1710
|
+
const task = createTaskExtended("offer", { offeredTo: worker });
|
|
1711
|
+
getDb()
|
|
1712
|
+
.prepare("UPDATE agent_tasks SET agentId = ?, status = 'reviewing' WHERE id = ?")
|
|
1713
|
+
.run(lead, task.id);
|
|
1714
|
+
|
|
1715
|
+
expect(getActiveTaskCount(worker)).toBe(1);
|
|
1716
|
+
expect(getActiveTaskCount(lead)).toBe(0);
|
|
1717
|
+
expect(hasCapacity(lead)).toBe(true);
|
|
1718
|
+
});
|
|
1719
|
+
});
|
|
1720
|
+
|
|
1721
|
+
describe("pool eligibility during activation", () => {
|
|
1722
|
+
test("an agent that has not re-registered since the flag was enabled is not assigned work", async () => {
|
|
1723
|
+
const id = makeAgent(2);
|
|
1724
|
+
// Registered while the flag was off, so it has no runtime row yet.
|
|
1725
|
+
await register(id, { maxTasks: 1 });
|
|
1726
|
+
|
|
1727
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1728
|
+
const task = createTaskExtended("pool-work");
|
|
1729
|
+
|
|
1730
|
+
await runHeartbeatSweep();
|
|
1731
|
+
|
|
1732
|
+
// Its polls return nothing until it re-registers, so assigning would
|
|
1733
|
+
// strand the task on it.
|
|
1734
|
+
expect(getTaskById(task.id)?.agentId ?? null).toBeNull();
|
|
1735
|
+
expect(getTaskById(task.id)?.status).toBe("unassigned");
|
|
1736
|
+
});
|
|
1737
|
+
|
|
1738
|
+
test("once it re-registers with a runtime, it becomes eligible again", async () => {
|
|
1739
|
+
const id = makeAgent(2);
|
|
1740
|
+
await register(id, { maxTasks: 1 });
|
|
1741
|
+
|
|
1742
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1743
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
1744
|
+
const task = createTaskExtended("pool-work");
|
|
1745
|
+
|
|
1746
|
+
await runHeartbeatSweep();
|
|
1747
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
1748
|
+
});
|
|
1749
|
+
});
|
|
1750
|
+
|
|
1751
|
+
describe("readiness after a credential wait", () => {
|
|
1752
|
+
test("re-registering the same runtime id revives a runtime that expired during the wait", async () => {
|
|
1753
|
+
const id = makeAgent(2);
|
|
1754
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1755
|
+
// Boot registration, then a long credential wait: the worker is not usable
|
|
1756
|
+
// capacity, so its runtime is allowed to expire and be pruned.
|
|
1757
|
+
const rt = crypto.randomUUID();
|
|
1758
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1759
|
+
makeRuntimeStale(rt);
|
|
1760
|
+
expireStaleRuntimeInstances();
|
|
1761
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
1762
|
+
|
|
1763
|
+
// Credentials arrive; the worker re-registers before polling, reusing the
|
|
1764
|
+
// per-boot identity rather than minting a new one.
|
|
1765
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1766
|
+
|
|
1767
|
+
expect(getRuntimeInstanceById(rt)?.status).toBe("active");
|
|
1768
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1769
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(1);
|
|
1770
|
+
});
|
|
1771
|
+
|
|
1772
|
+
test("the revived runtime can immediately receive work", async () => {
|
|
1773
|
+
const id = makeAgent(2);
|
|
1774
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1775
|
+
const rt = crypto.randomUUID();
|
|
1776
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1777
|
+
makeRuntimeStale(rt);
|
|
1778
|
+
expireStaleRuntimeInstances();
|
|
1779
|
+
|
|
1780
|
+
// Before re-registration the gate correctly withholds work.
|
|
1781
|
+
createTaskExtended("post-credential-work", { offeredTo: id });
|
|
1782
|
+
expect((await pollAgent(id, rt))?.trigger ?? null).toBeNull();
|
|
1783
|
+
|
|
1784
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1785
|
+
expect((await pollAgent(id, rt))?.trigger?.type).toBe("task_offered");
|
|
1786
|
+
});
|
|
1787
|
+
|
|
1788
|
+
test("re-register then report recovers readiness on a runtime that went stale waiting", async () => {
|
|
1789
|
+
const id = makeAgent(2);
|
|
1790
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1791
|
+
const rt = crypto.randomUUID();
|
|
1792
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1793
|
+
expect(await reportReady(id, rt, false)).toBe(200);
|
|
1794
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
1795
|
+
|
|
1796
|
+
// The credential wait outlives the stale window; the sweep has not
|
|
1797
|
+
// pruned the row yet.
|
|
1798
|
+
makeRuntimeStale(rt);
|
|
1799
|
+
|
|
1800
|
+
// The runner's post-wait sequence: revive the same per-boot identity
|
|
1801
|
+
// FIRST, then report readiness against the now-live row.
|
|
1802
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1803
|
+
expect(await reportReady(id, rt, true)).toBe(200);
|
|
1804
|
+
|
|
1805
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1806
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(true);
|
|
1807
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1808
|
+
});
|
|
1809
|
+
|
|
1810
|
+
test("the recovered agent reports busy when it still holds active work", async () => {
|
|
1811
|
+
const id = makeAgent(2);
|
|
1812
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1813
|
+
const rt = crypto.randomUUID();
|
|
1814
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1815
|
+
await reportReady(id, rt, false);
|
|
1816
|
+
const task = createTaskExtended("held-work", { agentId: id });
|
|
1817
|
+
startTask(task.id);
|
|
1818
|
+
makeRuntimeStale(rt);
|
|
1819
|
+
|
|
1820
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1821
|
+
await reportReady(id, rt, true);
|
|
1822
|
+
|
|
1823
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(true);
|
|
1824
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
1825
|
+
});
|
|
1826
|
+
|
|
1827
|
+
test("a transient registration failure delays but does not drop the recovery", async () => {
|
|
1828
|
+
const id = makeAgent(2);
|
|
1829
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1830
|
+
const rt = crypto.randomUUID();
|
|
1831
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1832
|
+
await reportReady(id, rt, false);
|
|
1833
|
+
makeRuntimeStale(rt);
|
|
1834
|
+
|
|
1835
|
+
// The runner's recovery: retry the strict registration of the SAME
|
|
1836
|
+
// per-boot identity until it succeeds, and only then report ready.
|
|
1837
|
+
let attempts = 0;
|
|
1838
|
+
await retryBootStep(
|
|
1839
|
+
async () => {
|
|
1840
|
+
attempts++;
|
|
1841
|
+
if (attempts < 2) throw new Error("transient network failure");
|
|
1842
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1843
|
+
},
|
|
1844
|
+
{ sleep: async () => {}, log: () => {} },
|
|
1845
|
+
);
|
|
1846
|
+
expect(await reportReady(id, rt, true)).toBe(200);
|
|
1847
|
+
|
|
1848
|
+
expect(attempts).toBe(2);
|
|
1849
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1850
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(true);
|
|
1851
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1852
|
+
});
|
|
1853
|
+
|
|
1854
|
+
test("recovery after several transient failures still ends busy with active work", async () => {
|
|
1855
|
+
const id = makeAgent(2);
|
|
1856
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1857
|
+
const rt = crypto.randomUUID();
|
|
1858
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1859
|
+
await reportReady(id, rt, false);
|
|
1860
|
+
const task = createTaskExtended("held-work", { agentId: id });
|
|
1861
|
+
startTask(task.id);
|
|
1862
|
+
makeRuntimeStale(rt);
|
|
1863
|
+
|
|
1864
|
+
let attempts = 0;
|
|
1865
|
+
await retryBootStep(
|
|
1866
|
+
async () => {
|
|
1867
|
+
attempts++;
|
|
1868
|
+
if (attempts < 3) throw new Error("transient network failure");
|
|
1869
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1870
|
+
},
|
|
1871
|
+
{ sleep: async () => {}, log: () => {} },
|
|
1872
|
+
);
|
|
1873
|
+
await reportReady(id, rt, true);
|
|
1874
|
+
|
|
1875
|
+
expect(attempts).toBe(3);
|
|
1876
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1877
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(true);
|
|
1878
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
1879
|
+
});
|
|
1880
|
+
|
|
1881
|
+
test("persistent registration failure leaves the runtime unrecovered and undispatchable", async () => {
|
|
1882
|
+
const id = makeAgent(2);
|
|
1883
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1884
|
+
const rt = crypto.randomUUID();
|
|
1885
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1886
|
+
await reportReady(id, rt, false);
|
|
1887
|
+
makeRuntimeStale(rt);
|
|
1888
|
+
|
|
1889
|
+
await expect(
|
|
1890
|
+
retryBootStep(
|
|
1891
|
+
async () => {
|
|
1892
|
+
throw new Error("registration endpoint down");
|
|
1893
|
+
},
|
|
1894
|
+
{ attempts: 3, sleep: async () => {}, log: () => {} },
|
|
1895
|
+
),
|
|
1896
|
+
).rejects.toThrow("registration endpoint down");
|
|
1897
|
+
|
|
1898
|
+
// Nothing was revived: the ready report still targets a stale row and the
|
|
1899
|
+
// dispatch gate still withholds work — polling has not resumed as if
|
|
1900
|
+
// recovery succeeded (the runner exits instead of entering the loop).
|
|
1901
|
+
await reportReady(id, rt, true);
|
|
1902
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(false);
|
|
1903
|
+
createTaskExtended("unreachable-work", { offeredTo: id });
|
|
1904
|
+
expect((await pollAgent(id, rt))?.trigger ?? null).toBeNull();
|
|
1905
|
+
});
|
|
1906
|
+
|
|
1907
|
+
test("the boot readiness write retries through a transient failure", async () => {
|
|
1908
|
+
const id = makeAgent(2);
|
|
1909
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1910
|
+
const rt = crypto.randomUUID();
|
|
1911
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1912
|
+
await reportReady(id, rt, false);
|
|
1913
|
+
makeRuntimeStale(rt);
|
|
1914
|
+
|
|
1915
|
+
// The runner's recovery sequence: registration first, then the readiness
|
|
1916
|
+
// write — BOTH retried, because after a stale revival the write is the
|
|
1917
|
+
// only transition out of waiting_for_credentials.
|
|
1918
|
+
await retryBootStep(() => register(id, { maxTasks: 1, runtimeInstanceId: rt }).then(() => {}), {
|
|
1919
|
+
sleep: async () => {},
|
|
1920
|
+
log: () => {},
|
|
1921
|
+
});
|
|
1922
|
+
let attempts = 0;
|
|
1923
|
+
await retryBootStep(
|
|
1924
|
+
async () => {
|
|
1925
|
+
attempts++;
|
|
1926
|
+
if (attempts < 2) throw new Error("transient network failure");
|
|
1927
|
+
await sendCredStatusReport(baseUrl, API_KEY, id, rt, { ready: true, missing: [] });
|
|
1928
|
+
},
|
|
1929
|
+
{ sleep: async () => {}, log: () => {} },
|
|
1930
|
+
);
|
|
1931
|
+
|
|
1932
|
+
expect(attempts).toBe(2);
|
|
1933
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(true);
|
|
1934
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
1935
|
+
});
|
|
1936
|
+
|
|
1937
|
+
test("a rejected readiness write surfaces instead of being swallowed", async () => {
|
|
1938
|
+
// Multi-runtime mode 400s a report without runtime identity; the strict
|
|
1939
|
+
// sender must propagate that instead of reporting success.
|
|
1940
|
+
const id = makeAgent(2);
|
|
1941
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1942
|
+
const rt = crypto.randomUUID();
|
|
1943
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1944
|
+
await reportReady(id, rt, false);
|
|
1945
|
+
|
|
1946
|
+
await expect(
|
|
1947
|
+
sendCredStatusReport(baseUrl, API_KEY, id, undefined, { ready: true, missing: [] }),
|
|
1948
|
+
).rejects.toThrow("credential-status report failed: 400");
|
|
1949
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
1950
|
+
});
|
|
1951
|
+
|
|
1952
|
+
test("a ready report against a stale row is dropped and revival preserves the waiting state", async () => {
|
|
1953
|
+
// Pins the server contract behind the runner's ordering: reporting
|
|
1954
|
+
// BEFORE re-registering loses the recovery.
|
|
1955
|
+
const id = makeAgent(2);
|
|
1956
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1957
|
+
const rt = crypto.randomUUID();
|
|
1958
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1959
|
+
await reportReady(id, rt, false);
|
|
1960
|
+
makeRuntimeStale(rt);
|
|
1961
|
+
|
|
1962
|
+
// Wrong order: the report matches no live row and drops silently.
|
|
1963
|
+
await reportReady(id, rt, true);
|
|
1964
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1965
|
+
|
|
1966
|
+
expect(getRuntimeInstanceById(rt)?.credentialReady).toBe(false);
|
|
1967
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
1968
|
+
});
|
|
1969
|
+
|
|
1970
|
+
test("re-registering before any expiry is idempotent and adds no second row", async () => {
|
|
1971
|
+
const id = makeAgent(2);
|
|
1972
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
1973
|
+
const rt = crypto.randomUUID();
|
|
1974
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1975
|
+
const before = getRuntimeInstanceById(rt);
|
|
1976
|
+
|
|
1977
|
+
const { status } = await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
1978
|
+
expect(status).toBe(200);
|
|
1979
|
+
expect(runtimeInstancesFor(id)).toHaveLength(1);
|
|
1980
|
+
expect(getRuntimeInstanceById(rt)?.status).toBe("active");
|
|
1981
|
+
expect(getRuntimeInstanceById(rt)?.createdAt).toBe(before?.createdAt ?? "");
|
|
1982
|
+
});
|
|
1983
|
+
|
|
1984
|
+
test("with the flag off, the same re-registration writes no runtime rows", async () => {
|
|
1985
|
+
const id = makeAgent(2);
|
|
1986
|
+
const rt = crypto.randomUUID();
|
|
1987
|
+
await register(id, { maxTasks: 3, runtimeInstanceId: rt });
|
|
1988
|
+
await register(id, { maxTasks: 3, runtimeInstanceId: rt });
|
|
1989
|
+
|
|
1990
|
+
expect(runtimeInstancesFor(id)).toHaveLength(0);
|
|
1991
|
+
expect(getAgentById(id)?.maxTasks).toBe(3);
|
|
1992
|
+
});
|
|
1993
|
+
});
|
|
1994
|
+
|
|
1995
|
+
async function reportReady(
|
|
1996
|
+
agentId: string,
|
|
1997
|
+
runtimeInstanceId: string | undefined,
|
|
1998
|
+
ready: boolean,
|
|
1999
|
+
): Promise<number> {
|
|
2000
|
+
const headers: Record<string, string> = {
|
|
2001
|
+
"Content-Type": "application/json",
|
|
2002
|
+
"X-Agent-ID": agentId,
|
|
2003
|
+
Authorization: `Bearer ${API_KEY}`,
|
|
2004
|
+
};
|
|
2005
|
+
if (runtimeInstanceId) headers["X-Runtime-Instance-ID"] = runtimeInstanceId;
|
|
2006
|
+
const res = await fetch(`${baseUrl}/api/agents/${agentId}/credential-status`, {
|
|
2007
|
+
method: "PUT",
|
|
2008
|
+
headers,
|
|
2009
|
+
body: JSON.stringify({ ready, missing: ready ? [] : ["ANTHROPIC_API_KEY"] }),
|
|
2010
|
+
});
|
|
2011
|
+
await res.text();
|
|
2012
|
+
return res.status;
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
describe("credential readiness is per runtime", () => {
|
|
2016
|
+
async function twoRuntimes(id: string) {
|
|
2017
|
+
const a = crypto.randomUUID();
|
|
2018
|
+
const b = crypto.randomUUID();
|
|
2019
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: a });
|
|
2020
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: b });
|
|
2021
|
+
return { a, b };
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
test("a waiting runtime does not disable its ready sibling", async () => {
|
|
2025
|
+
const id = makeAgent(2);
|
|
2026
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2027
|
+
const { a, b } = await twoRuntimes(id);
|
|
2028
|
+
|
|
2029
|
+
expect(await reportReady(id, a, true)).toBe(200);
|
|
2030
|
+
expect(await reportReady(id, b, false)).toBe(200);
|
|
2031
|
+
|
|
2032
|
+
expect(hasReadyLiveRuntime(id)).toBe(true);
|
|
2033
|
+
expect(getAgentById(id)?.status).not.toBe("waiting_for_credentials");
|
|
2034
|
+
// A can still be handed work.
|
|
2035
|
+
createTaskExtended("work", { offeredTo: id });
|
|
2036
|
+
expect((await pollAgent(id, a))?.trigger?.type).toBe("task_offered");
|
|
2037
|
+
});
|
|
2038
|
+
|
|
2039
|
+
test("a ready report does not pull a busy agent back to idle", async () => {
|
|
2040
|
+
const id = makeAgent(2);
|
|
2041
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2042
|
+
const { a, b } = await twoRuntimes(id);
|
|
2043
|
+
await reportReady(id, a, true);
|
|
2044
|
+
|
|
2045
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
2046
|
+
startTask(task.id);
|
|
2047
|
+
getDb().prepare("UPDATE agents SET status = 'busy' WHERE id = ?").run(id);
|
|
2048
|
+
|
|
2049
|
+
await reportReady(id, b, true);
|
|
2050
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
2051
|
+
});
|
|
2052
|
+
|
|
2053
|
+
test("all live runtimes waiting parks the agent", async () => {
|
|
2054
|
+
const id = makeAgent(2);
|
|
2055
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2056
|
+
const { a, b } = await twoRuntimes(id);
|
|
2057
|
+
await reportReady(id, a, false);
|
|
2058
|
+
await reportReady(id, b, false);
|
|
2059
|
+
|
|
2060
|
+
expect(hasReadyLiveRuntime(id)).toBe(false);
|
|
2061
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2062
|
+
});
|
|
2063
|
+
|
|
2064
|
+
test("losing the only ready runtime falls back to the waiting sibling's state", async () => {
|
|
2065
|
+
const id = makeAgent(2);
|
|
2066
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2067
|
+
const { a, b } = await twoRuntimes(id);
|
|
2068
|
+
await reportReady(id, a, true);
|
|
2069
|
+
await reportReady(id, b, false);
|
|
2070
|
+
expect(getAgentById(id)?.status).not.toBe("waiting_for_credentials");
|
|
2071
|
+
|
|
2072
|
+
await closeRuntime(id, a);
|
|
2073
|
+
expect(hasReadyLiveRuntime(id)).toBe(false);
|
|
2074
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2075
|
+
});
|
|
2076
|
+
|
|
2077
|
+
test("a stale ready runtime stops counting toward readiness", async () => {
|
|
2078
|
+
const id = makeAgent(2);
|
|
2079
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2080
|
+
const { a, b } = await twoRuntimes(id);
|
|
2081
|
+
await reportReady(id, a, true);
|
|
2082
|
+
await reportReady(id, b, false);
|
|
2083
|
+
|
|
2084
|
+
makeRuntimeStale(a);
|
|
2085
|
+
expect(hasReadyLiveRuntime(id)).toBe(false);
|
|
2086
|
+
});
|
|
2087
|
+
|
|
2088
|
+
test("losing the last runtime leaves the agent offline regardless of readiness", async () => {
|
|
2089
|
+
const id = makeAgent(2);
|
|
2090
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2091
|
+
const { a, b } = await twoRuntimes(id);
|
|
2092
|
+
await reportReady(id, a, true);
|
|
2093
|
+
await reportReady(id, b, true);
|
|
2094
|
+
|
|
2095
|
+
await closeRuntime(id, a);
|
|
2096
|
+
await closeRuntime(id, b);
|
|
2097
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
2098
|
+
expect(hasReadyLiveRuntime(id)).toBe(false);
|
|
2099
|
+
});
|
|
2100
|
+
|
|
2101
|
+
test("a runtime that never reports counts as ready (credential checks opted out)", async () => {
|
|
2102
|
+
const id = makeAgent(2);
|
|
2103
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2104
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
2105
|
+
expect(hasReadyLiveRuntime(id)).toBe(true);
|
|
2106
|
+
});
|
|
2107
|
+
|
|
2108
|
+
test("a readiness report without runtime identity is rejected and corrupts nothing", async () => {
|
|
2109
|
+
const id = makeAgent(2);
|
|
2110
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2111
|
+
const { a } = await twoRuntimes(id);
|
|
2112
|
+
await reportReady(id, a, true);
|
|
2113
|
+
|
|
2114
|
+
expect(await reportReady(id, undefined, false)).toBe(400);
|
|
2115
|
+
expect(hasReadyLiveRuntime(id)).toBe(true);
|
|
2116
|
+
expect(getAgentById(id)?.status).not.toBe("waiting_for_credentials");
|
|
2117
|
+
});
|
|
2118
|
+
|
|
2119
|
+
test("a foreign runtime id cannot change this agent's readiness", async () => {
|
|
2120
|
+
const owner = makeAgent(2);
|
|
2121
|
+
const other = makeAgent(2);
|
|
2122
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2123
|
+
const { a } = await twoRuntimes(owner);
|
|
2124
|
+
await reportReady(owner, a, true);
|
|
2125
|
+
await register(other, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
2126
|
+
|
|
2127
|
+
expect(await reportReady(other, a, false)).toBe(200);
|
|
2128
|
+
expect(hasReadyLiveRuntime(owner)).toBe(true);
|
|
2129
|
+
});
|
|
2130
|
+
|
|
2131
|
+
test("with the flag off, credential reporting keeps legacy semantics", async () => {
|
|
2132
|
+
const id = makeAgent(2);
|
|
2133
|
+
expect(await reportReady(id, undefined, false)).toBe(200);
|
|
2134
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2135
|
+
expect(await reportReady(id, undefined, true)).toBe(200);
|
|
2136
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
2137
|
+
});
|
|
2138
|
+
});
|
|
2139
|
+
|
|
2140
|
+
describe("remediation respects runtime liveness", () => {
|
|
2141
|
+
test("recovering a stale task leaves a zero-runtime agent offline", async () => {
|
|
2142
|
+
const id = makeAgent(2);
|
|
2143
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2144
|
+
const rt = crypto.randomUUID();
|
|
2145
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2146
|
+
|
|
2147
|
+
const task = createTaskExtended("crashed-work", { agentId: id });
|
|
2148
|
+
startTask(task.id);
|
|
2149
|
+
await startSessionFor(id, task.id, rt);
|
|
2150
|
+
getDb()
|
|
2151
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
2152
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), task.id);
|
|
2153
|
+
// A crashed process stops both its runtime ping and its session heartbeat.
|
|
2154
|
+
makeRuntimeStale(rt);
|
|
2155
|
+
makeSessionStale(task.id);
|
|
2156
|
+
|
|
2157
|
+
await runHeartbeatSweep();
|
|
2158
|
+
|
|
2159
|
+
// The task recovers, but nothing is left to serve the agent.
|
|
2160
|
+
expect(getTaskById(task.id)?.status).not.toBe("in_progress");
|
|
2161
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
2162
|
+
});
|
|
2163
|
+
|
|
2164
|
+
test("a live sibling keeps the agent available through remediation", async () => {
|
|
2165
|
+
const id = makeAgent(2);
|
|
2166
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2167
|
+
const dead = crypto.randomUUID();
|
|
2168
|
+
const live = crypto.randomUUID();
|
|
2169
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: dead });
|
|
2170
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
2171
|
+
|
|
2172
|
+
const task = createTaskExtended("crashed-work", { agentId: id });
|
|
2173
|
+
startTask(task.id);
|
|
2174
|
+
await startSessionFor(id, task.id, dead);
|
|
2175
|
+
getDb()
|
|
2176
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
2177
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), task.id);
|
|
2178
|
+
// A crashed process stops both its runtime ping and its session heartbeat.
|
|
2179
|
+
makeRuntimeStale(dead);
|
|
2180
|
+
makeSessionStale(task.id);
|
|
2181
|
+
|
|
2182
|
+
await runHeartbeatSweep();
|
|
2183
|
+
expect(getAgentById(id)?.status).not.toBe("offline");
|
|
2184
|
+
});
|
|
2185
|
+
|
|
2186
|
+
test("with the flag off, remediation restores idle as before", async () => {
|
|
2187
|
+
const id = makeAgent(2);
|
|
2188
|
+
const task = createTaskExtended("crashed-work", { agentId: id });
|
|
2189
|
+
startTask(task.id);
|
|
2190
|
+
getDb()
|
|
2191
|
+
.prepare("UPDATE agent_tasks SET lastUpdatedAt = ? WHERE id = ?")
|
|
2192
|
+
.run(new Date(Date.now() - 30 * 60000).toISOString(), task.id);
|
|
2193
|
+
|
|
2194
|
+
await runHeartbeatSweep();
|
|
2195
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
2196
|
+
});
|
|
2197
|
+
});
|
|
2198
|
+
|
|
2199
|
+
describe("poll refreshes runtime liveness", () => {
|
|
2200
|
+
test("polling advances last_seen_at", async () => {
|
|
2201
|
+
const id = makeAgent(2);
|
|
2202
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2203
|
+
const rt = crypto.randomUUID();
|
|
2204
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2205
|
+
const before = getRuntimeInstanceById(rt);
|
|
2206
|
+
|
|
2207
|
+
await Bun.sleep(10);
|
|
2208
|
+
await pollAgent(id, rt);
|
|
2209
|
+
const after = getRuntimeInstanceById(rt);
|
|
2210
|
+
expect(after && before && after.lastSeenAt > before.lastSeenAt).toBe(true);
|
|
2211
|
+
});
|
|
2212
|
+
|
|
2213
|
+
test("a worker polling keeps itself live under a short stale threshold", async () => {
|
|
2214
|
+
const id = makeAgent(2);
|
|
2215
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2216
|
+
process.env.RUNTIME_STALE_THRESHOLD_MIN = "1";
|
|
2217
|
+
const rt = crypto.randomUUID();
|
|
2218
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2219
|
+
|
|
2220
|
+
// Almost past the cutoff, then a poll — as the inner loop would do.
|
|
2221
|
+
makeRuntimeStale(rt, 0.9);
|
|
2222
|
+
await pollAgent(id, rt);
|
|
2223
|
+
expect(expireStaleRuntimeInstances().expired).toBe(0);
|
|
2224
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(1);
|
|
2225
|
+
delete process.env.RUNTIME_STALE_THRESHOLD_MIN;
|
|
2226
|
+
});
|
|
2227
|
+
|
|
2228
|
+
test("polling cannot revive an expired runtime", async () => {
|
|
2229
|
+
const id = makeAgent(2);
|
|
2230
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2231
|
+
const rt = crypto.randomUUID();
|
|
2232
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2233
|
+
makeRuntimeStale(rt);
|
|
2234
|
+
|
|
2235
|
+
expect((await pollAgent(id, rt))?.trigger ?? null).toBeNull();
|
|
2236
|
+
// Still stale: the refresh only matches an already-live row.
|
|
2237
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(0);
|
|
2238
|
+
expireStaleRuntimeInstances();
|
|
2239
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
2240
|
+
expect((await pollAgent(id, rt))?.trigger ?? null).toBeNull();
|
|
2241
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
2242
|
+
});
|
|
2243
|
+
|
|
2244
|
+
test("a foreign runtime id refreshes nothing", async () => {
|
|
2245
|
+
const id1 = makeAgent(2);
|
|
2246
|
+
const id2 = makeAgent(2);
|
|
2247
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2248
|
+
const r1 = crypto.randomUUID();
|
|
2249
|
+
await register(id1, { maxTasks: 1, runtimeInstanceId: r1 });
|
|
2250
|
+
await register(id2, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
2251
|
+
const before = getRuntimeInstanceById(r1);
|
|
2252
|
+
|
|
2253
|
+
await Bun.sleep(10);
|
|
2254
|
+
await pollAgent(id2, r1);
|
|
2255
|
+
expect(getRuntimeInstanceById(r1)?.lastSeenAt).toBe(before?.lastSeenAt ?? "");
|
|
2256
|
+
});
|
|
2257
|
+
});
|
|
2258
|
+
|
|
2259
|
+
/** Invoke MCP poll-task the way a worker process would, via request context. */
|
|
2260
|
+
async function pollTask(agentId: string, runtimeInstanceId?: string, onNotify?: () => void) {
|
|
2261
|
+
const handler = mcpServer.handlers.get("poll-task");
|
|
2262
|
+
if (!handler) throw new Error("poll-task not registered");
|
|
2263
|
+
const headers: Record<string, string> = { "x-agent-id": agentId };
|
|
2264
|
+
if (runtimeInstanceId) headers["x-runtime-instance-id"] = runtimeInstanceId;
|
|
2265
|
+
return (await handler(
|
|
2266
|
+
{},
|
|
2267
|
+
{
|
|
2268
|
+
sessionId: "test-session",
|
|
2269
|
+
requestInfo: { headers },
|
|
2270
|
+
// Fires between dispatch attempts — tests use it to mutate state
|
|
2271
|
+
// mid-long-poll.
|
|
2272
|
+
sendNotification: async () => {
|
|
2273
|
+
onNotify?.();
|
|
2274
|
+
},
|
|
2275
|
+
},
|
|
2276
|
+
)) as { structuredContent?: { task?: { id?: string } } };
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
function startedTaskId(result: { structuredContent?: { task?: { id?: string } } }) {
|
|
2280
|
+
return result?.structuredContent?.task?.id ?? null;
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
describe("expiry reconciles the surviving runtime set", () => {
|
|
2284
|
+
async function readyAndWaiting(id: string) {
|
|
2285
|
+
const ready = crypto.randomUUID();
|
|
2286
|
+
const waiting = crypto.randomUUID();
|
|
2287
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: ready });
|
|
2288
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: waiting });
|
|
2289
|
+
await reportReady(id, ready, true);
|
|
2290
|
+
await reportReady(id, waiting, false);
|
|
2291
|
+
return { ready, waiting };
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
test("expiring the only ready runtime parks the agent", async () => {
|
|
2295
|
+
const id = makeAgent(2);
|
|
2296
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2297
|
+
const { ready } = await readyAndWaiting(id);
|
|
2298
|
+
expect(getAgentById(id)?.status).not.toBe("waiting_for_credentials");
|
|
2299
|
+
|
|
2300
|
+
makeRuntimeStale(ready);
|
|
2301
|
+
expireStaleRuntimeInstances();
|
|
2302
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2303
|
+
});
|
|
2304
|
+
|
|
2305
|
+
test("expiring a waiting runtime leaves a ready sibling available", async () => {
|
|
2306
|
+
const id = makeAgent(2);
|
|
2307
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2308
|
+
const { waiting } = await readyAndWaiting(id);
|
|
2309
|
+
|
|
2310
|
+
makeRuntimeStale(waiting);
|
|
2311
|
+
expireStaleRuntimeInstances();
|
|
2312
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
2313
|
+
});
|
|
2314
|
+
|
|
2315
|
+
test("a busy agent stays busy when a waiting runtime expires", async () => {
|
|
2316
|
+
const id = makeAgent(2);
|
|
2317
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2318
|
+
const { waiting } = await readyAndWaiting(id);
|
|
2319
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
2320
|
+
startTask(task.id);
|
|
2321
|
+
|
|
2322
|
+
makeRuntimeStale(waiting);
|
|
2323
|
+
expireStaleRuntimeInstances();
|
|
2324
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
2325
|
+
});
|
|
2326
|
+
|
|
2327
|
+
test("expiring the last runtime is still offline", async () => {
|
|
2328
|
+
const id = makeAgent(2);
|
|
2329
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2330
|
+
const { ready, waiting } = await readyAndWaiting(id);
|
|
2331
|
+
makeRuntimeStale(ready);
|
|
2332
|
+
makeRuntimeStale(waiting);
|
|
2333
|
+
expireStaleRuntimeInstances();
|
|
2334
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
2335
|
+
});
|
|
2336
|
+
|
|
2337
|
+
test("the sweep does not assign pool work when only a waiting runtime survives", async () => {
|
|
2338
|
+
const id = makeAgent(2);
|
|
2339
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2340
|
+
const { ready } = await readyAndWaiting(id);
|
|
2341
|
+
makeRuntimeStale(ready);
|
|
2342
|
+
const task = createTaskExtended("pool-work");
|
|
2343
|
+
|
|
2344
|
+
await runHeartbeatSweep();
|
|
2345
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2346
|
+
expect(getTaskById(task.id)?.agentId ?? null).toBeNull();
|
|
2347
|
+
});
|
|
2348
|
+
});
|
|
2349
|
+
|
|
2350
|
+
describe("registration reconciles the agent", () => {
|
|
2351
|
+
test("a new runtime lifts a waiting agent out of waiting", async () => {
|
|
2352
|
+
const id = makeAgent(2);
|
|
2353
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2354
|
+
const waiting = crypto.randomUUID();
|
|
2355
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: waiting });
|
|
2356
|
+
await reportReady(id, waiting, false);
|
|
2357
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2358
|
+
|
|
2359
|
+
// Credential checks disabled on the newcomer: never reports, counts ready.
|
|
2360
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
2361
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
2362
|
+
});
|
|
2363
|
+
|
|
2364
|
+
test("registration does not force idle over active work", async () => {
|
|
2365
|
+
const id = makeAgent(2);
|
|
2366
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2367
|
+
const waiting = crypto.randomUUID();
|
|
2368
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: waiting });
|
|
2369
|
+
await reportReady(id, waiting, false);
|
|
2370
|
+
const task = createTaskExtended("work", { agentId: id });
|
|
2371
|
+
startTask(task.id);
|
|
2372
|
+
|
|
2373
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: crypto.randomUUID() });
|
|
2374
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
2375
|
+
});
|
|
2376
|
+
|
|
2377
|
+
test("registering another waiting runtime keeps the agent parked", async () => {
|
|
2378
|
+
const id = makeAgent(2);
|
|
2379
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2380
|
+
const a = crypto.randomUUID();
|
|
2381
|
+
const b = crypto.randomUUID();
|
|
2382
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: a });
|
|
2383
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: b });
|
|
2384
|
+
await reportReady(id, a, false);
|
|
2385
|
+
await reportReady(id, b, false);
|
|
2386
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2387
|
+
|
|
2388
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: b });
|
|
2389
|
+
expect(getAgentById(id)?.status).toBe("waiting_for_credentials");
|
|
2390
|
+
expect(runtimeInstancesFor(id)).toHaveLength(2);
|
|
2391
|
+
});
|
|
2392
|
+
|
|
2393
|
+
test("an offline agent comes back when a runtime registers", async () => {
|
|
2394
|
+
const id = makeAgent(2);
|
|
2395
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2396
|
+
const rt = crypto.randomUUID();
|
|
2397
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2398
|
+
await closeRuntime(id, rt);
|
|
2399
|
+
expect(getAgentById(id)?.status).toBe("offline");
|
|
2400
|
+
|
|
2401
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: crypto.randomUUID() });
|
|
2402
|
+
expect(getAgentById(id)?.status).toBe("idle");
|
|
2403
|
+
});
|
|
2404
|
+
|
|
2405
|
+
test("with the flag off, registration does not touch status", async () => {
|
|
2406
|
+
const id = makeAgent(2);
|
|
2407
|
+
getDb().prepare("UPDATE agents SET status = 'busy' WHERE id = ?").run(id);
|
|
2408
|
+
await register(id, { maxTasks: 2, runtimeInstanceId: crypto.randomUUID() });
|
|
2409
|
+
expect(getAgentById(id)?.status).toBe("busy");
|
|
2410
|
+
});
|
|
2411
|
+
});
|
|
2412
|
+
|
|
2413
|
+
describe("MCP poll-task requires a live runtime", () => {
|
|
2414
|
+
test("with the flag off it dispatches as before", async () => {
|
|
2415
|
+
const id = makeAgent(1);
|
|
2416
|
+
const task = createTaskExtended("mcp-work", { agentId: id });
|
|
2417
|
+
expect(startedTaskId(await pollTask(id))).toBe(task.id);
|
|
2418
|
+
});
|
|
2419
|
+
|
|
2420
|
+
test("a valid live runtime can dispatch", async () => {
|
|
2421
|
+
const id = makeAgent(1);
|
|
2422
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2423
|
+
const rt = crypto.randomUUID();
|
|
2424
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2425
|
+
const task = createTaskExtended("mcp-work", { agentId: id });
|
|
2426
|
+
expect(startedTaskId(await pollTask(id, rt))).toBe(task.id);
|
|
2427
|
+
});
|
|
2428
|
+
|
|
2429
|
+
test("missing, unknown and foreign runtime identities get no work", async () => {
|
|
2430
|
+
const id = makeAgent(1);
|
|
2431
|
+
const other = makeAgent(1);
|
|
2432
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2433
|
+
const rt = crypto.randomUUID();
|
|
2434
|
+
const foreign = crypto.randomUUID();
|
|
2435
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2436
|
+
await register(other, { maxTasks: 1, runtimeInstanceId: foreign });
|
|
2437
|
+
const task = createTaskExtended("mcp-work", { agentId: id });
|
|
2438
|
+
|
|
2439
|
+
expect(startedTaskId(await pollTask(id))).toBeNull();
|
|
2440
|
+
expect(startedTaskId(await pollTask(id, crypto.randomUUID()))).toBeNull();
|
|
2441
|
+
expect(startedTaskId(await pollTask(id, foreign))).toBeNull();
|
|
2442
|
+
expect(getTaskById(task.id)?.status).toBe("pending");
|
|
2443
|
+
});
|
|
2444
|
+
|
|
2445
|
+
test("an expired runtime cannot dispatch and is not revived", async () => {
|
|
2446
|
+
const id = makeAgent(1);
|
|
2447
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2448
|
+
const rt = crypto.randomUUID();
|
|
2449
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2450
|
+
createTaskExtended("mcp-work", { agentId: id });
|
|
2451
|
+
makeRuntimeStale(rt);
|
|
2452
|
+
|
|
2453
|
+
expect(startedTaskId(await pollTask(id, rt))).toBeNull();
|
|
2454
|
+
expect(countActiveRuntimeInstancesForAgent(id)).toBe(0);
|
|
2455
|
+
expireStaleRuntimeInstances();
|
|
2456
|
+
expect(startedTaskId(await pollTask(id, rt))).toBeNull();
|
|
2457
|
+
expect(getRuntimeInstanceById(rt)).toBeNull();
|
|
2458
|
+
});
|
|
2459
|
+
|
|
2460
|
+
test("a retired process cannot work beside its replacement", async () => {
|
|
2461
|
+
const id = makeAgent(1);
|
|
2462
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2463
|
+
const oldRt = crypto.randomUUID();
|
|
2464
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: oldRt });
|
|
2465
|
+
makeRuntimeStale(oldRt);
|
|
2466
|
+
expireStaleRuntimeInstances();
|
|
2467
|
+
const newRt = crypto.randomUUID();
|
|
2468
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: newRt });
|
|
2469
|
+
const task = createTaskExtended("mcp-work", { agentId: id });
|
|
2470
|
+
|
|
2471
|
+
expect(startedTaskId(await pollTask(id, oldRt))).toBeNull();
|
|
2472
|
+
expect(startedTaskId(await pollTask(id, newRt))).toBe(task.id);
|
|
2473
|
+
});
|
|
2474
|
+
|
|
2475
|
+
test("a poll-task call refreshes its runtime's liveness", async () => {
|
|
2476
|
+
const id = makeAgent(1);
|
|
2477
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2478
|
+
const rt = crypto.randomUUID();
|
|
2479
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2480
|
+
// A task makes the call return immediately instead of entering the wait.
|
|
2481
|
+
createTaskExtended("mcp-work", { agentId: id });
|
|
2482
|
+
const before = getRuntimeInstanceById(rt);
|
|
2483
|
+
|
|
2484
|
+
await Bun.sleep(10);
|
|
2485
|
+
await pollTask(id, rt);
|
|
2486
|
+
const after = getRuntimeInstanceById(rt);
|
|
2487
|
+
expect(after && before && after.lastSeenAt > before.lastSeenAt).toBe(true);
|
|
2488
|
+
});
|
|
2489
|
+
});
|
|
2490
|
+
|
|
2491
|
+
describe("logical capacity is shared across dispatch entrypoints", () => {
|
|
2492
|
+
test("maxTasks=1 admits one task across concurrent HTTP and MCP polls", async () => {
|
|
2493
|
+
const id = makeAgent(1);
|
|
2494
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2495
|
+
const httpRt = crypto.randomUUID();
|
|
2496
|
+
const mcpRt = crypto.randomUUID();
|
|
2497
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: httpRt });
|
|
2498
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: mcpRt });
|
|
2499
|
+
// One task per entrypoint's preferred shape, both against the same limit.
|
|
2500
|
+
createTaskExtended("offered-work", { offeredTo: id });
|
|
2501
|
+
createTaskExtended("pending-work", { agentId: id });
|
|
2502
|
+
|
|
2503
|
+
const [httpResult, mcpResult] = await Promise.all([pollAgent(id, httpRt), pollTask(id, mcpRt)]);
|
|
2504
|
+
|
|
2505
|
+
const admitted = (httpResult?.trigger?.type ? 1 : 0) + (startedTaskId(mcpResult) ? 1 : 0);
|
|
2506
|
+
expect(admitted).toBe(1);
|
|
2507
|
+
expect(getActiveTaskCount(id)).toBe(1);
|
|
2508
|
+
});
|
|
2509
|
+
|
|
2510
|
+
test("maxTasks=1 admits one pending task across concurrent MCP polls", async () => {
|
|
2511
|
+
const id = makeAgent(1);
|
|
2512
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2513
|
+
const rtA = crypto.randomUUID();
|
|
2514
|
+
const rtB = crypto.randomUUID();
|
|
2515
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtA });
|
|
2516
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtB });
|
|
2517
|
+
// Two already-pending assignments: both runtimes pass the liveness gate,
|
|
2518
|
+
// so only the in-transaction capacity check keeps the second one out.
|
|
2519
|
+
const first = createTaskExtended("pending-one", { agentId: id });
|
|
2520
|
+
const second = createTaskExtended("pending-two", { agentId: id });
|
|
2521
|
+
|
|
2522
|
+
const [a, b] = await Promise.all([pollTask(id, rtA), pollTask(id, rtB)]);
|
|
2523
|
+
|
|
2524
|
+
const started = [startedTaskId(a), startedTaskId(b)].filter(Boolean);
|
|
2525
|
+
expect(started).toHaveLength(1);
|
|
2526
|
+
expect(getActiveTaskCount(id)).toBe(1);
|
|
2527
|
+
// The blocked task stays pending for whenever capacity frees up.
|
|
2528
|
+
const statuses = [getTaskById(first.id)?.status, getTaskById(second.id)?.status].sort();
|
|
2529
|
+
expect(statuses).toEqual(["in_progress", "pending"]);
|
|
2530
|
+
});
|
|
2531
|
+
|
|
2532
|
+
test("maxTasks=2 lets two live runtimes start both pending tasks", async () => {
|
|
2533
|
+
const id = makeAgent(2);
|
|
2534
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2535
|
+
const rtA = crypto.randomUUID();
|
|
2536
|
+
const rtB = crypto.randomUUID();
|
|
2537
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtA });
|
|
2538
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtB });
|
|
2539
|
+
createTaskExtended("pending-one", { agentId: id });
|
|
2540
|
+
createTaskExtended("pending-two", { agentId: id });
|
|
2541
|
+
|
|
2542
|
+
const [a, b] = await Promise.all([pollTask(id, rtA), pollTask(id, rtB)]);
|
|
2543
|
+
|
|
2544
|
+
expect(startedTaskId(a)).not.toBeNull();
|
|
2545
|
+
expect(startedTaskId(b)).not.toBeNull();
|
|
2546
|
+
expect(startedTaskId(a)).not.toBe(startedTaskId(b));
|
|
2547
|
+
expect(getActiveTaskCount(id)).toBe(2);
|
|
2548
|
+
});
|
|
2549
|
+
});
|
|
2550
|
+
|
|
2551
|
+
describe("MCP poll-task revalidates the runtime at dispatch", () => {
|
|
2552
|
+
test("a runtime retired mid-poll cannot start work that appears later; a live sibling can", async () => {
|
|
2553
|
+
const id = makeAgent(1);
|
|
2554
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2555
|
+
const rtA = crypto.randomUUID();
|
|
2556
|
+
const rtB = crypto.randomUUID();
|
|
2557
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtA });
|
|
2558
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rtB });
|
|
2559
|
+
|
|
2560
|
+
// A enters the long poll with nothing to do; while it waits, its runtime
|
|
2561
|
+
// is retired and a pending task appears.
|
|
2562
|
+
let taskId: string | undefined;
|
|
2563
|
+
let mutated = false;
|
|
2564
|
+
const result = await pollTask(id, rtA, () => {
|
|
2565
|
+
if (mutated) return;
|
|
2566
|
+
mutated = true;
|
|
2567
|
+
makeRuntimeStale(rtA);
|
|
2568
|
+
taskId = createTaskExtended("mid-poll-work", { agentId: id }).id;
|
|
2569
|
+
});
|
|
2570
|
+
|
|
2571
|
+
// The retired poller exits without acquiring the task and without
|
|
2572
|
+
// advancing the empty-poll exit counter.
|
|
2573
|
+
expect(startedTaskId(result)).toBeNull();
|
|
2574
|
+
expect(taskId && getTaskById(taskId)?.status).toBe("pending");
|
|
2575
|
+
expect(getAgentById(id)?.emptyPollCount ?? 0).toBe(0);
|
|
2576
|
+
|
|
2577
|
+
// The live sibling picks it up normally.
|
|
2578
|
+
expect(taskId && startedTaskId(await pollTask(id, rtB))).toBe(taskId);
|
|
2579
|
+
expect(getActiveTaskCount(id)).toBe(1);
|
|
2580
|
+
});
|
|
2581
|
+
|
|
2582
|
+
test("a runtime closed mid-poll is not revived by the in-flight call", async () => {
|
|
2583
|
+
const id = makeAgent(1);
|
|
2584
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2585
|
+
const rt = crypto.randomUUID();
|
|
2586
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2587
|
+
|
|
2588
|
+
let taskId: string | undefined;
|
|
2589
|
+
let mutated = false;
|
|
2590
|
+
const result = await pollTask(id, rt, () => {
|
|
2591
|
+
if (mutated) return;
|
|
2592
|
+
mutated = true;
|
|
2593
|
+
getDb().prepare("UPDATE runtime_instances SET status = 'offline' WHERE id = ?").run(rt);
|
|
2594
|
+
taskId = createTaskExtended("post-close-work", { agentId: id }).id;
|
|
2595
|
+
});
|
|
2596
|
+
|
|
2597
|
+
expect(startedTaskId(result)).toBeNull();
|
|
2598
|
+
expect(taskId && getTaskById(taskId)?.status).toBe("pending");
|
|
2599
|
+
// Still offline — the dispatch revalidation cannot resurrect it.
|
|
2600
|
+
expect(getRuntimeInstanceById(rt)?.status).toBe("offline");
|
|
2601
|
+
});
|
|
2602
|
+
});
|
|
2603
|
+
|
|
2604
|
+
describe("MCP task-action claim requires a live runtime", () => {
|
|
2605
|
+
async function claim(agentId: string, taskId: string, runtimeInstanceId?: string) {
|
|
2606
|
+
const handler = mcpServer.handlers.get("task-action");
|
|
2607
|
+
if (!handler) throw new Error("task-action not registered");
|
|
2608
|
+
const headers: Record<string, string> = { "x-agent-id": agentId };
|
|
2609
|
+
if (runtimeInstanceId) headers["x-runtime-instance-id"] = runtimeInstanceId;
|
|
2610
|
+
return (await handler(
|
|
2611
|
+
{ action: "claim", taskId },
|
|
2612
|
+
{
|
|
2613
|
+
sessionId: "test-session",
|
|
2614
|
+
requestInfo: { headers },
|
|
2615
|
+
sendNotification: async () => {},
|
|
2616
|
+
},
|
|
2617
|
+
)) as { isError?: boolean };
|
|
2618
|
+
}
|
|
2619
|
+
|
|
2620
|
+
test("an expired runtime cannot claim, a live one can", async () => {
|
|
2621
|
+
const id = makeAgent(1);
|
|
2622
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2623
|
+
const dead = crypto.randomUUID();
|
|
2624
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: dead });
|
|
2625
|
+
makeRuntimeStale(dead);
|
|
2626
|
+
expireStaleRuntimeInstances();
|
|
2627
|
+
const task = createTaskExtended("pool-claim");
|
|
2628
|
+
|
|
2629
|
+
await claim(id, task.id, dead);
|
|
2630
|
+
expect(getTaskById(task.id)?.agentId ?? null).toBeNull();
|
|
2631
|
+
|
|
2632
|
+
const live = crypto.randomUUID();
|
|
2633
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: live });
|
|
2634
|
+
await claim(id, task.id, live);
|
|
2635
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
2636
|
+
});
|
|
2637
|
+
|
|
2638
|
+
test("with the flag off, claiming works without runtime identity", async () => {
|
|
2639
|
+
const id = makeAgent(1);
|
|
2640
|
+
const task = createTaskExtended("pool-claim");
|
|
2641
|
+
await claim(id, task.id);
|
|
2642
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
2643
|
+
});
|
|
2644
|
+
});
|
|
2645
|
+
|
|
2646
|
+
describe("MCP task-action accept requires a live runtime", () => {
|
|
2647
|
+
async function accept(agentId: string, taskId: string, runtimeInstanceId?: string) {
|
|
2648
|
+
const handler = mcpServer.handlers.get("task-action");
|
|
2649
|
+
if (!handler) throw new Error("task-action not registered");
|
|
2650
|
+
const headers: Record<string, string> = { "x-agent-id": agentId };
|
|
2651
|
+
if (runtimeInstanceId) headers["x-runtime-instance-id"] = runtimeInstanceId;
|
|
2652
|
+
return (await handler(
|
|
2653
|
+
{ action: "accept", taskId },
|
|
2654
|
+
{
|
|
2655
|
+
sessionId: "test-session",
|
|
2656
|
+
requestInfo: { headers },
|
|
2657
|
+
sendNotification: async () => {},
|
|
2658
|
+
},
|
|
2659
|
+
)) as { isError?: boolean };
|
|
2660
|
+
}
|
|
2661
|
+
|
|
2662
|
+
test("an expired runtime cannot accept and is not revived", async () => {
|
|
2663
|
+
const id = makeAgent(1);
|
|
2664
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2665
|
+
const dead = crypto.randomUUID();
|
|
2666
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: dead });
|
|
2667
|
+
makeRuntimeStale(dead);
|
|
2668
|
+
expireStaleRuntimeInstances();
|
|
2669
|
+
const task = createTaskExtended("offered-work", { offeredTo: id });
|
|
2670
|
+
|
|
2671
|
+
await accept(id, task.id, dead);
|
|
2672
|
+
expect(getTaskById(task.id)?.status).toBe("offered");
|
|
2673
|
+
expect(getRuntimeInstanceById(dead)).toBeNull();
|
|
2674
|
+
});
|
|
2675
|
+
|
|
2676
|
+
test("missing and foreign runtime identities cannot accept; the owning live one can", async () => {
|
|
2677
|
+
const id = makeAgent(1);
|
|
2678
|
+
const other = makeAgent(1);
|
|
2679
|
+
process.env.MULTI_RUNTIME_ENABLED = "true";
|
|
2680
|
+
const rt = crypto.randomUUID();
|
|
2681
|
+
const foreign = crypto.randomUUID();
|
|
2682
|
+
await register(id, { maxTasks: 1, runtimeInstanceId: rt });
|
|
2683
|
+
await register(other, { maxTasks: 1, runtimeInstanceId: foreign });
|
|
2684
|
+
const task = createTaskExtended("offered-work", { offeredTo: id });
|
|
2685
|
+
|
|
2686
|
+
await accept(id, task.id);
|
|
2687
|
+
expect(getTaskById(task.id)?.status).toBe("offered");
|
|
2688
|
+
await accept(id, task.id, foreign);
|
|
2689
|
+
expect(getTaskById(task.id)?.status).toBe("offered");
|
|
2690
|
+
|
|
2691
|
+
await accept(id, task.id, rt);
|
|
2692
|
+
expect(getTaskById(task.id)?.status).toBe("pending");
|
|
2693
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
2694
|
+
});
|
|
2695
|
+
|
|
2696
|
+
test("with the flag off, accepting works without runtime identity", async () => {
|
|
2697
|
+
const id = makeAgent(1);
|
|
2698
|
+
const task = createTaskExtended("offered-work", { offeredTo: id });
|
|
2699
|
+
await accept(id, task.id);
|
|
2700
|
+
expect(getTaskById(task.id)?.status).toBe("pending");
|
|
2701
|
+
expect(getTaskById(task.id)?.agentId).toBe(id);
|
|
2702
|
+
});
|
|
2703
|
+
});
|