@relaycast/engine 10.0.2 → 10.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db/migrations/0068_spawn_canceled_index.sql +19 -0
- package/dist/db/migrations/0069_nodes_online_liveness_index.sql +15 -0
- package/dist/engine/node.d.ts.map +1 -1
- package/dist/engine/node.js +320 -130
- package/dist/engine/node.js.map +1 -1
- package/dist/engine/nodeProvider.d.ts +16 -2
- package/dist/engine/nodeProvider.d.ts.map +1 -1
- package/dist/engine/nodeProvider.js +67 -5
- package/dist/engine/nodeProvider.js.map +1 -1
- package/dist/ports/database.d.ts +1 -1
- package/dist/ports/database.d.ts.map +1 -1
- package/dist/ports/database.js +6 -0
- package/dist/ports/database.js.map +1 -1
- package/package.json +3 -3
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
-- `agent.register` runs a migration-tombstone check on every registration
|
|
2
|
+
-- (rejectMigrationCanceledSpawnRegistration): did a spawn invocation for this
|
|
3
|
+
-- (node, provider, agent-name) tuple get cancelled during the provider
|
|
4
|
+
-- migration? Only rows with error = 'invocation_origin_unavailable' can match
|
|
5
|
+
-- (~2.9k in production) — but the planner was satisfying the query out of the
|
|
6
|
+
-- status index instead, seeking the whole status = 'failed' bucket (~216k
|
|
7
|
+
-- rows, ~1.8s per registration) and blowing the node-control liveness
|
|
8
|
+
-- deadline that surfaces as d1_pressure on every agent spawn.
|
|
9
|
+
--
|
|
10
|
+
-- A partial index on the tombstone predicates was the first cut, but the
|
|
11
|
+
-- check binds status and error through parameters, and SQLite cannot prove a
|
|
12
|
+
-- bound parameter satisfies a partial WHERE clause — the same lesson 0065
|
|
13
|
+
-- recorded for the task deadline sweep. This composite is non-partial: the
|
|
14
|
+
-- (workspace_id, dispatched_node_id) equality seek narrows to the handful of
|
|
15
|
+
-- invocations a node has ever seen, and the same seek also serves the
|
|
16
|
+
-- spawn-reservation EXISTS inside reserveNodeAgentSlot (~541 rows measured).
|
|
17
|
+
-- No data changes.
|
|
18
|
+
CREATE INDEX IF NOT EXISTS idx_action_invocations_spawn_canceled
|
|
19
|
+
ON action_invocations(workspace_id, dispatched_node_id);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
-- The cloud cron's `reconcileStaleOnlineNodes` candidate scan runs a global
|
|
2
|
+
-- liveness query with no workspace predicate:
|
|
3
|
+
-- SELECT id, workspace_id FROM nodes
|
|
4
|
+
-- WHERE status = 'online'
|
|
5
|
+
-- AND (last_heartbeat_at IS NULL OR last_heartbeat_at < ?) LIMIT ?
|
|
6
|
+
-- Every existing nodes index leads with workspace_id, so prod answers it with
|
|
7
|
+
-- a full table scan (~24k rows) on every maintenance tick, on the same
|
|
8
|
+
-- database whose overload the sweep exists to relieve. A partial covering
|
|
9
|
+
-- index over only the online rows turns it into an index-only scan of the
|
|
10
|
+
-- live set (tens to hundreds of rows, not tens of thousands). The OR keeps
|
|
11
|
+
-- it a scan of the index rather than a seek, which is fine at partial-index
|
|
12
|
+
-- cardinality. No data changes.
|
|
13
|
+
CREATE INDEX IF NOT EXISTS idx_nodes_online_liveness
|
|
14
|
+
ON nodes(last_heartbeat_at, workspace_id, id)
|
|
15
|
+
WHERE status = 'online';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../../src/engine/node.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,wBAAwB,EACxB,yBAAyB,EAEzB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAqB,MAAM,EAA8D,KAAK,EAAE,MAAM,iBAAiB,CAAC;
|
|
1
|
+
{"version":3,"file":"node.d.ts","sourceRoot":"","sources":["../../src/engine/node.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,wBAAwB,EACxB,yBAAyB,EAEzB,yBAAyB,EACzB,mBAAmB,EACnB,yBAAyB,EACzB,wBAAwB,EACxB,qBAAqB,EACrB,eAAe,EACf,sBAAsB,EACvB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAqB,MAAM,EAA8D,KAAK,EAAE,MAAM,iBAAiB,CAAC;AAM/H,OAAO,KAAK,EAAgC,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AACnF,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AA0BnE,OAAO,KAAK,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAG7F,KAAK,EAAE,GAAG,UAAU,CAAC,OAAO,KAAK,CAAC,CAAC;AACnC,KAAK,OAAO,GAAG,OAAO,KAAK,CAAC,YAAY,CAAC;AACzC,KAAK,QAAQ,GAAG,OAAO,MAAM,CAAC,YAAY,CAAC;AAC3C,KAAK,QAAQ,GAAG,IAAI,GAAG,WAAW,GAAG,MAAM,CAAC;AAC5C,KAAK,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAIpC,oFAAoF;AACpF,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED,MAAM,WAAW,4BAA4B;IAC3C,EAAE,EAAE,QAAQ,CAAC;IACb,QAAQ,EAAE,sBAAsB,CAAC;IACjC,cAAc,CAAC,EAAE,wBAAwB,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,GAAG,EAAE,MAAM,CAAC;CACb;AAUD,KAAK,cAAc,GAAG,MAAM,GAAG,eAAe,CAAC;AAE/C,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE5D;AA8ED,eAAO,MAAM,4BAA4B,WAAW,CAAC;AAoJrD,iBAAS,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;EA6B7C;AAyDD,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IACJ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,IAAI,CAAC,EAAE,QAAQ,CAAC;IAChB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IAC1C,YAAY,CAAC,EAAE,cAAc,EAAE,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,EACD,aAAa,GAAE;IACb,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE;QACjB,gBAAgB,EAAE,MAAM,CAAC;QACzB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;CACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+JP;AAED,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAGjE;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAO5E;AAED,wBAAsB,WAAW,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAM5E;AASD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,iBAAiB,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAuD3B;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,QAAQ,CAMzG;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,QAAQ,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,EAClH,iBAAiB,CAAC,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;UAyB3B;AAWD,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,CAAC;IACpC,UAAU,EAAE,yBAAyB,EAAE,CAAC;IACxC,QAAQ,EAAE,qBAAqB,CAAC;CACjC;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,mBAAmB,EAAE,MAAM,EAC3B,OAAO,EAAE,wBAAwB,EACjC,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,kBAAkB,CAAC,CAqG7B;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;UAiInC;AAED,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;IAAE,IAAI,CAAC,EAAE,wBAAwB,CAAC;IAAC,MAAM,CAAC,EAAE,iBAAiB,CAAA;CAAE,iBAiD1E;AAED;;;;GAIG;AACH,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,uBAAuB,EAAE,OAAO,EAChC,IAAI,CAAC,EAAE,wBAAwB,iBAsBhC;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE,wBAAwB,iBAgDhC;AAED,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,IAAI,CAAC,EAAE,wBAAwB,GAC9B,OAAO,CAAC,MAAM,CAAC,CAmBjB;AAwRD,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,MAAM,GAAG,gBAAgB,CAAC,EACvD,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;UAG7E;AA+ID,wBAAsB,6BAA6B,CACjD,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,iBAahB;AAED,iBAAS,gBAAgB,CAAC,GAAG,EAAE;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,EAAE,IAAI,CAAC;IAChB,SAAS,EAAE,IAAI,GAAG,IAAI,CAAC;CACxB;;;;;;;;;;;;;;EAqBA;AAED,wBAAsB,eAAe,CACnC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE;IAAE,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5D,OAAO,CAAC;IACT,OAAO,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;IAC7C,IAAI,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE,CAAC,CAyID;AAED;;;;;;;GAOG;AACH,wBAAsB,0BAA0B,CAC9C,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC9D,OAAO,CAAC,IAAI,CAAC,CAYf;AAED,wBAAsB,mBAAmB,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBA6BzG;AAED,wBAAsB,cAAc,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM;;;;;;;;;;;;;;YA2BjF;AAED,wBAAsB,oBAAoB,CACxC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,yBAAyB,EAClC,OAAO,GAAE;IAAE,uBAAuB,CAAC,EAAE,OAAO,CAAA;CAAO,GAClD,OAAO,CAAC,sBAAsB,CAAC,CAqQjC;AAED;;;;GAIG;AACH,wBAAsB,mBAAmB,CACvC,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,wBAAwB,EACjC,OAAO,GAAE;IAAE,uBAAuB,CAAC,EAAE,OAAO,CAAA;CAAO,GAClD,OAAO,CAAC,sBAAsB,CAAC,CAyHjC;AAED,wBAAsB,sBAAsB,CAC1C,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAC7C,IAAI,CAAC,EAAE,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;UAmDhC;AAQD,wBAAsB,kBAAkB,CACtC,EAAE,EAAE,EAAE,EACN,QAAQ,EAAE,sBAAsB,EAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,EACpB,eAAe,EAAE,mBAAmB,EAAE,EACtC,cAAc,CAAC,EAAE,wBAAwB;;;;;;;;;;GAmV1C;AAED,gFAAgF;AAChF,eAAO,MAAM,+BAA+B,MAAM,CAAC;AACnD,iFAAiF;AACjF,eAAO,MAAM,2BAA2B,MAAM,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,MAAM,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;IAC9B;;;;;OAKG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;;;;;;;OAcG;IACH,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,UAAU,CAAC,OAAO,UAAU,CAAC,EAAE,CAAC;IACvC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AA0FD,wBAAsB,SAAS,CAC7B,EAAE,EAAE,EAAE,EACN,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,oBAAoB,CAAC,CA+C/B;AAED,wBAAsB,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;UAG5E;AAwGD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,4BAA4B,iBAuahF"}
|
package/dist/engine/node.js
CHANGED
|
@@ -6,12 +6,13 @@ import { actionInvocations, agents, agentNodeBindings, channelMembers, channels,
|
|
|
6
6
|
import { randomHex, sha256Hex } from '../lib/crypto.js';
|
|
7
7
|
import { codedError } from '../lib/httpError.js';
|
|
8
8
|
import { acceptTaskInvocation, completeTaskInvocation } from './taskInvocation.js';
|
|
9
|
-
import { runAtomic, runAtomicWrites } from '../ports/database.js';
|
|
9
|
+
import { databaseConstraintKind, runAtomic, runAtomicWrites } from '../ports/database.js';
|
|
10
|
+
import { retryD1Write } from '../lib/d1Retry.js';
|
|
10
11
|
import { generateId } from './snowflake.js';
|
|
11
12
|
import { assertRegistrableAgentName } from './agent.js';
|
|
12
13
|
import { rotateAgentIdentity } from './agentIdentity.js';
|
|
13
14
|
import { isNodeLive, isReusableForMachineMatch, nodeHasCapacity, NODE_LIVENESS_TTL_MS } from './placement.js';
|
|
14
|
-
import { DEFAULT_PROVIDER_NAME, capabilityKind, heartbeatProvider, isProviderLive, markProviderOffline, materializeProviderActions, recomputeNodeAggregate, removeProvider, upsertProvider, } from './nodeProvider.js';
|
|
15
|
+
import { DEFAULT_PROVIDER_NAME, assertActionTaskMode, capabilityKind, heartbeatProvider, isProviderLive, markProviderOffline, materializeProviderActions, providerActionsMaterialized, recomputeNodeAggregate, removeProvider, upsertProvider, } from './nodeProvider.js';
|
|
15
16
|
import { serializeNodeOp } from './nodeLock.js';
|
|
16
17
|
import { completeNodeInvocation, dispatchCapacitySpawn, rescheduleInvocationsForLostNode, rescheduleNodeInvocation, } from './action.js';
|
|
17
18
|
import { emitInvocationCompletionEffects, emitAgentExitedEffects, emitNodeStatusEffects, fleetInvocationId } from './invocationCompletion.js';
|
|
@@ -158,6 +159,30 @@ function withRegisteredProtocolCapabilities(heartbeatCapabilities, registeredCap
|
|
|
158
159
|
...registeredCapabilities.filter(isNegotiatedProtocolCapability),
|
|
159
160
|
];
|
|
160
161
|
}
|
|
162
|
+
function stableJson(value) {
|
|
163
|
+
if (Array.isArray(value))
|
|
164
|
+
return `[${value.map(stableJson).join(',')}]`;
|
|
165
|
+
if (value !== null && typeof value === 'object') {
|
|
166
|
+
const entries = Object.entries(value)
|
|
167
|
+
.filter(([, field]) => field !== undefined)
|
|
168
|
+
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
|
|
169
|
+
.map(([key, field]) => `${JSON.stringify(key)}:${stableJson(field)}`);
|
|
170
|
+
return `{${entries.join(',')}}`;
|
|
171
|
+
}
|
|
172
|
+
return JSON.stringify(value) ?? 'null';
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Order-insensitive structural equality on capability sets. The persisted
|
|
176
|
+
* manifest is a JSON round-trip of what the client sent, so key order and
|
|
177
|
+
* absent-vs-undefined fields must not count as a roster change.
|
|
178
|
+
*/
|
|
179
|
+
function sameCapabilities(a, b) {
|
|
180
|
+
if (a.length !== b.length)
|
|
181
|
+
return false;
|
|
182
|
+
const left = a.map(stableJson).sort();
|
|
183
|
+
const right = b.map(stableJson).sort();
|
|
184
|
+
return left.every((entry, index) => entry === right[index]);
|
|
185
|
+
}
|
|
161
186
|
/**
|
|
162
187
|
* Whether the provider's LIVE connection negotiated cursor-gated delivery at
|
|
163
188
|
* `node.register` — i.e. whether it was denied the register-time flush and so
|
|
@@ -718,33 +743,65 @@ export async function heartbeatNode(db, workspaceId, nodeId, providerName, messa
|
|
|
718
743
|
.returning();
|
|
719
744
|
return updated ? publicNode(updated) : null;
|
|
720
745
|
}
|
|
746
|
+
let updated;
|
|
721
747
|
await runAtomic(db, async (tx) => {
|
|
748
|
+
const now = new Date();
|
|
722
749
|
if (message.capabilities !== undefined) {
|
|
723
750
|
const [existingProvider] = await tx
|
|
724
|
-
.select(
|
|
725
|
-
instanceId: nodeProviders.instanceId,
|
|
726
|
-
maxAgents: nodeProviders.maxAgents,
|
|
727
|
-
version: nodeProviders.version,
|
|
728
|
-
capabilities: nodeProviders.capabilities,
|
|
729
|
-
})
|
|
751
|
+
.select()
|
|
730
752
|
.from(nodeProviders)
|
|
731
753
|
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName)));
|
|
732
754
|
const caps = withRegisteredProtocolCapabilities(normalizeCapabilities(message.capabilities), existingProvider?.capabilities ?? []);
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
755
|
+
assertActionTaskMode(caps);
|
|
756
|
+
const maxAgents = message.max_agents ?? existingProvider?.maxAgents ?? 0;
|
|
757
|
+
// heartbeatProvider's load rule folded in here: a numeric load only
|
|
758
|
+
// counts as a measurement when the provider opts in via load_reported
|
|
759
|
+
// and is capacity-bounded (zero is the unbounded sentinel).
|
|
760
|
+
const reportedLoad = maxAgents > 0 && message.load_reported === true && typeof message.load === 'number'
|
|
761
|
+
? message.load
|
|
762
|
+
: undefined;
|
|
763
|
+
// One write carries both the roster refresh (capabilities/max_agents/
|
|
764
|
+
// version) and the liveness signal (load/active_agents/handlers_live)
|
|
765
|
+
// that upsertProvider + heartbeatProvider used to write separately.
|
|
766
|
+
const providerWrite = {
|
|
736
767
|
capabilities: caps,
|
|
737
|
-
maxAgents
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
});
|
|
741
|
-
await heartbeatProvider(tx, workspaceId, nodeId, providerName, {
|
|
742
|
-
load: message.load,
|
|
743
|
-
loadReported: message.load_reported,
|
|
768
|
+
maxAgents,
|
|
769
|
+
load: reportedLoad ?? 0,
|
|
770
|
+
loadReported: reportedLoad !== undefined,
|
|
744
771
|
activeAgents: message.active_agents,
|
|
745
772
|
handlersLive: message.handlers_live,
|
|
746
|
-
|
|
747
|
-
|
|
773
|
+
status: 'online',
|
|
774
|
+
lastHeartbeatAt: now,
|
|
775
|
+
};
|
|
776
|
+
if (existingProvider) {
|
|
777
|
+
await tx
|
|
778
|
+
.update(nodeProviders)
|
|
779
|
+
.set({ ...providerWrite, version: message.version ?? existingProvider.version })
|
|
780
|
+
.where(eq(nodeProviders.id, existingProvider.id));
|
|
781
|
+
}
|
|
782
|
+
else {
|
|
783
|
+
await tx.insert(nodeProviders).values({
|
|
784
|
+
id: `nprov_${generateId()}`,
|
|
785
|
+
workspaceId,
|
|
786
|
+
nodeId,
|
|
787
|
+
name: providerName,
|
|
788
|
+
instanceId: `${providerName}:heartbeat`,
|
|
789
|
+
version: message.version ?? 'unknown',
|
|
790
|
+
createdAt: now,
|
|
791
|
+
...providerWrite,
|
|
792
|
+
});
|
|
793
|
+
}
|
|
794
|
+
// Re-materializing an identical manifest rewrites the same action rows
|
|
795
|
+
// on every beat (~4 statements per action capability, every ~12s per
|
|
796
|
+
// provider). The prune/upsert pass runs only when the roster actually
|
|
797
|
+
// changes or the materialized rows drifted (a cheap one-read check that
|
|
798
|
+
// keeps the heartbeat self-healing); every node.register still goes
|
|
799
|
+
// through materializeProviderActions unconditionally.
|
|
800
|
+
if (existingProvider === undefined
|
|
801
|
+
|| !sameCapabilities(existingProvider.capabilities ?? [], caps)
|
|
802
|
+
|| !(await providerActionsMaterialized(tx, workspaceId, nodeId, providerName, caps.filter((cap) => capabilityKind(cap) === 'action')))) {
|
|
803
|
+
await materializeProviderActions(tx, workspaceId, nodeId, providerName, caps);
|
|
804
|
+
}
|
|
748
805
|
}
|
|
749
806
|
else {
|
|
750
807
|
await heartbeatProvider(tx, workspaceId, nodeId, providerName, {
|
|
@@ -757,19 +814,15 @@ export async function heartbeatNode(db, workspaceId, nodeId, providerName, messa
|
|
|
757
814
|
if (message.name !== undefined) {
|
|
758
815
|
await tx.update(nodes).set({ name: message.name }).where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
759
816
|
}
|
|
760
|
-
|
|
761
|
-
//
|
|
762
|
-
//
|
|
763
|
-
//
|
|
764
|
-
await tx
|
|
765
|
-
.
|
|
766
|
-
|
|
767
|
-
|
|
817
|
+
// provenLiveAt rides the same UPDATE as the aggregate, preserving the
|
|
818
|
+
// ordering the separate stamp used to guarantee: the aggregate derives
|
|
819
|
+
// lastHeartbeatAt from provider rows, which registration also writes,
|
|
820
|
+
// whereas this column is only ever set by an arriving frame.
|
|
821
|
+
updated = await recomputeNodeAggregate(tx, workspaceId, nodeId, {
|
|
822
|
+
...(message.version !== undefined ? { version: message.version } : {}),
|
|
823
|
+
provenLiveAt: now,
|
|
824
|
+
});
|
|
768
825
|
});
|
|
769
|
-
const [updated] = await db
|
|
770
|
-
.select()
|
|
771
|
-
.from(nodes)
|
|
772
|
-
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId)));
|
|
773
826
|
return updated ? publicNode(updated) : null;
|
|
774
827
|
}
|
|
775
828
|
export async function markNodeOffline(db, registry, workspaceId, nodeId, effects) {
|
|
@@ -905,19 +958,6 @@ export async function sweepOfflineNodes(db, registry, deps) {
|
|
|
905
958
|
}
|
|
906
959
|
return swept;
|
|
907
960
|
}
|
|
908
|
-
async function autoJoinGeneral(db, workspaceId, agentId) {
|
|
909
|
-
const [generalChannel] = await db
|
|
910
|
-
.select()
|
|
911
|
-
.from(channels)
|
|
912
|
-
.where(and(eq(channels.workspaceId, workspaceId), eq(channels.name, 'general')));
|
|
913
|
-
if (generalChannel) {
|
|
914
|
-
await db.insert(channelMembers).values({
|
|
915
|
-
channelId: generalChannel.id,
|
|
916
|
-
agentId,
|
|
917
|
-
role: 'member',
|
|
918
|
-
}).onConflictDoNothing();
|
|
919
|
-
}
|
|
920
|
-
}
|
|
921
961
|
/**
|
|
922
962
|
* Statements that bind `agent` to `nodeId` and move its routable location onto
|
|
923
963
|
* that node. Built, not executed: a caller that owns the whole move hands them
|
|
@@ -993,7 +1033,22 @@ async function activeBindingNodeIdsForAgent(db, workspaceId, agentId) {
|
|
|
993
1033
|
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agentId), eq(agentNodeBindings.status, 'active')));
|
|
994
1034
|
return rows.map((row) => row.nodeId);
|
|
995
1035
|
}
|
|
996
|
-
|
|
1036
|
+
/**
|
|
1037
|
+
* The capacity-admission statement (at most one reserved slot). Built, not
|
|
1038
|
+
* executed, so a caller that owns the whole registration can commit it in the
|
|
1039
|
+
* same atomic unit as the identity insert the reservation is for.
|
|
1040
|
+
*
|
|
1041
|
+
* A refused admission must *fail*, not merely match no rows: batched
|
|
1042
|
+
* statements cannot inspect each other's results, so a soft refusal lets the
|
|
1043
|
+
* insert in the same unit commit a ghost agent until a compensating delete
|
|
1044
|
+
* lands (and in that window a same-named registration reads a false
|
|
1045
|
+
* `agent_already_exists`). Writing NULL into `active_agents` when the
|
|
1046
|
+
* capacity predicate fails makes the refusal a statement error that aborts
|
|
1047
|
+
* the whole atomic group on every adapter — the same NOT NULL sentinel the
|
|
1048
|
+
* workspace growth guards use. `databaseConstraintKind` reports it as
|
|
1049
|
+
* 'node_capacity'.
|
|
1050
|
+
*/
|
|
1051
|
+
function reserveNodeAgentSlotStatement(db, workspaceId, node, claim) {
|
|
997
1052
|
const reservationCorrelation = claim?.invocationId
|
|
998
1053
|
? eq(actionInvocations.id, claim.invocationId)
|
|
999
1054
|
: claim
|
|
@@ -1015,18 +1070,35 @@ async function reserveNodeAgentSlot(db, workspaceId, node, claim) {
|
|
|
1015
1070
|
AND json_extract(${actionInvocations.input}, '$.name') = ${claim.agentName}
|
|
1016
1071
|
)`
|
|
1017
1072
|
: sql `0`;
|
|
1018
|
-
|
|
1073
|
+
return db
|
|
1019
1074
|
.update(nodes)
|
|
1020
1075
|
.set({
|
|
1021
|
-
activeAgents: sql `${nodes.activeAgents} + 1`,
|
|
1076
|
+
activeAgents: sql `CASE WHEN ${or(eq(nodes.maxAgents, 0), and(ownsSpawnReservation, sql `${nodes.activeAgents} < ${nodes.maxAgents}`), and(sql `NOT (${ownsSpawnReservation})`, sql `${nodes.activeAgents} + ${nodes.reservedAgents} < ${nodes.maxAgents}`))} THEN ${nodes.activeAgents} + 1 ELSE NULL END`,
|
|
1022
1077
|
})
|
|
1023
|
-
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, node.id)
|
|
1078
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, node.id)))
|
|
1024
1079
|
.returning({ id: nodes.id });
|
|
1025
|
-
|
|
1026
|
-
|
|
1080
|
+
}
|
|
1081
|
+
/** Translate the admission sentinel (and the node-row-gone empty update) into the capacity refusal callers expect. */
|
|
1082
|
+
async function reserveNodeAgentSlot(db, workspaceId, node, claim) {
|
|
1083
|
+
try {
|
|
1084
|
+
const [reserved] = await reserveNodeAgentSlotStatement(db, workspaceId, node, claim);
|
|
1085
|
+
if (!reserved) {
|
|
1086
|
+
throw codedError(`Node "${node.name}" is at capacity`, 'node_capacity_exceeded', 409);
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
catch (err) {
|
|
1090
|
+
if (databaseConstraintKind(err) === 'node_capacity') {
|
|
1091
|
+
throw codedError(`Node "${node.name}" is at capacity`, 'node_capacity_exceeded', 409);
|
|
1092
|
+
}
|
|
1093
|
+
throw err;
|
|
1027
1094
|
}
|
|
1028
1095
|
}
|
|
1029
|
-
|
|
1096
|
+
/**
|
|
1097
|
+
* The migration-tombstone read for a pending registration. Built, not
|
|
1098
|
+
* executed: registration batches it with the rest of its read phase so the
|
|
1099
|
+
* whole phase costs one D1 request.
|
|
1100
|
+
*/
|
|
1101
|
+
function canceledSpawnTombstoneQuery(db, workspaceId, registration) {
|
|
1030
1102
|
// Provider ownership added in migration 0030 is intentionally NULL for
|
|
1031
1103
|
// legacy rows whose multi-provider owner could not be inferred. Treat that
|
|
1032
1104
|
// unknown owner conservatively as matching any provider on the exact node
|
|
@@ -1036,17 +1108,10 @@ async function rejectMigrationCanceledSpawnRegistration(db, workspaceId, registr
|
|
|
1036
1108
|
const correlation = registration.invocationId
|
|
1037
1109
|
? and(eq(actionInvocations.id, registration.invocationId), eq(actionInvocations.dispatchedNodeId, registration.nodeId), providerCorrelation, sql `json_extract(${actionInvocations.input}, '$.name') = ${registration.agentName}`)
|
|
1038
1110
|
: and(eq(actionInvocations.dispatchedNodeId, registration.nodeId), providerCorrelation, sql `json_extract(${actionInvocations.input}, '$.name') = ${registration.agentName}`);
|
|
1039
|
-
|
|
1111
|
+
return db
|
|
1040
1112
|
.select({ id: actionInvocations.id })
|
|
1041
1113
|
.from(actionInvocations)
|
|
1042
1114
|
.where(and(eq(actionInvocations.workspaceId, workspaceId), correlation, eq(actionInvocations.invocationOrigin, 'legacy_unknown'), eq(actionInvocations.status, 'failed'), eq(actionInvocations.error, 'invocation_origin_unavailable'), or(eq(actionInvocations.actionName, 'spawn'), sql `${actionInvocations.actionName} LIKE 'spawn:%'`)));
|
|
1043
|
-
if (canceled) {
|
|
1044
|
-
// An ID-less legacy registration cannot distinguish a stale canceled
|
|
1045
|
-
// worker from a newer reservation for the same tuple. Fail closed until
|
|
1046
|
-
// the replacement echoes its invocation id; otherwise the stale process
|
|
1047
|
-
// can consume the replacement generation's capacity and identity.
|
|
1048
|
-
throw codedError(`Spawn invocation "${canceled.id}" was canceled during migration`, 'spawn_invocation_canceled', 409);
|
|
1049
|
-
}
|
|
1050
1115
|
}
|
|
1051
1116
|
/**
|
|
1052
1117
|
* The statement (at most one) that gives back one agent slot on each of
|
|
@@ -1428,102 +1493,227 @@ export async function listNodeAgents(db, workspaceId, nodeName) {
|
|
|
1428
1493
|
}
|
|
1429
1494
|
export async function registerAgentViaNode(db, workspaceId, nodeId, providerName, message, options = {}) {
|
|
1430
1495
|
assertRegistrableAgentName(message.name);
|
|
1496
|
+
const wantsAutoJoin = message.auto_join_general !== false;
|
|
1431
1497
|
const registered = await runAtomic(db, async (tx) => {
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1498
|
+
// Every awaited statement is a separate D1 round-trip — and this path runs
|
|
1499
|
+
// under the node-control liveness deadline whose misses surface as
|
|
1500
|
+
// `d1_pressure`. The reads have no interdependencies and the writes split
|
|
1501
|
+
// into an admission pair (slot reserve + identity insert) and a landing
|
|
1502
|
+
// set (membership + bindings), so on a batch-capable handle each group
|
|
1503
|
+
// ships as one atomic `batch` call. Inside an interactive transaction
|
|
1504
|
+
// (Node adapters) the tx handle exposes no batch, and the same groups run
|
|
1505
|
+
// sequentially — with results still pushed into the sinks as they land,
|
|
1506
|
+
// so the catch below compensates exactly the writes that committed, no
|
|
1507
|
+
// more and no less.
|
|
1508
|
+
const hasBatch = typeof tx.batch === 'function';
|
|
1509
|
+
const runStatements = async (statements, sink) => {
|
|
1510
|
+
if (hasBatch) {
|
|
1511
|
+
const results = await runAtomicWrites(tx, statements);
|
|
1512
|
+
// A batch is all-or-nothing: populate the sink only once it has
|
|
1513
|
+
// committed every statement.
|
|
1514
|
+
sink.push(...results);
|
|
1515
|
+
return results;
|
|
1516
|
+
}
|
|
1517
|
+
for (const statement of statements)
|
|
1518
|
+
sink.push(await statement);
|
|
1519
|
+
return sink;
|
|
1520
|
+
};
|
|
1521
|
+
const [canceledRows, nodeRows, providerRows, existingRows, channelRows,] = await runStatements([
|
|
1522
|
+
canceledSpawnTombstoneQuery(tx, workspaceId, {
|
|
1523
|
+
invocationId: message.invocation_id,
|
|
1524
|
+
nodeId,
|
|
1525
|
+
providerName,
|
|
1526
|
+
agentName: message.name,
|
|
1527
|
+
}),
|
|
1528
|
+
tx
|
|
1529
|
+
.select()
|
|
1530
|
+
.from(nodes)
|
|
1531
|
+
.where(and(eq(nodes.workspaceId, workspaceId), eq(nodes.id, nodeId))),
|
|
1532
|
+
tx
|
|
1533
|
+
.select({ capabilities: nodeProviders.capabilities })
|
|
1534
|
+
.from(nodeProviders)
|
|
1535
|
+
.where(and(eq(nodeProviders.workspaceId, workspaceId), eq(nodeProviders.nodeId, nodeId), eq(nodeProviders.name, providerName))),
|
|
1536
|
+
tx
|
|
1537
|
+
.select({ id: agents.id })
|
|
1538
|
+
.from(agents)
|
|
1539
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, message.name))),
|
|
1540
|
+
tx
|
|
1541
|
+
.select()
|
|
1542
|
+
.from(channels)
|
|
1543
|
+
.where(and(eq(channels.workspaceId, workspaceId), eq(channels.name, 'general'))),
|
|
1544
|
+
], []);
|
|
1545
|
+
const [canceled] = canceledRows;
|
|
1546
|
+
if (canceled) {
|
|
1547
|
+
// An ID-less legacy registration cannot distinguish a stale canceled
|
|
1548
|
+
// worker from a newer reservation for the same tuple. Fail closed until
|
|
1549
|
+
// the replacement echoes its invocation id; otherwise the stale process
|
|
1550
|
+
// can consume the replacement generation's capacity and identity.
|
|
1551
|
+
throw codedError(`Spawn invocation "${canceled.id}" was canceled during migration`, 'spawn_invocation_canceled', 409);
|
|
1552
|
+
}
|
|
1553
|
+
const [node] = nodeRows;
|
|
1442
1554
|
if (!node)
|
|
1443
1555
|
throw codedError(`Node "${nodeId}" not found`, 'node_not_found', 404);
|
|
1556
|
+
const [provider] = providerRows;
|
|
1444
1557
|
const cursorHandshake = (options.deliveryCursorSupported ?? true)
|
|
1445
|
-
&&
|
|
1558
|
+
&& (provider?.capabilities?.some((capability) => capability.name === FLEET_DELIVERY_CURSOR_CAPABILITY) ?? false);
|
|
1446
1559
|
// Preserve the established identity conflict even when this node is full.
|
|
1447
1560
|
// A concurrent create after this read is still caught by the unique insert
|
|
1448
1561
|
// below, whose reserved slot is then compensated in the catch path.
|
|
1449
|
-
const [existing] =
|
|
1450
|
-
.select({ id: agents.id })
|
|
1451
|
-
.from(agents)
|
|
1452
|
-
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.name, message.name)));
|
|
1562
|
+
const [existing] = existingRows;
|
|
1453
1563
|
if (existing) {
|
|
1454
1564
|
throw codedError(`Agent "${message.name}" already exists; use agent.recover with proof of the immutable id`, 'agent_already_exists', 409);
|
|
1455
1565
|
}
|
|
1566
|
+
const [generalChannel] = channelRows;
|
|
1456
1567
|
const token = `at_live_${randomHex(16)}`;
|
|
1457
1568
|
const tokenHash = await sha256Hex(token);
|
|
1458
1569
|
const now = new Date().toISOString();
|
|
1459
1570
|
const agentId = generateId();
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
//
|
|
1463
|
-
//
|
|
1464
|
-
//
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1571
|
+
const admissionResults = [];
|
|
1572
|
+
// Landing writes are all idempotent (conflict-ignore inserts, upserted
|
|
1573
|
+
// binding, guarded retirement), so the ambiguous-commit recovery below can
|
|
1574
|
+
// safely replay them when the admission group committed but its response
|
|
1575
|
+
// was lost.
|
|
1576
|
+
const landingWrites = () => [
|
|
1577
|
+
...(wantsAutoJoin && generalChannel
|
|
1578
|
+
? [tx
|
|
1579
|
+
.insert(channelMembers)
|
|
1580
|
+
.values({
|
|
1581
|
+
channelId: generalChannel.id,
|
|
1582
|
+
agentId,
|
|
1583
|
+
role: 'member',
|
|
1584
|
+
})
|
|
1585
|
+
.onConflictDoNothing()]
|
|
1586
|
+
: []),
|
|
1587
|
+
...agentNodeBindingWrites(tx, workspaceId, { id: agentId, locationNodeId: nodeId }, nodeId, {
|
|
1588
|
+
sessionRef: message.session_ref ?? null,
|
|
1589
|
+
deactivateExisting: true,
|
|
1590
|
+
}),
|
|
1591
|
+
];
|
|
1592
|
+
const registeredReply = (agent) => ({
|
|
1593
|
+
agent_id: agent.id,
|
|
1594
|
+
name: agent.name,
|
|
1595
|
+
token,
|
|
1596
|
+
...(cursorHandshake ? { delivery_ack_seq: agent.deliveryAckSeq } : {}),
|
|
1469
1597
|
});
|
|
1470
|
-
|
|
1598
|
+
// D1 has no interactive transaction, so the slot and the identity commit
|
|
1599
|
+
// as one batch; any later failure compensates whichever of them landed
|
|
1600
|
+
// rather than leaving a ghost agent or a leaked slot.
|
|
1471
1601
|
try {
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1602
|
+
await runStatements([
|
|
1603
|
+
reserveNodeAgentSlotStatement(tx, workspaceId, node, {
|
|
1604
|
+
invocationId: message.invocation_id,
|
|
1605
|
+
providerName,
|
|
1606
|
+
agentName: message.name,
|
|
1607
|
+
}),
|
|
1608
|
+
tx
|
|
1609
|
+
.insert(agents)
|
|
1610
|
+
.values({
|
|
1611
|
+
id: agentId,
|
|
1612
|
+
workspaceId,
|
|
1613
|
+
name: message.name,
|
|
1614
|
+
handle: `@${message.name}`,
|
|
1615
|
+
type: 'agent',
|
|
1616
|
+
tokenHash,
|
|
1617
|
+
status: 'active',
|
|
1618
|
+
persona: null,
|
|
1619
|
+
metadata: { fleet: { node_id: nodeId, invocation_id: message.invocation_id ?? null, registered_at: now } },
|
|
1620
|
+
locationType: 'via_node',
|
|
1621
|
+
locationNodeId: nodeId,
|
|
1622
|
+
providerName,
|
|
1623
|
+
originNodeId: nodeId,
|
|
1624
|
+
resumable: message.resumable ?? false,
|
|
1625
|
+
sessionRef: message.session_ref ?? null,
|
|
1626
|
+
})
|
|
1627
|
+
.onConflictDoNothing({ target: [agents.workspaceId, agents.name] })
|
|
1628
|
+
.returning(),
|
|
1629
|
+
], admissionResults);
|
|
1630
|
+
const [reserved] = (admissionResults[0] ?? []);
|
|
1631
|
+
const [agent] = (admissionResults[1] ?? []);
|
|
1632
|
+
if (!reserved) {
|
|
1633
|
+
throw codedError(`Node "${node.name}" is at capacity`, 'node_capacity_exceeded', 409);
|
|
1634
|
+
}
|
|
1635
|
+
if (!agent) {
|
|
1494
1636
|
throw codedError(`Agent "${message.name}" already exists; use agent.recover with proof of the immutable id`, 'agent_already_exists', 409);
|
|
1495
1637
|
}
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
await autoJoinGeneral(tx, workspaceId, result.id);
|
|
1499
|
-
await upsertAgentNodeBinding(tx, workspaceId, result, nodeId, {
|
|
1500
|
-
sessionRef: message.session_ref ?? null,
|
|
1501
|
-
deactivateExisting: true,
|
|
1502
|
-
});
|
|
1503
|
-
return {
|
|
1504
|
-
agent_id: result.id,
|
|
1505
|
-
name: result.name,
|
|
1506
|
-
token,
|
|
1507
|
-
...(cursorHandshake ? { delivery_ack_seq: result.deliveryAckSeq } : {}),
|
|
1508
|
-
};
|
|
1638
|
+
await runStatements(landingWrites(), []);
|
|
1639
|
+
return registeredReply(agent);
|
|
1509
1640
|
}
|
|
1510
1641
|
catch (err) {
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
}
|
|
1642
|
+
// The reserve statement writes NULL into active_agents on a refused
|
|
1643
|
+
// admission, aborting the group before the identity insert can run —
|
|
1644
|
+
// nothing landed, so there is nothing to compensate.
|
|
1645
|
+
if (databaseConstraintKind(err) === 'node_capacity') {
|
|
1646
|
+
throw codedError(`Node "${node.name}" is at capacity`, 'node_capacity_exceeded', 409);
|
|
1517
1647
|
}
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1648
|
+
// A batch is all-or-nothing, but its response is not: D1 can commit the
|
|
1649
|
+
// admission group and then drop the reply, leaving the sink empty while
|
|
1650
|
+
// the slot and identity are durable. A row bearing this freshly
|
|
1651
|
+
// generated id can only have come from our own insert, so probe for it
|
|
1652
|
+
// (with bounded transient retries) before deciding nothing landed —
|
|
1653
|
+
// otherwise the committed agent strands the name behind a permanent
|
|
1654
|
+
// agent_already_exists while leaking the slot it holds.
|
|
1655
|
+
if (admissionResults.length === 0) {
|
|
1656
|
+
let landed;
|
|
1657
|
+
try {
|
|
1658
|
+
[landed] = await retryD1Write(() => db
|
|
1659
|
+
.select()
|
|
1660
|
+
.from(agents)
|
|
1661
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agentId)))
|
|
1662
|
+
.limit(1), 3);
|
|
1663
|
+
}
|
|
1664
|
+
catch {
|
|
1665
|
+
// Probe inconclusive — fall through and surface the original error.
|
|
1521
1666
|
}
|
|
1667
|
+
if (landed) {
|
|
1668
|
+
try {
|
|
1669
|
+
await runStatements(landingWrites(), []);
|
|
1670
|
+
return registeredReply(landed);
|
|
1671
|
+
}
|
|
1672
|
+
catch (landingErr) {
|
|
1673
|
+
// The replayed landing can hit the same lost-reply ambiguity —
|
|
1674
|
+
// a committed landing leaves this agent's binding active on the
|
|
1675
|
+
// node, so probe it before deciding the outcome.
|
|
1676
|
+
let bound = false;
|
|
1677
|
+
try {
|
|
1678
|
+
const [row] = await retryD1Write(() => db
|
|
1679
|
+
.select({ id: agentNodeBindings.id })
|
|
1680
|
+
.from(agentNodeBindings)
|
|
1681
|
+
.where(and(eq(agentNodeBindings.workspaceId, workspaceId), eq(agentNodeBindings.agentId, agentId), eq(agentNodeBindings.nodeId, nodeId), eq(agentNodeBindings.status, 'active')))
|
|
1682
|
+
.limit(1), 3);
|
|
1683
|
+
bound = !!row;
|
|
1684
|
+
}
|
|
1685
|
+
catch {
|
|
1686
|
+
// Probe inconclusive — treat as not landed and roll back.
|
|
1687
|
+
}
|
|
1688
|
+
if (bound)
|
|
1689
|
+
return registeredReply(landed);
|
|
1690
|
+
// Landing never committed: roll the admitted identity back so a
|
|
1691
|
+
// retry registers cleanly instead of meeting agent_already_exists.
|
|
1692
|
+
await runStatements([
|
|
1693
|
+
tx
|
|
1694
|
+
.delete(agents)
|
|
1695
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agentId))),
|
|
1696
|
+
...releaseNodeAgentSlotWrites(tx, workspaceId, [nodeId]),
|
|
1697
|
+
], []);
|
|
1698
|
+
throw landingErr;
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
}
|
|
1702
|
+
const compensations = [];
|
|
1703
|
+
if (admissionResults[1]?.[0]) {
|
|
1704
|
+
compensations.push(tx
|
|
1705
|
+
.delete(agents)
|
|
1706
|
+
.where(and(eq(agents.workspaceId, workspaceId), eq(agents.id, agentId))));
|
|
1707
|
+
}
|
|
1708
|
+
if (admissionResults[0]?.[0]) {
|
|
1709
|
+
compensations.push(...releaseNodeAgentSlotWrites(tx, workspaceId, [nodeId]));
|
|
1522
1710
|
}
|
|
1711
|
+
if (compensations.length > 0)
|
|
1712
|
+
await runStatements(compensations, []);
|
|
1523
1713
|
throw err;
|
|
1524
1714
|
}
|
|
1525
1715
|
});
|
|
1526
|
-
if (
|
|
1716
|
+
if (wantsAutoJoin)
|
|
1527
1717
|
await invalidateChannelCache(workspaceId, 'general');
|
|
1528
1718
|
return registered;
|
|
1529
1719
|
}
|