@electric-ax/agents-server 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +404 -68
- package/dist/index.cjs +421 -67
- package/dist/index.d.cts +97 -11
- package/dist/index.d.ts +97 -11
- package/dist/index.js +414 -69
- package/drizzle/0009_entity_signal_statuses.sql +3 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +6 -6
- package/src/db/schema.ts +1 -1
- package/src/electric-agents-types.ts +76 -1
- package/src/entity-manager.ts +256 -33
- package/src/entity-registry.ts +57 -20
- package/src/entrypoint-lib.ts +5 -0
- package/src/index.ts +33 -0
- package/src/routing/context.ts +6 -0
- package/src/routing/dispatch-policy.ts +6 -0
- package/src/routing/durable-streams-router.ts +62 -13
- package/src/routing/entities-router.ts +57 -1
- package/src/routing/internal-router.ts +147 -23
- package/src/routing/runners-router.ts +1 -1
- package/src/runtime.ts +5 -1
- package/src/server.ts +18 -0
- package/src/stream-client.ts +10 -4
- package/src/webhook-signing.ts +173 -0
package/dist/entrypoint.js
CHANGED
|
@@ -4,7 +4,7 @@ import { DurableStreamTestServer } from "@durable-streams/server";
|
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
import { createServerAdapter } from "@whatwg-node/server";
|
|
6
6
|
import { Agent } from "undici";
|
|
7
|
-
import { appendPathToUrl, assertTags, buildTagsIndex, createEntityRegistry, createRuntimeHandler, entityStateSchema, getCronStreamPath, getCronStreamPathFromSpec, getEntitiesStreamPath, getNextCronFireAt, getSharedStateStreamPath, manifestChildKey, manifestSharedStateKey, manifestSourceKey, normalizeTags, parseCronStreamPath, resolveCronScheduleSpec, sourceRefForTags } from "@electric-ax/agents-runtime";
|
|
7
|
+
import { appendPathToUrl, assertTags, buildTagsIndex, createEntityRegistry, createRuntimeHandler, entityStateSchema, getCronStreamPath, getCronStreamPathFromSpec, getEntitiesStreamPath, getNextCronFireAt, getSharedStateStreamPath, manifestChildKey, manifestSharedStateKey, manifestSourceKey, normalizeTags, parseCronStreamPath, resolveCronScheduleSpec, sourceRefForTags, verifyWebhookSignature } from "@electric-ax/agents-runtime";
|
|
8
8
|
import fs, { existsSync } from "node:fs";
|
|
9
9
|
import path, { dirname, resolve } from "node:path";
|
|
10
10
|
import { drizzle } from "drizzle-orm/postgres-js";
|
|
@@ -19,7 +19,7 @@ import { SpanKind, SpanStatusCode, context, propagation, trace } from "@opentele
|
|
|
19
19
|
import { Type } from "@sinclair/typebox";
|
|
20
20
|
import pino from "pino";
|
|
21
21
|
import Ajv from "ajv";
|
|
22
|
-
import { createHash, randomUUID } from "node:crypto";
|
|
22
|
+
import { createHash, createPrivateKey, createPublicKey, generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
|
23
23
|
import fastq from "fastq";
|
|
24
24
|
import { ShapeStream, isChangeMessage, isControlMessage } from "@electric-sql/client";
|
|
25
25
|
|
|
@@ -90,7 +90,7 @@ const entities = pgTable(`entities`, {
|
|
|
90
90
|
index(`idx_entities_parent`).on(table.tenantId, table.parent),
|
|
91
91
|
index(`idx_entities_created_by`).on(table.tenantId, table.createdBy),
|
|
92
92
|
index(`entities_tags_index_gin`).using(`gin`, table.tagsIndex),
|
|
93
|
-
check(`chk_entities_status`, sql`${table.status} IN ('spawning', 'running', 'idle', 'stopped')`)
|
|
93
|
+
check(`chk_entities_status`, sql`${table.status} IN ('spawning', 'running', 'idle', 'paused', 'stopping', 'stopped', 'killed')`)
|
|
94
94
|
]);
|
|
95
95
|
const users = pgTable(`users`, {
|
|
96
96
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
@@ -367,12 +367,25 @@ function responseHeaders(response) {
|
|
|
367
367
|
|
|
368
368
|
//#endregion
|
|
369
369
|
//#region src/electric-agents-types.ts
|
|
370
|
+
const ENTITY_SIGNALS = [
|
|
371
|
+
`SIGINT`,
|
|
372
|
+
`SIGHUP`,
|
|
373
|
+
`SIGTERM`,
|
|
374
|
+
`SIGKILL`,
|
|
375
|
+
`SIGSTOP`,
|
|
376
|
+
`SIGCONT`,
|
|
377
|
+
`SIGUSR`
|
|
378
|
+
];
|
|
370
379
|
const VALID_ENTITY_STATUSES = new Set([
|
|
371
380
|
`spawning`,
|
|
372
381
|
`running`,
|
|
373
382
|
`idle`,
|
|
374
|
-
`
|
|
383
|
+
`paused`,
|
|
384
|
+
`stopping`,
|
|
385
|
+
`stopped`,
|
|
386
|
+
`killed`
|
|
375
387
|
]);
|
|
388
|
+
const VALID_ENTITY_SIGNALS = new Set(ENTITY_SIGNALS);
|
|
376
389
|
function assertEntityStatus(s) {
|
|
377
390
|
if (!VALID_ENTITY_STATUSES.has(s)) throw new Error(`Invalid entity status: "${s}"`);
|
|
378
391
|
return s;
|
|
@@ -393,6 +406,12 @@ function assertRunnerAdminStatus(s) {
|
|
|
393
406
|
if (!VALID_RUNNER_ADMIN_STATUSES.has(s)) throw new Error(`Invalid runner admin status: "${s}"`);
|
|
394
407
|
return s;
|
|
395
408
|
}
|
|
409
|
+
function isTerminalEntityStatus(status$1) {
|
|
410
|
+
return status$1 === `stopped` || status$1 === `killed`;
|
|
411
|
+
}
|
|
412
|
+
function rejectsNormalWrites(status$1) {
|
|
413
|
+
return status$1 === `stopping` || isTerminalEntityStatus(status$1);
|
|
414
|
+
}
|
|
396
415
|
/** Strip internal fields (write_token, subscription_id) from an entity. */
|
|
397
416
|
function toPublicEntity(entity) {
|
|
398
417
|
return {
|
|
@@ -414,6 +433,7 @@ const ErrCodeUnauthorized = `UNAUTHORIZED`;
|
|
|
414
433
|
const ErrCodeNotFound = `NOT_FOUND`;
|
|
415
434
|
const ErrCodeNotRunning = `NOT_RUNNING`;
|
|
416
435
|
const ErrCodeInvalidRequest = `INVALID_REQUEST`;
|
|
436
|
+
const ErrCodeInvalidSignal = `INVALID_SIGNAL`;
|
|
417
437
|
const ErrCodeUnknownEntityType = `UNKNOWN_ENTITY_TYPE`;
|
|
418
438
|
const ErrCodeSchemaValidationFailed = `SCHEMA_VALIDATION_FAILED`;
|
|
419
439
|
const ErrCodeUnknownMessageType = `UNKNOWN_MESSAGE_TYPE`;
|
|
@@ -1378,6 +1398,87 @@ function isLoopbackHostname(hostname) {
|
|
|
1378
1398
|
return hostname === `localhost` || hostname === `127.0.0.1` || hostname === `::1`;
|
|
1379
1399
|
}
|
|
1380
1400
|
|
|
1401
|
+
//#endregion
|
|
1402
|
+
//#region src/webhook-signing.ts
|
|
1403
|
+
const encoder = new TextEncoder();
|
|
1404
|
+
const defaultWebhookSigner = createEd25519WebhookSigner();
|
|
1405
|
+
function createEd25519WebhookSigner(options = {}) {
|
|
1406
|
+
const privateKey = options.privateKey ? importPrivateKey(options.privateKey) : generateKeyPairSync(`ed25519`).privateKey;
|
|
1407
|
+
if (privateKey.asymmetricKeyType !== `ed25519`) throw new Error(`Webhook signing key must be an Ed25519 private key`);
|
|
1408
|
+
const publicJwk = buildPublicJwk(privateKey, options.kid);
|
|
1409
|
+
return {
|
|
1410
|
+
sign: (body) => signWebhookBody(privateKey, publicJwk.kid, body),
|
|
1411
|
+
jwks: () => ({ keys: [{ ...publicJwk }] })
|
|
1412
|
+
};
|
|
1413
|
+
}
|
|
1414
|
+
function getDefaultWebhookSigner() {
|
|
1415
|
+
return defaultWebhookSigner;
|
|
1416
|
+
}
|
|
1417
|
+
async function webhookSigningMetadata(signer, streamRootUrl) {
|
|
1418
|
+
const jwks = await signer.jwks();
|
|
1419
|
+
const key = jwks.keys[0];
|
|
1420
|
+
if (!key) throw new Error(`Webhook signer did not provide any public keys`);
|
|
1421
|
+
return {
|
|
1422
|
+
alg: `ed25519`,
|
|
1423
|
+
kid: key.kid,
|
|
1424
|
+
jwks_url: appendPathToUrl(streamRootUrl, `/__ds/jwks.json`)
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
function signWebhookBody(privateKey, kid, body) {
|
|
1428
|
+
const timestamp$1 = Math.floor(Date.now() / 1e3);
|
|
1429
|
+
const payload = bytesWithTimestamp(timestamp$1, body);
|
|
1430
|
+
const signature = sign(null, payload, privateKey).toString(`base64url`);
|
|
1431
|
+
return `t=${timestamp$1},kid=${kid},ed25519=${signature}`;
|
|
1432
|
+
}
|
|
1433
|
+
function bytesWithTimestamp(timestamp$1, body) {
|
|
1434
|
+
const prefix = encoder.encode(`${timestamp$1}.`);
|
|
1435
|
+
const bodyBytes = typeof body === `string` ? encoder.encode(body) : body;
|
|
1436
|
+
return Buffer.concat([Buffer.from(prefix), Buffer.from(bodyBytes)]);
|
|
1437
|
+
}
|
|
1438
|
+
function importPrivateKey(input) {
|
|
1439
|
+
if (isKeyObject(input)) return input;
|
|
1440
|
+
if (typeof input === `string`) {
|
|
1441
|
+
const trimmed = input.trim();
|
|
1442
|
+
if (trimmed.startsWith(`{`)) return createPrivateKey({
|
|
1443
|
+
key: JSON.parse(trimmed),
|
|
1444
|
+
format: `jwk`
|
|
1445
|
+
});
|
|
1446
|
+
return createPrivateKey(trimmed.replace(/\\n/g, `\n`));
|
|
1447
|
+
}
|
|
1448
|
+
if (Buffer.isBuffer(input)) return createPrivateKey(input);
|
|
1449
|
+
return createPrivateKey({
|
|
1450
|
+
key: input,
|
|
1451
|
+
format: `jwk`
|
|
1452
|
+
});
|
|
1453
|
+
}
|
|
1454
|
+
function isKeyObject(input) {
|
|
1455
|
+
return typeof input === `object` && `type` in input && input.type === `private`;
|
|
1456
|
+
}
|
|
1457
|
+
function buildPublicJwk(privateKey, kid) {
|
|
1458
|
+
const exported = createPublicKey(privateKey).export({ format: `jwk` });
|
|
1459
|
+
if (exported.kty !== `OKP` || exported.crv !== `Ed25519` || !exported.x) throw new Error(`Failed to export Ed25519 webhook signing key`);
|
|
1460
|
+
return {
|
|
1461
|
+
kty: `OKP`,
|
|
1462
|
+
crv: `Ed25519`,
|
|
1463
|
+
x: exported.x,
|
|
1464
|
+
kid: kid ?? deriveKeyId({
|
|
1465
|
+
kty: exported.kty,
|
|
1466
|
+
crv: exported.crv,
|
|
1467
|
+
x: exported.x
|
|
1468
|
+
}),
|
|
1469
|
+
use: `sig`,
|
|
1470
|
+
alg: `EdDSA`
|
|
1471
|
+
};
|
|
1472
|
+
}
|
|
1473
|
+
function deriveKeyId(jwk) {
|
|
1474
|
+
const thumbprintInput = JSON.stringify({
|
|
1475
|
+
crv: jwk.crv,
|
|
1476
|
+
kty: jwk.kty,
|
|
1477
|
+
x: jwk.x
|
|
1478
|
+
});
|
|
1479
|
+
return `ds_${createHash(`sha256`).update(thumbprintInput).digest(`base64url`)}`;
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1381
1482
|
//#endregion
|
|
1382
1483
|
//#region src/routing/durable-streams-router.ts
|
|
1383
1484
|
const subscriptionProxyBodySchema = Type.Object({ webhook: Type.Optional(Type.Object({ url: Type.String() }, { additionalProperties: true })) }, { additionalProperties: true });
|
|
@@ -1394,6 +1495,7 @@ durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId`, deleteSubscri
|
|
|
1394
1495
|
durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/streams`, postSubscriptionStreams);
|
|
1395
1496
|
durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId/streams/:streamPath+`, deleteSubscriptionStream);
|
|
1396
1497
|
for (const action of subscriptionControlActions) durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/${action}`, subscriptionAction(action));
|
|
1498
|
+
durableStreamsRouter.get(`/__ds/jwks.json`, webhookJwks);
|
|
1397
1499
|
durableStreamsRouter.all(`/__ds`, controlPassThrough);
|
|
1398
1500
|
durableStreamsRouter.all(`/__ds/*`, controlPassThrough);
|
|
1399
1501
|
durableStreamsRouter.post(`*`, streamAppend);
|
|
@@ -1402,12 +1504,16 @@ function bodyFromBytes$1(body) {
|
|
|
1402
1504
|
return body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength);
|
|
1403
1505
|
}
|
|
1404
1506
|
function responseFromUpstream$1(response, body) {
|
|
1405
|
-
|
|
1507
|
+
const responseBody = forbidsResponseBody$1(response.status) ? null : body !== void 0 ? bodyFromBytes$1(body) : response.body;
|
|
1508
|
+
return new Response(responseBody, {
|
|
1406
1509
|
status: response.status,
|
|
1407
1510
|
statusText: response.statusText,
|
|
1408
1511
|
headers: responseHeaders(response)
|
|
1409
1512
|
});
|
|
1410
1513
|
}
|
|
1514
|
+
function forbidsResponseBody$1(status$1) {
|
|
1515
|
+
return status$1 === 204 || status$1 === 205 || status$1 === 304;
|
|
1516
|
+
}
|
|
1411
1517
|
async function forwardToDurableStreams(ctx, request, body, route = `stream`, urlOverride, durableStreamsBearerMode = `overwrite`) {
|
|
1412
1518
|
const headers = new Headers(request.headers);
|
|
1413
1519
|
headers.delete(`host`);
|
|
@@ -1441,28 +1547,32 @@ function rewriteSubscriptionBodyForBackend(payload, service, routingAdapter) {
|
|
|
1441
1547
|
return next;
|
|
1442
1548
|
});
|
|
1443
1549
|
}
|
|
1444
|
-
function rewriteSubscriptionResponseForClient(bytes, response,
|
|
1550
|
+
async function rewriteSubscriptionResponseForClient(bytes, response, ctx, routingAdapter) {
|
|
1445
1551
|
if (!response.headers.get(`content-type`)?.includes(`application/json`)) return bytes;
|
|
1446
1552
|
const payload = decodeJson(bytes);
|
|
1447
1553
|
if (!payload) return bytes;
|
|
1448
|
-
if (typeof payload.pattern === `string`) payload.pattern = routingAdapter.toRuntimeStreamPath(service, payload.pattern);
|
|
1554
|
+
if (typeof payload.pattern === `string`) payload.pattern = routingAdapter.toRuntimeStreamPath(ctx.service, payload.pattern);
|
|
1449
1555
|
if (Array.isArray(payload.streams)) payload.streams = payload.streams.map((stream) => {
|
|
1450
|
-
if (typeof stream === `string`) return routingAdapter.toRuntimeStreamPath(service, stream);
|
|
1556
|
+
if (typeof stream === `string`) return routingAdapter.toRuntimeStreamPath(ctx.service, stream);
|
|
1451
1557
|
if (stream && typeof stream === `object` && typeof stream.path === `string`) return {
|
|
1452
1558
|
...stream,
|
|
1453
|
-
path: routingAdapter.toRuntimeStreamPath(service, stream.path)
|
|
1559
|
+
path: routingAdapter.toRuntimeStreamPath(ctx.service, stream.path)
|
|
1454
1560
|
};
|
|
1455
1561
|
return stream;
|
|
1456
1562
|
});
|
|
1457
|
-
if (typeof payload.wake_stream === `string`) payload.wake_stream = routingAdapter.toRuntimeStreamPath(service, payload.wake_stream);
|
|
1458
|
-
if (typeof payload.stream === `string`) payload.stream = routingAdapter.toRuntimeStreamPath(service, payload.stream);
|
|
1563
|
+
if (typeof payload.wake_stream === `string`) payload.wake_stream = routingAdapter.toRuntimeStreamPath(ctx.service, payload.wake_stream);
|
|
1564
|
+
if (typeof payload.stream === `string`) payload.stream = routingAdapter.toRuntimeStreamPath(ctx.service, payload.stream);
|
|
1459
1565
|
if (Array.isArray(payload.acks)) payload.acks = payload.acks.map((ack) => {
|
|
1460
1566
|
if (!ack || typeof ack !== `object`) return ack;
|
|
1461
1567
|
const next = { ...ack };
|
|
1462
|
-
if (typeof next.stream === `string`) next.stream = routingAdapter.toRuntimeStreamPath(service, next.stream);
|
|
1463
|
-
if (typeof next.path === `string`) next.path = routingAdapter.toRuntimeStreamPath(service, next.path);
|
|
1568
|
+
if (typeof next.stream === `string`) next.stream = routingAdapter.toRuntimeStreamPath(ctx.service, next.stream);
|
|
1569
|
+
if (typeof next.path === `string`) next.path = routingAdapter.toRuntimeStreamPath(ctx.service, next.path);
|
|
1464
1570
|
return next;
|
|
1465
1571
|
});
|
|
1572
|
+
if (payload.webhook && typeof payload.webhook === `object` && !Array.isArray(payload.webhook)) {
|
|
1573
|
+
const webhook = payload.webhook;
|
|
1574
|
+
webhook.signing = await webhookSigningMetadata(resolveWebhookSigner$1(ctx), ctx.publicUrl);
|
|
1575
|
+
}
|
|
1466
1576
|
return new TextEncoder().encode(JSON.stringify(payload));
|
|
1467
1577
|
}
|
|
1468
1578
|
function decodeJson(bytes) {
|
|
@@ -1481,6 +1591,9 @@ function routeParam$2(request, name) {
|
|
|
1481
1591
|
function subscriptionRoutingAdapter(ctx) {
|
|
1482
1592
|
return resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl);
|
|
1483
1593
|
}
|
|
1594
|
+
function resolveWebhookSigner$1(ctx) {
|
|
1595
|
+
return ctx.webhookSigner ?? getDefaultWebhookSigner();
|
|
1596
|
+
}
|
|
1484
1597
|
async function rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter) {
|
|
1485
1598
|
const body = await readRequestBody(request);
|
|
1486
1599
|
if (body.length === 0) return {
|
|
@@ -1509,7 +1622,7 @@ async function rewriteSubscriptionRequestBody(request, ctx, subscriptionId, rout
|
|
|
1509
1622
|
async function forwardSubscriptionRequest(request, ctx, routingAdapter, opts = {}) {
|
|
1510
1623
|
const upstream = await forwardToDurableStreams(ctx, request, opts.body, `control`, opts.requestUrl, opts.bearerMode ?? `overwrite`);
|
|
1511
1624
|
let responseBytes = upstream.body ? new Uint8Array(await upstream.arrayBuffer()) : new Uint8Array();
|
|
1512
|
-
responseBytes = rewriteSubscriptionResponseForClient(responseBytes, upstream, ctx
|
|
1625
|
+
responseBytes = await rewriteSubscriptionResponseForClient(responseBytes, upstream, ctx, routingAdapter);
|
|
1513
1626
|
return {
|
|
1514
1627
|
upstream,
|
|
1515
1628
|
response: responseFromUpstream$1(upstream, responseBytes)
|
|
@@ -1582,6 +1695,15 @@ async function controlPassThrough(request, ctx) {
|
|
|
1582
1695
|
const upstream = await forwardToDurableStreams(ctx, request, void 0, `control`);
|
|
1583
1696
|
return responseFromUpstream$1(upstream);
|
|
1584
1697
|
}
|
|
1698
|
+
async function webhookJwks(_request, ctx) {
|
|
1699
|
+
return new Response(JSON.stringify(await resolveWebhookSigner$1(ctx).jwks()), {
|
|
1700
|
+
status: 200,
|
|
1701
|
+
headers: {
|
|
1702
|
+
"content-type": `application/jwk-set+json`,
|
|
1703
|
+
"cache-control": `public, max-age=300`
|
|
1704
|
+
}
|
|
1705
|
+
});
|
|
1706
|
+
}
|
|
1585
1707
|
async function streamAppend(request, ctx) {
|
|
1586
1708
|
return await electricAgentsStreamAppendRouter.fetch(createStreamAppendRouteRequest(request), ctx.runtime, (req, body) => forwardFetchRequest({
|
|
1587
1709
|
request: {
|
|
@@ -1966,7 +2088,7 @@ var PostgresRegistry = class {
|
|
|
1966
2088
|
const heartbeatAt = input.heartbeatAt ?? new Date();
|
|
1967
2089
|
await this.db.update(consumerClaims).set({
|
|
1968
2090
|
lastHeartbeatAt: heartbeatAt,
|
|
1969
|
-
leaseExpiresAt: input.leaseExpiresAt
|
|
2091
|
+
...input.leaseExpiresAt !== void 0 ? { leaseExpiresAt: input.leaseExpiresAt } : {},
|
|
1970
2092
|
updatedAt: heartbeatAt
|
|
1971
2093
|
}).where(and(eq(consumerClaims.tenantId, this.tenantId), eq(consumerClaims.consumerId, input.consumerId), eq(consumerClaims.epoch, input.epoch)));
|
|
1972
2094
|
}
|
|
@@ -1979,17 +2101,24 @@ var PostgresRegistry = class {
|
|
|
1979
2101
|
updatedAt: releasedAt
|
|
1980
2102
|
}).where(and(eq(consumerClaims.tenantId, this.tenantId), eq(consumerClaims.consumerId, input.consumerId), eq(consumerClaims.epoch, input.epoch))).returning();
|
|
1981
2103
|
const claim = rows[0] ? this.rowToConsumerClaim(rows[0]) : null;
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
2104
|
+
let entityCleared = false;
|
|
2105
|
+
if (claim) {
|
|
2106
|
+
const cleared = await this.db.update(entityDispatchState).set({
|
|
2107
|
+
activeConsumerId: null,
|
|
2108
|
+
activeRunnerId: null,
|
|
2109
|
+
activeEpoch: null,
|
|
2110
|
+
activeClaimedAt: null,
|
|
2111
|
+
activeLeaseExpiresAt: null,
|
|
2112
|
+
lastReleasedAt: releasedAt,
|
|
2113
|
+
lastCompletedAt: releasedAt,
|
|
2114
|
+
updatedAt: releasedAt
|
|
2115
|
+
}).where(and(eq(entityDispatchState.tenantId, this.tenantId), eq(entityDispatchState.entityUrl, claim.entity_url), eq(entityDispatchState.activeConsumerId, input.consumerId), eq(entityDispatchState.activeEpoch, input.epoch))).returning({ entityUrl: entityDispatchState.entityUrl });
|
|
2116
|
+
entityCleared = cleared.length > 0;
|
|
2117
|
+
}
|
|
2118
|
+
return {
|
|
2119
|
+
claim,
|
|
2120
|
+
entityCleared
|
|
2121
|
+
};
|
|
1993
2122
|
}
|
|
1994
2123
|
async getActiveClaimsForRunner(runnerId) {
|
|
1995
2124
|
const rows = await this.db.select().from(consumerClaims).where(and(eq(consumerClaims.tenantId, this.tenantId), eq(consumerClaims.runnerId, runnerId), eq(consumerClaims.status, `active`)));
|
|
@@ -2165,7 +2294,7 @@ var PostgresRegistry = class {
|
|
|
2165
2294
|
};
|
|
2166
2295
|
}
|
|
2167
2296
|
async updateStatus(entityUrl, status$1) {
|
|
2168
|
-
const whereClause = status$1
|
|
2297
|
+
const whereClause = isTerminalEntityStatus(status$1) ? this.entityWhere(entityUrl) : and(this.entityWhere(entityUrl), ne(entities.status, `stopped`), ne(entities.status, `killed`));
|
|
2169
2298
|
await this.db.update(entities).set({
|
|
2170
2299
|
status: status$1,
|
|
2171
2300
|
updatedAt: Date.now()
|
|
@@ -2173,13 +2302,17 @@ var PostgresRegistry = class {
|
|
|
2173
2302
|
}
|
|
2174
2303
|
async updateStatusWithTxid(entityUrl, status$1) {
|
|
2175
2304
|
return await this.db.transaction(async (tx) => {
|
|
2176
|
-
const
|
|
2177
|
-
await tx.update(entities).set({
|
|
2305
|
+
const rows = await tx.update(entities).set({
|
|
2178
2306
|
status: status$1,
|
|
2179
2307
|
updatedAt: Date.now()
|
|
2180
|
-
}).where(
|
|
2181
|
-
|
|
2182
|
-
|
|
2308
|
+
}).where(and(this.entityWhere(entityUrl), ne(entities.status, `stopped`), ne(entities.status, `killed`))).returning({ txid: sql`pg_current_xact_id()::xid::text` });
|
|
2309
|
+
return rows[0] ? parseInt(rows[0].txid) : null;
|
|
2310
|
+
});
|
|
2311
|
+
}
|
|
2312
|
+
async touchEntityWithTxid(entityUrl) {
|
|
2313
|
+
return await this.db.transaction(async (tx) => {
|
|
2314
|
+
const rows = await tx.update(entities).set({ updatedAt: Date.now() }).where(and(eq(entities.url, entityUrl), ne(entities.status, `stopped`), ne(entities.status, `killed`))).returning({ txid: sql`pg_current_xact_id()::xid::text` });
|
|
2315
|
+
return rows[0] ? parseInt(rows[0].txid) : null;
|
|
2183
2316
|
});
|
|
2184
2317
|
}
|
|
2185
2318
|
async setEntityTag(url, key, value) {
|
|
@@ -2596,6 +2729,7 @@ function createInitialQueuePosition(date) {
|
|
|
2596
2729
|
}
|
|
2597
2730
|
const DEFAULT_FORK_WAIT_TIMEOUT_MS = 12e4;
|
|
2598
2731
|
const DEFAULT_FORK_WAIT_POLL_MS = 250;
|
|
2732
|
+
const SERVER_SIGNAL_SENDER = `/_electric/server`;
|
|
2599
2733
|
function sleep(ms) {
|
|
2600
2734
|
return new Promise((resolve$1) => setTimeout(resolve$1, ms));
|
|
2601
2735
|
}
|
|
@@ -3047,16 +3181,16 @@ var EntityManager = class {
|
|
|
3047
3181
|
if (!root) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3048
3182
|
if (root.parent) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Only top-level entities can be forked`, 400);
|
|
3049
3183
|
const subtree = await this.listEntitySubtree(root);
|
|
3050
|
-
const stopped = subtree.find((entity) => entity.status
|
|
3051
|
-
if (stopped) throw new ElectricAgentsError(ErrCodeNotRunning, `Cannot fork
|
|
3052
|
-
let active = subtree.filter((entity) => entity.status !== `idle`);
|
|
3184
|
+
const stopped = subtree.find((entity) => isTerminalEntityStatus(entity.status));
|
|
3185
|
+
if (stopped) throw new ElectricAgentsError(ErrCodeNotRunning, `Cannot fork terminal entity "${stopped.url}"`, 409);
|
|
3186
|
+
let active = subtree.filter((entity) => entity.status !== `idle` && entity.status !== `paused`);
|
|
3053
3187
|
if (active.length === 0) {
|
|
3054
3188
|
this.addForkLocks(this.forkWorkLockedEntities, subtree.map((entity) => entity.url), workLocks);
|
|
3055
3189
|
const lockedRoot = await this.registry.getEntity(rootUrl);
|
|
3056
3190
|
if (!lockedRoot) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3057
3191
|
const lockedSubtree = await this.listEntitySubtree(lockedRoot);
|
|
3058
3192
|
this.addForkLocks(this.forkWorkLockedEntities, lockedSubtree.map((entity) => entity.url), workLocks);
|
|
3059
|
-
const lockedActive = lockedSubtree.filter((entity) => entity.status !== `idle`);
|
|
3193
|
+
const lockedActive = lockedSubtree.filter((entity) => entity.status !== `idle` && entity.status !== `paused`);
|
|
3060
3194
|
if (lockedActive.length === 0) return lockedSubtree;
|
|
3061
3195
|
this.releaseForkLocks(this.forkWorkLockedEntities, workLocks);
|
|
3062
3196
|
active = lockedActive;
|
|
@@ -3512,6 +3646,11 @@ var EntityManager = class {
|
|
|
3512
3646
|
if (req.position) value.position = req.position;
|
|
3513
3647
|
else if (value.mode === `queued` || value.mode === `paused`) value.position = createInitialQueuePosition(new Date(now));
|
|
3514
3648
|
if (value.status === `processed`) value.processed_at = now;
|
|
3649
|
+
const wakePausedEntity = entity.status === `paused` && req.mode !== `paused`;
|
|
3650
|
+
if (wakePausedEntity) {
|
|
3651
|
+
await this.registry.updateStatus(entityUrl, `idle`);
|
|
3652
|
+
await this.entityBridgeManager?.onEntityChanged(entityUrl);
|
|
3653
|
+
}
|
|
3515
3654
|
const envelope = entityStateSchema.inbox.insert({
|
|
3516
3655
|
key,
|
|
3517
3656
|
value
|
|
@@ -3539,7 +3678,7 @@ var EntityManager = class {
|
|
|
3539
3678
|
async updateInboxMessage(entityUrl, key, req) {
|
|
3540
3679
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3541
3680
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3542
|
-
if (entity.status
|
|
3681
|
+
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
3543
3682
|
const now = new Date().toISOString();
|
|
3544
3683
|
const value = {};
|
|
3545
3684
|
if (`payload` in req) value.payload = req.payload;
|
|
@@ -3560,7 +3699,7 @@ var EntityManager = class {
|
|
|
3560
3699
|
async deleteInboxMessage(entityUrl, key) {
|
|
3561
3700
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3562
3701
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3563
|
-
if (entity.status
|
|
3702
|
+
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
3564
3703
|
const envelope = entityStateSchema.inbox.delete({ key });
|
|
3565
3704
|
await this.streamClient.append(entity.streams.main, this.encodeChangeEvent(envelope));
|
|
3566
3705
|
}
|
|
@@ -3568,7 +3707,7 @@ var EntityManager = class {
|
|
|
3568
3707
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3569
3708
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3570
3709
|
if (!this.isValidWriteToken(entity, token)) throw new ElectricAgentsError(ErrCodeUnauthorized, `Invalid write token`, 401);
|
|
3571
|
-
if (entity.status
|
|
3710
|
+
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
3572
3711
|
if (typeof req.value !== `string`) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Tag values must be strings`, 400);
|
|
3573
3712
|
const result = await this.registry.setEntityTag(entityUrl, key, req.value);
|
|
3574
3713
|
const updated = result.entity;
|
|
@@ -3580,7 +3719,7 @@ var EntityManager = class {
|
|
|
3580
3719
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3581
3720
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3582
3721
|
if (!this.isValidWriteToken(entity, token)) throw new ElectricAgentsError(ErrCodeUnauthorized, `Invalid write token`, 401);
|
|
3583
|
-
if (entity.status
|
|
3722
|
+
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
3584
3723
|
const result = await this.registry.removeEntityTag(entityUrl, key);
|
|
3585
3724
|
const updated = result.entity;
|
|
3586
3725
|
if (!updated) throw new ElectricAgentsError(ErrCodeEntityPersistFailed, `Entity not found after tag delete`, 500);
|
|
@@ -3833,26 +3972,131 @@ var EntityManager = class {
|
|
|
3833
3972
|
}
|
|
3834
3973
|
};
|
|
3835
3974
|
}
|
|
3836
|
-
async
|
|
3975
|
+
async signal(entityUrl, req) {
|
|
3837
3976
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3838
3977
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3839
|
-
|
|
3840
|
-
|
|
3841
|
-
const
|
|
3842
|
-
|
|
3843
|
-
const
|
|
3844
|
-
|
|
3845
|
-
|
|
3978
|
+
if (isTerminalEntityStatus(entity.status)) throw new ElectricAgentsError(ErrCodeInvalidSignal, `Cannot signal a ${entity.status} entity`, 409);
|
|
3979
|
+
const now = new Date();
|
|
3980
|
+
const previousState = entity.status;
|
|
3981
|
+
const handling = this.serverHandlingForSignal(previousState, req.signal);
|
|
3982
|
+
const txid = handling.status === previousState ? await this.registry.touchEntityWithTxid(entityUrl) : await this.registry.updateStatusWithTxid(entityUrl, handling.status);
|
|
3983
|
+
if (txid === null) throw new ElectricAgentsError(ErrCodeInvalidSignal, `Cannot signal entity because it is already terminal`, 409);
|
|
3984
|
+
const key = `sig-${now.getTime()}-${randomUUID().slice(0, 8)}`;
|
|
3985
|
+
const signalValue = {
|
|
3986
|
+
signal: req.signal,
|
|
3987
|
+
status: handling.handled ? `handled` : `unhandled`,
|
|
3988
|
+
sender: SERVER_SIGNAL_SENDER,
|
|
3989
|
+
timestamp: now.toISOString()
|
|
3990
|
+
};
|
|
3991
|
+
if (req.reason !== void 0) signalValue.reason = req.reason;
|
|
3992
|
+
if (req.payload !== void 0) signalValue.payload = req.payload;
|
|
3993
|
+
if (handling.handled) {
|
|
3994
|
+
signalValue.handled_at = now.toISOString();
|
|
3995
|
+
signalValue.handled_by = SERVER_SIGNAL_SENDER;
|
|
3996
|
+
signalValue.outcome = handling.outcome;
|
|
3997
|
+
signalValue.previous_state = previousState;
|
|
3998
|
+
signalValue.new_state = handling.status;
|
|
3999
|
+
}
|
|
4000
|
+
const signalEvent = {
|
|
4001
|
+
type: `signal`,
|
|
4002
|
+
key,
|
|
4003
|
+
value: signalValue,
|
|
4004
|
+
headers: {
|
|
4005
|
+
operation: `insert`,
|
|
4006
|
+
timestamp: now.toISOString(),
|
|
4007
|
+
txid: String(txid)
|
|
4008
|
+
}
|
|
4009
|
+
};
|
|
4010
|
+
const shouldCloseStreams = isTerminalEntityStatus(handling.status);
|
|
4011
|
+
await this.appendSignalEvent(entity, signalEvent, shouldCloseStreams);
|
|
4012
|
+
if (!shouldCloseStreams) await this.evaluateWakes(entityUrl, signalEvent);
|
|
4013
|
+
if (handling.unregisterWakes) {
|
|
4014
|
+
await this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId);
|
|
4015
|
+
await this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId);
|
|
4016
|
+
}
|
|
4017
|
+
if (handling.status !== previousState && this.entityBridgeManager) await this.entityBridgeManager.onEntityChanged(entityUrl);
|
|
4018
|
+
return {
|
|
4019
|
+
url: entityUrl,
|
|
4020
|
+
signal: req.signal,
|
|
4021
|
+
previous_state: previousState,
|
|
4022
|
+
new_state: handling.status,
|
|
4023
|
+
created_at: now.getTime(),
|
|
4024
|
+
txid
|
|
4025
|
+
};
|
|
4026
|
+
}
|
|
4027
|
+
async kill(entityUrl) {
|
|
4028
|
+
const response = await this.signal(entityUrl, {
|
|
4029
|
+
signal: `SIGKILL`,
|
|
4030
|
+
reason: `Legacy kill command`
|
|
3846
4031
|
});
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
4032
|
+
return { txid: response.txid };
|
|
4033
|
+
}
|
|
4034
|
+
serverHandlingForSignal(status$1, signal) {
|
|
4035
|
+
if (signal === `SIGKILL`) return {
|
|
4036
|
+
status: `killed`,
|
|
4037
|
+
handled: true,
|
|
4038
|
+
outcome: `transitioned`,
|
|
4039
|
+
unregisterWakes: true
|
|
4040
|
+
};
|
|
4041
|
+
if (signal === `SIGTERM`) {
|
|
4042
|
+
if (status$1 === `idle` || status$1 === `paused`) return {
|
|
4043
|
+
status: `stopped`,
|
|
4044
|
+
handled: true,
|
|
4045
|
+
outcome: `transitioned`,
|
|
4046
|
+
unregisterWakes: true
|
|
4047
|
+
};
|
|
4048
|
+
if (status$1 === `running`) return {
|
|
4049
|
+
status: `stopping`,
|
|
4050
|
+
handled: false,
|
|
4051
|
+
outcome: `transitioned`,
|
|
4052
|
+
unregisterWakes: false
|
|
4053
|
+
};
|
|
4054
|
+
}
|
|
4055
|
+
if (status$1 === `paused` && signal !== `SIGCONT`) return {
|
|
4056
|
+
status: status$1,
|
|
4057
|
+
handled: true,
|
|
4058
|
+
outcome: `ignored`,
|
|
4059
|
+
unregisterWakes: false
|
|
4060
|
+
};
|
|
4061
|
+
if (signal === `SIGSTOP` && (status$1 === `idle` || status$1 === `running`)) return {
|
|
4062
|
+
status: `paused`,
|
|
4063
|
+
handled: status$1 === `idle`,
|
|
4064
|
+
outcome: `transitioned`,
|
|
4065
|
+
unregisterWakes: false
|
|
4066
|
+
};
|
|
4067
|
+
if (signal === `SIGCONT` && status$1 === `paused`) return {
|
|
4068
|
+
status: `idle`,
|
|
4069
|
+
handled: false,
|
|
4070
|
+
outcome: `transitioned`,
|
|
4071
|
+
unregisterWakes: false
|
|
4072
|
+
};
|
|
4073
|
+
return {
|
|
4074
|
+
status: status$1,
|
|
4075
|
+
handled: false,
|
|
4076
|
+
outcome: `ignored`,
|
|
4077
|
+
unregisterWakes: false
|
|
4078
|
+
};
|
|
4079
|
+
}
|
|
4080
|
+
async appendSignalEvent(entity, signalEvent, closeStreams) {
|
|
4081
|
+
const signalData = this.encodeChangeEvent(signalEvent);
|
|
4082
|
+
if (!closeStreams) {
|
|
4083
|
+
await this.streamClient.append(entity.streams.main, signalData);
|
|
4084
|
+
return;
|
|
4085
|
+
}
|
|
4086
|
+
const errorCloseEvent = {
|
|
4087
|
+
type: `signal`,
|
|
4088
|
+
key: signalEvent.key,
|
|
4089
|
+
value: signalEvent.value,
|
|
4090
|
+
headers: signalEvent.headers
|
|
4091
|
+
};
|
|
4092
|
+
const errorSignalData = this.encodeChangeEvent(errorCloseEvent);
|
|
4093
|
+
for (const [streamPath, data] of [[entity.streams.main, signalData], [entity.streams.error, errorSignalData]]) try {
|
|
4094
|
+
await this.streamClient.append(streamPath, data, { close: true });
|
|
3850
4095
|
} catch (err) {
|
|
3851
4096
|
const message = err instanceof Error ? err.message : String(err);
|
|
3852
4097
|
if (/closed/i.test(message) || /not found/i.test(message) || /404/.test(message) || /409/.test(message)) continue;
|
|
3853
4098
|
throw err;
|
|
3854
4099
|
}
|
|
3855
|
-
return { txid };
|
|
3856
4100
|
}
|
|
3857
4101
|
async validateWriteEvent(entity, event) {
|
|
3858
4102
|
if (!entity.type) return null;
|
|
@@ -3968,7 +4212,7 @@ var EntityManager = class {
|
|
|
3968
4212
|
async validateSendRequest(entityUrl, req) {
|
|
3969
4213
|
const entity = await this.registry.getEntity(entityUrl);
|
|
3970
4214
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
3971
|
-
if (entity.status
|
|
4215
|
+
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
3972
4216
|
if (req.type && entity.type) {
|
|
3973
4217
|
const { inboxSchemas } = await this.getEffectiveSchemas(entity);
|
|
3974
4218
|
if (inboxSchemas) {
|
|
@@ -4141,6 +4385,9 @@ async function assertDispatchPolicyAllowed(ctx, policy) {
|
|
|
4141
4385
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
4142
4386
|
if (runner.owner_principal !== ctx.principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
4143
4387
|
}
|
|
4388
|
+
function shouldLinkDispatchBeforeInitialMessage(policy) {
|
|
4389
|
+
return policy?.targets[0] !== void 0;
|
|
4390
|
+
}
|
|
4144
4391
|
async function linkEntityDispatchSubscription(ctx, entity) {
|
|
4145
4392
|
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
4146
4393
|
const target = dispatchPolicy?.targets[0];
|
|
@@ -4266,6 +4513,20 @@ const forkBodySchema = Type.Object({
|
|
|
4266
4513
|
waitTimeoutMs: Type.Optional(Type.Number())
|
|
4267
4514
|
});
|
|
4268
4515
|
const setTagBodySchema = Type.Object({ value: Type.String() });
|
|
4516
|
+
const entitySignalSchema = Type.Union([
|
|
4517
|
+
Type.Literal(`SIGINT`),
|
|
4518
|
+
Type.Literal(`SIGHUP`),
|
|
4519
|
+
Type.Literal(`SIGTERM`),
|
|
4520
|
+
Type.Literal(`SIGKILL`),
|
|
4521
|
+
Type.Literal(`SIGSTOP`),
|
|
4522
|
+
Type.Literal(`SIGCONT`),
|
|
4523
|
+
Type.Literal(`SIGUSR`)
|
|
4524
|
+
]);
|
|
4525
|
+
const signalBodySchema = Type.Object({
|
|
4526
|
+
signal: entitySignalSchema,
|
|
4527
|
+
reason: Type.Optional(Type.String()),
|
|
4528
|
+
payload: Type.Optional(Type.Unknown())
|
|
4529
|
+
});
|
|
4269
4530
|
const scheduleBodySchema = Type.Union([Type.Object({
|
|
4270
4531
|
scheduleType: Type.Literal(`cron`),
|
|
4271
4532
|
expression: Type.String(),
|
|
@@ -4289,6 +4550,7 @@ entitiesRouter.put(`/:type/:instanceId`, withSpawnableEntityType, withSchema(spa
|
|
|
4289
4550
|
entitiesRouter.get(`/:type/:instanceId`, withExistingEntity, getEntity);
|
|
4290
4551
|
entitiesRouter.head(`/:type/:instanceId`, withExistingEntity, headEntity);
|
|
4291
4552
|
entitiesRouter.delete(`/:type/:instanceId`, withExistingEntity, killEntity);
|
|
4553
|
+
entitiesRouter.post(`/:type/:instanceId/signal`, withExistingEntity, withSchema(signalBodySchema), signalEntity);
|
|
4292
4554
|
entitiesRouter.post(`/:type/:instanceId/send`, withExistingEntity, withSchema(sendBodySchema), sendEntity);
|
|
4293
4555
|
entitiesRouter.patch(`/:type/:instanceId/inbox/:messageKey`, withExistingEntity, withSchema(inboxMessageBodySchema), updateInboxMessage);
|
|
4294
4556
|
entitiesRouter.delete(`/:type/:instanceId/inbox/:messageKey`, withExistingEntity, deleteInboxMessage);
|
|
@@ -4492,11 +4754,13 @@ async function spawnEntity(request, ctx) {
|
|
|
4492
4754
|
wake: parsed.wake,
|
|
4493
4755
|
created_by: principal.url
|
|
4494
4756
|
});
|
|
4495
|
-
|
|
4757
|
+
const linkBeforeInitialMessage = parsed.initialMessage !== void 0 && shouldLinkDispatchBeforeInitialMessage(dispatchPolicy);
|
|
4758
|
+
if (linkBeforeInitialMessage) await linkEntityDispatchSubscription(ctx, entity);
|
|
4496
4759
|
if (parsed.initialMessage !== void 0) await ctx.entityManager.send(entity.url, {
|
|
4497
4760
|
from: principal.url,
|
|
4498
4761
|
payload: parsed.initialMessage
|
|
4499
4762
|
});
|
|
4763
|
+
if (!linkBeforeInitialMessage) await linkEntityDispatchSubscription(ctx, entity);
|
|
4500
4764
|
return json({
|
|
4501
4765
|
...toPublicEntity(entity),
|
|
4502
4766
|
txid: entity.txid
|
|
@@ -4520,6 +4784,22 @@ async function killEntity(request, ctx) {
|
|
|
4520
4784
|
ctx.runtime.claimWriteTokens.clearStream(ctx.service, entity.streams.main);
|
|
4521
4785
|
return json(result);
|
|
4522
4786
|
}
|
|
4787
|
+
async function signalEntity(request, ctx) {
|
|
4788
|
+
const principalMutationError = rejectPrincipalEntityMutation(request, `signaled`);
|
|
4789
|
+
if (principalMutationError) return principalMutationError;
|
|
4790
|
+
const parsed = routeBody(request);
|
|
4791
|
+
const { entityUrl, entity } = requireExistingEntityRoute(request);
|
|
4792
|
+
const result = await ctx.entityManager.signal(entityUrl, {
|
|
4793
|
+
signal: parsed.signal,
|
|
4794
|
+
reason: parsed.reason,
|
|
4795
|
+
payload: parsed.payload
|
|
4796
|
+
});
|
|
4797
|
+
if (result.new_state === `stopped` || result.new_state === `killed`) {
|
|
4798
|
+
await unlinkEntityDispatchSubscription(ctx, entity);
|
|
4799
|
+
ctx.runtime.claimWriteTokens.clearStream(ctx.service, entity.streams.main);
|
|
4800
|
+
}
|
|
4801
|
+
return json(result);
|
|
4802
|
+
}
|
|
4523
4803
|
|
|
4524
4804
|
//#endregion
|
|
4525
4805
|
//#region src/routing/entity-types-router.ts
|
|
@@ -5000,7 +5280,7 @@ async function notificationFromClaim(ctx, input) {
|
|
|
5000
5280
|
const primaryStream = withLeadingSlash(primary.path);
|
|
5001
5281
|
const entity = await ctx.entityManager.registry.getEntityByStream(primaryStream);
|
|
5002
5282
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Claim stream is not attached to an entity`, 404);
|
|
5003
|
-
if (entity.status === `stopped`) {
|
|
5283
|
+
if (entity.status === `stopped` || entity.status === `paused`) {
|
|
5004
5284
|
await ctx.streamClient.releaseSubscription(input.subscriptionId, input.claim.token, {
|
|
5005
5285
|
wake_id: input.claim.wake_id,
|
|
5006
5286
|
generation: input.claim.generation
|
|
@@ -5117,12 +5397,16 @@ function bodyFromBytes(body) {
|
|
|
5117
5397
|
return body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength);
|
|
5118
5398
|
}
|
|
5119
5399
|
function responseFromUpstream(response, body) {
|
|
5120
|
-
|
|
5400
|
+
const responseBody = forbidsResponseBody(response.status) ? null : body !== void 0 ? bodyFromBytes(body) : response.body;
|
|
5401
|
+
return new Response(responseBody, {
|
|
5121
5402
|
status: response.status,
|
|
5122
5403
|
statusText: response.statusText,
|
|
5123
5404
|
headers: responseHeaders(response)
|
|
5124
5405
|
});
|
|
5125
5406
|
}
|
|
5407
|
+
function forbidsResponseBody(status$1) {
|
|
5408
|
+
return status$1 === 204 || status$1 === 205 || status$1 === 304;
|
|
5409
|
+
}
|
|
5126
5410
|
function forwardHeadersFromRequest(request) {
|
|
5127
5411
|
const headers = new Headers(request.headers);
|
|
5128
5412
|
headers.delete(`host`);
|
|
@@ -5131,6 +5415,45 @@ function forwardHeadersFromRequest(request) {
|
|
|
5131
5415
|
function durableStreamsSubscriptionCallback(value) {
|
|
5132
5416
|
return value.startsWith(DS_SUBSCRIPTION_CALLBACK_PREFIX) ? value.slice(DS_SUBSCRIPTION_CALLBACK_PREFIX.length) : null;
|
|
5133
5417
|
}
|
|
5418
|
+
function resolveWebhookSigner(ctx) {
|
|
5419
|
+
return ctx.webhookSigner ?? getDefaultWebhookSigner();
|
|
5420
|
+
}
|
|
5421
|
+
function durableStreamsWebhookJwksUrl(ctx) {
|
|
5422
|
+
if (!ctx.durableStreamsRouting) return appendPathToUrl(ctx.durableStreamsUrl, `/__ds/jwks.json`);
|
|
5423
|
+
return resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl).controlUrl({
|
|
5424
|
+
durableStreamsUrl: ctx.durableStreamsUrl,
|
|
5425
|
+
serviceId: ctx.service,
|
|
5426
|
+
requestUrl: appendPathToUrl(ctx.publicUrl, `/__ds/jwks.json`)
|
|
5427
|
+
}).toString();
|
|
5428
|
+
}
|
|
5429
|
+
function durableStreamsJwksFetchClient(ctx) {
|
|
5430
|
+
return async (input, init) => {
|
|
5431
|
+
const headers = new Headers(init?.headers);
|
|
5432
|
+
await applyDurableStreamsBearer(headers, ctx.durableStreamsBearer, { overwrite: false });
|
|
5433
|
+
const nextInit = {
|
|
5434
|
+
...init ?? {},
|
|
5435
|
+
headers
|
|
5436
|
+
};
|
|
5437
|
+
if (ctx.durableStreamsDispatcher) nextInit.dispatcher = ctx.durableStreamsDispatcher;
|
|
5438
|
+
return await fetch(input, nextInit);
|
|
5439
|
+
};
|
|
5440
|
+
}
|
|
5441
|
+
function resolveDurableStreamsWebhookSignature(ctx) {
|
|
5442
|
+
if (ctx.durableStreamsWebhookSignature === false) return false;
|
|
5443
|
+
return {
|
|
5444
|
+
jwksUrl: ctx.durableStreamsWebhookSignature?.jwksUrl ?? durableStreamsWebhookJwksUrl(ctx),
|
|
5445
|
+
toleranceSeconds: ctx.durableStreamsWebhookSignature?.toleranceSeconds,
|
|
5446
|
+
cacheTtlMs: ctx.durableStreamsWebhookSignature?.cacheTtlMs,
|
|
5447
|
+
fetchClient: ctx.durableStreamsWebhookSignature?.fetchClient ?? durableStreamsJwksFetchClient(ctx)
|
|
5448
|
+
};
|
|
5449
|
+
}
|
|
5450
|
+
async function verifyDurableStreamsWebhook(request, ctx, body) {
|
|
5451
|
+
const config = resolveDurableStreamsWebhookSignature(ctx);
|
|
5452
|
+
if (config === false) return null;
|
|
5453
|
+
const verification = await verifyWebhookSignature(body, request.headers.get(`webhook-signature`), config);
|
|
5454
|
+
if (verification.ok) return null;
|
|
5455
|
+
return apiError(verification.status, verification.status === 401 ? ErrCodeUnauthorized : `WEBHOOK_SIGNATURE_UNAVAILABLE`, verification.error);
|
|
5456
|
+
}
|
|
5134
5457
|
function claimTokenFromRequest(request) {
|
|
5135
5458
|
const electricClaimToken = request.headers.get(`electric-claim-token`)?.trim();
|
|
5136
5459
|
if (electricClaimToken) return electricClaimToken;
|
|
@@ -5164,7 +5487,10 @@ async function webhookForward(request, ctx) {
|
|
|
5164
5487
|
const rootSpan = getRequestSpan(request);
|
|
5165
5488
|
rootSpan?.updateName(`webhook-forward`);
|
|
5166
5489
|
rootSpan?.setAttribute(`electric_agents.webhook.subscription_id`, subscriptionId);
|
|
5167
|
-
const
|
|
5490
|
+
const body = await readRequestBody(request);
|
|
5491
|
+
const signatureError = await verifyDurableStreamsWebhook(request, ctx, body);
|
|
5492
|
+
if (signatureError) return signatureError;
|
|
5493
|
+
const targetWebhookUrl = await tracer.startActiveSpan(`db.lookupSubscription`, async (span) => {
|
|
5168
5494
|
try {
|
|
5169
5495
|
const rows = await ctx.pgDb.select().from(subscriptionWebhooks).where(and(eq(subscriptionWebhooks.tenantId, ctx.service), eq(subscriptionWebhooks.subscriptionId, subscriptionId))).limit(1);
|
|
5170
5496
|
return rows[0]?.webhookUrl ?? null;
|
|
@@ -5172,7 +5498,6 @@ async function webhookForward(request, ctx) {
|
|
|
5172
5498
|
span.end();
|
|
5173
5499
|
}
|
|
5174
5500
|
});
|
|
5175
|
-
const [targetWebhookUrl, body] = await Promise.all([lookupPromise, readRequestBody(request)]);
|
|
5176
5501
|
if (!targetWebhookUrl) return apiError(404, ErrCodeSubscriptionNotFound, `Unknown webhook subscription`);
|
|
5177
5502
|
const parsedBodyResult = validateOptionalJsonBody(webhookForwardBodySchema, body, request.headers.get(`content-type`));
|
|
5178
5503
|
if (!parsedBodyResult.ok) return parsedBodyResult.response;
|
|
@@ -5223,7 +5548,7 @@ async function webhookForward(request, ctx) {
|
|
|
5223
5548
|
serverLog.warn(`[webhook-forward] consumerCallbacks upsert failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
|
|
5224
5549
|
}) : void 0;
|
|
5225
5550
|
const [entity, enriched] = await Promise.all([entityPromise, enrichPromise]);
|
|
5226
|
-
if (entity?.status === `stopped`) {
|
|
5551
|
+
if (entity?.status === `stopped` || entity?.status === `paused`) {
|
|
5227
5552
|
if (upsertPromise) await upsertPromise;
|
|
5228
5553
|
return json({ done: true });
|
|
5229
5554
|
}
|
|
@@ -5261,6 +5586,7 @@ async function webhookForward(request, ctx) {
|
|
|
5261
5586
|
const headers = forwardHeadersFromRequest(request);
|
|
5262
5587
|
headers.set(`content-type`, `application/json`);
|
|
5263
5588
|
headers.delete(`content-length`);
|
|
5589
|
+
headers.set(`webhook-signature`, await resolveWebhookSigner(ctx).sign(forwardBody));
|
|
5264
5590
|
let upstream;
|
|
5265
5591
|
try {
|
|
5266
5592
|
upstream = await tracer.startActiveSpan(`fetch.agent-handler`, async (span) => {
|
|
@@ -5348,8 +5674,9 @@ async function callbackForward(request, ctx) {
|
|
|
5348
5674
|
serverLog.info(`[callback-forward] done received for stream=${target.primaryStream} consumer=${consumerId}`);
|
|
5349
5675
|
const stillOwnsClaim = ctx.runtime.claimWriteTokens.owns(ctx.service, target.primaryStream, consumerId);
|
|
5350
5676
|
const entity = await ctx.entityManager.registry.getEntityByStream(target.primaryStream);
|
|
5351
|
-
|
|
5352
|
-
|
|
5677
|
+
let entityCleared = false;
|
|
5678
|
+
if (epoch !== void 0) {
|
|
5679
|
+
const result = await ctx.entityManager.registry.materializeReleasedClaim?.({
|
|
5353
5680
|
consumerId,
|
|
5354
5681
|
epoch,
|
|
5355
5682
|
ackedStreams: Array.isArray(requestBody?.acks) ? requestBody.acks.flatMap((ack) => {
|
|
@@ -5361,13 +5688,15 @@ async function callbackForward(request, ctx) {
|
|
|
5361
5688
|
}] : [];
|
|
5362
5689
|
}) : void 0
|
|
5363
5690
|
});
|
|
5364
|
-
|
|
5365
|
-
|
|
5691
|
+
entityCleared = result?.entityCleared ?? false;
|
|
5692
|
+
}
|
|
5693
|
+
if (entity && (entityCleared || stillOwnsClaim)) {
|
|
5694
|
+
await ctx.entityManager.registry.updateStatus(entity.url, entity.status === `stopping` ? `stopped` : `idle`);
|
|
5366
5695
|
await ctx.entityBridgeManager.onEntityChanged(entity.url);
|
|
5367
|
-
serverLog.info(`[callback-forward] status updated
|
|
5368
|
-
} else if (
|
|
5369
|
-
|
|
5370
|
-
else serverLog.
|
|
5696
|
+
serverLog.info(`[callback-forward] status updated after done for ${entity.url}`);
|
|
5697
|
+
} else if (!entity) serverLog.warn(`[callback-forward] done received but no entity found for stream=${target.primaryStream}`);
|
|
5698
|
+
if (stillOwnsClaim) ctx.runtime.claimWriteTokens.clearStream(ctx.service, target.primaryStream);
|
|
5699
|
+
else if (entity) serverLog.info(`[callback-forward] done arrived after in-memory token evicted (stream=${target.primaryStream} consumer=${consumerId})`);
|
|
5371
5700
|
} else if (requestBody?.done === true) serverLog.warn(`[callback-forward] done received but skipped: upstream.ok=${upstream.ok} primaryStream=${target.primaryStream ?? `null`} consumer=${consumerId}`);
|
|
5372
5701
|
} catch (err) {
|
|
5373
5702
|
serverLog.error(`[callback-forward] error processing done for consumer=${consumerId}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -6717,7 +7046,8 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
6717
7046
|
const primaryStream = `${entityUrl}/main`;
|
|
6718
7047
|
const callbacks = await this.db.select().from(consumerCallbacks).where(and(eq(consumerCallbacks.tenantId, this.serviceId), eq(consumerCallbacks.primaryStream, primaryStream))).limit(1);
|
|
6719
7048
|
if (callbacks.length > 0) return;
|
|
6720
|
-
await this.manager.registry.
|
|
7049
|
+
const entity = await this.manager.registry.getEntity(entityUrl);
|
|
7050
|
+
await this.manager.registry.updateStatus(entityUrl, entity?.status === `stopping` ? `stopped` : `idle`);
|
|
6721
7051
|
await this.entityBridgeManager.onEntityChanged(entityUrl);
|
|
6722
7052
|
}
|
|
6723
7053
|
};
|
|
@@ -7560,11 +7890,13 @@ var ElectricAgentsServer = class {
|
|
|
7560
7890
|
shuttingDown = false;
|
|
7561
7891
|
streamsAgent;
|
|
7562
7892
|
standaloneRuntime;
|
|
7893
|
+
webhookSigner;
|
|
7563
7894
|
streamClient;
|
|
7564
7895
|
options;
|
|
7565
7896
|
constructor(options) {
|
|
7566
7897
|
if (!options.durableStreamsUrl && !options.durableStreamsServer) throw new Error(`Either durableStreamsUrl or durableStreamsServer is required`);
|
|
7567
7898
|
this.options = options;
|
|
7899
|
+
this.webhookSigner = options.webhookSigner ?? createEd25519WebhookSigner({ privateKey: options.webhookSigningKey });
|
|
7568
7900
|
this.streamClient = options.durableStreamsUrl ? new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer }) : null;
|
|
7569
7901
|
}
|
|
7570
7902
|
get url() {
|
|
@@ -7720,6 +8052,8 @@ var ElectricAgentsServer = class {
|
|
|
7720
8052
|
durableStreamsBearer: this.options.durableStreamsBearer,
|
|
7721
8053
|
durableStreamsRouting: this.options.durableStreamsRouting,
|
|
7722
8054
|
durableStreamsDispatcher: this.streamsAgent,
|
|
8055
|
+
durableStreamsWebhookSignature: this.options.durableStreamsWebhookSignature,
|
|
8056
|
+
webhookSigner: this.webhookSigner,
|
|
7723
8057
|
electricUrl: this.options.electricUrl,
|
|
7724
8058
|
electricSecret: this.options.electricSecret,
|
|
7725
8059
|
ownAgentHandlerPaths: this.mockAgentBootstrap ? [MOCK_AGENT_HANDLER_PATH] : void 0,
|
|
@@ -7790,6 +8124,7 @@ function resolveElectricAgentsEntrypointOptions(env = process.env, cwd = process
|
|
|
7790
8124
|
const postgresUrl = validateUrl(`Postgres URL`, readRequiredEnv(env, [`ELECTRIC_AGENTS_DATABASE_URL`, `DATABASE_URL`], `Postgres connection URL`));
|
|
7791
8125
|
const electricUrl = readEnv(env, [`ELECTRIC_AGENTS_ELECTRIC_URL`, `ELECTRIC_URL`]);
|
|
7792
8126
|
const electricSecret = readEnv(env, [`ELECTRIC_AGENTS_ELECTRIC_SECRET`]);
|
|
8127
|
+
const webhookSigningKey = readEnv(env, [`ELECTRIC_AGENTS_WEBHOOK_SIGNING_PRIVATE_KEY`, `WEBHOOK_SIGNING_PRIVATE_KEY`]);
|
|
7793
8128
|
const baseUrl = readEnv(env, [`ELECTRIC_AGENTS_BASE_URL`, `BASE_URL`]);
|
|
7794
8129
|
return {
|
|
7795
8130
|
service: readEnv(env, [`ELECTRIC_AGENTS_SERVICE`, `SERVICE`]),
|
|
@@ -7800,6 +8135,7 @@ function resolveElectricAgentsEntrypointOptions(env = process.env, cwd = process
|
|
|
7800
8135
|
postgresUrl,
|
|
7801
8136
|
electricUrl: electricUrl ? validateUrl(`Electric URL`, electricUrl) : void 0,
|
|
7802
8137
|
electricSecret,
|
|
8138
|
+
webhookSigningKey,
|
|
7803
8139
|
host: readEnv(env, [`ELECTRIC_AGENTS_HOST`, `HOST`]) ?? DEFAULT_HOST,
|
|
7804
8140
|
port: readPort(env),
|
|
7805
8141
|
workingDirectory: readEnv(env, [`ELECTRIC_AGENTS_WORKING_DIRECTORY`, `WORKING_DIRECTORY`]) ?? cwd,
|