@electric-ax/agents-server 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +375 -115
- package/dist/index.cjs +1434 -1188
- package/dist/index.d.cts +270 -160
- package/dist/index.d.ts +270 -160
- package/dist/index.js +1434 -1188
- package/drizzle/0007_runner_diagnostics_and_principal.sql +22 -0
- package/drizzle/0008_runner_runtime_diagnostics.sql +50 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +4 -4
- package/src/db/schema.ts +33 -10
- package/src/electric-agents-types.ts +49 -3
- package/src/entity-registry.ts +136 -26
- package/src/host.ts +4 -5
- package/src/principal.ts +23 -11
- package/src/routing/context.ts +1 -1
- package/src/routing/dispatch-policy.ts +123 -20
- package/src/routing/durable-streams-routing-adapter.ts +6 -7
- package/src/routing/electric-proxy-router.ts +1 -0
- package/src/routing/entities-router.ts +4 -4
- package/src/routing/hooks.ts +8 -1
- package/src/routing/runners-router.ts +257 -20
- package/src/runtime.ts +4 -7
- package/src/server.ts +20 -15
- package/src/standalone-runtime.ts +4 -7
- package/src/stream-client.ts +16 -59
- package/src/utils/server-utils.ts +22 -4
package/dist/entrypoint.js
CHANGED
|
@@ -43,6 +43,7 @@ __export(schema_exports, {
|
|
|
43
43
|
entityDispatchState: () => entityDispatchState,
|
|
44
44
|
entityManifestSources: () => entityManifestSources,
|
|
45
45
|
entityTypes: () => entityTypes,
|
|
46
|
+
runnerRuntimeDiagnostics: () => runnerRuntimeDiagnostics,
|
|
46
47
|
runners: () => runners,
|
|
47
48
|
scheduledTasks: () => scheduledTasks,
|
|
48
49
|
subscriptionWebhooks: () => subscriptionWebhooks,
|
|
@@ -111,25 +112,35 @@ const users = pgTable(`users`, {
|
|
|
111
112
|
const runners = pgTable(`runners`, {
|
|
112
113
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
113
114
|
id: text(`id`).notNull(),
|
|
114
|
-
|
|
115
|
+
ownerPrincipal: text(`owner_principal`).notNull(),
|
|
115
116
|
label: text(`label`).notNull(),
|
|
116
117
|
kind: text(`kind`).notNull().default(`local`),
|
|
117
118
|
adminStatus: text(`admin_status`).notNull().default(`enabled`),
|
|
118
119
|
wakeStream: text(`wake_stream`).notNull(),
|
|
119
|
-
wakeStreamOffset: text(`wake_stream_offset`),
|
|
120
|
-
lastSeenAt: timestamp(`last_seen_at`, { withTimezone: true }),
|
|
121
|
-
livenessLeaseExpiresAt: timestamp(`liveness_lease_expires_at`, { withTimezone: true }),
|
|
122
120
|
createdAt: timestamp(`created_at`, { withTimezone: true }).notNull().defaultNow(),
|
|
123
121
|
updatedAt: timestamp(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
124
122
|
}, (table) => [
|
|
125
123
|
primaryKey({ columns: [table.tenantId, table.id] }),
|
|
126
124
|
unique(`uq_runners_wake_stream`).on(table.tenantId, table.wakeStream),
|
|
127
|
-
index(`
|
|
125
|
+
index(`idx_runners_owner_principal`).on(table.tenantId, table.ownerPrincipal),
|
|
128
126
|
index(`idx_runners_admin_status`).on(table.tenantId, table.adminStatus),
|
|
129
|
-
index(`idx_runners_liveness_lease_expires_at`).on(table.tenantId, table.livenessLeaseExpiresAt),
|
|
130
127
|
check(`chk_runners_kind`, sql`${table.kind} IN ('local', 'cloud-worker', 'sandbox', 'ci', 'server')`),
|
|
131
128
|
check(`chk_runners_admin_status`, sql`${table.adminStatus} IN ('enabled', 'disabled')`)
|
|
132
129
|
]);
|
|
130
|
+
const runnerRuntimeDiagnostics = pgTable(`runner_runtime_diagnostics`, {
|
|
131
|
+
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
132
|
+
runnerId: text(`runner_id`).notNull(),
|
|
133
|
+
ownerPrincipal: text(`owner_principal`).notNull(),
|
|
134
|
+
wakeStreamOffset: text(`wake_stream_offset`),
|
|
135
|
+
lastSeenAt: timestamp(`last_seen_at`, { withTimezone: true }).notNull(),
|
|
136
|
+
livenessLeaseExpiresAt: timestamp(`liveness_lease_expires_at`, { withTimezone: true }).notNull(),
|
|
137
|
+
diagnostics: jsonb(`diagnostics`),
|
|
138
|
+
updatedAt: timestamp(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
139
|
+
}, (table) => [
|
|
140
|
+
primaryKey({ columns: [table.tenantId, table.runnerId] }),
|
|
141
|
+
index(`idx_runner_runtime_diagnostics_owner`).on(table.tenantId, table.ownerPrincipal),
|
|
142
|
+
index(`idx_runner_runtime_diagnostics_liveness`).on(table.tenantId, table.livenessLeaseExpiresAt)
|
|
143
|
+
]);
|
|
133
144
|
const entityDispatchState = pgTable(`entity_dispatch_state`, {
|
|
134
145
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
135
146
|
entityUrl: text(`entity_url`).notNull(),
|
|
@@ -434,11 +445,9 @@ function electricUrlWithPath(electricUrl, path$1) {
|
|
|
434
445
|
//#endregion
|
|
435
446
|
//#region src/routing/durable-streams-routing-adapter.ts
|
|
436
447
|
function appendSearch(target, source) {
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
}
|
|
440
|
-
function removeServiceQuery(target) {
|
|
441
|
-
target.searchParams.delete(`service`);
|
|
448
|
+
source.searchParams.forEach((value, key) => {
|
|
449
|
+
if (key !== `service`) target.searchParams.append(key, value);
|
|
450
|
+
});
|
|
442
451
|
return target;
|
|
443
452
|
}
|
|
444
453
|
function withoutTrailingSlash(pathname) {
|
|
@@ -449,7 +458,7 @@ function appendRequestPathToStreamRoot(input) {
|
|
|
449
458
|
const path$1 = incomingUrl.pathname.replace(/^\/+/, ``);
|
|
450
459
|
const target = new URL(input.durableStreamsUrl);
|
|
451
460
|
target.pathname = path$1 ? `${withoutTrailingSlash(target.pathname)}/${path$1}` : withoutTrailingSlash(target.pathname);
|
|
452
|
-
return
|
|
461
|
+
return appendSearch(target, incomingUrl);
|
|
453
462
|
}
|
|
454
463
|
const streamRootDurableStreamsRoutingAdapter = {
|
|
455
464
|
streamUrl: appendRequestPathToStreamRoot,
|
|
@@ -539,23 +548,17 @@ async function applyDurableStreamsBearer(headers, bearer, opts = {}) {
|
|
|
539
548
|
const value = await resolveDurableStreamsBearer(bearer);
|
|
540
549
|
if (value) headers.set(`authorization`, value);
|
|
541
550
|
}
|
|
551
|
+
function appendPathToBaseUrl(baseUrl, path$1) {
|
|
552
|
+
const url = new URL(baseUrl);
|
|
553
|
+
const basePath = url.pathname.replace(/\/+$/, ``);
|
|
554
|
+
const childPath = path$1.replace(/^\/+/, ``);
|
|
555
|
+
url.pathname = childPath ? `${basePath === `/` ? `` : basePath}/${childPath}` : basePath || `/`;
|
|
556
|
+
return url.toString().replace(/\/+$/, ``);
|
|
557
|
+
}
|
|
542
558
|
function durableStreamsBearerHeaders(bearer) {
|
|
543
559
|
if (!bearer) return void 0;
|
|
544
560
|
return { authorization: async () => await resolveDurableStreamsBearer(bearer) ?? `` };
|
|
545
561
|
}
|
|
546
|
-
function durableStreamsServiceUrl(baseUrl, serviceId, options = {}) {
|
|
547
|
-
const url = new URL(baseUrl);
|
|
548
|
-
if (/\/v1\/streams\/[^/]+\/?$/.test(url.pathname)) return baseUrl.replace(/\/+$/, ``);
|
|
549
|
-
if (/\/v1\/stream\/[^/]+\/?$/.test(url.pathname)) return baseUrl.replace(/\/+$/, ``);
|
|
550
|
-
const scope = options.scope ?? `service`;
|
|
551
|
-
const encodedServiceId = encodeURIComponent(serviceId);
|
|
552
|
-
const path$1 = url.pathname.replace(/\/+$/, ``) || `/`;
|
|
553
|
-
if (path$1.endsWith(`/v1/streams`)) url.pathname = `${path$1}/${encodedServiceId}`;
|
|
554
|
-
else if (path$1.endsWith(`/v1/stream`)) url.pathname = scope === `service` ? `${path$1}/${encodedServiceId}` : path$1;
|
|
555
|
-
else if (scope === `stream-root`) url.pathname = `${path$1 === `/` ? `` : path$1}/v1/stream`;
|
|
556
|
-
else url.pathname = `${path$1 === `/` ? `` : path$1}/v1/stream/${encodedServiceId}`;
|
|
557
|
-
return url.toString().replace(/\/+$/, ``);
|
|
558
|
-
}
|
|
559
562
|
function isNotFoundError(err) {
|
|
560
563
|
return err instanceof DurableStreamError && err.code === ErrCodeNotFound || err instanceof FetchError && err.status === 404;
|
|
561
564
|
}
|
|
@@ -577,7 +580,7 @@ var StreamClient = class {
|
|
|
577
580
|
this.options = options;
|
|
578
581
|
}
|
|
579
582
|
streamUrl(path$1) {
|
|
580
|
-
return
|
|
583
|
+
return appendPathToBaseUrl(this.baseUrl, path$1);
|
|
581
584
|
}
|
|
582
585
|
streamHeaders() {
|
|
583
586
|
return durableStreamsBearerHeaders(this.options.bearer);
|
|
@@ -594,9 +597,7 @@ var StreamClient = class {
|
|
|
594
597
|
return normalizeSubscriptionPath(path$1);
|
|
595
598
|
}
|
|
596
599
|
subscriptionUrl(subscriptionId) {
|
|
597
|
-
|
|
598
|
-
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/__ds/subscriptions/${encodeURIComponent(subscriptionId)}`;
|
|
599
|
-
return url.toString();
|
|
600
|
+
return appendPathToBaseUrl(this.baseUrl, `/__ds/subscriptions/${encodeURIComponent(subscriptionId)}`);
|
|
600
601
|
}
|
|
601
602
|
subscriptionChildUrl(subscriptionId, ...segments) {
|
|
602
603
|
const url = new URL(this.subscriptionUrl(subscriptionId));
|
|
@@ -625,7 +626,7 @@ var StreamClient = class {
|
|
|
625
626
|
});
|
|
626
627
|
const headers = {
|
|
627
628
|
"content-type": `application/json`,
|
|
628
|
-
"Stream-Forked-From": sourcePath
|
|
629
|
+
"Stream-Forked-From": new URL(this.streamUrl(sourcePath)).pathname
|
|
629
630
|
};
|
|
630
631
|
injectTraceHeaders(headers);
|
|
631
632
|
const response = await fetch(this.streamUrl(path$1), {
|
|
@@ -978,15 +979,6 @@ var StreamClient = class {
|
|
|
978
979
|
if (!text$1.trim()) return {};
|
|
979
980
|
return this.subscriptionResponseBody(JSON.parse(text$1));
|
|
980
981
|
}
|
|
981
|
-
async getConsumerState(consumerId) {
|
|
982
|
-
const res = await fetch(`${this.baseUrl}/consumers/${encodeURIComponent(consumerId)}`, {
|
|
983
|
-
method: `GET`,
|
|
984
|
-
headers: await this.requestHeaders()
|
|
985
|
-
});
|
|
986
|
-
if (res.status === 404) return null;
|
|
987
|
-
if (!res.ok) throw new Error(`Consumer query failed: ${res.status} ${await res.text()}`);
|
|
988
|
-
return res.json();
|
|
989
|
-
}
|
|
990
982
|
};
|
|
991
983
|
|
|
992
984
|
//#endregion
|
|
@@ -1049,8 +1041,11 @@ function buildElectricProxyTarget(options) {
|
|
|
1049
1041
|
target.searchParams.set(`columns`, `"tenant_id","name","description","creation_schema","inbox_schemas","state_schemas","serve_endpoint","default_dispatch_policy","revision","created_at","updated_at"`);
|
|
1050
1042
|
applyTenantShapeWhere(target, options.tenantId);
|
|
1051
1043
|
} else if (table === `runners`) {
|
|
1052
|
-
target.searchParams.set(`columns`, `"tenant_id","id","
|
|
1053
|
-
applyTenantShapeWhere(target, options.tenantId);
|
|
1044
|
+
target.searchParams.set(`columns`, `"tenant_id","id","owner_principal","label","kind","admin_status","wake_stream","created_at","updated_at"`);
|
|
1045
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral$2(options.principalUrl ?? ``)}`]);
|
|
1046
|
+
} else if (table === `runner_runtime_diagnostics`) {
|
|
1047
|
+
target.searchParams.set(`columns`, `"tenant_id","runner_id","owner_principal","wake_stream_offset","last_seen_at","liveness_lease_expires_at","diagnostics","updated_at"`);
|
|
1048
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral$2(options.principalUrl ?? ``)}`]);
|
|
1054
1049
|
} else if (table === `entity_dispatch_state`) {
|
|
1055
1050
|
target.searchParams.set(`columns`, `"tenant_id","entity_url","pending_source_streams","pending_reason","pending_since","outstanding_wake_id","outstanding_wake_target","outstanding_wake_created_at","active_consumer_id","active_runner_id","active_epoch","active_claimed_at","active_lease_expires_at","last_wake_id","last_claimed_at","last_released_at","last_completed_at","last_error","updated_at"`);
|
|
1056
1051
|
applyTenantShapeWhere(target, options.tenantId);
|
|
@@ -1096,8 +1091,8 @@ function decodeJsonObject(body) {
|
|
|
1096
1091
|
} catch {}
|
|
1097
1092
|
return null;
|
|
1098
1093
|
}
|
|
1099
|
-
function applyTenantShapeWhere(target, tenantId) {
|
|
1100
|
-
const tenantWhere = `tenant_id = ${sqlStringLiteral$2(tenantId)}
|
|
1094
|
+
function applyTenantShapeWhere(target, tenantId, extraConditions = []) {
|
|
1095
|
+
const tenantWhere = [`tenant_id = ${sqlStringLiteral$2(tenantId)}`, ...extraConditions].join(` AND `);
|
|
1101
1096
|
const existingWhere = target.searchParams.get(`where`);
|
|
1102
1097
|
target.searchParams.set(`where`, existingWhere ? `${tenantWhere} AND (${existingWhere})` : tenantWhere);
|
|
1103
1098
|
}
|
|
@@ -1640,7 +1635,8 @@ async function proxyElectric(request, ctx) {
|
|
|
1640
1635
|
incomingUrl: new URL(request.url),
|
|
1641
1636
|
electricUrl: ctx.electricUrl,
|
|
1642
1637
|
electricSecret: ctx.electricSecret,
|
|
1643
|
-
tenantId: ctx.service
|
|
1638
|
+
tenantId: ctx.service,
|
|
1639
|
+
principalUrl: ctx.principal.url
|
|
1644
1640
|
});
|
|
1645
1641
|
const headers = new Headers(request.headers);
|
|
1646
1642
|
headers.delete(`host`);
|
|
@@ -1671,7 +1667,7 @@ const PRINCIPAL_KINDS = new Set([
|
|
|
1671
1667
|
]);
|
|
1672
1668
|
function parsePrincipalKey(input) {
|
|
1673
1669
|
const colon = input.indexOf(`:`);
|
|
1674
|
-
if (colon <= 0) throw new Error(`Invalid principal
|
|
1670
|
+
if (colon <= 0) throw new Error(`Invalid principal identifier`);
|
|
1675
1671
|
const kind = input.slice(0, colon);
|
|
1676
1672
|
const id = input.slice(colon + 1);
|
|
1677
1673
|
if (!PRINCIPAL_KINDS.has(kind)) throw new Error(`Invalid principal kind`);
|
|
@@ -1687,20 +1683,29 @@ function parsePrincipalKey(input) {
|
|
|
1687
1683
|
function principalUrl(key) {
|
|
1688
1684
|
return parsePrincipalKey(key).url;
|
|
1689
1685
|
}
|
|
1690
|
-
function
|
|
1686
|
+
function parsePrincipalUrl(url) {
|
|
1691
1687
|
if (!url.startsWith(`/principal/`)) return null;
|
|
1692
1688
|
const segment = url.slice(`/principal/`.length);
|
|
1693
1689
|
if (!segment || segment.includes(`/`)) return null;
|
|
1694
1690
|
try {
|
|
1695
|
-
|
|
1696
|
-
|
|
1691
|
+
return parsePrincipalKey(decodeURIComponent(segment));
|
|
1692
|
+
} catch {
|
|
1693
|
+
return null;
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
function parsePrincipalInput(input) {
|
|
1697
|
+
const urlPrincipal = parsePrincipalUrl(input);
|
|
1698
|
+
if (urlPrincipal) return urlPrincipal;
|
|
1699
|
+
try {
|
|
1700
|
+
return parsePrincipalKey(input);
|
|
1697
1701
|
} catch {
|
|
1698
1702
|
return null;
|
|
1699
1703
|
}
|
|
1700
1704
|
}
|
|
1701
1705
|
function getPrincipalFromRequest(request) {
|
|
1702
1706
|
const value = request.headers.get(ELECTRIC_PRINCIPAL_HEADER);
|
|
1703
|
-
|
|
1707
|
+
if (!value) return null;
|
|
1708
|
+
return parsePrincipalInput(value);
|
|
1704
1709
|
}
|
|
1705
1710
|
function getDevPrincipal() {
|
|
1706
1711
|
return parsePrincipalKey(`system:dev-local`);
|
|
@@ -1713,9 +1718,8 @@ const BUILT_IN_SYSTEM_PRINCIPAL_IDS = new Set([
|
|
|
1713
1718
|
function isBuiltInSystemPrincipalUrl(url) {
|
|
1714
1719
|
if (!url?.startsWith(`/principal/`)) return false;
|
|
1715
1720
|
try {
|
|
1716
|
-
const
|
|
1717
|
-
if (!
|
|
1718
|
-
const principal = parsePrincipalKey(key);
|
|
1721
|
+
const principal = parsePrincipalUrl(url);
|
|
1722
|
+
if (!principal) return false;
|
|
1719
1723
|
return principal.kind === `system` && BUILT_IN_SYSTEM_PRINCIPAL_IDS.has(principal.id);
|
|
1720
1724
|
} catch {
|
|
1721
1725
|
return false;
|
|
@@ -1723,12 +1727,11 @@ function isBuiltInSystemPrincipalUrl(url) {
|
|
|
1723
1727
|
}
|
|
1724
1728
|
function principalFromCreatedBy(createdBy) {
|
|
1725
1729
|
if (!createdBy) return void 0;
|
|
1726
|
-
const
|
|
1727
|
-
if (!
|
|
1730
|
+
const principal = parsePrincipalUrl(createdBy);
|
|
1731
|
+
if (!principal) return {
|
|
1728
1732
|
url: createdBy,
|
|
1729
1733
|
key: null
|
|
1730
1734
|
};
|
|
1731
|
-
const principal = parsePrincipalKey(key);
|
|
1732
1735
|
return {
|
|
1733
1736
|
url: principal.url,
|
|
1734
1737
|
key: principal.key,
|
|
@@ -1827,7 +1830,7 @@ var PostgresRegistry = class {
|
|
|
1827
1830
|
await this.db.insert(runners).values({
|
|
1828
1831
|
tenantId: this.tenantId,
|
|
1829
1832
|
id: input.id,
|
|
1830
|
-
|
|
1833
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
1831
1834
|
label: input.label,
|
|
1832
1835
|
kind: input.kind ?? `local`,
|
|
1833
1836
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -1836,7 +1839,7 @@ var PostgresRegistry = class {
|
|
|
1836
1839
|
}).onConflictDoUpdate({
|
|
1837
1840
|
target: [runners.tenantId, runners.id],
|
|
1838
1841
|
set: {
|
|
1839
|
-
|
|
1842
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
1840
1843
|
label: input.label,
|
|
1841
1844
|
kind: input.kind ?? `local`,
|
|
1842
1845
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -1854,20 +1857,46 @@ var PostgresRegistry = class {
|
|
|
1854
1857
|
}
|
|
1855
1858
|
async listRunners(filter) {
|
|
1856
1859
|
const conditions = [eq(runners.tenantId, this.tenantId)];
|
|
1857
|
-
if (filter?.
|
|
1860
|
+
if (filter?.ownerPrincipal) conditions.push(eq(runners.ownerPrincipal, filter.ownerPrincipal));
|
|
1858
1861
|
const rows = await this.db.select().from(runners).where(and(...conditions)).orderBy(desc(runners.createdAt));
|
|
1859
1862
|
return rows.map((row) => this.rowToRunner(row));
|
|
1860
1863
|
}
|
|
1861
1864
|
async heartbeatRunner(input) {
|
|
1862
1865
|
const now = input.heartbeatAt ?? new Date();
|
|
1863
1866
|
const leaseExpiresAt = input.livenessLeaseExpiresAt ?? new Date(now.getTime() + (input.leaseMs ?? DEFAULT_RUNNER_LEASE_MS));
|
|
1864
|
-
|
|
1867
|
+
await this.db.insert(runnerRuntimeDiagnostics).values({
|
|
1868
|
+
tenantId: this.tenantId,
|
|
1869
|
+
runnerId: input.runnerId,
|
|
1870
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
1865
1871
|
lastSeenAt: now,
|
|
1866
1872
|
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
1867
|
-
|
|
1873
|
+
wakeStreamOffset: input.wakeStreamOffset,
|
|
1874
|
+
diagnostics: input.diagnostics,
|
|
1868
1875
|
updatedAt: now
|
|
1869
|
-
}).
|
|
1870
|
-
|
|
1876
|
+
}).onConflictDoUpdate({
|
|
1877
|
+
target: [runnerRuntimeDiagnostics.tenantId, runnerRuntimeDiagnostics.runnerId],
|
|
1878
|
+
set: {
|
|
1879
|
+
lastSeenAt: now,
|
|
1880
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
1881
|
+
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
1882
|
+
...input.wakeStreamOffset !== void 0 ? { wakeStreamOffset: input.wakeStreamOffset } : {},
|
|
1883
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {},
|
|
1884
|
+
updatedAt: now
|
|
1885
|
+
}
|
|
1886
|
+
});
|
|
1887
|
+
const runner = await this.getRunner(input.runnerId);
|
|
1888
|
+
if (!runner) return null;
|
|
1889
|
+
return {
|
|
1890
|
+
...runner,
|
|
1891
|
+
last_seen_at: now.toISOString(),
|
|
1892
|
+
liveness_lease_expires_at: leaseExpiresAt.toISOString(),
|
|
1893
|
+
...input.wakeStreamOffset !== void 0 ? { wake_stream_offset: input.wakeStreamOffset } : {},
|
|
1894
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {}
|
|
1895
|
+
};
|
|
1896
|
+
}
|
|
1897
|
+
async getRunnerDiagnostics(runnerId) {
|
|
1898
|
+
const rows = await this.db.select().from(runnerRuntimeDiagnostics).where(and(eq(runnerRuntimeDiagnostics.tenantId, this.tenantId), eq(runnerRuntimeDiagnostics.runnerId, runnerId))).limit(1);
|
|
1899
|
+
return rows[0] ? this.rowToRunnerRuntimeDiagnostics(rows[0]) : null;
|
|
1871
1900
|
}
|
|
1872
1901
|
async setRunnerAdminStatus(runnerId, adminStatus) {
|
|
1873
1902
|
const rows = await this.db.update(runners).set({
|
|
@@ -1962,6 +1991,27 @@ var PostgresRegistry = class {
|
|
|
1962
1991
|
}).where(and(eq(entityDispatchState.tenantId, this.tenantId), eq(entityDispatchState.entityUrl, claim.entity_url), eq(entityDispatchState.activeConsumerId, input.consumerId), eq(entityDispatchState.activeEpoch, input.epoch)));
|
|
1963
1992
|
return claim;
|
|
1964
1993
|
}
|
|
1994
|
+
async getActiveClaimsForRunner(runnerId) {
|
|
1995
|
+
const rows = await this.db.select().from(consumerClaims).where(and(eq(consumerClaims.tenantId, this.tenantId), eq(consumerClaims.runnerId, runnerId), eq(consumerClaims.status, `active`)));
|
|
1996
|
+
return rows.map((row) => this.rowToConsumerClaim(row));
|
|
1997
|
+
}
|
|
1998
|
+
async getDispatchStatsForRunner(runnerId) {
|
|
1999
|
+
const rows = await this.db.select().from(entityDispatchState).where(and(eq(entityDispatchState.tenantId, this.tenantId), eq(entityDispatchState.activeRunnerId, runnerId)));
|
|
2000
|
+
let activeClaim = 0;
|
|
2001
|
+
let outstandingWake = 0;
|
|
2002
|
+
let pendingWork = 0;
|
|
2003
|
+
for (const row of rows) {
|
|
2004
|
+
if (row.activeConsumerId) activeClaim++;
|
|
2005
|
+
if (row.outstandingWakeId && !row.activeConsumerId) outstandingWake++;
|
|
2006
|
+
const pending = row.pendingSourceStreams;
|
|
2007
|
+
if (pending && pending.length > 0) pendingWork++;
|
|
2008
|
+
}
|
|
2009
|
+
return {
|
|
2010
|
+
entities_with_active_claim: activeClaim,
|
|
2011
|
+
entities_with_outstanding_wake: outstandingWake,
|
|
2012
|
+
entities_with_pending_work: pendingWork
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
1965
2015
|
entityTypeWhere(name) {
|
|
1966
2016
|
return and(eq(entityTypes.tenantId, this.tenantId), eq(entityTypes.name, name));
|
|
1967
2017
|
}
|
|
@@ -2427,23 +2477,28 @@ var PostgresRegistry = class {
|
|
|
2427
2477
|
};
|
|
2428
2478
|
}
|
|
2429
2479
|
rowToRunner(row) {
|
|
2430
|
-
const now = Date.now();
|
|
2431
|
-
const livenessExpiry = row.livenessLeaseExpiresAt?.getTime();
|
|
2432
2480
|
return {
|
|
2433
2481
|
id: row.id,
|
|
2434
|
-
|
|
2482
|
+
owner_principal: row.ownerPrincipal,
|
|
2435
2483
|
label: row.label,
|
|
2436
2484
|
kind: assertRunnerKind(row.kind),
|
|
2437
2485
|
admin_status: assertRunnerAdminStatus(row.adminStatus),
|
|
2438
|
-
liveness: livenessExpiry !== void 0 && livenessExpiry > now ? `online` : `offline`,
|
|
2439
|
-
last_seen_at: row.lastSeenAt?.toISOString(),
|
|
2440
|
-
liveness_lease_expires_at: row.livenessLeaseExpiresAt?.toISOString(),
|
|
2441
2486
|
wake_stream: row.wakeStream,
|
|
2442
|
-
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
2443
2487
|
created_at: row.createdAt.toISOString(),
|
|
2444
2488
|
updated_at: row.updatedAt.toISOString()
|
|
2445
2489
|
};
|
|
2446
2490
|
}
|
|
2491
|
+
rowToRunnerRuntimeDiagnostics(row) {
|
|
2492
|
+
return {
|
|
2493
|
+
runner_id: row.runnerId,
|
|
2494
|
+
owner_principal: row.ownerPrincipal,
|
|
2495
|
+
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
2496
|
+
last_seen_at: row.lastSeenAt.toISOString(),
|
|
2497
|
+
liveness_lease_expires_at: row.livenessLeaseExpiresAt.toISOString(),
|
|
2498
|
+
diagnostics: row.diagnostics ?? void 0,
|
|
2499
|
+
updated_at: row.updatedAt.toISOString()
|
|
2500
|
+
};
|
|
2501
|
+
}
|
|
2447
2502
|
rowToConsumerClaim(row) {
|
|
2448
2503
|
return {
|
|
2449
2504
|
consumer_id: row.consumerId,
|
|
@@ -3982,6 +4037,7 @@ var ElectricAgentsError = class extends Error {
|
|
|
3982
4037
|
|
|
3983
4038
|
//#endregion
|
|
3984
4039
|
//#region src/routing/dispatch-policy.ts
|
|
4040
|
+
const linkedDispatchSubscriptions = new WeakMap();
|
|
3985
4041
|
function subscriptionIdForDispatchTarget(target) {
|
|
3986
4042
|
if (target.subscription_id) return target.subscription_id;
|
|
3987
4043
|
if (target.type === `runner`) return `runner:${target.runnerId}`;
|
|
@@ -3990,7 +4046,7 @@ function subscriptionIdForDispatchTarget(target) {
|
|
|
3990
4046
|
}
|
|
3991
4047
|
function subscriptionIdForEntityDispatchTarget(target, entityUrl) {
|
|
3992
4048
|
const base = subscriptionIdForDispatchTarget(target);
|
|
3993
|
-
if (!target.subscription_id) return base;
|
|
4049
|
+
if (!target.subscription_id && target.type !== `runner`) return base;
|
|
3994
4050
|
const digest = createHash(`sha256`).update(entityUrl).digest(`hex`);
|
|
3995
4051
|
return `${base}:${digest.slice(0, 16)}`;
|
|
3996
4052
|
}
|
|
@@ -4034,24 +4090,74 @@ function sameDispatchDestination(a, b) {
|
|
|
4034
4090
|
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url;
|
|
4035
4091
|
return false;
|
|
4036
4092
|
}
|
|
4093
|
+
function subscriptionHasStream(ctx, existing, streamPath) {
|
|
4094
|
+
const normalizedStream = streamPath.replace(/^\/+/, ``);
|
|
4095
|
+
const backendStream = `${ctx.service}/${normalizedStream}`;
|
|
4096
|
+
return existing.streams?.some((stream) => {
|
|
4097
|
+
const path$1 = typeof stream === `string` ? stream : stream.path;
|
|
4098
|
+
if (!path$1) return false;
|
|
4099
|
+
const normalized = path$1.replace(/^\/+/, ``);
|
|
4100
|
+
return normalized === normalizedStream || normalized === backendStream;
|
|
4101
|
+
}) ?? false;
|
|
4102
|
+
}
|
|
4103
|
+
function dispatchLinkCacheKey(ctx, subscriptionId, streamPath) {
|
|
4104
|
+
return `${ctx.service}:${subscriptionId}:${streamPath}`;
|
|
4105
|
+
}
|
|
4106
|
+
function getDispatchLinkCache(ctx) {
|
|
4107
|
+
let cache = linkedDispatchSubscriptions.get(ctx.streamClient);
|
|
4108
|
+
if (!cache) {
|
|
4109
|
+
cache = new Set();
|
|
4110
|
+
linkedDispatchSubscriptions.set(ctx.streamClient, cache);
|
|
4111
|
+
}
|
|
4112
|
+
return cache;
|
|
4113
|
+
}
|
|
4114
|
+
function isSubscriptionAlreadyExistsError(err) {
|
|
4115
|
+
if (!(err instanceof DurableStreamsSubscriptionError)) return false;
|
|
4116
|
+
if (err.status === 409) return true;
|
|
4117
|
+
return err.code === `SUBSCRIPTION_ALREADY_EXISTS` || err.code === `ALREADY_EXISTS` || /already exists/i.test(err.errorMessage ?? err.body ?? err.message);
|
|
4118
|
+
}
|
|
4119
|
+
async function ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, input, existing) {
|
|
4120
|
+
if (!existing) try {
|
|
4121
|
+
await ctx.streamClient.putSubscription(subscriptionId, input);
|
|
4122
|
+
return;
|
|
4123
|
+
} catch (err) {
|
|
4124
|
+
if (!isSubscriptionAlreadyExistsError(err)) throw err;
|
|
4125
|
+
existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
4126
|
+
if (!existing) {
|
|
4127
|
+
serverLog.warn(`[dispatch-policy] subscription create raced with existing subscription but it could not be read`, {
|
|
4128
|
+
subscriptionId,
|
|
4129
|
+
stream: streamPath
|
|
4130
|
+
});
|
|
4131
|
+
return;
|
|
4132
|
+
}
|
|
4133
|
+
}
|
|
4134
|
+
if (!subscriptionHasStream(ctx, existing, streamPath)) await ctx.streamClient.addSubscriptionStreams(subscriptionId, [streamPath]);
|
|
4135
|
+
}
|
|
4037
4136
|
async function assertDispatchPolicyAllowed(ctx, policy) {
|
|
4038
4137
|
const target = policy?.targets[0];
|
|
4039
4138
|
if (!target || target.type !== `runner`) return;
|
|
4139
|
+
if (!ctx.principal) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires an authenticated owner`, 401);
|
|
4040
4140
|
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
4041
4141
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
4042
|
-
if (
|
|
4142
|
+
if (runner.owner_principal !== ctx.principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
4043
4143
|
}
|
|
4044
4144
|
async function linkEntityDispatchSubscription(ctx, entity) {
|
|
4045
4145
|
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
4046
4146
|
const target = dispatchPolicy?.targets[0];
|
|
4047
4147
|
if (!target) return;
|
|
4048
|
-
|
|
4148
|
+
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
4149
|
+
const cacheKey = dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main);
|
|
4150
|
+
const cache = getDispatchLinkCache(ctx);
|
|
4151
|
+
if (cache.has(cacheKey)) return;
|
|
4152
|
+
await linkStreamToTargetSubscription(ctx, target, entity, subscriptionId);
|
|
4153
|
+
cache.add(cacheKey);
|
|
4049
4154
|
}
|
|
4050
4155
|
async function unlinkEntityDispatchSubscription(ctx, entity) {
|
|
4051
4156
|
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
4052
4157
|
const target = dispatchPolicy?.targets[0];
|
|
4053
4158
|
if (!target) return;
|
|
4054
4159
|
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
4160
|
+
getDispatchLinkCache(ctx).delete(dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main));
|
|
4055
4161
|
await ctx.streamClient.removeSubscriptionStream(subscriptionId, entity.streams.main).catch((err) => {
|
|
4056
4162
|
serverLog.warn(`[dispatch-policy] failed to remove stream from subscription`, {
|
|
4057
4163
|
subscriptionId,
|
|
@@ -4059,37 +4165,32 @@ async function unlinkEntityDispatchSubscription(ctx, entity) {
|
|
|
4059
4165
|
}, err);
|
|
4060
4166
|
});
|
|
4061
4167
|
}
|
|
4062
|
-
async function linkStreamToTargetSubscription(ctx, target, entity) {
|
|
4168
|
+
async function linkStreamToTargetSubscription(ctx, target, entity, subscriptionId) {
|
|
4063
4169
|
const streamPath = entity.streams.main;
|
|
4064
|
-
|
|
4170
|
+
await ctx.streamClient.ensure(streamPath, { contentType: `application/json` });
|
|
4065
4171
|
const existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
4066
4172
|
if (target.type === `runner`) {
|
|
4067
4173
|
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
4068
4174
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
4069
4175
|
const wakeStream = runner.wake_stream || runnerWakeStream(target.runnerId);
|
|
4070
4176
|
await ctx.streamClient.ensure(wakeStream, { contentType: `application/json` });
|
|
4071
|
-
|
|
4072
|
-
|
|
4073
|
-
|
|
4074
|
-
|
|
4075
|
-
|
|
4076
|
-
|
|
4077
|
-
});
|
|
4078
|
-
return;
|
|
4079
|
-
}
|
|
4080
|
-
await ctx.streamClient.addSubscriptionStreams(subscriptionId, [streamPath]);
|
|
4177
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
4178
|
+
type: `pull-wake`,
|
|
4179
|
+
streams: [streamPath],
|
|
4180
|
+
wake_stream: wakeStream,
|
|
4181
|
+
description: `Electric Agents runner ${target.runnerId}`
|
|
4182
|
+
}, existing);
|
|
4081
4183
|
return;
|
|
4082
4184
|
}
|
|
4083
4185
|
const webhookUrl = rewriteLoopbackWebhookUrl(target.url);
|
|
4084
4186
|
if (!webhookUrl) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Webhook dispatch target must include a valid URL`, 400);
|
|
4085
4187
|
const forwardUrl = appendPathToUrl(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
4086
|
-
|
|
4188
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
4087
4189
|
type: `webhook`,
|
|
4088
4190
|
streams: [streamPath],
|
|
4089
4191
|
webhook: { url: forwardUrl },
|
|
4090
4192
|
description: `Electric Agents webhook ${subscriptionId}`
|
|
4091
|
-
});
|
|
4092
|
-
else await ctx.streamClient.addSubscriptionStreams(subscriptionId, [streamPath]);
|
|
4193
|
+
}, existing);
|
|
4093
4194
|
await ctx.pgDb.insert(subscriptionWebhooks).values({
|
|
4094
4195
|
tenantId: ctx.service,
|
|
4095
4196
|
subscriptionId,
|
|
@@ -4341,10 +4442,8 @@ async function sendEntity(request, ctx) {
|
|
|
4341
4442
|
if (parsed.from !== void 0 && parsed.from !== principal.url) return apiError(400, ErrCodeInvalidRequest, `Request from must match Electric-Principal`);
|
|
4342
4443
|
await ctx.entityManager.ensurePrincipal(principal);
|
|
4343
4444
|
const { entityUrl, entity } = requireExistingEntityRoute(request);
|
|
4344
|
-
|
|
4345
|
-
|
|
4346
|
-
await linkEntityDispatchSubscription(ctx, updatedEntity);
|
|
4347
|
-
}
|
|
4445
|
+
const dispatchEntity = entity.dispatch_policy ? entity : await backfillEntityDispatchPolicy(ctx, entity);
|
|
4446
|
+
await linkEntityDispatchSubscription(ctx, dispatchEntity);
|
|
4348
4447
|
if (parsed.afterMs && parsed.afterMs > 0) await ctx.entityManager.enqueueDelayedSend(entityUrl, {
|
|
4349
4448
|
from: principal.url,
|
|
4350
4449
|
payload: parsed.payload,
|
|
@@ -4561,7 +4660,13 @@ function applyCors(response) {
|
|
|
4561
4660
|
const headers = new Headers(response.headers);
|
|
4562
4661
|
headers.set(`access-control-allow-origin`, `*`);
|
|
4563
4662
|
headers.set(`access-control-allow-methods`, `GET, POST, PUT, PATCH, DELETE, OPTIONS`);
|
|
4564
|
-
headers.set(`access-control-allow-headers`,
|
|
4663
|
+
headers.set(`access-control-allow-headers`, [
|
|
4664
|
+
`content-type`,
|
|
4665
|
+
`authorization`,
|
|
4666
|
+
`electric-claim-token`,
|
|
4667
|
+
ELECTRIC_PRINCIPAL_HEADER,
|
|
4668
|
+
`ngrok-skip-browser-warning`
|
|
4669
|
+
].join(`, `));
|
|
4565
4670
|
headers.set(`access-control-expose-headers`, `*`);
|
|
4566
4671
|
return new Response(response.body, {
|
|
4567
4672
|
status: response.status,
|
|
@@ -4606,7 +4711,7 @@ function withLeadingSlash(path$1) {
|
|
|
4606
4711
|
//#region src/routing/runners-router.ts
|
|
4607
4712
|
const registerRunnerBodySchema = Type.Object({
|
|
4608
4713
|
id: Type.String(),
|
|
4609
|
-
|
|
4714
|
+
owner_principal: Type.Optional(Type.String()),
|
|
4610
4715
|
label: Type.String(),
|
|
4611
4716
|
kind: Type.Optional(Type.Union([
|
|
4612
4717
|
Type.Literal(`local`),
|
|
@@ -4622,7 +4727,8 @@ const heartbeatBodySchema = Type.Object({
|
|
|
4622
4727
|
lease_ms: Type.Optional(Type.Number()),
|
|
4623
4728
|
wake_stream_offset: Type.Optional(Type.String()),
|
|
4624
4729
|
wakeStreamOffset: Type.Optional(Type.String()),
|
|
4625
|
-
liveness_lease_expires_at: Type.Optional(Type.String())
|
|
4730
|
+
liveness_lease_expires_at: Type.Optional(Type.String()),
|
|
4731
|
+
diagnostics: Type.Optional(Type.Record(Type.String(), Type.Unknown()))
|
|
4626
4732
|
});
|
|
4627
4733
|
const claimBodySchema = Type.Object({
|
|
4628
4734
|
subscription_id: Type.Optional(Type.String()),
|
|
@@ -4630,9 +4736,39 @@ const claimBodySchema = Type.Object({
|
|
|
4630
4736
|
generation: Type.Optional(Type.Number()),
|
|
4631
4737
|
ts: Type.Optional(Type.Union([Type.String(), Type.Number()]))
|
|
4632
4738
|
}, { additionalProperties: true });
|
|
4739
|
+
const runnerClientStatuses = new Set([
|
|
4740
|
+
`stopped`,
|
|
4741
|
+
`starting`,
|
|
4742
|
+
`connecting`,
|
|
4743
|
+
`streaming`,
|
|
4744
|
+
`reconnecting`,
|
|
4745
|
+
`stopping`
|
|
4746
|
+
]);
|
|
4747
|
+
const runnerLastClaimResults = new Set([
|
|
4748
|
+
`claimed`,
|
|
4749
|
+
`no_work`,
|
|
4750
|
+
`error`
|
|
4751
|
+
]);
|
|
4752
|
+
const runnerStringOrNullDiagnostics = [
|
|
4753
|
+
`started_at`,
|
|
4754
|
+
`stream_connected_since`,
|
|
4755
|
+
`last_error`,
|
|
4756
|
+
`last_error_at`,
|
|
4757
|
+
`last_heartbeat_at`,
|
|
4758
|
+
`last_claim_at`,
|
|
4759
|
+
`last_dispatch_at`
|
|
4760
|
+
];
|
|
4761
|
+
const runnerNumberDiagnostics = [
|
|
4762
|
+
`reconnect_count`,
|
|
4763
|
+
`events_received`,
|
|
4764
|
+
`claims_succeeded`,
|
|
4765
|
+
`claims_skipped`,
|
|
4766
|
+
`claims_failed`
|
|
4767
|
+
];
|
|
4633
4768
|
const runnersRouter = Router({ base: `/_electric/runners` });
|
|
4634
4769
|
runnersRouter.post(`/`, withSchema(registerRunnerBodySchema), registerRunner);
|
|
4635
4770
|
runnersRouter.get(`/`, listRunners);
|
|
4771
|
+
runnersRouter.get(`/:id/health`, runnerHealth);
|
|
4636
4772
|
runnersRouter.get(`/:id`, getRunner);
|
|
4637
4773
|
runnersRouter.post(`/:id/heartbeat`, withSchema(heartbeatBodySchema), heartbeat);
|
|
4638
4774
|
runnersRouter.post(`/:id/enable`, setEnabled);
|
|
@@ -4645,14 +4781,41 @@ function routeParam$1(request, name) {
|
|
|
4645
4781
|
function firstQueryValue(value) {
|
|
4646
4782
|
return Array.isArray(value) ? value[0] : value;
|
|
4647
4783
|
}
|
|
4784
|
+
function requireAuthenticatedPrincipal(ctx) {
|
|
4785
|
+
if (ctx.principal) return ctx.principal;
|
|
4786
|
+
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner route requires an authenticated principal`, 401);
|
|
4787
|
+
}
|
|
4788
|
+
function canonicalOwnerPrincipal(input) {
|
|
4789
|
+
return parsePrincipalUrl(input)?.url ?? null;
|
|
4790
|
+
}
|
|
4791
|
+
function sanitizeRunnerDiagnostics(diagnostics) {
|
|
4792
|
+
if (!diagnostics) return void 0;
|
|
4793
|
+
const sanitized = {};
|
|
4794
|
+
if (typeof diagnostics.status === `string` && runnerClientStatuses.has(diagnostics.status)) sanitized.status = diagnostics.status;
|
|
4795
|
+
if (typeof diagnostics.stream_connected === `boolean`) sanitized.stream_connected = diagnostics.stream_connected;
|
|
4796
|
+
if (typeof diagnostics.last_heartbeat_ok === `boolean`) sanitized.last_heartbeat_ok = diagnostics.last_heartbeat_ok;
|
|
4797
|
+
if (diagnostics.last_claim_result === null || typeof diagnostics.last_claim_result === `string` && runnerLastClaimResults.has(diagnostics.last_claim_result)) sanitized.last_claim_result = diagnostics.last_claim_result;
|
|
4798
|
+
for (const key of runnerStringOrNullDiagnostics) {
|
|
4799
|
+
const value = diagnostics[key];
|
|
4800
|
+
if (typeof value === `string` || value === null) sanitized[key] = value;
|
|
4801
|
+
}
|
|
4802
|
+
for (const key of runnerNumberDiagnostics) {
|
|
4803
|
+
const value = diagnostics[key];
|
|
4804
|
+
if (typeof value === `number` && Number.isFinite(value) && value >= 0) sanitized[key] = value;
|
|
4805
|
+
}
|
|
4806
|
+
return Object.keys(sanitized).length > 0 ? sanitized : void 0;
|
|
4807
|
+
}
|
|
4648
4808
|
async function registerRunner(request, ctx) {
|
|
4649
4809
|
const parsed = routeBody(request);
|
|
4650
|
-
const
|
|
4651
|
-
|
|
4652
|
-
if (
|
|
4810
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
4811
|
+
const ownerPrincipal = parsed.owner_principal ?? principal.url;
|
|
4812
|
+
if (!ownerPrincipal) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal is required when no authenticated principal is present`, 400);
|
|
4813
|
+
const canonicalOwner = canonicalOwnerPrincipal(ownerPrincipal);
|
|
4814
|
+
if (!canonicalOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${ownerPrincipal}`, 400);
|
|
4815
|
+
if (canonicalOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
4653
4816
|
const runner = await ctx.entityManager.registry.createRunner({
|
|
4654
4817
|
id: parsed.id,
|
|
4655
|
-
|
|
4818
|
+
ownerPrincipal: canonicalOwner,
|
|
4656
4819
|
label: parsed.label,
|
|
4657
4820
|
kind: parsed.kind,
|
|
4658
4821
|
adminStatus: parsed.admin_status,
|
|
@@ -4662,26 +4825,32 @@ async function registerRunner(request, ctx) {
|
|
|
4662
4825
|
return json(runner, { status: 201 });
|
|
4663
4826
|
}
|
|
4664
4827
|
async function listRunners(request, ctx) {
|
|
4665
|
-
const
|
|
4666
|
-
|
|
4667
|
-
const
|
|
4828
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
4829
|
+
const requestedOwner = firstQueryValue(request.query.owner_principal);
|
|
4830
|
+
const canonicalRequestedOwner = requestedOwner ? canonicalOwnerPrincipal(requestedOwner) : void 0;
|
|
4831
|
+
if (requestedOwner && !canonicalRequestedOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${requestedOwner}`, 400);
|
|
4832
|
+
if (canonicalRequestedOwner && canonicalRequestedOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
4833
|
+
const runners$1 = await ctx.entityManager.registry.listRunners({ ownerPrincipal: principal.url });
|
|
4668
4834
|
return json(runners$1);
|
|
4669
4835
|
}
|
|
4670
4836
|
async function getRunner(request, ctx) {
|
|
4671
4837
|
const runner = await requireRunner(ctx, routeParam$1(request, `id`));
|
|
4672
|
-
assertRunnerOwnerIfAuthenticated(ctx, runner.
|
|
4838
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
4673
4839
|
return json(runner);
|
|
4674
4840
|
}
|
|
4675
4841
|
async function heartbeat(request, ctx) {
|
|
4676
4842
|
const runnerId = routeParam$1(request, `id`);
|
|
4843
|
+
requireAuthenticatedPrincipal(ctx);
|
|
4677
4844
|
const existing = await requireRunner(ctx, runnerId);
|
|
4678
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
4845
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
4679
4846
|
const parsed = routeBody(request);
|
|
4680
4847
|
const runner = await ctx.entityManager.registry.heartbeatRunner({
|
|
4681
4848
|
runnerId,
|
|
4849
|
+
ownerPrincipal: existing.owner_principal,
|
|
4682
4850
|
leaseMs: parsed.lease_ms,
|
|
4683
4851
|
wakeStreamOffset: parsed.wake_stream_offset ?? parsed.wakeStreamOffset,
|
|
4684
|
-
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0
|
|
4852
|
+
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0,
|
|
4853
|
+
diagnostics: sanitizeRunnerDiagnostics(parsed.diagnostics)
|
|
4685
4854
|
});
|
|
4686
4855
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
4687
4856
|
return json(runner);
|
|
@@ -4694,16 +4863,18 @@ async function setDisabled(request, ctx) {
|
|
|
4694
4863
|
}
|
|
4695
4864
|
async function setRunnerStatus(request, ctx, adminStatus) {
|
|
4696
4865
|
const runnerId = routeParam$1(request, `id`);
|
|
4866
|
+
requireAuthenticatedPrincipal(ctx);
|
|
4697
4867
|
const existing = await requireRunner(ctx, runnerId);
|
|
4698
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
4868
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
4699
4869
|
const runner = await ctx.entityManager.registry.setRunnerAdminStatus(runnerId, adminStatus);
|
|
4700
4870
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
4701
4871
|
return json(runner);
|
|
4702
4872
|
}
|
|
4703
4873
|
async function claimWake(request, ctx) {
|
|
4704
4874
|
const runnerId = routeParam$1(request, `id`);
|
|
4875
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
4705
4876
|
const runner = await requireRunner(ctx, runnerId);
|
|
4706
|
-
if (
|
|
4877
|
+
if (runner.owner_principal !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner claim requires the authenticated owner`, 403);
|
|
4707
4878
|
if (runner.admin_status !== `enabled`) throw new ElectricAgentsError(ErrCodeNotRunning, `Runner is disabled`, 409);
|
|
4708
4879
|
const parsed = routeBody(request);
|
|
4709
4880
|
const expectedSubscriptionId = subscriptionIdForDispatchTarget({
|
|
@@ -4734,11 +4905,95 @@ async function requireRunner(ctx, runnerId) {
|
|
|
4734
4905
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
4735
4906
|
return runner;
|
|
4736
4907
|
}
|
|
4737
|
-
function assertRunnerOwnerIfAuthenticated(ctx,
|
|
4738
|
-
|
|
4739
|
-
if (
|
|
4908
|
+
function assertRunnerOwnerIfAuthenticated(ctx, ownerPrincipal) {
|
|
4909
|
+
requireAuthenticatedPrincipal(ctx);
|
|
4910
|
+
if (ownerPrincipal === ctx.principal.url) return;
|
|
4740
4911
|
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner access requires the authenticated owner`, 403);
|
|
4741
4912
|
}
|
|
4913
|
+
async function runnerHealth(request, ctx) {
|
|
4914
|
+
const runnerId = routeParam$1(request, `id`);
|
|
4915
|
+
const runner = await requireRunner(ctx, runnerId);
|
|
4916
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
4917
|
+
const runtimeDiagnostics = await ctx.entityManager.registry.getRunnerDiagnostics(runnerId);
|
|
4918
|
+
const now = Date.now();
|
|
4919
|
+
const parsedLeaseExpiresAt = runtimeDiagnostics?.liveness_lease_expires_at ? new Date(runtimeDiagnostics.liveness_lease_expires_at).getTime() : null;
|
|
4920
|
+
const leaseExpiresAt = parsedLeaseExpiresAt !== null && Number.isFinite(parsedLeaseExpiresAt) ? parsedLeaseExpiresAt : null;
|
|
4921
|
+
let livenessStatus;
|
|
4922
|
+
if (runner.admin_status === `disabled`) livenessStatus = `offline`;
|
|
4923
|
+
else if (leaseExpiresAt !== null && leaseExpiresAt > now) livenessStatus = `online`;
|
|
4924
|
+
else if (leaseExpiresAt !== null) livenessStatus = `expired`;
|
|
4925
|
+
else livenessStatus = `offline`;
|
|
4926
|
+
const [activeClaims, dispatchStats] = await Promise.all([ctx.entityManager.registry.getActiveClaimsForRunner(runnerId), ctx.entityManager.registry.getDispatchStatsForRunner(runnerId)]);
|
|
4927
|
+
const clientDiagnostics = sanitizeRunnerDiagnostics(runtimeDiagnostics?.diagnostics) ?? null;
|
|
4928
|
+
const issues = [];
|
|
4929
|
+
let healthStatus = `healthy`;
|
|
4930
|
+
const escalate = (floor) => {
|
|
4931
|
+
if (floor === `unhealthy`) healthStatus = `unhealthy`;
|
|
4932
|
+
else if (healthStatus === `healthy`) healthStatus = `degraded`;
|
|
4933
|
+
};
|
|
4934
|
+
if (runner.admin_status === `disabled`) {
|
|
4935
|
+
escalate(`unhealthy`);
|
|
4936
|
+
issues.push(`Runner is disabled`);
|
|
4937
|
+
}
|
|
4938
|
+
if (livenessStatus === `expired`) {
|
|
4939
|
+
escalate(`unhealthy`);
|
|
4940
|
+
const ago = leaseExpiresAt ? Math.round((now - leaseExpiresAt) / 1e3) : 0;
|
|
4941
|
+
issues.push(`Heartbeat lease expired ${ago}s ago`);
|
|
4942
|
+
}
|
|
4943
|
+
if (livenessStatus === `offline` && runner.admin_status === `enabled`) {
|
|
4944
|
+
escalate(`degraded`);
|
|
4945
|
+
issues.push(`Runner has never sent a heartbeat`);
|
|
4946
|
+
}
|
|
4947
|
+
if (clientDiagnostics) {
|
|
4948
|
+
if (clientDiagnostics.stream_connected === false) {
|
|
4949
|
+
escalate(`degraded`);
|
|
4950
|
+
issues.push(`Client reports stream disconnected`);
|
|
4951
|
+
}
|
|
4952
|
+
if (clientDiagnostics.last_heartbeat_ok === false) {
|
|
4953
|
+
escalate(`degraded`);
|
|
4954
|
+
issues.push(`Client reports last heartbeat failed`);
|
|
4955
|
+
}
|
|
4956
|
+
if (typeof clientDiagnostics.reconnect_count === `number` && clientDiagnostics.reconnect_count > 5) {
|
|
4957
|
+
escalate(`degraded`);
|
|
4958
|
+
issues.push(`Client has reconnected ${clientDiagnostics.reconnect_count} times`);
|
|
4959
|
+
}
|
|
4960
|
+
} else if (runtimeDiagnostics?.last_seen_at) {
|
|
4961
|
+
escalate(`degraded`);
|
|
4962
|
+
issues.push(`No client diagnostics available`);
|
|
4963
|
+
}
|
|
4964
|
+
const body = {
|
|
4965
|
+
runner: {
|
|
4966
|
+
id: runner.id,
|
|
4967
|
+
admin_status: runner.admin_status,
|
|
4968
|
+
liveness_status: livenessStatus,
|
|
4969
|
+
lease_expires_at: leaseExpiresAt !== null ? runtimeDiagnostics?.liveness_lease_expires_at ?? null : null,
|
|
4970
|
+
lease_remaining_ms: leaseExpiresAt !== null ? Math.max(0, leaseExpiresAt - now) : null,
|
|
4971
|
+
wake_stream: runner.wake_stream,
|
|
4972
|
+
wake_stream_offset: runtimeDiagnostics?.wake_stream_offset ?? null,
|
|
4973
|
+
last_seen_at: runtimeDiagnostics?.last_seen_at ?? null,
|
|
4974
|
+
created_at: runner.created_at
|
|
4975
|
+
},
|
|
4976
|
+
client: clientDiagnostics,
|
|
4977
|
+
claims: {
|
|
4978
|
+
active_count: activeClaims.length,
|
|
4979
|
+
active: activeClaims.map((c) => ({
|
|
4980
|
+
consumer_id: c.consumer_id,
|
|
4981
|
+
epoch: c.epoch,
|
|
4982
|
+
entity_url: c.entity_url,
|
|
4983
|
+
stream_path: c.stream_path,
|
|
4984
|
+
claimed_at: c.claimed_at,
|
|
4985
|
+
last_heartbeat_at: c.last_heartbeat_at ?? null,
|
|
4986
|
+
lease_expires_at: c.lease_expires_at ?? null
|
|
4987
|
+
}))
|
|
4988
|
+
},
|
|
4989
|
+
dispatch: dispatchStats,
|
|
4990
|
+
health: {
|
|
4991
|
+
status: healthStatus,
|
|
4992
|
+
issues
|
|
4993
|
+
}
|
|
4994
|
+
};
|
|
4995
|
+
return json(body);
|
|
4996
|
+
}
|
|
4742
4997
|
async function notificationFromClaim(ctx, input) {
|
|
4743
4998
|
const primary = input.claim.streams.find((stream) => stream.has_pending === true) ?? input.claim.streams[0];
|
|
4744
4999
|
if (!primary?.path) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Claim response did not include a stream`, 502);
|
|
@@ -6223,7 +6478,7 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
6223
6478
|
this.service = this.serviceId;
|
|
6224
6479
|
this.db = options.db;
|
|
6225
6480
|
if (options.streamClient) this.streamClient = options.streamClient;
|
|
6226
|
-
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(
|
|
6481
|
+
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer });
|
|
6227
6482
|
else throw new Error(`Either durableStreamsUrl or streamClient is required`);
|
|
6228
6483
|
this.registry = options.registry ?? new PostgresRegistry(this.db, this.serviceId);
|
|
6229
6484
|
this.wakeRegistry = options.wakeRegistry;
|
|
@@ -7141,7 +7396,7 @@ var WakeRegistry = class {
|
|
|
7141
7396
|
//#region src/standalone-runtime.ts
|
|
7142
7397
|
async function startStandaloneAgentsRuntime(options) {
|
|
7143
7398
|
const serviceId = options.service ?? options.tenantId ?? DEFAULT_TENANT_ID;
|
|
7144
|
-
const streamClient = options.streamClient ?? (options.durableStreamsUrl ? new StreamClient(
|
|
7399
|
+
const streamClient = options.streamClient ?? (options.durableStreamsUrl ? new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer }) : void 0);
|
|
7145
7400
|
if (!streamClient) throw new Error(`Either durableStreamsUrl or streamClient is required`);
|
|
7146
7401
|
const registry = new PostgresRegistry(options.db, serviceId);
|
|
7147
7402
|
const wakeRegistry = options.wakeRegistry ?? new WakeRegistry(options.db, serviceId);
|
|
@@ -7289,6 +7544,11 @@ function createMockAgentBootstrap(options) {
|
|
|
7289
7544
|
registry
|
|
7290
7545
|
};
|
|
7291
7546
|
}
|
|
7547
|
+
function durableStreamTestServerBackendUrl(origin) {
|
|
7548
|
+
const url = new URL(origin);
|
|
7549
|
+
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/v1/stream`;
|
|
7550
|
+
return url.toString().replace(/\/+$/, ``);
|
|
7551
|
+
}
|
|
7292
7552
|
var ElectricAgentsServer = class {
|
|
7293
7553
|
server;
|
|
7294
7554
|
electricAgentsManager;
|
|
@@ -7305,7 +7565,7 @@ var ElectricAgentsServer = class {
|
|
|
7305
7565
|
constructor(options) {
|
|
7306
7566
|
if (!options.durableStreamsUrl && !options.durableStreamsServer) throw new Error(`Either durableStreamsUrl or durableStreamsServer is required`);
|
|
7307
7567
|
this.options = options;
|
|
7308
|
-
this.streamClient = options.durableStreamsUrl ? new StreamClient(
|
|
7568
|
+
this.streamClient = options.durableStreamsUrl ? new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer }) : null;
|
|
7309
7569
|
}
|
|
7310
7570
|
get url() {
|
|
7311
7571
|
if (!this._url) throw new Error(`Server not started`);
|
|
@@ -7325,8 +7585,8 @@ var ElectricAgentsServer = class {
|
|
|
7325
7585
|
serverLog.info(`[agent-server] starting durable streams server...`);
|
|
7326
7586
|
const streamsUrl = await this.options.durableStreamsServer.start();
|
|
7327
7587
|
serverLog.info(`[agent-server] durable streams server started at ${streamsUrl}`);
|
|
7328
|
-
this.options.durableStreamsUrl = streamsUrl;
|
|
7329
|
-
this.streamClient = new StreamClient(
|
|
7588
|
+
this.options.durableStreamsUrl = durableStreamTestServerBackendUrl(streamsUrl);
|
|
7589
|
+
this.streamClient = new StreamClient(this.options.durableStreamsUrl, { bearer: this.options.durableStreamsBearer });
|
|
7330
7590
|
}
|
|
7331
7591
|
this.streamsAgent = new Agent({
|
|
7332
7592
|
keepAliveTimeout: 6e4,
|
|
@@ -7456,7 +7716,7 @@ var ElectricAgentsServer = class {
|
|
|
7456
7716
|
principal,
|
|
7457
7717
|
publicUrl: this.publicUrl,
|
|
7458
7718
|
localUrl: this._url,
|
|
7459
|
-
durableStreamsUrl: this.
|
|
7719
|
+
durableStreamsUrl: this.options.durableStreamsUrl,
|
|
7460
7720
|
durableStreamsBearer: this.options.durableStreamsBearer,
|
|
7461
7721
|
durableStreamsRouting: this.options.durableStreamsRouting,
|
|
7462
7722
|
durableStreamsDispatcher: this.streamsAgent,
|