@electric-ax/agents-server 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +529 -248
- package/dist/index.cjs +1603 -1332
- package/dist/index.d.cts +274 -162
- package/dist/index.d.ts +274 -162
- package/dist/index.js +1601 -1332
- package/drizzle/0007_runner_diagnostics_and_principal.sql +22 -0
- package/drizzle/0008_runner_runtime_diagnostics.sql +50 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +6 -6
- package/src/db/schema.ts +33 -10
- package/src/electric-agents-types.ts +49 -3
- package/src/entity-registry.ts +136 -26
- package/src/host.ts +4 -5
- package/src/index.ts +5 -1
- package/src/principal.ts +23 -11
- package/src/routing/context.ts +1 -0
- package/src/routing/dispatch-policy.ts +123 -20
- package/src/routing/durable-streams-router.ts +286 -116
- package/src/routing/durable-streams-routing-adapter.ts +31 -64
- package/src/routing/electric-proxy-router.ts +1 -0
- package/src/routing/entities-router.ts +5 -5
- package/src/routing/hooks.ts +8 -1
- package/src/routing/internal-router.ts +6 -2
- package/src/routing/runners-router.ts +257 -19
- package/src/runtime.ts +4 -5
- package/src/server.ts +21 -15
- package/src/standalone-runtime.ts +4 -5
- package/src/stream-client.ts +18 -69
- package/src/utils/server-utils.ts +27 -8
|
@@ -1,8 +1,3 @@
|
|
|
1
|
-
import {
|
|
2
|
-
prefixTenantStreamPath,
|
|
3
|
-
stripTenantStreamPrefix,
|
|
4
|
-
} from './tenant-stream-paths.js'
|
|
5
|
-
|
|
6
1
|
export interface DurableStreamsRoutingInput {
|
|
7
2
|
durableStreamsUrl: string
|
|
8
3
|
serviceId: string
|
|
@@ -11,86 +6,58 @@ export interface DurableStreamsRoutingInput {
|
|
|
11
6
|
|
|
12
7
|
export interface DurableStreamsRoutingAdapter {
|
|
13
8
|
streamUrl(input: DurableStreamsRoutingInput): URL
|
|
14
|
-
|
|
9
|
+
controlUrl(input: DurableStreamsRoutingInput): URL
|
|
15
10
|
toBackendStreamPath(serviceId: string, streamPath: string): string
|
|
16
11
|
toRuntimeStreamPath(serviceId: string, streamPath: string): string
|
|
17
12
|
}
|
|
18
13
|
|
|
19
14
|
function appendSearch(target: URL, source: URL): URL {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
function removeServiceQuery(target: URL): URL {
|
|
25
|
-
target.searchParams.delete(`service`)
|
|
26
|
-
return target
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function logicalStreamPathFromRequest(
|
|
30
|
-
requestUrl: string,
|
|
31
|
-
serviceId: string
|
|
32
|
-
): { incomingUrl: URL; streamPath: string } {
|
|
33
|
-
const incomingUrl = new URL(requestUrl, `http://localhost`)
|
|
34
|
-
const segments = incomingUrl.pathname.split(`/`).filter(Boolean)
|
|
35
|
-
if (segments[0] === `v1` && segments[1] === `stream`) {
|
|
36
|
-
return {
|
|
37
|
-
incomingUrl,
|
|
38
|
-
streamPath: segments.length > 2 ? `/${segments.slice(3).join(`/`)}` : `/`,
|
|
15
|
+
source.searchParams.forEach((value, key) => {
|
|
16
|
+
if (key !== `service`) {
|
|
17
|
+
target.searchParams.append(key, value)
|
|
39
18
|
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return {
|
|
43
|
-
incomingUrl,
|
|
44
|
-
streamPath: incomingUrl.pathname || `/${serviceId}`,
|
|
45
|
-
}
|
|
19
|
+
})
|
|
20
|
+
return target
|
|
46
21
|
}
|
|
47
22
|
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
backendStreamPath: string
|
|
51
|
-
): URL {
|
|
52
|
-
const path = backendStreamPath.replace(/^\/+/, ``)
|
|
53
|
-
const target = new URL(`/v1/stream/${path}`, input.durableStreamsUrl)
|
|
54
|
-
return target
|
|
23
|
+
function withoutTrailingSlash(pathname: string): string {
|
|
24
|
+
return pathname.replace(/\/+$/, ``) || `/`
|
|
55
25
|
}
|
|
56
26
|
|
|
57
|
-
function
|
|
27
|
+
function appendRequestPathToStreamRoot(input: DurableStreamsRoutingInput): URL {
|
|
58
28
|
const incomingUrl = new URL(input.requestUrl, `http://localhost`)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
)
|
|
64
|
-
)
|
|
29
|
+
const path = incomingUrl.pathname.replace(/^\/+/, ``)
|
|
30
|
+
const target = new URL(input.durableStreamsUrl)
|
|
31
|
+
target.pathname = path
|
|
32
|
+
? `${withoutTrailingSlash(target.pathname)}/${path}`
|
|
33
|
+
: withoutTrailingSlash(target.pathname)
|
|
34
|
+
return appendSearch(target, incomingUrl)
|
|
65
35
|
}
|
|
66
36
|
|
|
67
|
-
export const
|
|
37
|
+
export const streamRootDurableStreamsRoutingAdapter: DurableStreamsRoutingAdapter =
|
|
68
38
|
{
|
|
69
|
-
streamUrl
|
|
70
|
-
const { incomingUrl, streamPath } = logicalStreamPathFromRequest(
|
|
71
|
-
input.requestUrl,
|
|
72
|
-
input.serviceId
|
|
73
|
-
)
|
|
74
|
-
const target = backendStreamUrl(
|
|
75
|
-
input,
|
|
76
|
-
prefixTenantStreamPath(streamPath, input.serviceId)
|
|
77
|
-
)
|
|
78
|
-
return removeServiceQuery(appendSearch(target, incomingUrl))
|
|
79
|
-
},
|
|
39
|
+
streamUrl: appendRequestPathToStreamRoot,
|
|
80
40
|
|
|
81
|
-
|
|
41
|
+
controlUrl: appendRequestPathToStreamRoot,
|
|
82
42
|
|
|
83
|
-
toBackendStreamPath(
|
|
84
|
-
return
|
|
43
|
+
toBackendStreamPath(_serviceId, streamPath) {
|
|
44
|
+
return streamPath.replace(/^\/+/, ``)
|
|
85
45
|
},
|
|
86
46
|
|
|
87
|
-
toRuntimeStreamPath(
|
|
88
|
-
return
|
|
47
|
+
toRuntimeStreamPath(_serviceId, streamPath) {
|
|
48
|
+
return streamPath.replace(/^\/+/, ``)
|
|
89
49
|
},
|
|
90
50
|
}
|
|
91
51
|
|
|
52
|
+
export const pathPrefixedSingleTenantDurableStreamsRoutingAdapter =
|
|
53
|
+
streamRootDurableStreamsRoutingAdapter
|
|
54
|
+
|
|
55
|
+
export const tenantRootDurableStreamsRoutingAdapter =
|
|
56
|
+
streamRootDurableStreamsRoutingAdapter
|
|
57
|
+
|
|
92
58
|
export function resolveDurableStreamsRoutingAdapter(
|
|
93
|
-
adapter?: DurableStreamsRoutingAdapter
|
|
59
|
+
adapter?: DurableStreamsRoutingAdapter,
|
|
60
|
+
_durableStreamsUrl?: string
|
|
94
61
|
): DurableStreamsRoutingAdapter {
|
|
95
|
-
return adapter ??
|
|
62
|
+
return adapter ?? streamRootDurableStreamsRoutingAdapter
|
|
96
63
|
}
|
|
@@ -530,10 +530,10 @@ async function sendEntity(
|
|
|
530
530
|
await ctx.entityManager.ensurePrincipal(principal)
|
|
531
531
|
const { entityUrl, entity } = requireExistingEntityRoute(request)
|
|
532
532
|
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
await
|
|
536
|
-
|
|
533
|
+
const dispatchEntity = entity.dispatch_policy
|
|
534
|
+
? entity
|
|
535
|
+
: await backfillEntityDispatchPolicy(ctx, entity)
|
|
536
|
+
await linkEntityDispatchSubscription(ctx, dispatchEntity)
|
|
537
537
|
|
|
538
538
|
if (parsed.afterMs && parsed.afterMs > 0) {
|
|
539
539
|
await ctx.entityManager.enqueueDelayedSend(
|
|
@@ -614,13 +614,13 @@ async function spawnEntity(
|
|
|
614
614
|
wake: parsed.wake,
|
|
615
615
|
created_by: principal.url,
|
|
616
616
|
})
|
|
617
|
+
await linkEntityDispatchSubscription(ctx, entity)
|
|
617
618
|
if (parsed.initialMessage !== undefined) {
|
|
618
619
|
await ctx.entityManager.send(entity.url, {
|
|
619
620
|
from: principal.url,
|
|
620
621
|
payload: parsed.initialMessage,
|
|
621
622
|
})
|
|
622
623
|
}
|
|
623
|
-
await linkEntityDispatchSubscription(ctx, entity)
|
|
624
624
|
|
|
625
625
|
return json(
|
|
626
626
|
{ ...toPublicEntity(entity), txid: entity.txid },
|
package/src/routing/hooks.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { SpanKind, SpanStatusCode } from '@opentelemetry/api'
|
|
2
2
|
import { apiError } from '../electric-agents-http.js'
|
|
3
3
|
import { ElectricAgentsError } from '../entity-manager.js'
|
|
4
|
+
import { ELECTRIC_PRINCIPAL_HEADER } from '../principal.js'
|
|
4
5
|
import { ATTR, extractTraceContext, tracer } from '../tracing.js'
|
|
5
6
|
import { serverLog } from '../utils/log.js'
|
|
6
7
|
import type { Span } from '@opentelemetry/api'
|
|
@@ -80,7 +81,13 @@ export function applyCors(
|
|
|
80
81
|
)
|
|
81
82
|
headers.set(
|
|
82
83
|
`access-control-allow-headers`,
|
|
83
|
-
|
|
84
|
+
[
|
|
85
|
+
`content-type`,
|
|
86
|
+
`authorization`,
|
|
87
|
+
`electric-claim-token`,
|
|
88
|
+
ELECTRIC_PRINCIPAL_HEADER,
|
|
89
|
+
`ngrok-skip-browser-warning`,
|
|
90
|
+
].join(`, `)
|
|
84
91
|
)
|
|
85
92
|
headers.set(`access-control-expose-headers`, `*`)
|
|
86
93
|
return new Response(response.body, {
|
|
@@ -294,7 +294,8 @@ async function webhookForward(
|
|
|
294
294
|
const parsedBody = parsedBodyResult.value as WebhookForwardBody | undefined
|
|
295
295
|
const newWebhook = newWebhookPayload(parsedBody)
|
|
296
296
|
const routingAdapter = resolveDurableStreamsRoutingAdapter(
|
|
297
|
-
ctx.durableStreamsRouting
|
|
297
|
+
ctx.durableStreamsRouting,
|
|
298
|
+
ctx.durableStreamsUrl
|
|
298
299
|
)
|
|
299
300
|
|
|
300
301
|
if (parsedBody) {
|
|
@@ -537,7 +538,10 @@ async function callbackForward(
|
|
|
537
538
|
ctx.service,
|
|
538
539
|
consumerId,
|
|
539
540
|
requestBody,
|
|
540
|
-
resolveDurableStreamsRoutingAdapter(
|
|
541
|
+
resolveDurableStreamsRoutingAdapter(
|
|
542
|
+
ctx.durableStreamsRouting,
|
|
543
|
+
ctx.durableStreamsUrl
|
|
544
|
+
)
|
|
541
545
|
)
|
|
542
546
|
|
|
543
547
|
let upstream: Response
|
|
@@ -9,11 +9,12 @@ import {
|
|
|
9
9
|
ErrCodeNotFound,
|
|
10
10
|
ErrCodeNotRunning,
|
|
11
11
|
ErrCodeUnauthorized,
|
|
12
|
+
type RunnerHealthResponse,
|
|
12
13
|
} from '../electric-agents-types.js'
|
|
13
14
|
import { routeBody, withSchema } from './schema.js'
|
|
14
15
|
import { subscriptionIdForDispatchTarget } from './dispatch-policy.js'
|
|
15
16
|
import { withLeadingSlash } from './tenant-stream-paths.js'
|
|
16
|
-
import { principalFromCreatedBy } from '../principal.js'
|
|
17
|
+
import { parsePrincipalUrl, principalFromCreatedBy } from '../principal.js'
|
|
17
18
|
import type { JsonRouteRequest } from './schema.js'
|
|
18
19
|
import type { RouterType } from 'itty-router'
|
|
19
20
|
import type { TenantContext } from './context.js'
|
|
@@ -35,7 +36,7 @@ export type RunnersRoutes = RouterType<
|
|
|
35
36
|
|
|
36
37
|
const registerRunnerBodySchema = Type.Object({
|
|
37
38
|
id: Type.String(),
|
|
38
|
-
|
|
39
|
+
owner_principal: Type.Optional(Type.String()),
|
|
39
40
|
label: Type.String(),
|
|
40
41
|
kind: Type.Optional(
|
|
41
42
|
Type.Union([
|
|
@@ -57,6 +58,7 @@ const heartbeatBodySchema = Type.Object({
|
|
|
57
58
|
wake_stream_offset: Type.Optional(Type.String()),
|
|
58
59
|
wakeStreamOffset: Type.Optional(Type.String()),
|
|
59
60
|
liveness_lease_expires_at: Type.Optional(Type.String()),
|
|
61
|
+
diagnostics: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
|
|
60
62
|
})
|
|
61
63
|
|
|
62
64
|
const claimBodySchema = Type.Object(
|
|
@@ -72,6 +74,35 @@ const claimBodySchema = Type.Object(
|
|
|
72
74
|
type RegisterRunnerBody = Static<typeof registerRunnerBodySchema>
|
|
73
75
|
type HeartbeatBody = Static<typeof heartbeatBodySchema>
|
|
74
76
|
type ClaimBody = Static<typeof claimBodySchema>
|
|
77
|
+
type RunnerClientDiagnostics = NonNullable<RunnerHealthResponse[`client`]>
|
|
78
|
+
|
|
79
|
+
const runnerClientStatuses = new Set<RunnerClientDiagnostics[`status`]>([
|
|
80
|
+
`stopped`,
|
|
81
|
+
`starting`,
|
|
82
|
+
`connecting`,
|
|
83
|
+
`streaming`,
|
|
84
|
+
`reconnecting`,
|
|
85
|
+
`stopping`,
|
|
86
|
+
])
|
|
87
|
+
const runnerLastClaimResults = new Set<
|
|
88
|
+
NonNullable<RunnerClientDiagnostics[`last_claim_result`]>
|
|
89
|
+
>([`claimed`, `no_work`, `error`])
|
|
90
|
+
const runnerStringOrNullDiagnostics = [
|
|
91
|
+
`started_at`,
|
|
92
|
+
`stream_connected_since`,
|
|
93
|
+
`last_error`,
|
|
94
|
+
`last_error_at`,
|
|
95
|
+
`last_heartbeat_at`,
|
|
96
|
+
`last_claim_at`,
|
|
97
|
+
`last_dispatch_at`,
|
|
98
|
+
] as const
|
|
99
|
+
const runnerNumberDiagnostics = [
|
|
100
|
+
`reconnect_count`,
|
|
101
|
+
`events_received`,
|
|
102
|
+
`claims_succeeded`,
|
|
103
|
+
`claims_skipped`,
|
|
104
|
+
`claims_failed`,
|
|
105
|
+
] as const
|
|
75
106
|
|
|
76
107
|
export const runnersRouter: RunnersRoutes = Router<
|
|
77
108
|
RunnersRouteRequest,
|
|
@@ -83,6 +114,7 @@ export const runnersRouter: RunnersRoutes = Router<
|
|
|
83
114
|
|
|
84
115
|
runnersRouter.post(`/`, withSchema(registerRunnerBodySchema), registerRunner)
|
|
85
116
|
runnersRouter.get(`/`, listRunners)
|
|
117
|
+
runnersRouter.get(`/:id/health`, runnerHealth)
|
|
86
118
|
runnersRouter.get(`/:id`, getRunner)
|
|
87
119
|
runnersRouter.post(`/:id/heartbeat`, withSchema(heartbeatBodySchema), heartbeat)
|
|
88
120
|
runnersRouter.post(`/:id/enable`, setEnabled)
|
|
@@ -100,30 +132,104 @@ function firstQueryValue(
|
|
|
100
132
|
return Array.isArray(value) ? value[0] : value
|
|
101
133
|
}
|
|
102
134
|
|
|
135
|
+
function requireAuthenticatedPrincipal(
|
|
136
|
+
ctx: TenantContext
|
|
137
|
+
): NonNullable<TenantContext[`principal`]> {
|
|
138
|
+
if (ctx.principal) return ctx.principal
|
|
139
|
+
throw new ElectricAgentsError(
|
|
140
|
+
ErrCodeUnauthorized,
|
|
141
|
+
`Runner route requires an authenticated principal`,
|
|
142
|
+
401
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function canonicalOwnerPrincipal(input: string): string | null {
|
|
147
|
+
return parsePrincipalUrl(input)?.url ?? null
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function sanitizeRunnerDiagnostics(
|
|
151
|
+
diagnostics: Record<string, unknown> | null | undefined
|
|
152
|
+
): RunnerClientDiagnostics | undefined {
|
|
153
|
+
if (!diagnostics) return undefined
|
|
154
|
+
const sanitized: Record<string, unknown> = {}
|
|
155
|
+
|
|
156
|
+
if (
|
|
157
|
+
typeof diagnostics.status === `string` &&
|
|
158
|
+
runnerClientStatuses.has(
|
|
159
|
+
diagnostics.status as RunnerClientDiagnostics[`status`]
|
|
160
|
+
)
|
|
161
|
+
) {
|
|
162
|
+
sanitized.status = diagnostics.status
|
|
163
|
+
}
|
|
164
|
+
if (typeof diagnostics.stream_connected === `boolean`) {
|
|
165
|
+
sanitized.stream_connected = diagnostics.stream_connected
|
|
166
|
+
}
|
|
167
|
+
if (typeof diagnostics.last_heartbeat_ok === `boolean`) {
|
|
168
|
+
sanitized.last_heartbeat_ok = diagnostics.last_heartbeat_ok
|
|
169
|
+
}
|
|
170
|
+
if (
|
|
171
|
+
diagnostics.last_claim_result === null ||
|
|
172
|
+
(typeof diagnostics.last_claim_result === `string` &&
|
|
173
|
+
runnerLastClaimResults.has(
|
|
174
|
+
diagnostics.last_claim_result as NonNullable<
|
|
175
|
+
RunnerClientDiagnostics[`last_claim_result`]
|
|
176
|
+
>
|
|
177
|
+
))
|
|
178
|
+
) {
|
|
179
|
+
sanitized.last_claim_result = diagnostics.last_claim_result
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
for (const key of runnerStringOrNullDiagnostics) {
|
|
183
|
+
const value = diagnostics[key]
|
|
184
|
+
if (typeof value === `string` || value === null) {
|
|
185
|
+
sanitized[key] = value
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
for (const key of runnerNumberDiagnostics) {
|
|
189
|
+
const value = diagnostics[key]
|
|
190
|
+
if (typeof value === `number` && Number.isFinite(value) && value >= 0) {
|
|
191
|
+
sanitized[key] = value
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return Object.keys(sanitized).length > 0
|
|
196
|
+
? (sanitized as RunnerClientDiagnostics)
|
|
197
|
+
: undefined
|
|
198
|
+
}
|
|
199
|
+
|
|
103
200
|
async function registerRunner(
|
|
104
201
|
request: RunnersRouteRequest,
|
|
105
202
|
ctx: TenantContext
|
|
106
203
|
): Promise<Response> {
|
|
107
204
|
const parsed = routeBody<RegisterRunnerBody>(request)
|
|
108
|
-
const
|
|
109
|
-
|
|
205
|
+
const principal = requireAuthenticatedPrincipal(ctx)
|
|
206
|
+
const ownerPrincipal = parsed.owner_principal ?? principal.url
|
|
207
|
+
if (!ownerPrincipal) {
|
|
208
|
+
throw new ElectricAgentsError(
|
|
209
|
+
ErrCodeInvalidRequest,
|
|
210
|
+
`owner_principal is required when no authenticated principal is present`,
|
|
211
|
+
400
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
const canonicalOwner = canonicalOwnerPrincipal(ownerPrincipal)
|
|
215
|
+
if (!canonicalOwner) {
|
|
110
216
|
throw new ElectricAgentsError(
|
|
111
217
|
ErrCodeInvalidRequest,
|
|
112
|
-
`
|
|
218
|
+
`owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${ownerPrincipal}`,
|
|
113
219
|
400
|
|
114
220
|
)
|
|
115
221
|
}
|
|
116
|
-
if (
|
|
222
|
+
if (canonicalOwner !== principal.url) {
|
|
117
223
|
throw new ElectricAgentsError(
|
|
118
224
|
ErrCodeUnauthorized,
|
|
119
|
-
`
|
|
225
|
+
`owner_principal must match the authenticated principal`,
|
|
120
226
|
403
|
|
121
227
|
)
|
|
122
228
|
}
|
|
123
229
|
|
|
124
230
|
const runner = await ctx.entityManager.registry.createRunner({
|
|
125
231
|
id: parsed.id,
|
|
126
|
-
|
|
232
|
+
ownerPrincipal: canonicalOwner,
|
|
127
233
|
label: parsed.label,
|
|
128
234
|
kind: parsed.kind,
|
|
129
235
|
adminStatus: parsed.admin_status,
|
|
@@ -139,16 +245,27 @@ async function listRunners(
|
|
|
139
245
|
request: RunnersRouteRequest,
|
|
140
246
|
ctx: TenantContext
|
|
141
247
|
): Promise<Response> {
|
|
142
|
-
const
|
|
143
|
-
|
|
248
|
+
const principal = requireAuthenticatedPrincipal(ctx)
|
|
249
|
+
const requestedOwner = firstQueryValue(request.query.owner_principal)
|
|
250
|
+
const canonicalRequestedOwner = requestedOwner
|
|
251
|
+
? canonicalOwnerPrincipal(requestedOwner)
|
|
252
|
+
: undefined
|
|
253
|
+
if (requestedOwner && !canonicalRequestedOwner) {
|
|
254
|
+
throw new ElectricAgentsError(
|
|
255
|
+
ErrCodeInvalidRequest,
|
|
256
|
+
`owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${requestedOwner}`,
|
|
257
|
+
400
|
|
258
|
+
)
|
|
259
|
+
}
|
|
260
|
+
if (canonicalRequestedOwner && canonicalRequestedOwner !== principal.url) {
|
|
144
261
|
throw new ElectricAgentsError(
|
|
145
262
|
ErrCodeUnauthorized,
|
|
146
|
-
`
|
|
263
|
+
`owner_principal must match the authenticated principal`,
|
|
147
264
|
403
|
|
148
265
|
)
|
|
149
266
|
}
|
|
150
267
|
const runners = await ctx.entityManager.registry.listRunners({
|
|
151
|
-
|
|
268
|
+
ownerPrincipal: principal.url,
|
|
152
269
|
})
|
|
153
270
|
return json(runners)
|
|
154
271
|
}
|
|
@@ -158,7 +275,7 @@ async function getRunner(
|
|
|
158
275
|
ctx: TenantContext
|
|
159
276
|
): Promise<Response> {
|
|
160
277
|
const runner = await requireRunner(ctx, routeParam(request, `id`))
|
|
161
|
-
assertRunnerOwnerIfAuthenticated(ctx, runner.
|
|
278
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal)
|
|
162
279
|
return json(runner)
|
|
163
280
|
}
|
|
164
281
|
|
|
@@ -167,16 +284,19 @@ async function heartbeat(
|
|
|
167
284
|
ctx: TenantContext
|
|
168
285
|
): Promise<Response> {
|
|
169
286
|
const runnerId = routeParam(request, `id`)
|
|
287
|
+
requireAuthenticatedPrincipal(ctx)
|
|
170
288
|
const existing = await requireRunner(ctx, runnerId)
|
|
171
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
289
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal)
|
|
172
290
|
const parsed = routeBody<HeartbeatBody>(request)
|
|
173
291
|
const runner = await ctx.entityManager.registry.heartbeatRunner({
|
|
174
292
|
runnerId,
|
|
293
|
+
ownerPrincipal: existing.owner_principal,
|
|
175
294
|
leaseMs: parsed.lease_ms,
|
|
176
295
|
wakeStreamOffset: parsed.wake_stream_offset ?? parsed.wakeStreamOffset,
|
|
177
296
|
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at
|
|
178
297
|
? new Date(parsed.liveness_lease_expires_at)
|
|
179
298
|
: undefined,
|
|
299
|
+
diagnostics: sanitizeRunnerDiagnostics(parsed.diagnostics),
|
|
180
300
|
})
|
|
181
301
|
if (!runner) {
|
|
182
302
|
throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404)
|
|
@@ -204,8 +324,9 @@ async function setRunnerStatus(
|
|
|
204
324
|
adminStatus: `enabled` | `disabled`
|
|
205
325
|
): Promise<Response> {
|
|
206
326
|
const runnerId = routeParam(request, `id`)
|
|
327
|
+
requireAuthenticatedPrincipal(ctx)
|
|
207
328
|
const existing = await requireRunner(ctx, runnerId)
|
|
208
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
329
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal)
|
|
209
330
|
const runner = await ctx.entityManager.registry.setRunnerAdminStatus(
|
|
210
331
|
runnerId,
|
|
211
332
|
adminStatus
|
|
@@ -221,8 +342,9 @@ async function claimWake(
|
|
|
221
342
|
ctx: TenantContext
|
|
222
343
|
): Promise<Response> {
|
|
223
344
|
const runnerId = routeParam(request, `id`)
|
|
345
|
+
const principal = requireAuthenticatedPrincipal(ctx)
|
|
224
346
|
const runner = await requireRunner(ctx, runnerId)
|
|
225
|
-
if (
|
|
347
|
+
if (runner.owner_principal !== principal.url) {
|
|
226
348
|
throw new ElectricAgentsError(
|
|
227
349
|
ErrCodeUnauthorized,
|
|
228
350
|
`Runner claim requires the authenticated owner`,
|
|
@@ -296,10 +418,10 @@ async function requireRunner(ctx: TenantContext, runnerId: string) {
|
|
|
296
418
|
|
|
297
419
|
function assertRunnerOwnerIfAuthenticated(
|
|
298
420
|
ctx: TenantContext,
|
|
299
|
-
|
|
421
|
+
ownerPrincipal: string
|
|
300
422
|
): void {
|
|
301
|
-
|
|
302
|
-
if (
|
|
423
|
+
requireAuthenticatedPrincipal(ctx)
|
|
424
|
+
if (ownerPrincipal === ctx.principal.url) return
|
|
303
425
|
throw new ElectricAgentsError(
|
|
304
426
|
ErrCodeUnauthorized,
|
|
305
427
|
`Runner access requires the authenticated owner`,
|
|
@@ -307,6 +429,122 @@ function assertRunnerOwnerIfAuthenticated(
|
|
|
307
429
|
)
|
|
308
430
|
}
|
|
309
431
|
|
|
432
|
+
async function runnerHealth(
|
|
433
|
+
request: RunnersRouteRequest,
|
|
434
|
+
ctx: TenantContext
|
|
435
|
+
): Promise<Response> {
|
|
436
|
+
const runnerId = routeParam(request, `id`)
|
|
437
|
+
const runner = await requireRunner(ctx, runnerId)
|
|
438
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal)
|
|
439
|
+
const runtimeDiagnostics =
|
|
440
|
+
await ctx.entityManager.registry.getRunnerDiagnostics(runnerId)
|
|
441
|
+
|
|
442
|
+
const now = Date.now()
|
|
443
|
+
const parsedLeaseExpiresAt = runtimeDiagnostics?.liveness_lease_expires_at
|
|
444
|
+
? new Date(runtimeDiagnostics.liveness_lease_expires_at).getTime()
|
|
445
|
+
: null
|
|
446
|
+
const leaseExpiresAt =
|
|
447
|
+
parsedLeaseExpiresAt !== null && Number.isFinite(parsedLeaseExpiresAt)
|
|
448
|
+
? parsedLeaseExpiresAt
|
|
449
|
+
: null
|
|
450
|
+
|
|
451
|
+
let livenessStatus: `online` | `offline` | `expired`
|
|
452
|
+
if (runner.admin_status === `disabled`) {
|
|
453
|
+
livenessStatus = `offline`
|
|
454
|
+
} else if (leaseExpiresAt !== null && leaseExpiresAt > now) {
|
|
455
|
+
livenessStatus = `online`
|
|
456
|
+
} else if (leaseExpiresAt !== null) {
|
|
457
|
+
livenessStatus = `expired`
|
|
458
|
+
} else {
|
|
459
|
+
livenessStatus = `offline`
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
const [activeClaims, dispatchStats] = await Promise.all([
|
|
463
|
+
ctx.entityManager.registry.getActiveClaimsForRunner(runnerId),
|
|
464
|
+
ctx.entityManager.registry.getDispatchStatsForRunner(runnerId),
|
|
465
|
+
])
|
|
466
|
+
|
|
467
|
+
const clientDiagnostics =
|
|
468
|
+
sanitizeRunnerDiagnostics(runtimeDiagnostics?.diagnostics) ?? null
|
|
469
|
+
const issues: Array<string> = []
|
|
470
|
+
let healthStatus: `healthy` | `degraded` | `unhealthy` = `healthy`
|
|
471
|
+
|
|
472
|
+
const escalate = (floor: `degraded` | `unhealthy`): void => {
|
|
473
|
+
if (floor === `unhealthy`) healthStatus = `unhealthy`
|
|
474
|
+
else if (healthStatus === `healthy`) healthStatus = `degraded`
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
if (runner.admin_status === `disabled`) {
|
|
478
|
+
escalate(`unhealthy`)
|
|
479
|
+
issues.push(`Runner is disabled`)
|
|
480
|
+
}
|
|
481
|
+
if (livenessStatus === `expired`) {
|
|
482
|
+
escalate(`unhealthy`)
|
|
483
|
+
const ago = leaseExpiresAt ? Math.round((now - leaseExpiresAt) / 1000) : 0
|
|
484
|
+
issues.push(`Heartbeat lease expired ${ago}s ago`)
|
|
485
|
+
}
|
|
486
|
+
if (livenessStatus === `offline` && runner.admin_status === `enabled`) {
|
|
487
|
+
escalate(`degraded`)
|
|
488
|
+
issues.push(`Runner has never sent a heartbeat`)
|
|
489
|
+
}
|
|
490
|
+
if (clientDiagnostics) {
|
|
491
|
+
if (clientDiagnostics.stream_connected === false) {
|
|
492
|
+
escalate(`degraded`)
|
|
493
|
+
issues.push(`Client reports stream disconnected`)
|
|
494
|
+
}
|
|
495
|
+
if (clientDiagnostics.last_heartbeat_ok === false) {
|
|
496
|
+
escalate(`degraded`)
|
|
497
|
+
issues.push(`Client reports last heartbeat failed`)
|
|
498
|
+
}
|
|
499
|
+
if (
|
|
500
|
+
typeof clientDiagnostics.reconnect_count === `number` &&
|
|
501
|
+
clientDiagnostics.reconnect_count > 5
|
|
502
|
+
) {
|
|
503
|
+
escalate(`degraded`)
|
|
504
|
+
issues.push(
|
|
505
|
+
`Client has reconnected ${clientDiagnostics.reconnect_count} times`
|
|
506
|
+
)
|
|
507
|
+
}
|
|
508
|
+
} else if (runtimeDiagnostics?.last_seen_at) {
|
|
509
|
+
escalate(`degraded`)
|
|
510
|
+
issues.push(`No client diagnostics available`)
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const body: RunnerHealthResponse = {
|
|
514
|
+
runner: {
|
|
515
|
+
id: runner.id,
|
|
516
|
+
admin_status: runner.admin_status,
|
|
517
|
+
liveness_status: livenessStatus,
|
|
518
|
+
lease_expires_at:
|
|
519
|
+
leaseExpiresAt !== null
|
|
520
|
+
? (runtimeDiagnostics?.liveness_lease_expires_at ?? null)
|
|
521
|
+
: null,
|
|
522
|
+
lease_remaining_ms:
|
|
523
|
+
leaseExpiresAt !== null ? Math.max(0, leaseExpiresAt - now) : null,
|
|
524
|
+
wake_stream: runner.wake_stream,
|
|
525
|
+
wake_stream_offset: runtimeDiagnostics?.wake_stream_offset ?? null,
|
|
526
|
+
last_seen_at: runtimeDiagnostics?.last_seen_at ?? null,
|
|
527
|
+
created_at: runner.created_at,
|
|
528
|
+
},
|
|
529
|
+
client: clientDiagnostics,
|
|
530
|
+
claims: {
|
|
531
|
+
active_count: activeClaims.length,
|
|
532
|
+
active: activeClaims.map((c) => ({
|
|
533
|
+
consumer_id: c.consumer_id,
|
|
534
|
+
epoch: c.epoch,
|
|
535
|
+
entity_url: c.entity_url,
|
|
536
|
+
stream_path: c.stream_path,
|
|
537
|
+
claimed_at: c.claimed_at,
|
|
538
|
+
last_heartbeat_at: c.last_heartbeat_at ?? null,
|
|
539
|
+
lease_expires_at: c.lease_expires_at ?? null,
|
|
540
|
+
})),
|
|
541
|
+
},
|
|
542
|
+
dispatch: dispatchStats,
|
|
543
|
+
health: { status: healthStatus, issues },
|
|
544
|
+
}
|
|
545
|
+
return json(body)
|
|
546
|
+
}
|
|
547
|
+
|
|
310
548
|
async function notificationFromClaim(
|
|
311
549
|
ctx: TenantContext,
|
|
312
550
|
input: {
|
package/src/runtime.ts
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
import { SchemaValidator } from './electric-agents/schema-validator.js'
|
|
12
12
|
import { serverLog } from './utils/log.js'
|
|
13
13
|
import { isPermanentElectricAgentsError } from './scheduler.js'
|
|
14
|
-
import { StreamClient
|
|
14
|
+
import { StreamClient } from './stream-client.js'
|
|
15
15
|
import { DEFAULT_TENANT_ID } from './tenant.js'
|
|
16
16
|
import type { DrizzleDB } from './db/index.js'
|
|
17
17
|
import type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
|
|
@@ -63,10 +63,9 @@ export class ElectricAgentsTenantRuntime {
|
|
|
63
63
|
if (options.streamClient) {
|
|
64
64
|
this.streamClient = options.streamClient
|
|
65
65
|
} else if (options.durableStreamsUrl) {
|
|
66
|
-
this.streamClient = new StreamClient(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
)
|
|
66
|
+
this.streamClient = new StreamClient(options.durableStreamsUrl, {
|
|
67
|
+
bearer: options.durableStreamsBearer,
|
|
68
|
+
})
|
|
70
69
|
} else {
|
|
71
70
|
throw new Error(`Either durableStreamsUrl or streamClient is required`)
|
|
72
71
|
}
|