@electric-ax/agents-server 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/LICENSE +177 -0
  2. package/dist/chunk-Cl8Af3a2.js +11 -0
  3. package/dist/entrypoint.js +7319 -0
  4. package/dist/index.cjs +7090 -0
  5. package/dist/index.d.cts +4262 -0
  6. package/dist/index.d.ts +4263 -0
  7. package/dist/index.js +7053 -0
  8. package/drizzle/0000_baseline.sql +97 -0
  9. package/drizzle/0001_entity_tags_and_bridges.sql +45 -0
  10. package/drizzle/0002_tag_outbox_hardening.sql +14 -0
  11. package/drizzle/0003_entity_manifest_sources.sql +11 -0
  12. package/drizzle/0004_tenant_scoping.sql +139 -0
  13. package/drizzle/0005_pull_wake_control_plane.sql +156 -0
  14. package/drizzle/meta/0000_snapshot.json +593 -0
  15. package/drizzle/meta/_journal.json +48 -0
  16. package/package.json +89 -0
  17. package/src/authenticated-user-format.ts +17 -0
  18. package/src/claim-write-token-store.ts +74 -0
  19. package/src/db/index.ts +53 -0
  20. package/src/db/schema.ts +490 -0
  21. package/src/dev-asserted-auth.ts +46 -0
  22. package/src/dispatch-policy-schema.ts +52 -0
  23. package/src/electric-agents/adapter-types.ts +70 -0
  24. package/src/electric-agents/default-entity-schemas.ts +1 -0
  25. package/src/electric-agents/schema-validator.ts +143 -0
  26. package/src/electric-agents-http.ts +46 -0
  27. package/src/electric-agents-types.ts +335 -0
  28. package/src/entity-bridge-manager.ts +694 -0
  29. package/src/entity-manager.ts +2601 -0
  30. package/src/entity-projector.ts +765 -0
  31. package/src/entity-registry.ts +1162 -0
  32. package/src/entrypoint-lib.ts +295 -0
  33. package/src/entrypoint.ts +11 -0
  34. package/src/host.ts +323 -0
  35. package/src/index.ts +49 -0
  36. package/src/manifest-side-effects.ts +183 -0
  37. package/src/routing/agent-ui-router.ts +81 -0
  38. package/src/routing/context.ts +35 -0
  39. package/src/routing/cron-router.ts +45 -0
  40. package/src/routing/dispatch-policy.ts +248 -0
  41. package/src/routing/durable-streams-router.ts +407 -0
  42. package/src/routing/durable-streams-routing-adapter.ts +96 -0
  43. package/src/routing/electric-proxy-router.ts +61 -0
  44. package/src/routing/entities-router.ts +484 -0
  45. package/src/routing/entity-types-router.ts +229 -0
  46. package/src/routing/global-router.ts +33 -0
  47. package/src/routing/hooks.ts +123 -0
  48. package/src/routing/internal-router.ts +741 -0
  49. package/src/routing/oss-server-router.ts +56 -0
  50. package/src/routing/runners-router.ts +416 -0
  51. package/src/routing/schema.ts +141 -0
  52. package/src/routing/stream-append.ts +196 -0
  53. package/src/routing/tenant-stream-paths.ts +26 -0
  54. package/src/runtime-registry.ts +49 -0
  55. package/src/runtime.ts +537 -0
  56. package/src/scheduler.ts +788 -0
  57. package/src/schema-validation.ts +15 -0
  58. package/src/server.ts +374 -0
  59. package/src/standalone-runtime.ts +188 -0
  60. package/src/stream-client.ts +842 -0
  61. package/src/tag-stream-outbox-drainer.ts +188 -0
  62. package/src/tenant.ts +25 -0
  63. package/src/tracing.ts +57 -0
  64. package/src/utils/electric-url.ts +15 -0
  65. package/src/utils/log.ts +95 -0
  66. package/src/utils/server-utils.ts +245 -0
  67. package/src/utils/webhook-url.ts +33 -0
  68. package/src/wake-registry.ts +946 -0
@@ -0,0 +1,2601 @@
1
+ import { randomUUID } from 'node:crypto'
2
+ import fastq from 'fastq'
3
+ import {
4
+ assertTags,
5
+ entityStateSchema,
6
+ getCronStreamPath,
7
+ getSharedStateStreamPath,
8
+ getNextCronFireAt,
9
+ manifestChildKey,
10
+ manifestSharedStateKey,
11
+ manifestSourceKey,
12
+ resolveCronScheduleSpec,
13
+ } from '@electric-ax/agents-runtime'
14
+ import {
15
+ ErrCodeDuplicateURL,
16
+ ErrCodeEntityPersistFailed,
17
+ ErrCodeForkInProgress,
18
+ ErrCodeForkWaitTimeout,
19
+ ErrCodeInvalidRequest,
20
+ ErrCodeNotFound,
21
+ ErrCodeNotRunning,
22
+ ErrCodeSchemaKeyExists,
23
+ ErrCodeSchemaValidationFailed,
24
+ ErrCodeUnauthorized,
25
+ ErrCodeUnknownEntityType,
26
+ ErrCodeUnknownEventType,
27
+ ErrCodeUnknownMessageType,
28
+ } from './electric-agents-types.js'
29
+ import { parseDispatchPolicy } from './dispatch-policy-schema.js'
30
+ import { EntityAlreadyExistsError } from './entity-registry.js'
31
+ import { serverLog } from './utils/log.js'
32
+ import {
33
+ buildManifestWakeRegistration,
34
+ extractManifestCronSpec,
35
+ } from './manifest-side-effects.js'
36
+ import { DEFAULT_TENANT_ID } from './tenant.js'
37
+ import { ATTR, withSpan } from './tracing.js'
38
+ import type { queueAsPromised } from 'fastq'
39
+ import type { SchedulerClient } from './scheduler.js'
40
+ import type { WakeEvalResult, WakeRegistry } from './wake-registry.js'
41
+ import type { WakeMessage } from '@electric-ax/agents-runtime'
42
+ import type { PostgresRegistry } from './entity-registry.js'
43
+ import type { SchemaValidator } from './electric-agents/schema-validator.js'
44
+ import type { StreamClient } from './stream-client.js'
45
+ import type {
46
+ DispatchPolicy,
47
+ DispatchTarget,
48
+ ElectricAgentsEntity,
49
+ ElectricAgentsEntityType,
50
+ RegisterEntityTypeRequest,
51
+ SendRequest,
52
+ SetTagRequest,
53
+ TypedSpawnRequest,
54
+ } from './electric-agents-types.js'
55
+ import type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
56
+
57
+ type SpawnPersistResult = [
58
+ PromiseSettledResult<void>,
59
+ PromiseSettledResult<void>,
60
+ PromiseSettledResult<number>,
61
+ ]
62
+ type SpawnPersistJob = () => Promise<SpawnPersistResult>
63
+ type WriteTokenValidator = (
64
+ entity: ElectricAgentsEntity,
65
+ token: string
66
+ ) => boolean
67
+
68
+ type ForkSubtreeOptions = {
69
+ rootInstanceId?: string
70
+ waitTimeoutMs?: number
71
+ waitPollMs?: number
72
+ }
73
+
74
+ type ForkEntityPlan = {
75
+ source: ElectricAgentsEntity
76
+ fork: ElectricAgentsEntity
77
+ }
78
+
79
+ type ForkStateSnapshot = {
80
+ manifestsByEntity: Map<string, Map<string, Record<string, unknown>>>
81
+ childStatusesByEntity: Map<string, Map<string, Record<string, unknown>>>
82
+ replayWatermarksByEntity: Map<string, Map<string, Record<string, unknown>>>
83
+ sharedStateIds: Set<string>
84
+ }
85
+
86
+ type ForkResult = {
87
+ root: ElectricAgentsEntity
88
+ entities: Array<ElectricAgentsEntity>
89
+ }
90
+
91
+ const DEFAULT_FORK_WAIT_TIMEOUT_MS = 120_000
92
+ const DEFAULT_FORK_WAIT_POLL_MS = 250
93
+
94
+ function applyTypeDefaultSubscriptionScope(
95
+ policy: DispatchPolicy,
96
+ typeDefault: DispatchPolicy | undefined
97
+ ): DispatchPolicy {
98
+ const target = policy.targets[0]
99
+ const defaultTarget = typeDefault?.targets[0]
100
+ if (!target || !defaultTarget?.subscription_id) return policy
101
+ if (!sameDispatchDestination(target, defaultTarget)) return policy
102
+ if (target.subscription_id === defaultTarget.subscription_id) return policy
103
+
104
+ return {
105
+ targets: [{ ...target, subscription_id: defaultTarget.subscription_id }],
106
+ }
107
+ }
108
+
109
+ function sameDispatchDestination(
110
+ a: DispatchTarget,
111
+ b: DispatchTarget
112
+ ): boolean {
113
+ if (a.type !== b.type) return false
114
+ if (a.type === `runner` && b.type === `runner`) {
115
+ return a.runnerId === b.runnerId
116
+ }
117
+ if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url
118
+ return false
119
+ }
120
+
121
+ function sleep(ms: number): Promise<void> {
122
+ return new Promise((resolve) => setTimeout(resolve, ms))
123
+ }
124
+
125
+ function omitUndefined<T extends Record<string, unknown>>(value: T): T {
126
+ return Object.fromEntries(
127
+ Object.entries(value).filter(([, entry]) => entry !== undefined)
128
+ ) as T
129
+ }
130
+
131
+ function isRecord(value: unknown): value is Record<string, unknown> {
132
+ return typeof value === `object` && value !== null && !Array.isArray(value)
133
+ }
134
+
135
+ function cloneRecord<T extends Record<string, unknown>>(value: T): T {
136
+ return JSON.parse(JSON.stringify(value)) as T
137
+ }
138
+
139
+ /**
140
+ * Orchestrates the Electric Agents entity lifecycle: register types, spawn, send, kill.
141
+ *
142
+ * Entity identity is the URL (/{type}/{instance_id}). Entity tags and
143
+ * lifecycle state are persisted directly in Postgres. Durable streams remain
144
+ * the append-only transport for inbox/state events.
145
+ */
146
+ export class EntityManager {
147
+ readonly registry: PostgresRegistry
148
+ private readonly tenantId: string
149
+ private streamClient: StreamClient
150
+ private validator: SchemaValidator
151
+ private scheduler: SchedulerClient | null = null
152
+ private entityBridgeManager: EntityBridgeCoordinator | null = null
153
+ private writeTokenValidator: WriteTokenValidator | null = null
154
+ readonly wakeRegistry: WakeRegistry
155
+ private forkWorkLockedEntities = new Map<string, number>()
156
+ private forkWriteLockedEntities = new Map<string, number>()
157
+ private forkWriteLockedStreams = new Map<string, number>()
158
+ private spawnPersistQueue: queueAsPromised<
159
+ SpawnPersistJob,
160
+ SpawnPersistResult
161
+ >
162
+ private readonly stopWakeRegistryOnShutdown: boolean
163
+
164
+ constructor(opts: {
165
+ registry: PostgresRegistry
166
+ streamClient: StreamClient
167
+ validator: SchemaValidator
168
+ wakeRegistry: WakeRegistry
169
+ scheduler?: SchedulerClient
170
+ entityBridgeManager?: EntityBridgeCoordinator
171
+ writeTokenValidator?: WriteTokenValidator
172
+ spawnConcurrency?: number
173
+ stopWakeRegistryOnShutdown?: boolean
174
+ }) {
175
+ this.registry = opts.registry
176
+ this.tenantId = opts.registry.tenantId ?? DEFAULT_TENANT_ID
177
+ this.streamClient = opts.streamClient
178
+ this.validator = opts.validator
179
+ this.wakeRegistry = opts.wakeRegistry
180
+ this.scheduler = opts.scheduler ?? null
181
+ this.entityBridgeManager = opts.entityBridgeManager ?? null
182
+ this.writeTokenValidator = opts.writeTokenValidator ?? null
183
+ this.stopWakeRegistryOnShutdown = opts.stopWakeRegistryOnShutdown ?? true
184
+
185
+ const spawnConcurrency =
186
+ opts.spawnConcurrency ??
187
+ Number(process.env.ELECTRIC_AGENTS_SPAWN_CONCURRENCY ?? 16)
188
+ this.spawnPersistQueue = fastq.promise<
189
+ unknown,
190
+ SpawnPersistJob,
191
+ SpawnPersistResult
192
+ >(async (job) => job(), spawnConcurrency)
193
+
194
+ this.wakeRegistry.setTimeoutCallback((result) => {
195
+ void this.deliverWakeResult(result)
196
+ }, this.tenantId)
197
+ this.wakeRegistry.setDebounceCallback((result) => {
198
+ void this.deliverWakeResult(result)
199
+ }, this.tenantId)
200
+ }
201
+
202
+ async rebuildWakeRegistry(
203
+ electricUrl?: string,
204
+ electricSecret?: string
205
+ ): Promise<void> {
206
+ if (electricUrl) {
207
+ await this.wakeRegistry.startSync(electricUrl, electricSecret)
208
+ return
209
+ }
210
+
211
+ await this.wakeRegistry.loadRegistrations()
212
+ }
213
+
214
+ setWriteTokenValidator(validator: WriteTokenValidator): void {
215
+ this.writeTokenValidator = validator
216
+ }
217
+
218
+ isValidWriteToken(entity: ElectricAgentsEntity, token: string): boolean {
219
+ return this.writeTokenValidator
220
+ ? this.writeTokenValidator(entity, token)
221
+ : token === entity.write_token
222
+ }
223
+
224
+ private encodeChangeEvent(event: Record<string, unknown>): Uint8Array {
225
+ return new TextEncoder().encode(JSON.stringify(event))
226
+ }
227
+
228
+ // ==========================================================================
229
+ // Entity Type Registration
230
+ // ==========================================================================
231
+
232
+ async registerEntityType(
233
+ req: RegisterEntityTypeRequest
234
+ ): Promise<ElectricAgentsEntityType> {
235
+ if (!req.name) {
236
+ throw new ElectricAgentsError(
237
+ ErrCodeInvalidRequest,
238
+ `Missing required field: name`,
239
+ 400
240
+ )
241
+ }
242
+ if (req.name.startsWith(`_`)) {
243
+ throw new ElectricAgentsError(
244
+ ErrCodeInvalidRequest,
245
+ `Entity type names starting with "_" are reserved`,
246
+ 400
247
+ )
248
+ }
249
+ if (!req.description) {
250
+ throw new ElectricAgentsError(
251
+ ErrCodeInvalidRequest,
252
+ `Missing required field: description`,
253
+ 400
254
+ )
255
+ }
256
+
257
+ // Validate schema subset for each provided schema.
258
+ this.validateSchema(req.creation_schema)
259
+ this.validateSchemaMap(req.inbox_schemas)
260
+ this.validateSchemaMap(req.state_schemas)
261
+ const defaultDispatchPolicy = req.default_dispatch_policy
262
+ ? this.validateDispatchPolicy(req.default_dispatch_policy, {
263
+ label: `default_dispatch_policy`,
264
+ })
265
+ : undefined
266
+
267
+ const existing = await this.registry.getEntityType(req.name)
268
+ const now = new Date().toISOString()
269
+ const entityType: ElectricAgentsEntityType = {
270
+ name: req.name,
271
+ description: req.description,
272
+ creation_schema: req.creation_schema,
273
+ inbox_schemas: req.inbox_schemas,
274
+ state_schemas: req.state_schemas,
275
+ serve_endpoint: req.serve_endpoint,
276
+ default_dispatch_policy: defaultDispatchPolicy,
277
+ revision: existing ? existing.revision + 1 : 1,
278
+ created_at: existing?.created_at ?? now,
279
+ updated_at: now,
280
+ }
281
+
282
+ await this.registry.createEntityType(entityType)
283
+
284
+ const stored = await this.registry.getEntityType(req.name)
285
+ if (!stored) {
286
+ throw new Error(`Failed to read back entity type "${req.name}"`)
287
+ }
288
+
289
+ return stored
290
+ }
291
+
292
+ async deleteEntityType(name: string): Promise<void> {
293
+ const existing = await this.registry.getEntityType(name)
294
+ if (!existing) {
295
+ throw new ElectricAgentsError(
296
+ ErrCodeNotFound,
297
+ `Entity type "${name}" not found`,
298
+ 404
299
+ )
300
+ }
301
+
302
+ await this.registry.deleteEntityType(name)
303
+ }
304
+
305
+ // ==========================================================================
306
+ // Spawn
307
+ // ==========================================================================
308
+
309
+ /**
310
+ * Spawn a new entity of the given type with durable streams.
311
+ */
312
+ async spawn(
313
+ typeName: string,
314
+ req: TypedSpawnRequest
315
+ ): Promise<ElectricAgentsEntity & { txid: number }> {
316
+ return await withSpan(`electric_agents.spawn`, async (span) => {
317
+ span.setAttributes({
318
+ [ATTR.ENTITY_TYPE]: typeName,
319
+ ...(req.parent ? { [ATTR.PARENT_URL]: req.parent } : {}),
320
+ })
321
+ const entity = await this.spawnInner(typeName, req)
322
+ span.setAttribute(ATTR.ENTITY_URL, entity.url)
323
+ return entity
324
+ })
325
+ }
326
+
327
+ private async spawnInner(
328
+ typeName: string,
329
+ req: TypedSpawnRequest
330
+ ): Promise<ElectricAgentsEntity & { txid: number }> {
331
+ if (typeName.startsWith(`_`)) {
332
+ throw new ElectricAgentsError(
333
+ ErrCodeInvalidRequest,
334
+ `Entity type names starting with "_" are reserved`,
335
+ 400
336
+ )
337
+ }
338
+
339
+ // Look up the entity type from the registry.
340
+ const entityType = await this.registry.getEntityType(typeName)
341
+ if (!entityType) {
342
+ throw new ElectricAgentsError(
343
+ ErrCodeUnknownEntityType,
344
+ `Entity type "${typeName}" not found`,
345
+ 404
346
+ )
347
+ }
348
+
349
+ // Validate args against creation_schema if declared.
350
+ if (entityType.creation_schema && req.args) {
351
+ const valErr = this.validator.validate(
352
+ entityType.creation_schema,
353
+ req.args
354
+ )
355
+ if (valErr) {
356
+ throw new ElectricAgentsError(
357
+ valErr.code,
358
+ valErr.message,
359
+ 422,
360
+ valErr.details
361
+ )
362
+ }
363
+ }
364
+
365
+ const initialTags = this.validateTags(req.tags ?? {})
366
+
367
+ const instanceId = req.instance_id || randomUUID()
368
+ if (instanceId.includes(`/`)) {
369
+ throw new ElectricAgentsError(
370
+ ErrCodeInvalidRequest,
371
+ `instance_id must not contain forward slashes`,
372
+ 400
373
+ )
374
+ }
375
+
376
+ const writeToken = randomUUID()
377
+
378
+ const entityURL = `/${typeName}/${instanceId}`
379
+ const mainPath = `${entityURL}/main`
380
+ const errorPath = `${entityURL}/error`
381
+
382
+ const subscriptionId = `${typeName}-handler`
383
+
384
+ const spawnT0 = performance.now()
385
+
386
+ const existingByURL = await this.registry.getEntity(entityURL)
387
+ if (existingByURL) {
388
+ throw new ElectricAgentsError(
389
+ ErrCodeDuplicateURL,
390
+ `Entity already exists at URL "${entityURL}"`,
391
+ 409
392
+ )
393
+ }
394
+
395
+ let parentEntity: ElectricAgentsEntity | null = null
396
+ if (req.parent) {
397
+ parentEntity = await this.registry.getEntity(req.parent)
398
+ if (!parentEntity) {
399
+ throw new ElectricAgentsError(
400
+ ErrCodeNotFound,
401
+ `Parent entity "${req.parent}" not found`,
402
+ 404
403
+ )
404
+ }
405
+ }
406
+
407
+ const dispatchPolicy = req.dispatch_policy
408
+ ? this.validateDispatchPolicy(req.dispatch_policy, {
409
+ label: `dispatch_policy`,
410
+ })
411
+ : parentEntity?.dispatch_policy
412
+ ? applyTypeDefaultSubscriptionScope(
413
+ parentEntity.dispatch_policy,
414
+ entityType.default_dispatch_policy
415
+ )
416
+ : entityType.default_dispatch_policy
417
+
418
+ const now = Date.now()
419
+ const entityData: ElectricAgentsEntity = {
420
+ type: typeName,
421
+ status: `idle`,
422
+ url: entityURL,
423
+ streams: {
424
+ main: mainPath,
425
+ error: errorPath,
426
+ },
427
+ subscription_id: subscriptionId,
428
+ dispatch_policy: dispatchPolicy,
429
+ write_token: writeToken,
430
+ tags: initialTags,
431
+ spawn_args: req.args,
432
+ type_revision: entityType.revision,
433
+ inbox_schemas: entityType.inbox_schemas,
434
+ state_schemas: entityType.state_schemas,
435
+ created_at: now,
436
+ updated_at: now,
437
+ }
438
+ if (req.parent) {
439
+ entityData.parent = req.parent
440
+ }
441
+
442
+ if (req.wake) {
443
+ await this.wakeRegistry.register({
444
+ tenantId: this.tenantId,
445
+ subscriberUrl: req.wake.subscriberUrl,
446
+ sourceUrl: entityURL,
447
+ condition: req.wake.condition,
448
+ debounceMs: req.wake.debounceMs,
449
+ timeoutMs: req.wake.timeoutMs,
450
+ oneShot: false,
451
+ includeResponse: req.wake.includeResponse,
452
+ })
453
+ }
454
+
455
+ const contentType = `application/json`
456
+
457
+ const createdEvent = entityStateSchema.entityCreated.insert({
458
+ key: `entity-created`,
459
+ value: {
460
+ entity_type: typeName,
461
+ timestamp: new Date().toISOString(),
462
+ args: req.args ?? {},
463
+ ...(req.parent ? { parent_url: req.parent } : {}),
464
+ },
465
+ } as any)
466
+
467
+ const initialEvents: Array<Record<string, unknown>> = [
468
+ createdEvent as Record<string, unknown>,
469
+ ]
470
+
471
+ if (req.initialMessage !== undefined) {
472
+ const msgNow = new Date().toISOString()
473
+ const inboxEvent = entityStateSchema.inbox.insert({
474
+ key: `msg-in-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
475
+ value: {
476
+ from: req.parent ?? `spawn`,
477
+ payload: req.initialMessage,
478
+ timestamp: msgNow,
479
+ },
480
+ } as any)
481
+ initialEvents.push(inboxEvent as Record<string, unknown>)
482
+ }
483
+
484
+ // JSON-mode streams: server flattens one level. Append auto-wraps the
485
+ // body in [...] but create does not, so we wrap it ourselves.
486
+ const initialBody = `[${initialEvents.map((e) => JSON.stringify(e)).join(`,`)}]`
487
+
488
+ const queueEnterT0 = performance.now()
489
+ const queueWaiting = this.spawnPersistQueue.length()
490
+ const queueRunning = this.spawnPersistQueue.running()
491
+ const [mainStreamResult, errorStreamResult, entityResult] =
492
+ await this.spawnPersistQueue.push(async () => {
493
+ // Create entity first so it's visible in the DB before stream
494
+ // creation can trigger webhooks that look up the entity.
495
+ let entityTxid: number
496
+ try {
497
+ entityTxid = await withSpan(`db.createEntity`, () =>
498
+ this.registry.createEntity(entityData)
499
+ )
500
+ } catch (err) {
501
+ return [
502
+ { status: `fulfilled`, value: undefined },
503
+ { status: `fulfilled`, value: undefined },
504
+ { status: `rejected`, reason: err },
505
+ ] as SpawnPersistResult
506
+ }
507
+
508
+ const [mainStreamResult, errorStreamResult] = await Promise.allSettled([
509
+ this.streamClient.create(mainPath, {
510
+ contentType,
511
+ body: initialBody,
512
+ }),
513
+ this.streamClient.create(errorPath, { contentType }),
514
+ ])
515
+
516
+ return [
517
+ mainStreamResult,
518
+ errorStreamResult,
519
+ { status: `fulfilled`, value: entityTxid },
520
+ ] as SpawnPersistResult
521
+ })
522
+ const parallelMs = +(performance.now() - queueEnterT0).toFixed(2)
523
+
524
+ if (
525
+ mainStreamResult.status === `rejected` ||
526
+ errorStreamResult.status === `rejected` ||
527
+ entityResult.status === `rejected`
528
+ ) {
529
+ const entityReason =
530
+ entityResult.status === `rejected` ? entityResult.reason : null
531
+ const streamReason =
532
+ mainStreamResult.status === `rejected`
533
+ ? mainStreamResult.reason
534
+ : errorStreamResult.status === `rejected`
535
+ ? errorStreamResult.reason
536
+ : null
537
+ const isDuplicate = entityReason instanceof EntityAlreadyExistsError
538
+ const isStreamConflict =
539
+ !!streamReason &&
540
+ typeof streamReason === `object` &&
541
+ ((`status` in streamReason && streamReason.status === 409) ||
542
+ (`code` in streamReason && streamReason.code === `CONFLICT_SEQ`))
543
+
544
+ const rollbacks: Array<Promise<unknown>> = []
545
+ // On duplicate, the winning spawn owns both the stream and the row —
546
+ // don't roll back either. For any other failure, clean up what succeeded.
547
+ if (!isDuplicate && !isStreamConflict) {
548
+ if (mainStreamResult.status === `fulfilled`) {
549
+ rollbacks.push(this.streamClient.delete(mainPath))
550
+ }
551
+ if (errorStreamResult.status === `fulfilled`) {
552
+ rollbacks.push(this.streamClient.delete(errorPath))
553
+ }
554
+ if (entityResult.status === `fulfilled`) {
555
+ rollbacks.push(this.registry.deleteEntity(entityURL))
556
+ }
557
+ if (req.wake) {
558
+ rollbacks.push(
559
+ this.wakeRegistry.unregisterBySubscriberAndSource(
560
+ req.wake.subscriberUrl,
561
+ entityURL,
562
+ this.tenantId
563
+ )
564
+ )
565
+ }
566
+ await Promise.allSettled(rollbacks)
567
+ }
568
+
569
+ if (isDuplicate || isStreamConflict) {
570
+ throw new ElectricAgentsError(
571
+ ErrCodeDuplicateURL,
572
+ `Entity already exists at URL "${entityURL}"`,
573
+ 409
574
+ )
575
+ }
576
+
577
+ const failure =
578
+ mainStreamResult.status === `rejected`
579
+ ? mainStreamResult.reason
580
+ : errorStreamResult.status === `rejected`
581
+ ? errorStreamResult.reason
582
+ : (entityResult as PromiseRejectedResult).reason
583
+ if (failure instanceof Error) throw failure
584
+ throw new ElectricAgentsError(
585
+ `SPAWN_FAILED`,
586
+ `Spawn failed: ${String(failure)}`,
587
+ 500
588
+ )
589
+ }
590
+
591
+ const txid = entityResult.value
592
+
593
+ serverLog.event(
594
+ {
595
+ event: `spawn`,
596
+ url: entityURL,
597
+ type: typeName,
598
+ parent: req.parent,
599
+ parallelMs,
600
+ totalMs: +(performance.now() - spawnT0).toFixed(2),
601
+ queueWaiting,
602
+ queueRunning,
603
+ },
604
+ `spawn done`
605
+ )
606
+ return { ...entityData, txid }
607
+ }
608
+
609
+ // ==========================================================================
610
+ // Fork
611
+ // ==========================================================================
612
+
613
+ async forkSubtree(
614
+ rootUrl: string,
615
+ opts: ForkSubtreeOptions = {}
616
+ ): Promise<ForkResult> {
617
+ return await withSpan(`electric_agents.forkSubtree`, async (span) => {
618
+ span.setAttribute(ATTR.ENTITY_URL, rootUrl)
619
+ const result = await this.forkSubtreeInner(rootUrl, opts)
620
+ span.setAttribute(`electric_agents.fork.root_url`, result.root.url)
621
+ span.setAttribute(
622
+ `electric_agents.fork.entity_count`,
623
+ result.entities.length
624
+ )
625
+ return result
626
+ })
627
+ }
628
+
629
+ private async forkSubtreeInner(
630
+ rootUrl: string,
631
+ opts: ForkSubtreeOptions
632
+ ): Promise<ForkResult> {
633
+ const forkT0 = performance.now()
634
+ const workLocks = new Set<string>()
635
+ const writeEntityLocks = new Set<string>()
636
+ const writeStreamLocks = new Set<string>()
637
+
638
+ try {
639
+ const sourceTree = await this.waitForIdleSubtree(rootUrl, opts, workLocks)
640
+ const sourceRoot = sourceTree[0]!
641
+ if (sourceRoot.parent) {
642
+ throw new ElectricAgentsError(
643
+ ErrCodeInvalidRequest,
644
+ `Only top-level entities can be forked`,
645
+ 400
646
+ )
647
+ }
648
+
649
+ const snapshot = await this.readForkStateSnapshot(sourceTree)
650
+ const suffix = randomUUID().slice(0, 8)
651
+ const entityUrlMap = await this.buildForkEntityUrlMap(sourceTree, {
652
+ suffix,
653
+ rootUrl,
654
+ rootInstanceId: opts.rootInstanceId,
655
+ })
656
+ const sharedStateIdMap = await this.buildForkSharedStateIdMap(
657
+ snapshot.sharedStateIds,
658
+ suffix
659
+ )
660
+ const stringMap = this.buildForkStringMap(entityUrlMap, sharedStateIdMap)
661
+ const entityPlans = this.buildForkEntityPlans(
662
+ sourceTree,
663
+ entityUrlMap,
664
+ stringMap
665
+ )
666
+
667
+ this.addForkLocks(
668
+ this.forkWriteLockedEntities,
669
+ sourceTree.map((entity) => entity.url),
670
+ writeEntityLocks
671
+ )
672
+ this.addForkLocks(
673
+ this.forkWriteLockedStreams,
674
+ [...snapshot.sharedStateIds].map((id) => getSharedStateStreamPath(id)),
675
+ writeStreamLocks
676
+ )
677
+
678
+ const createdStreams: Array<string> = []
679
+ const createdEntities: Array<string> = []
680
+ const activeManifestsByEntity = new Map<
681
+ string,
682
+ Map<string, Record<string, unknown>>
683
+ >()
684
+
685
+ try {
686
+ for (const plan of entityPlans) {
687
+ await this.streamClient.fork(
688
+ plan.fork.streams.main,
689
+ plan.source.streams.main
690
+ )
691
+ createdStreams.push(plan.fork.streams.main)
692
+ await this.streamClient.fork(
693
+ plan.fork.streams.error,
694
+ plan.source.streams.error
695
+ )
696
+ createdStreams.push(plan.fork.streams.error)
697
+ }
698
+
699
+ for (const [sourceId, forkId] of sharedStateIdMap) {
700
+ const sourcePath = getSharedStateStreamPath(sourceId)
701
+ const forkPath = getSharedStateStreamPath(forkId)
702
+ await this.streamClient.fork(forkPath, sourcePath)
703
+ createdStreams.push(forkPath)
704
+ }
705
+
706
+ for (const plan of entityPlans) {
707
+ const reconciliation = this.buildForkReconciliation(
708
+ plan,
709
+ snapshot,
710
+ entityUrlMap,
711
+ sharedStateIdMap,
712
+ stringMap
713
+ )
714
+ activeManifestsByEntity.set(plan.fork.url, reconciliation.manifests)
715
+ for (const event of reconciliation.events) {
716
+ await this.streamClient.append(
717
+ plan.fork.streams.main,
718
+ this.encodeChangeEvent(event)
719
+ )
720
+ }
721
+ }
722
+
723
+ for (const plan of entityPlans) {
724
+ await this.registry.createEntity(plan.fork)
725
+ createdEntities.push(plan.fork.url)
726
+ }
727
+
728
+ for (const plan of entityPlans) {
729
+ const manifests =
730
+ activeManifestsByEntity.get(plan.fork.url) ?? new Map()
731
+ await this.materializeForkManifestSideEffects(
732
+ plan.fork.url,
733
+ manifests
734
+ )
735
+ }
736
+
737
+ const root = entityPlans.find(
738
+ (plan) => plan.source.url === rootUrl
739
+ )!.fork
740
+ serverLog.event(
741
+ {
742
+ event: `fork`,
743
+ url: rootUrl,
744
+ forkUrl: root.url,
745
+ entities: entityPlans.length,
746
+ sharedStateStreams: sharedStateIdMap.size,
747
+ totalMs: +(performance.now() - forkT0).toFixed(2),
748
+ },
749
+ `fork done`
750
+ )
751
+ return { root, entities: entityPlans.map((plan) => plan.fork) }
752
+ } catch (err) {
753
+ await Promise.allSettled([
754
+ ...createdEntities.flatMap((entityUrl) => [
755
+ this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId),
756
+ this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId),
757
+ this.registry.deleteEntity(entityUrl),
758
+ ]),
759
+ ...Array.from(sharedStateIdMap.values()).map((id) =>
760
+ this.wakeRegistry.unregisterBySource(
761
+ getSharedStateStreamPath(id),
762
+ this.tenantId
763
+ )
764
+ ),
765
+ ...createdStreams.map((streamPath) =>
766
+ this.streamClient.delete(streamPath)
767
+ ),
768
+ ])
769
+ throw err
770
+ } finally {
771
+ this.releaseForkLocks(this.forkWriteLockedStreams, writeStreamLocks)
772
+ this.releaseForkLocks(this.forkWriteLockedEntities, writeEntityLocks)
773
+ }
774
+ } finally {
775
+ this.releaseForkLocks(this.forkWorkLockedEntities, workLocks)
776
+ }
777
+ }
778
+
779
+ isForkWorkLockedEntity(entityUrl: string): boolean {
780
+ return (this.forkWorkLockedEntities.get(entityUrl) ?? 0) > 0
781
+ }
782
+
783
+ isForkWriteLockedEntity(entityUrl: string): boolean {
784
+ return (this.forkWriteLockedEntities.get(entityUrl) ?? 0) > 0
785
+ }
786
+
787
+ isForkWriteLockedStream(streamPath: string): boolean {
788
+ return (this.forkWriteLockedStreams.get(streamPath) ?? 0) > 0
789
+ }
790
+
791
+ private assertEntityNotForkWorkLocked(entityUrl: string): void {
792
+ if (!this.isForkWorkLockedEntity(entityUrl)) return
793
+ throw new ElectricAgentsError(
794
+ ErrCodeForkInProgress,
795
+ `Entity subtree is being forked`,
796
+ 409
797
+ )
798
+ }
799
+
800
+ private addForkLocks(
801
+ locks: Map<string, number>,
802
+ keys: Array<string>,
803
+ held: Set<string>
804
+ ): void {
805
+ for (const key of keys) {
806
+ if (held.has(key)) continue
807
+ locks.set(key, (locks.get(key) ?? 0) + 1)
808
+ held.add(key)
809
+ }
810
+ }
811
+
812
+ private releaseForkLocks(
813
+ locks: Map<string, number>,
814
+ held: Set<string>
815
+ ): void {
816
+ for (const key of held) {
817
+ const count = locks.get(key) ?? 0
818
+ if (count <= 1) {
819
+ locks.delete(key)
820
+ } else {
821
+ locks.set(key, count - 1)
822
+ }
823
+ }
824
+ held.clear()
825
+ }
826
+
827
+ private async waitForIdleSubtree(
828
+ rootUrl: string,
829
+ opts: ForkSubtreeOptions,
830
+ workLocks: Set<string>
831
+ ): Promise<Array<ElectricAgentsEntity>> {
832
+ const timeoutMs = opts.waitTimeoutMs ?? DEFAULT_FORK_WAIT_TIMEOUT_MS
833
+ const pollMs = opts.waitPollMs ?? DEFAULT_FORK_WAIT_POLL_MS
834
+ if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
835
+ throw new ElectricAgentsError(
836
+ ErrCodeInvalidRequest,
837
+ `waitTimeoutMs must be a non-negative number`,
838
+ 400
839
+ )
840
+ }
841
+ if (!Number.isFinite(pollMs) || pollMs <= 0) {
842
+ throw new ElectricAgentsError(
843
+ ErrCodeInvalidRequest,
844
+ `waitPollMs must be a positive number`,
845
+ 400
846
+ )
847
+ }
848
+
849
+ const deadline = Date.now() + timeoutMs
850
+ while (true) {
851
+ const root = await this.registry.getEntity(rootUrl)
852
+ if (!root) {
853
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
854
+ }
855
+ if (root.parent) {
856
+ throw new ElectricAgentsError(
857
+ ErrCodeInvalidRequest,
858
+ `Only top-level entities can be forked`,
859
+ 400
860
+ )
861
+ }
862
+
863
+ const subtree = await this.listEntitySubtree(root)
864
+ const stopped = subtree.find((entity) => entity.status === `stopped`)
865
+ if (stopped) {
866
+ throw new ElectricAgentsError(
867
+ ErrCodeNotRunning,
868
+ `Cannot fork stopped entity "${stopped.url}"`,
869
+ 409
870
+ )
871
+ }
872
+
873
+ let active = subtree.filter((entity) => entity.status !== `idle`)
874
+ if (active.length === 0) {
875
+ this.addForkLocks(
876
+ this.forkWorkLockedEntities,
877
+ subtree.map((entity) => entity.url),
878
+ workLocks
879
+ )
880
+ const lockedRoot = await this.registry.getEntity(rootUrl)
881
+ if (!lockedRoot) {
882
+ throw new ElectricAgentsError(
883
+ ErrCodeNotFound,
884
+ `Entity not found`,
885
+ 404
886
+ )
887
+ }
888
+ const lockedSubtree = await this.listEntitySubtree(lockedRoot)
889
+ this.addForkLocks(
890
+ this.forkWorkLockedEntities,
891
+ lockedSubtree.map((entity) => entity.url),
892
+ workLocks
893
+ )
894
+ const lockedActive = lockedSubtree.filter(
895
+ (entity) => entity.status !== `idle`
896
+ )
897
+ if (lockedActive.length === 0) {
898
+ return lockedSubtree
899
+ }
900
+ this.releaseForkLocks(this.forkWorkLockedEntities, workLocks)
901
+ active = lockedActive
902
+ }
903
+
904
+ if (Date.now() >= deadline) {
905
+ throw new ElectricAgentsError(
906
+ ErrCodeForkWaitTimeout,
907
+ `Timed out waiting for subtree to become idle`,
908
+ 409,
909
+ { active: active.map((entity) => entity.url) }
910
+ )
911
+ }
912
+
913
+ await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now())))
914
+ }
915
+ }
916
+
917
+ private async listEntitySubtree(
918
+ root: ElectricAgentsEntity
919
+ ): Promise<Array<ElectricAgentsEntity>> {
920
+ const result: Array<ElectricAgentsEntity> = []
921
+ const queue: Array<ElectricAgentsEntity> = [root]
922
+ const seen = new Set<string>()
923
+
924
+ while (queue.length > 0) {
925
+ const entity = queue.shift()!
926
+ if (seen.has(entity.url)) continue
927
+ seen.add(entity.url)
928
+ result.push(entity)
929
+
930
+ const { entities: children } = await this.registry.listEntities({
931
+ parent: entity.url,
932
+ limit: 10_000,
933
+ })
934
+ for (const child of children) {
935
+ queue.push(child)
936
+ }
937
+ }
938
+
939
+ return result
940
+ }
941
+
942
+ private async readForkStateSnapshot(
943
+ entitiesToFork: Array<ElectricAgentsEntity>
944
+ ): Promise<ForkStateSnapshot> {
945
+ const manifestsByEntity = new Map<
946
+ string,
947
+ Map<string, Record<string, unknown>>
948
+ >()
949
+ const childStatusesByEntity = new Map<
950
+ string,
951
+ Map<string, Record<string, unknown>>
952
+ >()
953
+ const replayWatermarksByEntity = new Map<
954
+ string,
955
+ Map<string, Record<string, unknown>>
956
+ >()
957
+ const sharedStateIds = new Set<string>()
958
+
959
+ for (const entity of entitiesToFork) {
960
+ const events = await this.streamClient.readJson<Record<string, unknown>>(
961
+ entity.streams.main
962
+ )
963
+ const manifests = this.reduceStateRows(events, `manifest`)
964
+ const childStatuses = this.reduceStateRows(events, `child_status`)
965
+ const replayWatermarks = this.reduceStateRows(events, `replay_watermark`)
966
+
967
+ manifestsByEntity.set(entity.url, manifests)
968
+ childStatusesByEntity.set(entity.url, childStatuses)
969
+ replayWatermarksByEntity.set(entity.url, replayWatermarks)
970
+
971
+ for (const manifest of manifests.values()) {
972
+ this.collectSharedStateIds(manifest, sharedStateIds)
973
+ }
974
+ }
975
+
976
+ return {
977
+ manifestsByEntity,
978
+ childStatusesByEntity,
979
+ replayWatermarksByEntity,
980
+ sharedStateIds,
981
+ }
982
+ }
983
+
984
+ private reduceStateRows(
985
+ rawEvents: Array<unknown>,
986
+ eventType: string
987
+ ): Map<string, Record<string, unknown>> {
988
+ const rows = new Map<string, Record<string, unknown>>()
989
+ const events = rawEvents.flatMap((item) =>
990
+ Array.isArray(item) ? item : [item]
991
+ )
992
+
993
+ for (const event of events) {
994
+ if (!isRecord(event) || event.type !== eventType) continue
995
+ if (typeof event.key !== `string`) continue
996
+ const headers = isRecord(event.headers) ? event.headers : undefined
997
+ const operation = headers?.operation
998
+ if (operation === `delete`) {
999
+ rows.delete(event.key)
1000
+ continue
1001
+ }
1002
+ if (isRecord(event.value)) {
1003
+ rows.set(event.key, cloneRecord(event.value))
1004
+ }
1005
+ }
1006
+
1007
+ return rows
1008
+ }
1009
+
1010
+ private collectSharedStateIds(
1011
+ manifest: Record<string, unknown>,
1012
+ sharedStateIds: Set<string>
1013
+ ): void {
1014
+ if (manifest.kind === `shared-state` && typeof manifest.id === `string`) {
1015
+ sharedStateIds.add(manifest.id)
1016
+ return
1017
+ }
1018
+
1019
+ if (manifest.kind !== `source` || manifest.sourceType !== `db`) {
1020
+ return
1021
+ }
1022
+
1023
+ if (typeof manifest.sourceRef === `string`) {
1024
+ sharedStateIds.add(manifest.sourceRef)
1025
+ }
1026
+ const config = isRecord(manifest.config) ? manifest.config : undefined
1027
+ if (typeof config?.id === `string`) {
1028
+ sharedStateIds.add(config.id)
1029
+ }
1030
+ }
1031
+
1032
+ private async buildForkEntityUrlMap(
1033
+ entitiesToFork: Array<ElectricAgentsEntity>,
1034
+ opts: { suffix: string; rootUrl: string; rootInstanceId?: string }
1035
+ ): Promise<Map<string, string>> {
1036
+ const map = new Map<string, string>()
1037
+ const reserved = new Set<string>()
1038
+
1039
+ for (const entity of entitiesToFork) {
1040
+ const { type, instanceId } = this.parseEntityUrl(entity.url)
1041
+ const rootRequestedId =
1042
+ entity.url === opts.rootUrl ? opts.rootInstanceId : undefined
1043
+ const baseId = rootRequestedId ?? `${instanceId}-fork-${opts.suffix}`
1044
+ const forkUrl = await this.reserveForkEntityUrl(type, baseId, reserved, {
1045
+ exact: rootRequestedId !== undefined,
1046
+ })
1047
+ map.set(entity.url, forkUrl)
1048
+ }
1049
+
1050
+ return map
1051
+ }
1052
+
1053
+ private async reserveForkEntityUrl(
1054
+ type: string,
1055
+ baseId: string,
1056
+ reserved: Set<string>,
1057
+ opts?: { exact?: boolean }
1058
+ ): Promise<string> {
1059
+ if (!baseId || baseId.includes(`/`)) {
1060
+ throw new ElectricAgentsError(
1061
+ ErrCodeInvalidRequest,
1062
+ `Fork instance_id must not be empty or contain forward slashes`,
1063
+ 400
1064
+ )
1065
+ }
1066
+
1067
+ let attempt = 0
1068
+ while (true) {
1069
+ const instanceId = attempt === 0 ? baseId : `${baseId}-${attempt}`
1070
+ const url = `/${type}/${instanceId}`
1071
+ const exists = reserved.has(url) || (await this.registry.getEntity(url))
1072
+ if (!exists) {
1073
+ reserved.add(url)
1074
+ return url
1075
+ }
1076
+ if (opts?.exact) {
1077
+ throw new ElectricAgentsError(
1078
+ ErrCodeDuplicateURL,
1079
+ `Entity already exists at URL "${url}"`,
1080
+ 409
1081
+ )
1082
+ }
1083
+ attempt += 1
1084
+ }
1085
+ }
1086
+
1087
+ private async buildForkSharedStateIdMap(
1088
+ sourceIds: Set<string>,
1089
+ suffix: string
1090
+ ): Promise<Map<string, string>> {
1091
+ const map = new Map<string, string>()
1092
+ const reserved = new Set<string>()
1093
+
1094
+ for (const sourceId of [...sourceIds].sort()) {
1095
+ const baseId = `${sourceId}-fork-${suffix}`
1096
+ let attempt = 0
1097
+ while (true) {
1098
+ const candidate = attempt === 0 ? baseId : `${baseId}-${attempt}`
1099
+ const path = getSharedStateStreamPath(candidate)
1100
+ if (
1101
+ !reserved.has(candidate) &&
1102
+ !(await this.streamClient.exists(path))
1103
+ ) {
1104
+ reserved.add(candidate)
1105
+ map.set(sourceId, candidate)
1106
+ break
1107
+ }
1108
+ attempt += 1
1109
+ }
1110
+ }
1111
+
1112
+ return map
1113
+ }
1114
+
1115
+ private buildForkStringMap(
1116
+ entityUrlMap: Map<string, string>,
1117
+ sharedStateIdMap: Map<string, string>
1118
+ ): Map<string, string> {
1119
+ const stringMap = new Map<string, string>()
1120
+ for (const [sourceUrl, forkUrl] of entityUrlMap) {
1121
+ stringMap.set(sourceUrl, forkUrl)
1122
+ stringMap.set(`${sourceUrl}/main`, `${forkUrl}/main`)
1123
+ stringMap.set(`${sourceUrl}/error`, `${forkUrl}/error`)
1124
+ }
1125
+ for (const [sourceId, forkId] of sharedStateIdMap) {
1126
+ stringMap.set(sourceId, forkId)
1127
+ stringMap.set(
1128
+ getSharedStateStreamPath(sourceId),
1129
+ getSharedStateStreamPath(forkId)
1130
+ )
1131
+ }
1132
+ return stringMap
1133
+ }
1134
+
1135
+ private buildForkEntityPlans(
1136
+ entitiesToFork: Array<ElectricAgentsEntity>,
1137
+ entityUrlMap: Map<string, string>,
1138
+ stringMap: Map<string, string>
1139
+ ): Array<ForkEntityPlan> {
1140
+ const now = Date.now()
1141
+ return entitiesToFork.map((source) => {
1142
+ const forkUrl = entityUrlMap.get(source.url)
1143
+ if (!forkUrl) {
1144
+ throw new Error(`Missing fork URL for ${source.url}`)
1145
+ }
1146
+ const { type } = this.parseEntityUrl(forkUrl)
1147
+ const parent = source.parent ? entityUrlMap.get(source.parent) : undefined
1148
+ const spawnArgs = isRecord(source.spawn_args)
1149
+ ? (this.remapJsonValue(source.spawn_args, stringMap) as Record<
1150
+ string,
1151
+ unknown
1152
+ >)
1153
+ : source.spawn_args
1154
+
1155
+ const fork: ElectricAgentsEntity = {
1156
+ ...source,
1157
+ url: forkUrl,
1158
+ type,
1159
+ status: `idle`,
1160
+ streams: {
1161
+ main: `${forkUrl}/main`,
1162
+ error: `${forkUrl}/error`,
1163
+ },
1164
+ subscription_id: `${type}-handler`,
1165
+ write_token: randomUUID(),
1166
+ spawn_args: spawnArgs,
1167
+ parent,
1168
+ created_at: now,
1169
+ updated_at: now,
1170
+ }
1171
+ if (!parent) {
1172
+ delete fork.parent
1173
+ }
1174
+
1175
+ return { source, fork }
1176
+ })
1177
+ }
1178
+
1179
+ private buildForkReconciliation(
1180
+ plan: ForkEntityPlan,
1181
+ snapshot: ForkStateSnapshot,
1182
+ entityUrlMap: Map<string, string>,
1183
+ sharedStateIdMap: Map<string, string>,
1184
+ stringMap: Map<string, string>
1185
+ ): {
1186
+ events: Array<Record<string, unknown>>
1187
+ manifests: Map<string, Record<string, unknown>>
1188
+ } {
1189
+ const txid = `fork-${randomUUID()}`
1190
+ const headers = {
1191
+ txid,
1192
+ forkedFrom: plan.source.url,
1193
+ }
1194
+ const events: Array<Record<string, unknown>> = [
1195
+ entityStateSchema.entityCreated.update({
1196
+ key: `entity-created`,
1197
+ value: omitUndefined({
1198
+ entity_type: plan.fork.type,
1199
+ timestamp: new Date().toISOString(),
1200
+ args: plan.fork.spawn_args ?? {},
1201
+ parent_url: plan.fork.parent,
1202
+ }),
1203
+ headers,
1204
+ } as any) as Record<string, unknown>,
1205
+ ]
1206
+
1207
+ const activeManifests = new Map<string, Record<string, unknown>>()
1208
+ const sourceManifests =
1209
+ snapshot.manifestsByEntity.get(plan.source.url) ?? new Map()
1210
+ for (const [key, value] of sourceManifests) {
1211
+ const remapped = this.remapManifestEntry(
1212
+ key,
1213
+ value,
1214
+ entityUrlMap,
1215
+ sharedStateIdMap
1216
+ )
1217
+ activeManifests.set(remapped.key, remapped.value)
1218
+ if (!remapped.changed) {
1219
+ continue
1220
+ }
1221
+
1222
+ if (remapped.key !== key) {
1223
+ events.push(
1224
+ entityStateSchema.manifests.delete({
1225
+ key,
1226
+ headers,
1227
+ } as any) as Record<string, unknown>
1228
+ )
1229
+ events.push(
1230
+ entityStateSchema.manifests.insert({
1231
+ key: remapped.key,
1232
+ value: remapped.value as any,
1233
+ headers,
1234
+ } as any) as Record<string, unknown>
1235
+ )
1236
+ } else {
1237
+ events.push(
1238
+ entityStateSchema.manifests.update({
1239
+ key,
1240
+ value: remapped.value as any,
1241
+ headers,
1242
+ } as any) as Record<string, unknown>
1243
+ )
1244
+ }
1245
+ }
1246
+
1247
+ const childStatuses =
1248
+ snapshot.childStatusesByEntity.get(plan.source.url) ?? new Map()
1249
+ for (const [key, value] of childStatuses) {
1250
+ const remapped = this.remapChildStatus(value, entityUrlMap)
1251
+ if (!remapped) continue
1252
+ events.push(
1253
+ entityStateSchema.childStatus.update({
1254
+ key,
1255
+ value: remapped as any,
1256
+ headers,
1257
+ } as any) as Record<string, unknown>
1258
+ )
1259
+ }
1260
+
1261
+ const replayWatermarks =
1262
+ snapshot.replayWatermarksByEntity.get(plan.source.url) ?? new Map()
1263
+ for (const [key, value] of replayWatermarks) {
1264
+ const remapped = this.remapReplayWatermark(key, value, stringMap)
1265
+ if (!remapped) continue
1266
+ if (remapped.key !== key) {
1267
+ events.push(
1268
+ entityStateSchema.replayWatermarks.delete({
1269
+ key,
1270
+ headers,
1271
+ } as any) as Record<string, unknown>
1272
+ )
1273
+ events.push(
1274
+ entityStateSchema.replayWatermarks.insert({
1275
+ key: remapped.key,
1276
+ value: remapped.value as any,
1277
+ headers,
1278
+ } as any) as Record<string, unknown>
1279
+ )
1280
+ } else {
1281
+ events.push(
1282
+ entityStateSchema.replayWatermarks.update({
1283
+ key,
1284
+ value: remapped.value as any,
1285
+ headers,
1286
+ } as any) as Record<string, unknown>
1287
+ )
1288
+ }
1289
+ }
1290
+
1291
+ return { events, manifests: activeManifests }
1292
+ }
1293
+
1294
+ private remapManifestEntry(
1295
+ key: string,
1296
+ value: Record<string, unknown>,
1297
+ entityUrlMap: Map<string, string>,
1298
+ sharedStateIdMap: Map<string, string>
1299
+ ): {
1300
+ key: string
1301
+ value: Record<string, unknown>
1302
+ changed: boolean
1303
+ } {
1304
+ const next = cloneRecord(value)
1305
+
1306
+ if (next.kind === `child` && typeof next.entity_url === `string`) {
1307
+ const forkUrl = entityUrlMap.get(next.entity_url)
1308
+ if (!forkUrl) return { key, value: next, changed: false }
1309
+ const { instanceId } = this.parseEntityUrl(forkUrl)
1310
+ next.id = instanceId
1311
+ next.entity_url = forkUrl
1312
+ next.key = manifestChildKey(String(next.entity_type), instanceId)
1313
+ return { key: String(next.key), value: next, changed: true }
1314
+ }
1315
+
1316
+ if (next.kind === `shared-state` && typeof next.id === `string`) {
1317
+ const forkId = sharedStateIdMap.get(next.id)
1318
+ if (!forkId) return { key, value: next, changed: false }
1319
+ next.id = forkId
1320
+ next.key = manifestSharedStateKey(forkId)
1321
+ return { key: String(next.key), value: next, changed: true }
1322
+ }
1323
+
1324
+ if (next.kind === `source` && next.sourceType === `entity`) {
1325
+ const config = isRecord(next.config) ? next.config : {}
1326
+ const sourceUrl =
1327
+ typeof config.entityUrl === `string`
1328
+ ? config.entityUrl
1329
+ : typeof next.sourceRef === `string`
1330
+ ? next.sourceRef
1331
+ : undefined
1332
+ const forkUrl = sourceUrl ? entityUrlMap.get(sourceUrl) : undefined
1333
+ if (!forkUrl) return { key, value: next, changed: false }
1334
+ const { type } = this.parseEntityUrl(forkUrl)
1335
+ next.sourceRef = forkUrl
1336
+ next.key = manifestSourceKey(`entity`, forkUrl)
1337
+ next.config = {
1338
+ ...config,
1339
+ entityUrl: forkUrl,
1340
+ streamPath: `${forkUrl}/main`,
1341
+ entityType: type,
1342
+ }
1343
+ return { key: String(next.key), value: next, changed: true }
1344
+ }
1345
+
1346
+ if (next.kind === `source` && next.sourceType === `db`) {
1347
+ const config = isRecord(next.config) ? next.config : {}
1348
+ const sourceId =
1349
+ typeof next.sourceRef === `string`
1350
+ ? next.sourceRef
1351
+ : typeof config.id === `string`
1352
+ ? config.id
1353
+ : undefined
1354
+ const forkId = sourceId ? sharedStateIdMap.get(sourceId) : undefined
1355
+ if (!forkId) return { key, value: next, changed: false }
1356
+ next.sourceRef = forkId
1357
+ next.key = manifestSourceKey(`db`, forkId)
1358
+ next.config = {
1359
+ ...config,
1360
+ id: forkId,
1361
+ }
1362
+ return { key: String(next.key), value: next, changed: true }
1363
+ }
1364
+
1365
+ if (next.kind === `schedule` && next.scheduleType === `future_send`) {
1366
+ let changed = false
1367
+ if (typeof next.targetUrl === `string`) {
1368
+ const forkTarget = entityUrlMap.get(next.targetUrl)
1369
+ if (forkTarget) {
1370
+ next.targetUrl = forkTarget
1371
+ changed = true
1372
+ }
1373
+ }
1374
+ if (typeof next.from === `string`) {
1375
+ const forkFrom = entityUrlMap.get(next.from)
1376
+ if (forkFrom) {
1377
+ next.from = forkFrom
1378
+ changed = true
1379
+ }
1380
+ }
1381
+ return { key, value: next, changed }
1382
+ }
1383
+
1384
+ return { key, value: next, changed: false }
1385
+ }
1386
+
1387
+ private remapChildStatus(
1388
+ value: Record<string, unknown>,
1389
+ entityUrlMap: Map<string, string>
1390
+ ): Record<string, unknown> | null {
1391
+ if (typeof value.entity_url !== `string`) return null
1392
+ const forkUrl = entityUrlMap.get(value.entity_url)
1393
+ if (!forkUrl) return null
1394
+ const { type } = this.parseEntityUrl(forkUrl)
1395
+ return {
1396
+ ...value,
1397
+ entity_url: forkUrl,
1398
+ entity_type: type,
1399
+ }
1400
+ }
1401
+
1402
+ private remapReplayWatermark(
1403
+ key: string,
1404
+ value: Record<string, unknown>,
1405
+ stringMap: Map<string, string>
1406
+ ): { key: string; value: Record<string, unknown> } | null {
1407
+ if (typeof value.source_id !== `string`) return null
1408
+ const sourceId = value.source_id
1409
+ const forkSourceId = stringMap.get(sourceId)
1410
+ if (!forkSourceId) return null
1411
+ const next = { ...value, source_id: forkSourceId }
1412
+ return {
1413
+ key: key === sourceId ? forkSourceId : key,
1414
+ value: next,
1415
+ }
1416
+ }
1417
+
1418
+ private remapJsonValue(
1419
+ value: unknown,
1420
+ stringMap: Map<string, string>
1421
+ ): unknown {
1422
+ if (typeof value === `string`) {
1423
+ return stringMap.get(value) ?? value
1424
+ }
1425
+ if (Array.isArray(value)) {
1426
+ return value.map((item) => this.remapJsonValue(item, stringMap))
1427
+ }
1428
+ if (isRecord(value)) {
1429
+ return Object.fromEntries(
1430
+ Object.entries(value).map(([key, item]) => [
1431
+ key,
1432
+ this.remapJsonValue(item, stringMap),
1433
+ ])
1434
+ )
1435
+ }
1436
+ return value
1437
+ }
1438
+
1439
+ private async materializeForkManifestSideEffects(
1440
+ entityUrl: string,
1441
+ manifests: Map<string, Record<string, unknown>>
1442
+ ): Promise<void> {
1443
+ for (const [manifestKey, manifest] of manifests) {
1444
+ await this.syncEntitiesManifestSource(
1445
+ entityUrl,
1446
+ manifestKey,
1447
+ `upsert`,
1448
+ manifest
1449
+ )
1450
+
1451
+ const wake = buildManifestWakeRegistration(
1452
+ entityUrl,
1453
+ manifest,
1454
+ manifestKey
1455
+ )
1456
+ if (wake) {
1457
+ await this.wakeRegistry.register({
1458
+ ...wake,
1459
+ tenantId: this.tenantId,
1460
+ })
1461
+ }
1462
+
1463
+ const cronSpec = extractManifestCronSpec(manifest)
1464
+ if (cronSpec && this.scheduler) {
1465
+ await this.getOrCreateCronStream(cronSpec.expression, cronSpec.timezone)
1466
+ }
1467
+
1468
+ await this.syncManifestFutureSendSchedule(
1469
+ entityUrl,
1470
+ manifestKey,
1471
+ manifest
1472
+ )
1473
+ }
1474
+ }
1475
+
1476
+ private async syncManifestFutureSendSchedule(
1477
+ ownerEntityUrl: string,
1478
+ manifestKey: string,
1479
+ manifest: Record<string, unknown>
1480
+ ): Promise<void> {
1481
+ if (!this.scheduler) return
1482
+ if (
1483
+ manifest.kind !== `schedule` ||
1484
+ manifest.scheduleType !== `future_send` ||
1485
+ (manifest.status !== undefined && manifest.status !== `pending`)
1486
+ ) {
1487
+ return
1488
+ }
1489
+
1490
+ const fireAtRaw = manifest.fireAt
1491
+ const producerId = manifest.producerId
1492
+ const targetUrl = manifest.targetUrl
1493
+ if (
1494
+ typeof fireAtRaw !== `string` ||
1495
+ typeof producerId !== `string` ||
1496
+ typeof targetUrl !== `string`
1497
+ ) {
1498
+ serverLog.warn(
1499
+ `[agent-server] invalid forked future_send manifest entry for ${ownerEntityUrl}/${manifestKey}`
1500
+ )
1501
+ return
1502
+ }
1503
+
1504
+ const fireAt = new Date(fireAtRaw)
1505
+ if (Number.isNaN(fireAt.getTime())) {
1506
+ serverLog.warn(
1507
+ `[agent-server] invalid forked future_send fireAt for ${ownerEntityUrl}/${manifestKey}: ${fireAtRaw}`
1508
+ )
1509
+ return
1510
+ }
1511
+
1512
+ await this.scheduler.syncManifestDelayedSend(
1513
+ ownerEntityUrl,
1514
+ manifestKey,
1515
+ {
1516
+ entityUrl: targetUrl,
1517
+ from:
1518
+ typeof manifest.from === `string` ? manifest.from : ownerEntityUrl,
1519
+ payload: manifest.payload,
1520
+ key: `scheduled-${producerId}`,
1521
+ type:
1522
+ typeof manifest.messageType === `string`
1523
+ ? manifest.messageType
1524
+ : undefined,
1525
+ producerId,
1526
+ manifest: {
1527
+ ownerEntityUrl,
1528
+ key: manifestKey,
1529
+ entry: omitUndefined({
1530
+ ...manifest,
1531
+ key: manifestKey,
1532
+ kind: `schedule`,
1533
+ scheduleType: `future_send`,
1534
+ targetUrl,
1535
+ fireAt: fireAt.toISOString(),
1536
+ producerId,
1537
+ status: `pending`,
1538
+ }),
1539
+ },
1540
+ },
1541
+ fireAt
1542
+ )
1543
+ }
1544
+
1545
+ private parseEntityUrl(url: string): { type: string; instanceId: string } {
1546
+ const segments = url.split(`/`).filter(Boolean)
1547
+ if (segments.length !== 2 || !segments[0] || !segments[1]) {
1548
+ throw new ElectricAgentsError(
1549
+ ErrCodeInvalidRequest,
1550
+ `Invalid entity URL "${url}"`,
1551
+ 400
1552
+ )
1553
+ }
1554
+ return { type: segments[0], instanceId: segments[1] }
1555
+ }
1556
+
1557
+ // ==========================================================================
1558
+ // Send
1559
+ // ==========================================================================
1560
+
1561
+ /**
1562
+ * Deliver a message to an entity's main stream, with optional input schema
1563
+ * validation.
1564
+ */
1565
+ async send(
1566
+ entityUrl: string,
1567
+ req: SendRequest,
1568
+ opts?: { producerId?: string }
1569
+ ): Promise<void> {
1570
+ const entity = await this.validateSendRequest(entityUrl, req)
1571
+ if (
1572
+ this.isForkWorkLockedEntity(entityUrl) &&
1573
+ !(req.from && this.isForkWorkLockedEntity(req.from))
1574
+ ) {
1575
+ this.assertEntityNotForkWorkLocked(entityUrl)
1576
+ }
1577
+
1578
+ const now = new Date().toISOString()
1579
+ const key =
1580
+ req.key ??
1581
+ `msg-in-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
1582
+
1583
+ const value: Record<string, unknown> = {
1584
+ from: req.from,
1585
+ payload: req.payload,
1586
+ timestamp: now,
1587
+ }
1588
+ if (req.type) {
1589
+ value.message_type = req.type
1590
+ }
1591
+
1592
+ const envelope = entityStateSchema.inbox.insert({
1593
+ key,
1594
+ value,
1595
+ } as any)
1596
+
1597
+ const encoded = this.encodeChangeEvent(envelope as Record<string, unknown>)
1598
+ try {
1599
+ if (opts?.producerId) {
1600
+ await this.streamClient.appendIdempotent(entity.streams.main, encoded, {
1601
+ producerId: opts.producerId,
1602
+ })
1603
+ return
1604
+ }
1605
+
1606
+ await this.streamClient.append(entity.streams.main, encoded)
1607
+ } catch (err) {
1608
+ if (this.isClosedStreamError(err)) {
1609
+ throw new ElectricAgentsError(
1610
+ ErrCodeNotRunning,
1611
+ `Entity is stopped`,
1612
+ 409
1613
+ )
1614
+ }
1615
+ throw err
1616
+ }
1617
+ }
1618
+
1619
+ // ==========================================================================
1620
+ // Tag Updates
1621
+ // ==========================================================================
1622
+
1623
+ async setTag(
1624
+ entityUrl: string,
1625
+ key: string,
1626
+ req: SetTagRequest,
1627
+ token: string
1628
+ ): Promise<ElectricAgentsEntity> {
1629
+ const entity = await this.registry.getEntity(entityUrl)
1630
+ if (!entity) {
1631
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
1632
+ }
1633
+
1634
+ if (!this.isValidWriteToken(entity, token)) {
1635
+ throw new ElectricAgentsError(
1636
+ ErrCodeUnauthorized,
1637
+ `Invalid write token`,
1638
+ 401
1639
+ )
1640
+ }
1641
+ if (entity.status === `stopped`) {
1642
+ throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
1643
+ }
1644
+
1645
+ if (typeof req.value !== `string`) {
1646
+ throw new ElectricAgentsError(
1647
+ ErrCodeInvalidRequest,
1648
+ `Tag values must be strings`,
1649
+ 400
1650
+ )
1651
+ }
1652
+
1653
+ const result = await this.registry.setEntityTag(entityUrl, key, req.value)
1654
+ const updated = result.entity
1655
+ if (!updated) {
1656
+ throw new ElectricAgentsError(
1657
+ ErrCodeEntityPersistFailed,
1658
+ `Entity not found after tag write`,
1659
+ 500
1660
+ )
1661
+ }
1662
+
1663
+ if (result.changed && this.entityBridgeManager) {
1664
+ await this.entityBridgeManager.onEntityChanged(entityUrl)
1665
+ }
1666
+
1667
+ return updated
1668
+ }
1669
+
1670
+ async removeTag(
1671
+ entityUrl: string,
1672
+ key: string,
1673
+ token: string
1674
+ ): Promise<ElectricAgentsEntity> {
1675
+ const entity = await this.registry.getEntity(entityUrl)
1676
+ if (!entity) {
1677
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
1678
+ }
1679
+
1680
+ if (!this.isValidWriteToken(entity, token)) {
1681
+ throw new ElectricAgentsError(
1682
+ ErrCodeUnauthorized,
1683
+ `Invalid write token`,
1684
+ 401
1685
+ )
1686
+ }
1687
+ if (entity.status === `stopped`) {
1688
+ throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
1689
+ }
1690
+
1691
+ const result = await this.registry.removeEntityTag(entityUrl, key)
1692
+ const updated = result.entity
1693
+ if (!updated) {
1694
+ throw new ElectricAgentsError(
1695
+ ErrCodeEntityPersistFailed,
1696
+ `Entity not found after tag delete`,
1697
+ 500
1698
+ )
1699
+ }
1700
+
1701
+ if (result.changed && this.entityBridgeManager) {
1702
+ await this.entityBridgeManager.onEntityChanged(entityUrl)
1703
+ }
1704
+
1705
+ return updated
1706
+ }
1707
+
1708
+ async registerEntitiesSource(tags: Record<string, string>): Promise<{
1709
+ sourceRef: string
1710
+ streamUrl: string
1711
+ }> {
1712
+ if (!this.entityBridgeManager) {
1713
+ throw new Error(`Entity bridge manager not configured`)
1714
+ }
1715
+ return this.entityBridgeManager.register(this.validateTags(tags))
1716
+ }
1717
+
1718
+ async writeManifestEntry(
1719
+ entityUrl: string,
1720
+ key: string,
1721
+ operation: `insert` | `update` | `upsert` | `delete`,
1722
+ value?: Record<string, unknown>,
1723
+ opts?: { producerId?: string; txid?: string }
1724
+ ): Promise<void> {
1725
+ const entity = await this.registry.getEntity(entityUrl)
1726
+ if (!entity) {
1727
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
1728
+ }
1729
+
1730
+ const event: Record<string, unknown> = {
1731
+ type: `manifest`,
1732
+ key,
1733
+ headers: {
1734
+ operation,
1735
+ timestamp: new Date().toISOString(),
1736
+ ...(opts?.txid ? { txid: opts.txid } : {}),
1737
+ },
1738
+ }
1739
+ if (value !== undefined) {
1740
+ event.value = value
1741
+ }
1742
+
1743
+ const encoded = this.encodeChangeEvent(event)
1744
+ if (opts?.producerId) {
1745
+ await this.streamClient.appendIdempotent(entity.streams.main, encoded, {
1746
+ producerId: opts.producerId,
1747
+ })
1748
+ await this.syncEntitiesManifestSource(entityUrl, key, operation, value)
1749
+ return
1750
+ }
1751
+
1752
+ await this.streamClient.append(entity.streams.main, encoded)
1753
+ await this.syncEntitiesManifestSource(entityUrl, key, operation, value)
1754
+ }
1755
+
1756
+ async upsertCronSchedule(
1757
+ entityUrl: string,
1758
+ req: {
1759
+ id: string
1760
+ expression: string
1761
+ timezone?: string
1762
+ payload?: unknown
1763
+ debounceMs?: number
1764
+ timeoutMs?: number
1765
+ }
1766
+ ): Promise<{ txid: string }> {
1767
+ if (req.payload === undefined) {
1768
+ throw new ElectricAgentsError(
1769
+ ErrCodeInvalidRequest,
1770
+ `Missing required field: payload`,
1771
+ 400
1772
+ )
1773
+ }
1774
+
1775
+ const spec = resolveCronScheduleSpec(req.expression, req.timezone)
1776
+
1777
+ const manifestKey = `schedule:${req.id}`
1778
+ await this.wakeRegistry.unregisterByManifestKey(
1779
+ entityUrl,
1780
+ manifestKey,
1781
+ this.tenantId
1782
+ )
1783
+ await this.wakeRegistry.register({
1784
+ tenantId: this.tenantId,
1785
+ subscriberUrl: entityUrl,
1786
+ sourceUrl: getCronStreamPath(spec.expression, spec.timezone),
1787
+ condition: {
1788
+ on: `change`,
1789
+ },
1790
+ debounceMs: req.debounceMs,
1791
+ timeoutMs: req.timeoutMs,
1792
+ oneShot: false,
1793
+ manifestKey,
1794
+ })
1795
+ await this.getOrCreateCronStream(spec.expression, spec.timezone)
1796
+
1797
+ const txid = randomUUID()
1798
+ await this.writeManifestEntry(
1799
+ entityUrl,
1800
+ manifestKey,
1801
+ `upsert`,
1802
+ {
1803
+ key: manifestKey,
1804
+ kind: `schedule`,
1805
+ id: req.id,
1806
+ scheduleType: `cron`,
1807
+ expression: spec.expression,
1808
+ timezone: spec.timezone,
1809
+ payload: req.payload,
1810
+ wake: {
1811
+ on: `change`,
1812
+ ...(typeof req.debounceMs === `number`
1813
+ ? { debounceMs: req.debounceMs }
1814
+ : {}),
1815
+ ...(typeof req.timeoutMs === `number`
1816
+ ? { timeoutMs: req.timeoutMs }
1817
+ : {}),
1818
+ },
1819
+ },
1820
+ { txid }
1821
+ )
1822
+
1823
+ return { txid }
1824
+ }
1825
+
1826
+ async upsertFutureSendSchedule(
1827
+ ownerEntityUrl: string,
1828
+ req: {
1829
+ id: string
1830
+ payload: unknown
1831
+ targetUrl?: string
1832
+ fireAt: string
1833
+ from?: string
1834
+ messageType?: string
1835
+ }
1836
+ ): Promise<{ txid: string }> {
1837
+ if (!this.scheduler) {
1838
+ throw new Error(`Scheduler not configured`)
1839
+ }
1840
+
1841
+ const targetUrl = req.targetUrl ?? ownerEntityUrl
1842
+ const from = req.from ?? ownerEntityUrl
1843
+ const fireAt = new Date(req.fireAt)
1844
+ if (Number.isNaN(fireAt.getTime())) {
1845
+ throw new ElectricAgentsError(
1846
+ ErrCodeInvalidRequest,
1847
+ `Invalid fireAt timestamp: ${req.fireAt}`,
1848
+ 400
1849
+ )
1850
+ }
1851
+
1852
+ await this.validateSendRequest(targetUrl, {
1853
+ from,
1854
+ payload: req.payload,
1855
+ type: req.messageType,
1856
+ })
1857
+
1858
+ const manifestKey = `schedule:${req.id}`
1859
+ const producerId = `future-send-${randomUUID()}`
1860
+
1861
+ await this.wakeRegistry.unregisterByManifestKey(
1862
+ ownerEntityUrl,
1863
+ manifestKey,
1864
+ this.tenantId
1865
+ )
1866
+ await this.scheduler.syncManifestDelayedSend(
1867
+ ownerEntityUrl,
1868
+ manifestKey,
1869
+ {
1870
+ entityUrl: targetUrl,
1871
+ from,
1872
+ payload: req.payload,
1873
+ key: `scheduled-${producerId}`,
1874
+ type: req.messageType,
1875
+ producerId,
1876
+ manifest: {
1877
+ ownerEntityUrl,
1878
+ key: manifestKey,
1879
+ entry: {
1880
+ key: manifestKey,
1881
+ kind: `schedule`,
1882
+ id: req.id,
1883
+ scheduleType: `future_send`,
1884
+ fireAt: fireAt.toISOString(),
1885
+ targetUrl,
1886
+ payload: req.payload,
1887
+ producerId,
1888
+ ...(req.from ? { from: req.from } : {}),
1889
+ ...(req.messageType ? { messageType: req.messageType } : {}),
1890
+ status: `pending`,
1891
+ },
1892
+ },
1893
+ },
1894
+ fireAt
1895
+ )
1896
+
1897
+ const txid = randomUUID()
1898
+ await this.writeManifestEntry(
1899
+ ownerEntityUrl,
1900
+ manifestKey,
1901
+ `upsert`,
1902
+ {
1903
+ key: manifestKey,
1904
+ kind: `schedule`,
1905
+ id: req.id,
1906
+ scheduleType: `future_send`,
1907
+ fireAt: fireAt.toISOString(),
1908
+ targetUrl,
1909
+ payload: req.payload,
1910
+ producerId,
1911
+ ...(req.from ? { from: req.from } : {}),
1912
+ ...(req.messageType ? { messageType: req.messageType } : {}),
1913
+ status: `pending`,
1914
+ },
1915
+ { txid }
1916
+ )
1917
+
1918
+ return { txid }
1919
+ }
1920
+
1921
+ async deleteSchedule(
1922
+ entityUrl: string,
1923
+ req: { id: string }
1924
+ ): Promise<{ txid: string }> {
1925
+ const manifestKey = `schedule:${req.id}`
1926
+ if (this.scheduler) {
1927
+ await this.scheduler.cancelManifestDelayedSend(entityUrl, manifestKey)
1928
+ }
1929
+ await this.wakeRegistry.unregisterByManifestKey(
1930
+ entityUrl,
1931
+ manifestKey,
1932
+ this.tenantId
1933
+ )
1934
+
1935
+ const txid = randomUUID()
1936
+ await this.writeManifestEntry(entityUrl, manifestKey, `delete`, undefined, {
1937
+ txid,
1938
+ })
1939
+
1940
+ return { txid }
1941
+ }
1942
+
1943
+ // ==========================================================================
1944
+ // Wake Evaluation
1945
+ // ==========================================================================
1946
+
1947
+ /**
1948
+ * Register a wake subscription from a subscriber to a source entity.
1949
+ */
1950
+ async registerWake(opts: {
1951
+ subscriberUrl: string
1952
+ sourceUrl: string
1953
+ condition: `runFinished` | { on: `change`; collections?: Array<string> }
1954
+ debounceMs?: number
1955
+ timeoutMs?: number
1956
+ includeResponse?: boolean
1957
+ manifestKey?: string
1958
+ }): Promise<void> {
1959
+ await this.wakeRegistry.register({
1960
+ tenantId: this.tenantId,
1961
+ subscriberUrl: opts.subscriberUrl,
1962
+ sourceUrl: opts.sourceUrl,
1963
+ condition: opts.condition,
1964
+ oneShot: false,
1965
+ debounceMs: opts.debounceMs,
1966
+ timeoutMs: opts.timeoutMs,
1967
+ includeResponse: opts.includeResponse,
1968
+ manifestKey: opts.manifestKey,
1969
+ })
1970
+ }
1971
+
1972
+ async enqueueDelayedSend(
1973
+ entityUrl: string,
1974
+ req: SendRequest,
1975
+ fireAt: Date
1976
+ ): Promise<void> {
1977
+ if (!this.scheduler) {
1978
+ throw new Error(`Scheduler not configured`)
1979
+ }
1980
+
1981
+ await this.validateSendRequest(entityUrl, req)
1982
+
1983
+ await this.scheduler.enqueueDelayedSend(
1984
+ {
1985
+ entityUrl,
1986
+ from: req.from,
1987
+ payload: req.payload,
1988
+ key: req.key,
1989
+ type: req.type,
1990
+ },
1991
+ fireAt
1992
+ )
1993
+ }
1994
+
1995
+ /**
1996
+ * Evaluate an event against registered wake conditions and deliver results.
1997
+ */
1998
+ async evaluateWakes(
1999
+ sourceUrl: string,
2000
+ event: Record<string, unknown>
2001
+ ): Promise<void> {
2002
+ return await withSpan(`electric_agents.evaluateWakes`, async (span) => {
2003
+ span.setAttribute(ATTR.WAKE_SOURCE, sourceUrl)
2004
+ const results = this.wakeRegistry.evaluate(
2005
+ sourceUrl,
2006
+ event,
2007
+ this.tenantId
2008
+ )
2009
+ span.setAttribute(`electric_agents.wake.subscriber_count`, results.length)
2010
+ const settled = await Promise.allSettled(
2011
+ results.map((result) => this.deliverWakeResult(result))
2012
+ )
2013
+ for (const [index, result] of settled.entries()) {
2014
+ if (result.status === `rejected`) {
2015
+ serverLog.warn(
2016
+ `[agent-server] failed to deliver wake for ${results[index]!.subscriberUrl}:`,
2017
+ result.reason
2018
+ )
2019
+ }
2020
+ }
2021
+ })
2022
+ }
2023
+
2024
+ /**
2025
+ * Deliver a wake result: append WakeMessage to subscriber's stream and
2026
+ * trigger webhook notification.
2027
+ */
2028
+ private async deliverWakeResult(result: WakeEvalResult): Promise<void> {
2029
+ if (result.tenantId !== this.tenantId) return
2030
+
2031
+ return await withSpan(`electric_agents.deliverWake`, async (span) => {
2032
+ span.setAttributes({
2033
+ [ATTR.WAKE_SUBSCRIBER]: result.subscriberUrl,
2034
+ [ATTR.WAKE_SOURCE]: result.wakeMessage.source,
2035
+ [ATTR.WAKE_KIND]: result.wakeMessage.timeout ? `timeout` : `change`,
2036
+ })
2037
+ // Fetch subscriber and source entity in parallel — runFinished wakes need
2038
+ // both, plain wakes only need subscriber but the extra read is cheap.
2039
+ const needsSource = result.runFinishedStatus !== undefined
2040
+ const [subscriber, sourceEntity] = await Promise.all([
2041
+ this.registry.getEntity(result.subscriberUrl),
2042
+ needsSource
2043
+ ? this.registry.getEntity(result.wakeMessage.source)
2044
+ : Promise.resolve(null),
2045
+ ])
2046
+ if (!subscriber) return
2047
+ const wakeMessage = await this.buildWakeMessage(
2048
+ subscriber,
2049
+ result,
2050
+ sourceEntity
2051
+ )
2052
+ const wakeEvent = entityStateSchema.wakes.insert({
2053
+ key: `wake-${result.registrationDbId}-${result.sourceEventKey}`,
2054
+ value: wakeMessage,
2055
+ } as any)
2056
+ await this.streamClient.appendIdempotent(
2057
+ subscriber.streams.main,
2058
+ this.encodeChangeEvent(wakeEvent as Record<string, unknown>),
2059
+ {
2060
+ producerId: `wake-reg-${result.registrationDbId}-${result.sourceEventKey}`,
2061
+ }
2062
+ )
2063
+ })
2064
+ }
2065
+
2066
+ private async syncEntitiesManifestSource(
2067
+ entityUrl: string,
2068
+ manifestKey: string,
2069
+ operation: `insert` | `update` | `upsert` | `delete`,
2070
+ value?: Record<string, unknown>
2071
+ ): Promise<void> {
2072
+ const sourceRef =
2073
+ operation === `delete` ? undefined : this.extractEntitiesSourceRef(value)
2074
+ await this.registry.replaceEntityManifestSource(
2075
+ entityUrl,
2076
+ manifestKey,
2077
+ sourceRef
2078
+ )
2079
+ }
2080
+
2081
+ private extractEntitiesSourceRef(
2082
+ manifest?: Record<string, unknown>
2083
+ ): string | undefined {
2084
+ if (
2085
+ manifest?.kind === `source` &&
2086
+ manifest.sourceType === `entities` &&
2087
+ typeof manifest.sourceRef === `string`
2088
+ ) {
2089
+ return manifest.sourceRef
2090
+ }
2091
+ return undefined
2092
+ }
2093
+
2094
+ /**
2095
+ * Read a child entity's stream and extract concatenated text deltas
2096
+ * for a specific run, plus any error messages for that run.
2097
+ */
2098
+ private async extractRunResponse(
2099
+ entity: ElectricAgentsEntity,
2100
+ runKey: string,
2101
+ runStatus: `completed` | `failed`
2102
+ ): Promise<{ response?: string; error?: string }> {
2103
+ let events: Array<Record<string, unknown>>
2104
+ try {
2105
+ events = await this.streamClient.readJson<Record<string, unknown>>(
2106
+ entity.streams.main
2107
+ )
2108
+ } catch (err) {
2109
+ serverLog.warn(
2110
+ `[agent-server] failed to read child stream for ${entity.url} (${runKey}): ${err instanceof Error ? err.message : String(err)}`
2111
+ )
2112
+ return { error: `Failed to load child response` }
2113
+ }
2114
+
2115
+ const textDeltas: Array<string> = []
2116
+ const errors: Array<string> = []
2117
+
2118
+ for (const parsed of events) {
2119
+ const value = parsed.value as Record<string, unknown> | undefined
2120
+ if (!value) continue
2121
+
2122
+ if (parsed.type === `text_delta`) {
2123
+ if ((value.run_id as string) === runKey) {
2124
+ textDeltas.push((value.delta as string) || ``)
2125
+ }
2126
+ } else if (parsed.type === `error` && runStatus === `failed`) {
2127
+ if ((value.run_id as string) === runKey) {
2128
+ errors.push((value.message as string) || ``)
2129
+ }
2130
+ }
2131
+ }
2132
+
2133
+ const result: { response?: string; error?: string } = {}
2134
+
2135
+ const runText = textDeltas.join(``)
2136
+ if (runText.length > 0) {
2137
+ result.response = runText
2138
+ }
2139
+
2140
+ if (errors.length > 0) {
2141
+ result.error = errors.join(`\n`)
2142
+ }
2143
+
2144
+ return result
2145
+ }
2146
+
2147
+ private async buildWakeMessage(
2148
+ subscriber: ElectricAgentsEntity,
2149
+ result: WakeEvalResult,
2150
+ sourceEntity: ElectricAgentsEntity | null
2151
+ ): Promise<WakeMessage> {
2152
+ const wakeMessage: WakeMessage = {
2153
+ timestamp: new Date().toISOString(),
2154
+ ...result.wakeMessage,
2155
+ }
2156
+ if (!result.runFinishedStatus) {
2157
+ return wakeMessage
2158
+ }
2159
+
2160
+ if (!sourceEntity) {
2161
+ throw new Error(
2162
+ `[agent-server] runFinished wake source entity not found: ${result.wakeMessage.source}`
2163
+ )
2164
+ }
2165
+
2166
+ // `runFinished` is valid both for spawned children and explicitly observed
2167
+ // entities. Only child wakes get the richer sibling-status payload.
2168
+ if (sourceEntity.parent !== subscriber.url) {
2169
+ return wakeMessage
2170
+ }
2171
+
2172
+ const includeResponse = result.includeResponse !== false
2173
+ const changes = result.wakeMessage.changes
2174
+ const runKey = changes[changes.length - 1]?.key
2175
+ const { response, error } =
2176
+ includeResponse && runKey
2177
+ ? await this.extractRunResponse(
2178
+ sourceEntity,
2179
+ runKey,
2180
+ result.runFinishedStatus
2181
+ )
2182
+ : {}
2183
+
2184
+ return {
2185
+ ...wakeMessage,
2186
+ finished_child: {
2187
+ url: sourceEntity.url,
2188
+ type: sourceEntity.type,
2189
+ run_status: result.runFinishedStatus,
2190
+ ...(response !== undefined ? { response } : {}),
2191
+ ...(error !== undefined ? { error } : {}),
2192
+ },
2193
+ }
2194
+ }
2195
+
2196
+ // ==========================================================================
2197
+ // Kill
2198
+ // ==========================================================================
2199
+
2200
+ async kill(entityUrl: string): Promise<{ txid: number }> {
2201
+ const entity = await this.registry.getEntity(entityUrl)
2202
+ if (!entity) {
2203
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
2204
+ }
2205
+
2206
+ await this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId)
2207
+ await this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId)
2208
+
2209
+ const txid = await this.registry.updateStatusWithTxid(entityUrl, `stopped`)
2210
+ if (this.entityBridgeManager) {
2211
+ await this.entityBridgeManager.onEntityChanged(entityUrl)
2212
+ }
2213
+
2214
+ // Append entity_stopped to main/error streams and close them.
2215
+ const stoppedEvent = entityStateSchema.entityStopped.insert({
2216
+ key: `stopped`,
2217
+ value: {
2218
+ timestamp: new Date().toISOString(),
2219
+ },
2220
+ } as any)
2221
+ const eofData = this.encodeChangeEvent(
2222
+ stoppedEvent as Record<string, unknown>
2223
+ )
2224
+
2225
+ for (const streamPath of [entity.streams.main, entity.streams.error]) {
2226
+ try {
2227
+ await this.streamClient.append(streamPath, eofData, { close: true })
2228
+ } catch (err) {
2229
+ const message = err instanceof Error ? err.message : String(err)
2230
+ if (
2231
+ /closed/i.test(message) ||
2232
+ /not found/i.test(message) ||
2233
+ /404/.test(message) ||
2234
+ /409/.test(message)
2235
+ ) {
2236
+ continue
2237
+ }
2238
+ throw err
2239
+ }
2240
+ }
2241
+
2242
+ return { txid }
2243
+ }
2244
+
2245
+ // ==========================================================================
2246
+ // Write Validation
2247
+ // ==========================================================================
2248
+
2249
+ async validateWriteEvent(
2250
+ entity: ElectricAgentsEntity,
2251
+ event: Record<string, unknown>
2252
+ ): Promise<{ code: string; message: string; status: number } | null> {
2253
+ if (!entity.type) return null
2254
+
2255
+ const { stateSchemas } = await this.getEffectiveSchemas(entity)
2256
+ if (!stateSchemas) return null
2257
+
2258
+ const eventType = event.type as string | undefined
2259
+ if (!eventType) return null
2260
+
2261
+ if (!(eventType in stateSchemas)) {
2262
+ return {
2263
+ code: ErrCodeUnknownEventType,
2264
+ message: `Unknown event type "${eventType}"`,
2265
+ status: 422,
2266
+ }
2267
+ }
2268
+
2269
+ const schema = stateSchemas[eventType]
2270
+ if (schema) {
2271
+ const headers = event.headers as Record<string, unknown> | undefined
2272
+ const operation = headers?.operation
2273
+ const rawPayload =
2274
+ operation === `delete` && `old_value` in event
2275
+ ? event.old_value
2276
+ : event.value
2277
+ if (rawPayload === undefined) {
2278
+ return null
2279
+ }
2280
+ const payload =
2281
+ typeof rawPayload === `object` && rawPayload !== null
2282
+ ? (rawPayload as Record<string, unknown>)
2283
+ : rawPayload
2284
+ const valErr = this.validator.validate(schema, payload)
2285
+ if (valErr) {
2286
+ return {
2287
+ code: ErrCodeSchemaValidationFailed,
2288
+ message: valErr.message,
2289
+ status: 422,
2290
+ }
2291
+ }
2292
+ }
2293
+
2294
+ return null
2295
+ }
2296
+
2297
+ // ==========================================================================
2298
+ // Amend Schemas
2299
+ // ==========================================================================
2300
+
2301
+ /**
2302
+ * Add new input/output schema keys to an entity type directly in Postgres.
2303
+ */
2304
+ async amendSchemas(
2305
+ typeName: string,
2306
+ schemas: {
2307
+ inbox_schemas?: Record<string, Record<string, unknown>>
2308
+ state_schemas?: Record<string, Record<string, unknown>>
2309
+ }
2310
+ ): Promise<ElectricAgentsEntityType> {
2311
+ // Validate each provided schema via validateSchemaSubset.
2312
+ this.validateSchemaMap(schemas.inbox_schemas)
2313
+ this.validateSchemaMap(schemas.state_schemas)
2314
+
2315
+ // Look up current entity type.
2316
+ const existing = await this.registry.getEntityType(typeName)
2317
+ if (!existing) {
2318
+ throw new ElectricAgentsError(
2319
+ ErrCodeUnknownEntityType,
2320
+ `Entity type "${typeName}" not found`,
2321
+ 404
2322
+ )
2323
+ }
2324
+
2325
+ // Check for key overlap (additive only, no overwriting).
2326
+ if (schemas.inbox_schemas && existing.inbox_schemas) {
2327
+ for (const key of Object.keys(schemas.inbox_schemas)) {
2328
+ if (key in existing.inbox_schemas) {
2329
+ throw new ElectricAgentsError(
2330
+ ErrCodeSchemaKeyExists,
2331
+ `Cannot amend existing inbox schema key: ${key}`,
2332
+ 409
2333
+ )
2334
+ }
2335
+ }
2336
+ }
2337
+ if (schemas.state_schemas && existing.state_schemas) {
2338
+ for (const key of Object.keys(schemas.state_schemas)) {
2339
+ if (key in existing.state_schemas) {
2340
+ throw new ElectricAgentsError(
2341
+ ErrCodeSchemaKeyExists,
2342
+ `Cannot amend existing state schema key: ${key}`,
2343
+ 409
2344
+ )
2345
+ }
2346
+ }
2347
+ }
2348
+
2349
+ // Merge schemas.
2350
+ const mergedInbox = schemas.inbox_schemas
2351
+ ? { ...(existing.inbox_schemas ?? {}), ...schemas.inbox_schemas }
2352
+ : existing.inbox_schemas
2353
+ const mergedState = schemas.state_schemas
2354
+ ? { ...(existing.state_schemas ?? {}), ...schemas.state_schemas }
2355
+ : existing.state_schemas
2356
+
2357
+ const now = new Date().toISOString()
2358
+ const nextRevision = existing.revision + 1
2359
+
2360
+ const updatedType: ElectricAgentsEntityType = {
2361
+ name: existing.name,
2362
+ description: existing.description,
2363
+ creation_schema: existing.creation_schema,
2364
+ inbox_schemas: mergedInbox,
2365
+ state_schemas: mergedState,
2366
+ serve_endpoint: existing.serve_endpoint,
2367
+ revision: nextRevision,
2368
+ created_at: existing.created_at,
2369
+ updated_at: now,
2370
+ }
2371
+
2372
+ await this.registry.updateEntityTypeInPlace(updatedType)
2373
+
2374
+ return (await this.registry.getEntityType(typeName)) ?? updatedType
2375
+ }
2376
+
2377
+ // ==========================================================================
2378
+ // Webhook Enrichment
2379
+ // ==========================================================================
2380
+
2381
+ /**
2382
+ * Enrich webhook payload with entity context.
2383
+ * Called by ElectricAgentsServer during webhook forwarding to inject entity context.
2384
+ */
2385
+ async enrichPayload(
2386
+ payload: Record<string, unknown>,
2387
+ consumer: { primary_stream: string }
2388
+ ): Promise<Record<string, unknown>> {
2389
+ const entity = await this.registry.getEntityByStream(
2390
+ consumer.primary_stream
2391
+ )
2392
+ if (!entity) return payload
2393
+
2394
+ return {
2395
+ ...payload,
2396
+ entity: {
2397
+ type: entity.type,
2398
+ status: entity.status,
2399
+ url: entity.url,
2400
+ streams: entity.streams,
2401
+ tags: entity.tags,
2402
+ spawnArgs: entity.spawn_args,
2403
+ },
2404
+ triggerEvent: `message_received`,
2405
+ }
2406
+ }
2407
+
2408
+ private validateSchema(schema: Record<string, unknown> | undefined): void {
2409
+ if (!schema) return
2410
+ const err = this.validator.validateSchemaSubset(schema)
2411
+ if (err) {
2412
+ throw new ElectricAgentsError(err.code, err.message, 400)
2413
+ }
2414
+ }
2415
+
2416
+ private validateSchemaMap(
2417
+ schemas: Record<string, Record<string, unknown>> | undefined
2418
+ ): void {
2419
+ if (!schemas) return
2420
+ for (const schema of Object.values(schemas)) {
2421
+ this.validateSchema(schema)
2422
+ }
2423
+ }
2424
+
2425
+ private validateDispatchPolicy(
2426
+ input: unknown,
2427
+ opts: { label: string }
2428
+ ): DispatchPolicy {
2429
+ try {
2430
+ return parseDispatchPolicy(input, opts.label)
2431
+ } catch (error) {
2432
+ throw new ElectricAgentsError(
2433
+ ErrCodeInvalidRequest,
2434
+ error instanceof Error ? error.message : `Invalid dispatch policy`,
2435
+ 400
2436
+ )
2437
+ }
2438
+ }
2439
+
2440
+ private validateTags(input: unknown): Record<string, string> {
2441
+ try {
2442
+ return assertTags(input)
2443
+ } catch (error) {
2444
+ throw new ElectricAgentsError(
2445
+ ErrCodeInvalidRequest,
2446
+ error instanceof Error ? error.message : `Invalid tags`,
2447
+ 400
2448
+ )
2449
+ }
2450
+ }
2451
+
2452
+ private async validateSendRequest(
2453
+ entityUrl: string,
2454
+ req: SendRequest
2455
+ ): Promise<ElectricAgentsEntity> {
2456
+ const entity = await this.registry.getEntity(entityUrl)
2457
+ if (!entity) {
2458
+ throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
2459
+ }
2460
+ if (entity.status === `stopped`) {
2461
+ throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
2462
+ }
2463
+
2464
+ if (req.type && entity.type) {
2465
+ const { inboxSchemas } = await this.getEffectiveSchemas(entity)
2466
+ if (inboxSchemas) {
2467
+ const schema = inboxSchemas[req.type]
2468
+ if (!schema) {
2469
+ throw new ElectricAgentsError(
2470
+ ErrCodeUnknownMessageType,
2471
+ `Unknown message type "${req.type}"`,
2472
+ 422
2473
+ )
2474
+ }
2475
+ const valErr = this.validator.validate(schema, req.payload)
2476
+ if (valErr) {
2477
+ throw new ElectricAgentsError(
2478
+ valErr.code,
2479
+ valErr.message,
2480
+ 422,
2481
+ valErr.details
2482
+ )
2483
+ }
2484
+ }
2485
+ }
2486
+
2487
+ if (!req.from) {
2488
+ throw new ElectricAgentsError(
2489
+ ErrCodeInvalidRequest,
2490
+ `Missing required field: from`,
2491
+ 400
2492
+ )
2493
+ }
2494
+ if (req.payload === undefined) {
2495
+ throw new ElectricAgentsError(
2496
+ ErrCodeInvalidRequest,
2497
+ `Missing required field: payload`,
2498
+ 400
2499
+ )
2500
+ }
2501
+
2502
+ return entity
2503
+ }
2504
+
2505
+ private async getEffectiveSchemas(entity: ElectricAgentsEntity): Promise<{
2506
+ inboxSchemas?: Record<string, Record<string, unknown>>
2507
+ stateSchemas?: Record<string, Record<string, unknown>>
2508
+ }> {
2509
+ if (!entity.type) {
2510
+ return {
2511
+ inboxSchemas: entity.inbox_schemas,
2512
+ stateSchemas: entity.state_schemas,
2513
+ }
2514
+ }
2515
+
2516
+ const latestType = await this.registry.getEntityType(entity.type)
2517
+
2518
+ return {
2519
+ inboxSchemas: latestType?.inbox_schemas
2520
+ ? { ...(entity.inbox_schemas ?? {}), ...latestType.inbox_schemas }
2521
+ : entity.inbox_schemas,
2522
+ stateSchemas: latestType?.state_schemas
2523
+ ? { ...(entity.state_schemas ?? {}), ...latestType.state_schemas }
2524
+ : entity.state_schemas,
2525
+ }
2526
+ }
2527
+
2528
+ private isClosedStreamError(err: unknown): boolean {
2529
+ if (!(err instanceof Error)) {
2530
+ return false
2531
+ }
2532
+
2533
+ const status =
2534
+ `status` in err ? (err as { status?: unknown }).status : undefined
2535
+
2536
+ return (
2537
+ (status === 409 && /Stream is closed/i.test(err.message)) ||
2538
+ /Stream append failed:\s*409\s+Stream is closed/i.test(err.message) ||
2539
+ /HTTP Error 409\b.*Stream is closed/i.test(err.message)
2540
+ )
2541
+ }
2542
+
2543
+ /**
2544
+ * Ensure a virtual cron stream exists and schedule its next tick.
2545
+ * Returns the stream path (e.g. `/_cron/<base64url>`).
2546
+ */
2547
+ async getOrCreateCronStream(
2548
+ expression: string,
2549
+ timezone?: string
2550
+ ): Promise<string> {
2551
+ if (!this.scheduler) {
2552
+ throw new Error(`Scheduler not configured`)
2553
+ }
2554
+
2555
+ const spec = resolveCronScheduleSpec(expression, timezone)
2556
+ const streamPath = getCronStreamPath(spec.expression, spec.timezone)
2557
+
2558
+ // Ensure the backing stream exists
2559
+ const exists = await this.streamClient.exists(streamPath)
2560
+ if (!exists) {
2561
+ await this.streamClient.create(streamPath, {
2562
+ contentType: `application/json`,
2563
+ })
2564
+ }
2565
+
2566
+ const fireAt = getNextCronFireAt(spec.expression, spec.timezone)
2567
+ await this.scheduler.enqueueCronTick(
2568
+ spec.expression,
2569
+ spec.timezone,
2570
+ 0,
2571
+ streamPath,
2572
+ fireAt
2573
+ )
2574
+
2575
+ return streamPath
2576
+ }
2577
+
2578
+ async shutdown(): Promise<void> {
2579
+ if (this.stopWakeRegistryOnShutdown) {
2580
+ await this.wakeRegistry.stopSync()
2581
+ }
2582
+ this.registry.close()
2583
+ }
2584
+ }
2585
+
2586
+ export class ElectricAgentsError extends Error {
2587
+ readonly details?: unknown
2588
+
2589
+ constructor(
2590
+ readonly code: string,
2591
+ message: string,
2592
+ readonly status: number,
2593
+ details?: unknown
2594
+ ) {
2595
+ super(message)
2596
+ this.name = `ElectricAgentsError`
2597
+ if (details !== undefined) {
2598
+ this.details = details
2599
+ }
2600
+ }
2601
+ }