@electric-ax/agents-server 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -0
- package/dist/chunk-Cl8Af3a2.js +11 -0
- package/dist/entrypoint.js +7319 -0
- package/dist/index.cjs +7090 -0
- package/dist/index.d.cts +4262 -0
- package/dist/index.d.ts +4263 -0
- package/dist/index.js +7053 -0
- package/drizzle/0000_baseline.sql +97 -0
- package/drizzle/0001_entity_tags_and_bridges.sql +45 -0
- package/drizzle/0002_tag_outbox_hardening.sql +14 -0
- package/drizzle/0003_entity_manifest_sources.sql +11 -0
- package/drizzle/0004_tenant_scoping.sql +139 -0
- package/drizzle/0005_pull_wake_control_plane.sql +156 -0
- package/drizzle/meta/0000_snapshot.json +593 -0
- package/drizzle/meta/_journal.json +48 -0
- package/package.json +89 -0
- package/src/authenticated-user-format.ts +17 -0
- package/src/claim-write-token-store.ts +74 -0
- package/src/db/index.ts +53 -0
- package/src/db/schema.ts +490 -0
- package/src/dev-asserted-auth.ts +46 -0
- package/src/dispatch-policy-schema.ts +52 -0
- package/src/electric-agents/adapter-types.ts +70 -0
- package/src/electric-agents/default-entity-schemas.ts +1 -0
- package/src/electric-agents/schema-validator.ts +143 -0
- package/src/electric-agents-http.ts +46 -0
- package/src/electric-agents-types.ts +335 -0
- package/src/entity-bridge-manager.ts +694 -0
- package/src/entity-manager.ts +2601 -0
- package/src/entity-projector.ts +765 -0
- package/src/entity-registry.ts +1162 -0
- package/src/entrypoint-lib.ts +295 -0
- package/src/entrypoint.ts +11 -0
- package/src/host.ts +323 -0
- package/src/index.ts +49 -0
- package/src/manifest-side-effects.ts +183 -0
- package/src/routing/agent-ui-router.ts +81 -0
- package/src/routing/context.ts +35 -0
- package/src/routing/cron-router.ts +45 -0
- package/src/routing/dispatch-policy.ts +248 -0
- package/src/routing/durable-streams-router.ts +407 -0
- package/src/routing/durable-streams-routing-adapter.ts +96 -0
- package/src/routing/electric-proxy-router.ts +61 -0
- package/src/routing/entities-router.ts +484 -0
- package/src/routing/entity-types-router.ts +229 -0
- package/src/routing/global-router.ts +33 -0
- package/src/routing/hooks.ts +123 -0
- package/src/routing/internal-router.ts +741 -0
- package/src/routing/oss-server-router.ts +56 -0
- package/src/routing/runners-router.ts +416 -0
- package/src/routing/schema.ts +141 -0
- package/src/routing/stream-append.ts +196 -0
- package/src/routing/tenant-stream-paths.ts +26 -0
- package/src/runtime-registry.ts +49 -0
- package/src/runtime.ts +537 -0
- package/src/scheduler.ts +788 -0
- package/src/schema-validation.ts +15 -0
- package/src/server.ts +374 -0
- package/src/standalone-runtime.ts +188 -0
- package/src/stream-client.ts +842 -0
- package/src/tag-stream-outbox-drainer.ts +188 -0
- package/src/tenant.ts +25 -0
- package/src/tracing.ts +57 -0
- package/src/utils/electric-url.ts +15 -0
- package/src/utils/log.ts +95 -0
- package/src/utils/server-utils.ts +245 -0
- package/src/utils/webhook-url.ts +33 -0
- package/src/wake-registry.ts +946 -0
|
@@ -0,0 +1,2601 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import fastq from 'fastq'
|
|
3
|
+
import {
|
|
4
|
+
assertTags,
|
|
5
|
+
entityStateSchema,
|
|
6
|
+
getCronStreamPath,
|
|
7
|
+
getSharedStateStreamPath,
|
|
8
|
+
getNextCronFireAt,
|
|
9
|
+
manifestChildKey,
|
|
10
|
+
manifestSharedStateKey,
|
|
11
|
+
manifestSourceKey,
|
|
12
|
+
resolveCronScheduleSpec,
|
|
13
|
+
} from '@electric-ax/agents-runtime'
|
|
14
|
+
import {
|
|
15
|
+
ErrCodeDuplicateURL,
|
|
16
|
+
ErrCodeEntityPersistFailed,
|
|
17
|
+
ErrCodeForkInProgress,
|
|
18
|
+
ErrCodeForkWaitTimeout,
|
|
19
|
+
ErrCodeInvalidRequest,
|
|
20
|
+
ErrCodeNotFound,
|
|
21
|
+
ErrCodeNotRunning,
|
|
22
|
+
ErrCodeSchemaKeyExists,
|
|
23
|
+
ErrCodeSchemaValidationFailed,
|
|
24
|
+
ErrCodeUnauthorized,
|
|
25
|
+
ErrCodeUnknownEntityType,
|
|
26
|
+
ErrCodeUnknownEventType,
|
|
27
|
+
ErrCodeUnknownMessageType,
|
|
28
|
+
} from './electric-agents-types.js'
|
|
29
|
+
import { parseDispatchPolicy } from './dispatch-policy-schema.js'
|
|
30
|
+
import { EntityAlreadyExistsError } from './entity-registry.js'
|
|
31
|
+
import { serverLog } from './utils/log.js'
|
|
32
|
+
import {
|
|
33
|
+
buildManifestWakeRegistration,
|
|
34
|
+
extractManifestCronSpec,
|
|
35
|
+
} from './manifest-side-effects.js'
|
|
36
|
+
import { DEFAULT_TENANT_ID } from './tenant.js'
|
|
37
|
+
import { ATTR, withSpan } from './tracing.js'
|
|
38
|
+
import type { queueAsPromised } from 'fastq'
|
|
39
|
+
import type { SchedulerClient } from './scheduler.js'
|
|
40
|
+
import type { WakeEvalResult, WakeRegistry } from './wake-registry.js'
|
|
41
|
+
import type { WakeMessage } from '@electric-ax/agents-runtime'
|
|
42
|
+
import type { PostgresRegistry } from './entity-registry.js'
|
|
43
|
+
import type { SchemaValidator } from './electric-agents/schema-validator.js'
|
|
44
|
+
import type { StreamClient } from './stream-client.js'
|
|
45
|
+
import type {
|
|
46
|
+
DispatchPolicy,
|
|
47
|
+
DispatchTarget,
|
|
48
|
+
ElectricAgentsEntity,
|
|
49
|
+
ElectricAgentsEntityType,
|
|
50
|
+
RegisterEntityTypeRequest,
|
|
51
|
+
SendRequest,
|
|
52
|
+
SetTagRequest,
|
|
53
|
+
TypedSpawnRequest,
|
|
54
|
+
} from './electric-agents-types.js'
|
|
55
|
+
import type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
|
|
56
|
+
|
|
57
|
+
type SpawnPersistResult = [
|
|
58
|
+
PromiseSettledResult<void>,
|
|
59
|
+
PromiseSettledResult<void>,
|
|
60
|
+
PromiseSettledResult<number>,
|
|
61
|
+
]
|
|
62
|
+
type SpawnPersistJob = () => Promise<SpawnPersistResult>
|
|
63
|
+
type WriteTokenValidator = (
|
|
64
|
+
entity: ElectricAgentsEntity,
|
|
65
|
+
token: string
|
|
66
|
+
) => boolean
|
|
67
|
+
|
|
68
|
+
type ForkSubtreeOptions = {
|
|
69
|
+
rootInstanceId?: string
|
|
70
|
+
waitTimeoutMs?: number
|
|
71
|
+
waitPollMs?: number
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
type ForkEntityPlan = {
|
|
75
|
+
source: ElectricAgentsEntity
|
|
76
|
+
fork: ElectricAgentsEntity
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
type ForkStateSnapshot = {
|
|
80
|
+
manifestsByEntity: Map<string, Map<string, Record<string, unknown>>>
|
|
81
|
+
childStatusesByEntity: Map<string, Map<string, Record<string, unknown>>>
|
|
82
|
+
replayWatermarksByEntity: Map<string, Map<string, Record<string, unknown>>>
|
|
83
|
+
sharedStateIds: Set<string>
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
type ForkResult = {
|
|
87
|
+
root: ElectricAgentsEntity
|
|
88
|
+
entities: Array<ElectricAgentsEntity>
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const DEFAULT_FORK_WAIT_TIMEOUT_MS = 120_000
|
|
92
|
+
const DEFAULT_FORK_WAIT_POLL_MS = 250
|
|
93
|
+
|
|
94
|
+
function applyTypeDefaultSubscriptionScope(
|
|
95
|
+
policy: DispatchPolicy,
|
|
96
|
+
typeDefault: DispatchPolicy | undefined
|
|
97
|
+
): DispatchPolicy {
|
|
98
|
+
const target = policy.targets[0]
|
|
99
|
+
const defaultTarget = typeDefault?.targets[0]
|
|
100
|
+
if (!target || !defaultTarget?.subscription_id) return policy
|
|
101
|
+
if (!sameDispatchDestination(target, defaultTarget)) return policy
|
|
102
|
+
if (target.subscription_id === defaultTarget.subscription_id) return policy
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
targets: [{ ...target, subscription_id: defaultTarget.subscription_id }],
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function sameDispatchDestination(
|
|
110
|
+
a: DispatchTarget,
|
|
111
|
+
b: DispatchTarget
|
|
112
|
+
): boolean {
|
|
113
|
+
if (a.type !== b.type) return false
|
|
114
|
+
if (a.type === `runner` && b.type === `runner`) {
|
|
115
|
+
return a.runnerId === b.runnerId
|
|
116
|
+
}
|
|
117
|
+
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url
|
|
118
|
+
return false
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function sleep(ms: number): Promise<void> {
|
|
122
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function omitUndefined<T extends Record<string, unknown>>(value: T): T {
|
|
126
|
+
return Object.fromEntries(
|
|
127
|
+
Object.entries(value).filter(([, entry]) => entry !== undefined)
|
|
128
|
+
) as T
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
132
|
+
return typeof value === `object` && value !== null && !Array.isArray(value)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function cloneRecord<T extends Record<string, unknown>>(value: T): T {
|
|
136
|
+
return JSON.parse(JSON.stringify(value)) as T
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Orchestrates the Electric Agents entity lifecycle: register types, spawn, send, kill.
|
|
141
|
+
*
|
|
142
|
+
* Entity identity is the URL (/{type}/{instance_id}). Entity tags and
|
|
143
|
+
* lifecycle state are persisted directly in Postgres. Durable streams remain
|
|
144
|
+
* the append-only transport for inbox/state events.
|
|
145
|
+
*/
|
|
146
|
+
export class EntityManager {
|
|
147
|
+
readonly registry: PostgresRegistry
|
|
148
|
+
private readonly tenantId: string
|
|
149
|
+
private streamClient: StreamClient
|
|
150
|
+
private validator: SchemaValidator
|
|
151
|
+
private scheduler: SchedulerClient | null = null
|
|
152
|
+
private entityBridgeManager: EntityBridgeCoordinator | null = null
|
|
153
|
+
private writeTokenValidator: WriteTokenValidator | null = null
|
|
154
|
+
readonly wakeRegistry: WakeRegistry
|
|
155
|
+
private forkWorkLockedEntities = new Map<string, number>()
|
|
156
|
+
private forkWriteLockedEntities = new Map<string, number>()
|
|
157
|
+
private forkWriteLockedStreams = new Map<string, number>()
|
|
158
|
+
private spawnPersistQueue: queueAsPromised<
|
|
159
|
+
SpawnPersistJob,
|
|
160
|
+
SpawnPersistResult
|
|
161
|
+
>
|
|
162
|
+
private readonly stopWakeRegistryOnShutdown: boolean
|
|
163
|
+
|
|
164
|
+
constructor(opts: {
|
|
165
|
+
registry: PostgresRegistry
|
|
166
|
+
streamClient: StreamClient
|
|
167
|
+
validator: SchemaValidator
|
|
168
|
+
wakeRegistry: WakeRegistry
|
|
169
|
+
scheduler?: SchedulerClient
|
|
170
|
+
entityBridgeManager?: EntityBridgeCoordinator
|
|
171
|
+
writeTokenValidator?: WriteTokenValidator
|
|
172
|
+
spawnConcurrency?: number
|
|
173
|
+
stopWakeRegistryOnShutdown?: boolean
|
|
174
|
+
}) {
|
|
175
|
+
this.registry = opts.registry
|
|
176
|
+
this.tenantId = opts.registry.tenantId ?? DEFAULT_TENANT_ID
|
|
177
|
+
this.streamClient = opts.streamClient
|
|
178
|
+
this.validator = opts.validator
|
|
179
|
+
this.wakeRegistry = opts.wakeRegistry
|
|
180
|
+
this.scheduler = opts.scheduler ?? null
|
|
181
|
+
this.entityBridgeManager = opts.entityBridgeManager ?? null
|
|
182
|
+
this.writeTokenValidator = opts.writeTokenValidator ?? null
|
|
183
|
+
this.stopWakeRegistryOnShutdown = opts.stopWakeRegistryOnShutdown ?? true
|
|
184
|
+
|
|
185
|
+
const spawnConcurrency =
|
|
186
|
+
opts.spawnConcurrency ??
|
|
187
|
+
Number(process.env.ELECTRIC_AGENTS_SPAWN_CONCURRENCY ?? 16)
|
|
188
|
+
this.spawnPersistQueue = fastq.promise<
|
|
189
|
+
unknown,
|
|
190
|
+
SpawnPersistJob,
|
|
191
|
+
SpawnPersistResult
|
|
192
|
+
>(async (job) => job(), spawnConcurrency)
|
|
193
|
+
|
|
194
|
+
this.wakeRegistry.setTimeoutCallback((result) => {
|
|
195
|
+
void this.deliverWakeResult(result)
|
|
196
|
+
}, this.tenantId)
|
|
197
|
+
this.wakeRegistry.setDebounceCallback((result) => {
|
|
198
|
+
void this.deliverWakeResult(result)
|
|
199
|
+
}, this.tenantId)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async rebuildWakeRegistry(
|
|
203
|
+
electricUrl?: string,
|
|
204
|
+
electricSecret?: string
|
|
205
|
+
): Promise<void> {
|
|
206
|
+
if (electricUrl) {
|
|
207
|
+
await this.wakeRegistry.startSync(electricUrl, electricSecret)
|
|
208
|
+
return
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
await this.wakeRegistry.loadRegistrations()
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
setWriteTokenValidator(validator: WriteTokenValidator): void {
|
|
215
|
+
this.writeTokenValidator = validator
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
isValidWriteToken(entity: ElectricAgentsEntity, token: string): boolean {
|
|
219
|
+
return this.writeTokenValidator
|
|
220
|
+
? this.writeTokenValidator(entity, token)
|
|
221
|
+
: token === entity.write_token
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private encodeChangeEvent(event: Record<string, unknown>): Uint8Array {
|
|
225
|
+
return new TextEncoder().encode(JSON.stringify(event))
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// ==========================================================================
|
|
229
|
+
// Entity Type Registration
|
|
230
|
+
// ==========================================================================
|
|
231
|
+
|
|
232
|
+
async registerEntityType(
|
|
233
|
+
req: RegisterEntityTypeRequest
|
|
234
|
+
): Promise<ElectricAgentsEntityType> {
|
|
235
|
+
if (!req.name) {
|
|
236
|
+
throw new ElectricAgentsError(
|
|
237
|
+
ErrCodeInvalidRequest,
|
|
238
|
+
`Missing required field: name`,
|
|
239
|
+
400
|
|
240
|
+
)
|
|
241
|
+
}
|
|
242
|
+
if (req.name.startsWith(`_`)) {
|
|
243
|
+
throw new ElectricAgentsError(
|
|
244
|
+
ErrCodeInvalidRequest,
|
|
245
|
+
`Entity type names starting with "_" are reserved`,
|
|
246
|
+
400
|
|
247
|
+
)
|
|
248
|
+
}
|
|
249
|
+
if (!req.description) {
|
|
250
|
+
throw new ElectricAgentsError(
|
|
251
|
+
ErrCodeInvalidRequest,
|
|
252
|
+
`Missing required field: description`,
|
|
253
|
+
400
|
|
254
|
+
)
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Validate schema subset for each provided schema.
|
|
258
|
+
this.validateSchema(req.creation_schema)
|
|
259
|
+
this.validateSchemaMap(req.inbox_schemas)
|
|
260
|
+
this.validateSchemaMap(req.state_schemas)
|
|
261
|
+
const defaultDispatchPolicy = req.default_dispatch_policy
|
|
262
|
+
? this.validateDispatchPolicy(req.default_dispatch_policy, {
|
|
263
|
+
label: `default_dispatch_policy`,
|
|
264
|
+
})
|
|
265
|
+
: undefined
|
|
266
|
+
|
|
267
|
+
const existing = await this.registry.getEntityType(req.name)
|
|
268
|
+
const now = new Date().toISOString()
|
|
269
|
+
const entityType: ElectricAgentsEntityType = {
|
|
270
|
+
name: req.name,
|
|
271
|
+
description: req.description,
|
|
272
|
+
creation_schema: req.creation_schema,
|
|
273
|
+
inbox_schemas: req.inbox_schemas,
|
|
274
|
+
state_schemas: req.state_schemas,
|
|
275
|
+
serve_endpoint: req.serve_endpoint,
|
|
276
|
+
default_dispatch_policy: defaultDispatchPolicy,
|
|
277
|
+
revision: existing ? existing.revision + 1 : 1,
|
|
278
|
+
created_at: existing?.created_at ?? now,
|
|
279
|
+
updated_at: now,
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
await this.registry.createEntityType(entityType)
|
|
283
|
+
|
|
284
|
+
const stored = await this.registry.getEntityType(req.name)
|
|
285
|
+
if (!stored) {
|
|
286
|
+
throw new Error(`Failed to read back entity type "${req.name}"`)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
return stored
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
async deleteEntityType(name: string): Promise<void> {
|
|
293
|
+
const existing = await this.registry.getEntityType(name)
|
|
294
|
+
if (!existing) {
|
|
295
|
+
throw new ElectricAgentsError(
|
|
296
|
+
ErrCodeNotFound,
|
|
297
|
+
`Entity type "${name}" not found`,
|
|
298
|
+
404
|
|
299
|
+
)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
await this.registry.deleteEntityType(name)
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// ==========================================================================
|
|
306
|
+
// Spawn
|
|
307
|
+
// ==========================================================================
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Spawn a new entity of the given type with durable streams.
|
|
311
|
+
*/
|
|
312
|
+
async spawn(
|
|
313
|
+
typeName: string,
|
|
314
|
+
req: TypedSpawnRequest
|
|
315
|
+
): Promise<ElectricAgentsEntity & { txid: number }> {
|
|
316
|
+
return await withSpan(`electric_agents.spawn`, async (span) => {
|
|
317
|
+
span.setAttributes({
|
|
318
|
+
[ATTR.ENTITY_TYPE]: typeName,
|
|
319
|
+
...(req.parent ? { [ATTR.PARENT_URL]: req.parent } : {}),
|
|
320
|
+
})
|
|
321
|
+
const entity = await this.spawnInner(typeName, req)
|
|
322
|
+
span.setAttribute(ATTR.ENTITY_URL, entity.url)
|
|
323
|
+
return entity
|
|
324
|
+
})
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
private async spawnInner(
|
|
328
|
+
typeName: string,
|
|
329
|
+
req: TypedSpawnRequest
|
|
330
|
+
): Promise<ElectricAgentsEntity & { txid: number }> {
|
|
331
|
+
if (typeName.startsWith(`_`)) {
|
|
332
|
+
throw new ElectricAgentsError(
|
|
333
|
+
ErrCodeInvalidRequest,
|
|
334
|
+
`Entity type names starting with "_" are reserved`,
|
|
335
|
+
400
|
|
336
|
+
)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Look up the entity type from the registry.
|
|
340
|
+
const entityType = await this.registry.getEntityType(typeName)
|
|
341
|
+
if (!entityType) {
|
|
342
|
+
throw new ElectricAgentsError(
|
|
343
|
+
ErrCodeUnknownEntityType,
|
|
344
|
+
`Entity type "${typeName}" not found`,
|
|
345
|
+
404
|
|
346
|
+
)
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Validate args against creation_schema if declared.
|
|
350
|
+
if (entityType.creation_schema && req.args) {
|
|
351
|
+
const valErr = this.validator.validate(
|
|
352
|
+
entityType.creation_schema,
|
|
353
|
+
req.args
|
|
354
|
+
)
|
|
355
|
+
if (valErr) {
|
|
356
|
+
throw new ElectricAgentsError(
|
|
357
|
+
valErr.code,
|
|
358
|
+
valErr.message,
|
|
359
|
+
422,
|
|
360
|
+
valErr.details
|
|
361
|
+
)
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
const initialTags = this.validateTags(req.tags ?? {})
|
|
366
|
+
|
|
367
|
+
const instanceId = req.instance_id || randomUUID()
|
|
368
|
+
if (instanceId.includes(`/`)) {
|
|
369
|
+
throw new ElectricAgentsError(
|
|
370
|
+
ErrCodeInvalidRequest,
|
|
371
|
+
`instance_id must not contain forward slashes`,
|
|
372
|
+
400
|
|
373
|
+
)
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const writeToken = randomUUID()
|
|
377
|
+
|
|
378
|
+
const entityURL = `/${typeName}/${instanceId}`
|
|
379
|
+
const mainPath = `${entityURL}/main`
|
|
380
|
+
const errorPath = `${entityURL}/error`
|
|
381
|
+
|
|
382
|
+
const subscriptionId = `${typeName}-handler`
|
|
383
|
+
|
|
384
|
+
const spawnT0 = performance.now()
|
|
385
|
+
|
|
386
|
+
const existingByURL = await this.registry.getEntity(entityURL)
|
|
387
|
+
if (existingByURL) {
|
|
388
|
+
throw new ElectricAgentsError(
|
|
389
|
+
ErrCodeDuplicateURL,
|
|
390
|
+
`Entity already exists at URL "${entityURL}"`,
|
|
391
|
+
409
|
|
392
|
+
)
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
let parentEntity: ElectricAgentsEntity | null = null
|
|
396
|
+
if (req.parent) {
|
|
397
|
+
parentEntity = await this.registry.getEntity(req.parent)
|
|
398
|
+
if (!parentEntity) {
|
|
399
|
+
throw new ElectricAgentsError(
|
|
400
|
+
ErrCodeNotFound,
|
|
401
|
+
`Parent entity "${req.parent}" not found`,
|
|
402
|
+
404
|
|
403
|
+
)
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const dispatchPolicy = req.dispatch_policy
|
|
408
|
+
? this.validateDispatchPolicy(req.dispatch_policy, {
|
|
409
|
+
label: `dispatch_policy`,
|
|
410
|
+
})
|
|
411
|
+
: parentEntity?.dispatch_policy
|
|
412
|
+
? applyTypeDefaultSubscriptionScope(
|
|
413
|
+
parentEntity.dispatch_policy,
|
|
414
|
+
entityType.default_dispatch_policy
|
|
415
|
+
)
|
|
416
|
+
: entityType.default_dispatch_policy
|
|
417
|
+
|
|
418
|
+
const now = Date.now()
|
|
419
|
+
const entityData: ElectricAgentsEntity = {
|
|
420
|
+
type: typeName,
|
|
421
|
+
status: `idle`,
|
|
422
|
+
url: entityURL,
|
|
423
|
+
streams: {
|
|
424
|
+
main: mainPath,
|
|
425
|
+
error: errorPath,
|
|
426
|
+
},
|
|
427
|
+
subscription_id: subscriptionId,
|
|
428
|
+
dispatch_policy: dispatchPolicy,
|
|
429
|
+
write_token: writeToken,
|
|
430
|
+
tags: initialTags,
|
|
431
|
+
spawn_args: req.args,
|
|
432
|
+
type_revision: entityType.revision,
|
|
433
|
+
inbox_schemas: entityType.inbox_schemas,
|
|
434
|
+
state_schemas: entityType.state_schemas,
|
|
435
|
+
created_at: now,
|
|
436
|
+
updated_at: now,
|
|
437
|
+
}
|
|
438
|
+
if (req.parent) {
|
|
439
|
+
entityData.parent = req.parent
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
if (req.wake) {
|
|
443
|
+
await this.wakeRegistry.register({
|
|
444
|
+
tenantId: this.tenantId,
|
|
445
|
+
subscriberUrl: req.wake.subscriberUrl,
|
|
446
|
+
sourceUrl: entityURL,
|
|
447
|
+
condition: req.wake.condition,
|
|
448
|
+
debounceMs: req.wake.debounceMs,
|
|
449
|
+
timeoutMs: req.wake.timeoutMs,
|
|
450
|
+
oneShot: false,
|
|
451
|
+
includeResponse: req.wake.includeResponse,
|
|
452
|
+
})
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const contentType = `application/json`
|
|
456
|
+
|
|
457
|
+
const createdEvent = entityStateSchema.entityCreated.insert({
|
|
458
|
+
key: `entity-created`,
|
|
459
|
+
value: {
|
|
460
|
+
entity_type: typeName,
|
|
461
|
+
timestamp: new Date().toISOString(),
|
|
462
|
+
args: req.args ?? {},
|
|
463
|
+
...(req.parent ? { parent_url: req.parent } : {}),
|
|
464
|
+
},
|
|
465
|
+
} as any)
|
|
466
|
+
|
|
467
|
+
const initialEvents: Array<Record<string, unknown>> = [
|
|
468
|
+
createdEvent as Record<string, unknown>,
|
|
469
|
+
]
|
|
470
|
+
|
|
471
|
+
if (req.initialMessage !== undefined) {
|
|
472
|
+
const msgNow = new Date().toISOString()
|
|
473
|
+
const inboxEvent = entityStateSchema.inbox.insert({
|
|
474
|
+
key: `msg-in-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
475
|
+
value: {
|
|
476
|
+
from: req.parent ?? `spawn`,
|
|
477
|
+
payload: req.initialMessage,
|
|
478
|
+
timestamp: msgNow,
|
|
479
|
+
},
|
|
480
|
+
} as any)
|
|
481
|
+
initialEvents.push(inboxEvent as Record<string, unknown>)
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// JSON-mode streams: server flattens one level. Append auto-wraps the
|
|
485
|
+
// body in [...] but create does not, so we wrap it ourselves.
|
|
486
|
+
const initialBody = `[${initialEvents.map((e) => JSON.stringify(e)).join(`,`)}]`
|
|
487
|
+
|
|
488
|
+
const queueEnterT0 = performance.now()
|
|
489
|
+
const queueWaiting = this.spawnPersistQueue.length()
|
|
490
|
+
const queueRunning = this.spawnPersistQueue.running()
|
|
491
|
+
const [mainStreamResult, errorStreamResult, entityResult] =
|
|
492
|
+
await this.spawnPersistQueue.push(async () => {
|
|
493
|
+
// Create entity first so it's visible in the DB before stream
|
|
494
|
+
// creation can trigger webhooks that look up the entity.
|
|
495
|
+
let entityTxid: number
|
|
496
|
+
try {
|
|
497
|
+
entityTxid = await withSpan(`db.createEntity`, () =>
|
|
498
|
+
this.registry.createEntity(entityData)
|
|
499
|
+
)
|
|
500
|
+
} catch (err) {
|
|
501
|
+
return [
|
|
502
|
+
{ status: `fulfilled`, value: undefined },
|
|
503
|
+
{ status: `fulfilled`, value: undefined },
|
|
504
|
+
{ status: `rejected`, reason: err },
|
|
505
|
+
] as SpawnPersistResult
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
const [mainStreamResult, errorStreamResult] = await Promise.allSettled([
|
|
509
|
+
this.streamClient.create(mainPath, {
|
|
510
|
+
contentType,
|
|
511
|
+
body: initialBody,
|
|
512
|
+
}),
|
|
513
|
+
this.streamClient.create(errorPath, { contentType }),
|
|
514
|
+
])
|
|
515
|
+
|
|
516
|
+
return [
|
|
517
|
+
mainStreamResult,
|
|
518
|
+
errorStreamResult,
|
|
519
|
+
{ status: `fulfilled`, value: entityTxid },
|
|
520
|
+
] as SpawnPersistResult
|
|
521
|
+
})
|
|
522
|
+
const parallelMs = +(performance.now() - queueEnterT0).toFixed(2)
|
|
523
|
+
|
|
524
|
+
if (
|
|
525
|
+
mainStreamResult.status === `rejected` ||
|
|
526
|
+
errorStreamResult.status === `rejected` ||
|
|
527
|
+
entityResult.status === `rejected`
|
|
528
|
+
) {
|
|
529
|
+
const entityReason =
|
|
530
|
+
entityResult.status === `rejected` ? entityResult.reason : null
|
|
531
|
+
const streamReason =
|
|
532
|
+
mainStreamResult.status === `rejected`
|
|
533
|
+
? mainStreamResult.reason
|
|
534
|
+
: errorStreamResult.status === `rejected`
|
|
535
|
+
? errorStreamResult.reason
|
|
536
|
+
: null
|
|
537
|
+
const isDuplicate = entityReason instanceof EntityAlreadyExistsError
|
|
538
|
+
const isStreamConflict =
|
|
539
|
+
!!streamReason &&
|
|
540
|
+
typeof streamReason === `object` &&
|
|
541
|
+
((`status` in streamReason && streamReason.status === 409) ||
|
|
542
|
+
(`code` in streamReason && streamReason.code === `CONFLICT_SEQ`))
|
|
543
|
+
|
|
544
|
+
const rollbacks: Array<Promise<unknown>> = []
|
|
545
|
+
// On duplicate, the winning spawn owns both the stream and the row —
|
|
546
|
+
// don't roll back either. For any other failure, clean up what succeeded.
|
|
547
|
+
if (!isDuplicate && !isStreamConflict) {
|
|
548
|
+
if (mainStreamResult.status === `fulfilled`) {
|
|
549
|
+
rollbacks.push(this.streamClient.delete(mainPath))
|
|
550
|
+
}
|
|
551
|
+
if (errorStreamResult.status === `fulfilled`) {
|
|
552
|
+
rollbacks.push(this.streamClient.delete(errorPath))
|
|
553
|
+
}
|
|
554
|
+
if (entityResult.status === `fulfilled`) {
|
|
555
|
+
rollbacks.push(this.registry.deleteEntity(entityURL))
|
|
556
|
+
}
|
|
557
|
+
if (req.wake) {
|
|
558
|
+
rollbacks.push(
|
|
559
|
+
this.wakeRegistry.unregisterBySubscriberAndSource(
|
|
560
|
+
req.wake.subscriberUrl,
|
|
561
|
+
entityURL,
|
|
562
|
+
this.tenantId
|
|
563
|
+
)
|
|
564
|
+
)
|
|
565
|
+
}
|
|
566
|
+
await Promise.allSettled(rollbacks)
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
if (isDuplicate || isStreamConflict) {
|
|
570
|
+
throw new ElectricAgentsError(
|
|
571
|
+
ErrCodeDuplicateURL,
|
|
572
|
+
`Entity already exists at URL "${entityURL}"`,
|
|
573
|
+
409
|
|
574
|
+
)
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
const failure =
|
|
578
|
+
mainStreamResult.status === `rejected`
|
|
579
|
+
? mainStreamResult.reason
|
|
580
|
+
: errorStreamResult.status === `rejected`
|
|
581
|
+
? errorStreamResult.reason
|
|
582
|
+
: (entityResult as PromiseRejectedResult).reason
|
|
583
|
+
if (failure instanceof Error) throw failure
|
|
584
|
+
throw new ElectricAgentsError(
|
|
585
|
+
`SPAWN_FAILED`,
|
|
586
|
+
`Spawn failed: ${String(failure)}`,
|
|
587
|
+
500
|
|
588
|
+
)
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
const txid = entityResult.value
|
|
592
|
+
|
|
593
|
+
serverLog.event(
|
|
594
|
+
{
|
|
595
|
+
event: `spawn`,
|
|
596
|
+
url: entityURL,
|
|
597
|
+
type: typeName,
|
|
598
|
+
parent: req.parent,
|
|
599
|
+
parallelMs,
|
|
600
|
+
totalMs: +(performance.now() - spawnT0).toFixed(2),
|
|
601
|
+
queueWaiting,
|
|
602
|
+
queueRunning,
|
|
603
|
+
},
|
|
604
|
+
`spawn done`
|
|
605
|
+
)
|
|
606
|
+
return { ...entityData, txid }
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
// ==========================================================================
|
|
610
|
+
// Fork
|
|
611
|
+
// ==========================================================================
|
|
612
|
+
|
|
613
|
+
async forkSubtree(
|
|
614
|
+
rootUrl: string,
|
|
615
|
+
opts: ForkSubtreeOptions = {}
|
|
616
|
+
): Promise<ForkResult> {
|
|
617
|
+
return await withSpan(`electric_agents.forkSubtree`, async (span) => {
|
|
618
|
+
span.setAttribute(ATTR.ENTITY_URL, rootUrl)
|
|
619
|
+
const result = await this.forkSubtreeInner(rootUrl, opts)
|
|
620
|
+
span.setAttribute(`electric_agents.fork.root_url`, result.root.url)
|
|
621
|
+
span.setAttribute(
|
|
622
|
+
`electric_agents.fork.entity_count`,
|
|
623
|
+
result.entities.length
|
|
624
|
+
)
|
|
625
|
+
return result
|
|
626
|
+
})
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
private async forkSubtreeInner(
|
|
630
|
+
rootUrl: string,
|
|
631
|
+
opts: ForkSubtreeOptions
|
|
632
|
+
): Promise<ForkResult> {
|
|
633
|
+
const forkT0 = performance.now()
|
|
634
|
+
const workLocks = new Set<string>()
|
|
635
|
+
const writeEntityLocks = new Set<string>()
|
|
636
|
+
const writeStreamLocks = new Set<string>()
|
|
637
|
+
|
|
638
|
+
try {
|
|
639
|
+
const sourceTree = await this.waitForIdleSubtree(rootUrl, opts, workLocks)
|
|
640
|
+
const sourceRoot = sourceTree[0]!
|
|
641
|
+
if (sourceRoot.parent) {
|
|
642
|
+
throw new ElectricAgentsError(
|
|
643
|
+
ErrCodeInvalidRequest,
|
|
644
|
+
`Only top-level entities can be forked`,
|
|
645
|
+
400
|
|
646
|
+
)
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
const snapshot = await this.readForkStateSnapshot(sourceTree)
|
|
650
|
+
const suffix = randomUUID().slice(0, 8)
|
|
651
|
+
const entityUrlMap = await this.buildForkEntityUrlMap(sourceTree, {
|
|
652
|
+
suffix,
|
|
653
|
+
rootUrl,
|
|
654
|
+
rootInstanceId: opts.rootInstanceId,
|
|
655
|
+
})
|
|
656
|
+
const sharedStateIdMap = await this.buildForkSharedStateIdMap(
|
|
657
|
+
snapshot.sharedStateIds,
|
|
658
|
+
suffix
|
|
659
|
+
)
|
|
660
|
+
const stringMap = this.buildForkStringMap(entityUrlMap, sharedStateIdMap)
|
|
661
|
+
const entityPlans = this.buildForkEntityPlans(
|
|
662
|
+
sourceTree,
|
|
663
|
+
entityUrlMap,
|
|
664
|
+
stringMap
|
|
665
|
+
)
|
|
666
|
+
|
|
667
|
+
this.addForkLocks(
|
|
668
|
+
this.forkWriteLockedEntities,
|
|
669
|
+
sourceTree.map((entity) => entity.url),
|
|
670
|
+
writeEntityLocks
|
|
671
|
+
)
|
|
672
|
+
this.addForkLocks(
|
|
673
|
+
this.forkWriteLockedStreams,
|
|
674
|
+
[...snapshot.sharedStateIds].map((id) => getSharedStateStreamPath(id)),
|
|
675
|
+
writeStreamLocks
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
const createdStreams: Array<string> = []
|
|
679
|
+
const createdEntities: Array<string> = []
|
|
680
|
+
const activeManifestsByEntity = new Map<
|
|
681
|
+
string,
|
|
682
|
+
Map<string, Record<string, unknown>>
|
|
683
|
+
>()
|
|
684
|
+
|
|
685
|
+
try {
|
|
686
|
+
for (const plan of entityPlans) {
|
|
687
|
+
await this.streamClient.fork(
|
|
688
|
+
plan.fork.streams.main,
|
|
689
|
+
plan.source.streams.main
|
|
690
|
+
)
|
|
691
|
+
createdStreams.push(plan.fork.streams.main)
|
|
692
|
+
await this.streamClient.fork(
|
|
693
|
+
plan.fork.streams.error,
|
|
694
|
+
plan.source.streams.error
|
|
695
|
+
)
|
|
696
|
+
createdStreams.push(plan.fork.streams.error)
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
for (const [sourceId, forkId] of sharedStateIdMap) {
|
|
700
|
+
const sourcePath = getSharedStateStreamPath(sourceId)
|
|
701
|
+
const forkPath = getSharedStateStreamPath(forkId)
|
|
702
|
+
await this.streamClient.fork(forkPath, sourcePath)
|
|
703
|
+
createdStreams.push(forkPath)
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
for (const plan of entityPlans) {
|
|
707
|
+
const reconciliation = this.buildForkReconciliation(
|
|
708
|
+
plan,
|
|
709
|
+
snapshot,
|
|
710
|
+
entityUrlMap,
|
|
711
|
+
sharedStateIdMap,
|
|
712
|
+
stringMap
|
|
713
|
+
)
|
|
714
|
+
activeManifestsByEntity.set(plan.fork.url, reconciliation.manifests)
|
|
715
|
+
for (const event of reconciliation.events) {
|
|
716
|
+
await this.streamClient.append(
|
|
717
|
+
plan.fork.streams.main,
|
|
718
|
+
this.encodeChangeEvent(event)
|
|
719
|
+
)
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
for (const plan of entityPlans) {
|
|
724
|
+
await this.registry.createEntity(plan.fork)
|
|
725
|
+
createdEntities.push(plan.fork.url)
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
for (const plan of entityPlans) {
|
|
729
|
+
const manifests =
|
|
730
|
+
activeManifestsByEntity.get(plan.fork.url) ?? new Map()
|
|
731
|
+
await this.materializeForkManifestSideEffects(
|
|
732
|
+
plan.fork.url,
|
|
733
|
+
manifests
|
|
734
|
+
)
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
const root = entityPlans.find(
|
|
738
|
+
(plan) => plan.source.url === rootUrl
|
|
739
|
+
)!.fork
|
|
740
|
+
serverLog.event(
|
|
741
|
+
{
|
|
742
|
+
event: `fork`,
|
|
743
|
+
url: rootUrl,
|
|
744
|
+
forkUrl: root.url,
|
|
745
|
+
entities: entityPlans.length,
|
|
746
|
+
sharedStateStreams: sharedStateIdMap.size,
|
|
747
|
+
totalMs: +(performance.now() - forkT0).toFixed(2),
|
|
748
|
+
},
|
|
749
|
+
`fork done`
|
|
750
|
+
)
|
|
751
|
+
return { root, entities: entityPlans.map((plan) => plan.fork) }
|
|
752
|
+
} catch (err) {
|
|
753
|
+
await Promise.allSettled([
|
|
754
|
+
...createdEntities.flatMap((entityUrl) => [
|
|
755
|
+
this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId),
|
|
756
|
+
this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId),
|
|
757
|
+
this.registry.deleteEntity(entityUrl),
|
|
758
|
+
]),
|
|
759
|
+
...Array.from(sharedStateIdMap.values()).map((id) =>
|
|
760
|
+
this.wakeRegistry.unregisterBySource(
|
|
761
|
+
getSharedStateStreamPath(id),
|
|
762
|
+
this.tenantId
|
|
763
|
+
)
|
|
764
|
+
),
|
|
765
|
+
...createdStreams.map((streamPath) =>
|
|
766
|
+
this.streamClient.delete(streamPath)
|
|
767
|
+
),
|
|
768
|
+
])
|
|
769
|
+
throw err
|
|
770
|
+
} finally {
|
|
771
|
+
this.releaseForkLocks(this.forkWriteLockedStreams, writeStreamLocks)
|
|
772
|
+
this.releaseForkLocks(this.forkWriteLockedEntities, writeEntityLocks)
|
|
773
|
+
}
|
|
774
|
+
} finally {
|
|
775
|
+
this.releaseForkLocks(this.forkWorkLockedEntities, workLocks)
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
isForkWorkLockedEntity(entityUrl: string): boolean {
|
|
780
|
+
return (this.forkWorkLockedEntities.get(entityUrl) ?? 0) > 0
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
isForkWriteLockedEntity(entityUrl: string): boolean {
|
|
784
|
+
return (this.forkWriteLockedEntities.get(entityUrl) ?? 0) > 0
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
isForkWriteLockedStream(streamPath: string): boolean {
|
|
788
|
+
return (this.forkWriteLockedStreams.get(streamPath) ?? 0) > 0
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
private assertEntityNotForkWorkLocked(entityUrl: string): void {
|
|
792
|
+
if (!this.isForkWorkLockedEntity(entityUrl)) return
|
|
793
|
+
throw new ElectricAgentsError(
|
|
794
|
+
ErrCodeForkInProgress,
|
|
795
|
+
`Entity subtree is being forked`,
|
|
796
|
+
409
|
|
797
|
+
)
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
private addForkLocks(
|
|
801
|
+
locks: Map<string, number>,
|
|
802
|
+
keys: Array<string>,
|
|
803
|
+
held: Set<string>
|
|
804
|
+
): void {
|
|
805
|
+
for (const key of keys) {
|
|
806
|
+
if (held.has(key)) continue
|
|
807
|
+
locks.set(key, (locks.get(key) ?? 0) + 1)
|
|
808
|
+
held.add(key)
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
private releaseForkLocks(
|
|
813
|
+
locks: Map<string, number>,
|
|
814
|
+
held: Set<string>
|
|
815
|
+
): void {
|
|
816
|
+
for (const key of held) {
|
|
817
|
+
const count = locks.get(key) ?? 0
|
|
818
|
+
if (count <= 1) {
|
|
819
|
+
locks.delete(key)
|
|
820
|
+
} else {
|
|
821
|
+
locks.set(key, count - 1)
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
held.clear()
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
private async waitForIdleSubtree(
|
|
828
|
+
rootUrl: string,
|
|
829
|
+
opts: ForkSubtreeOptions,
|
|
830
|
+
workLocks: Set<string>
|
|
831
|
+
): Promise<Array<ElectricAgentsEntity>> {
|
|
832
|
+
const timeoutMs = opts.waitTimeoutMs ?? DEFAULT_FORK_WAIT_TIMEOUT_MS
|
|
833
|
+
const pollMs = opts.waitPollMs ?? DEFAULT_FORK_WAIT_POLL_MS
|
|
834
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
|
835
|
+
throw new ElectricAgentsError(
|
|
836
|
+
ErrCodeInvalidRequest,
|
|
837
|
+
`waitTimeoutMs must be a non-negative number`,
|
|
838
|
+
400
|
|
839
|
+
)
|
|
840
|
+
}
|
|
841
|
+
if (!Number.isFinite(pollMs) || pollMs <= 0) {
|
|
842
|
+
throw new ElectricAgentsError(
|
|
843
|
+
ErrCodeInvalidRequest,
|
|
844
|
+
`waitPollMs must be a positive number`,
|
|
845
|
+
400
|
|
846
|
+
)
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
const deadline = Date.now() + timeoutMs
|
|
850
|
+
while (true) {
|
|
851
|
+
const root = await this.registry.getEntity(rootUrl)
|
|
852
|
+
if (!root) {
|
|
853
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
854
|
+
}
|
|
855
|
+
if (root.parent) {
|
|
856
|
+
throw new ElectricAgentsError(
|
|
857
|
+
ErrCodeInvalidRequest,
|
|
858
|
+
`Only top-level entities can be forked`,
|
|
859
|
+
400
|
|
860
|
+
)
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
const subtree = await this.listEntitySubtree(root)
|
|
864
|
+
const stopped = subtree.find((entity) => entity.status === `stopped`)
|
|
865
|
+
if (stopped) {
|
|
866
|
+
throw new ElectricAgentsError(
|
|
867
|
+
ErrCodeNotRunning,
|
|
868
|
+
`Cannot fork stopped entity "${stopped.url}"`,
|
|
869
|
+
409
|
|
870
|
+
)
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
let active = subtree.filter((entity) => entity.status !== `idle`)
|
|
874
|
+
if (active.length === 0) {
|
|
875
|
+
this.addForkLocks(
|
|
876
|
+
this.forkWorkLockedEntities,
|
|
877
|
+
subtree.map((entity) => entity.url),
|
|
878
|
+
workLocks
|
|
879
|
+
)
|
|
880
|
+
const lockedRoot = await this.registry.getEntity(rootUrl)
|
|
881
|
+
if (!lockedRoot) {
|
|
882
|
+
throw new ElectricAgentsError(
|
|
883
|
+
ErrCodeNotFound,
|
|
884
|
+
`Entity not found`,
|
|
885
|
+
404
|
|
886
|
+
)
|
|
887
|
+
}
|
|
888
|
+
const lockedSubtree = await this.listEntitySubtree(lockedRoot)
|
|
889
|
+
this.addForkLocks(
|
|
890
|
+
this.forkWorkLockedEntities,
|
|
891
|
+
lockedSubtree.map((entity) => entity.url),
|
|
892
|
+
workLocks
|
|
893
|
+
)
|
|
894
|
+
const lockedActive = lockedSubtree.filter(
|
|
895
|
+
(entity) => entity.status !== `idle`
|
|
896
|
+
)
|
|
897
|
+
if (lockedActive.length === 0) {
|
|
898
|
+
return lockedSubtree
|
|
899
|
+
}
|
|
900
|
+
this.releaseForkLocks(this.forkWorkLockedEntities, workLocks)
|
|
901
|
+
active = lockedActive
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
if (Date.now() >= deadline) {
|
|
905
|
+
throw new ElectricAgentsError(
|
|
906
|
+
ErrCodeForkWaitTimeout,
|
|
907
|
+
`Timed out waiting for subtree to become idle`,
|
|
908
|
+
409,
|
|
909
|
+
{ active: active.map((entity) => entity.url) }
|
|
910
|
+
)
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
await sleep(Math.min(pollMs, Math.max(0, deadline - Date.now())))
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
private async listEntitySubtree(
|
|
918
|
+
root: ElectricAgentsEntity
|
|
919
|
+
): Promise<Array<ElectricAgentsEntity>> {
|
|
920
|
+
const result: Array<ElectricAgentsEntity> = []
|
|
921
|
+
const queue: Array<ElectricAgentsEntity> = [root]
|
|
922
|
+
const seen = new Set<string>()
|
|
923
|
+
|
|
924
|
+
while (queue.length > 0) {
|
|
925
|
+
const entity = queue.shift()!
|
|
926
|
+
if (seen.has(entity.url)) continue
|
|
927
|
+
seen.add(entity.url)
|
|
928
|
+
result.push(entity)
|
|
929
|
+
|
|
930
|
+
const { entities: children } = await this.registry.listEntities({
|
|
931
|
+
parent: entity.url,
|
|
932
|
+
limit: 10_000,
|
|
933
|
+
})
|
|
934
|
+
for (const child of children) {
|
|
935
|
+
queue.push(child)
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
return result
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
private async readForkStateSnapshot(
|
|
943
|
+
entitiesToFork: Array<ElectricAgentsEntity>
|
|
944
|
+
): Promise<ForkStateSnapshot> {
|
|
945
|
+
const manifestsByEntity = new Map<
|
|
946
|
+
string,
|
|
947
|
+
Map<string, Record<string, unknown>>
|
|
948
|
+
>()
|
|
949
|
+
const childStatusesByEntity = new Map<
|
|
950
|
+
string,
|
|
951
|
+
Map<string, Record<string, unknown>>
|
|
952
|
+
>()
|
|
953
|
+
const replayWatermarksByEntity = new Map<
|
|
954
|
+
string,
|
|
955
|
+
Map<string, Record<string, unknown>>
|
|
956
|
+
>()
|
|
957
|
+
const sharedStateIds = new Set<string>()
|
|
958
|
+
|
|
959
|
+
for (const entity of entitiesToFork) {
|
|
960
|
+
const events = await this.streamClient.readJson<Record<string, unknown>>(
|
|
961
|
+
entity.streams.main
|
|
962
|
+
)
|
|
963
|
+
const manifests = this.reduceStateRows(events, `manifest`)
|
|
964
|
+
const childStatuses = this.reduceStateRows(events, `child_status`)
|
|
965
|
+
const replayWatermarks = this.reduceStateRows(events, `replay_watermark`)
|
|
966
|
+
|
|
967
|
+
manifestsByEntity.set(entity.url, manifests)
|
|
968
|
+
childStatusesByEntity.set(entity.url, childStatuses)
|
|
969
|
+
replayWatermarksByEntity.set(entity.url, replayWatermarks)
|
|
970
|
+
|
|
971
|
+
for (const manifest of manifests.values()) {
|
|
972
|
+
this.collectSharedStateIds(manifest, sharedStateIds)
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
return {
|
|
977
|
+
manifestsByEntity,
|
|
978
|
+
childStatusesByEntity,
|
|
979
|
+
replayWatermarksByEntity,
|
|
980
|
+
sharedStateIds,
|
|
981
|
+
}
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
private reduceStateRows(
|
|
985
|
+
rawEvents: Array<unknown>,
|
|
986
|
+
eventType: string
|
|
987
|
+
): Map<string, Record<string, unknown>> {
|
|
988
|
+
const rows = new Map<string, Record<string, unknown>>()
|
|
989
|
+
const events = rawEvents.flatMap((item) =>
|
|
990
|
+
Array.isArray(item) ? item : [item]
|
|
991
|
+
)
|
|
992
|
+
|
|
993
|
+
for (const event of events) {
|
|
994
|
+
if (!isRecord(event) || event.type !== eventType) continue
|
|
995
|
+
if (typeof event.key !== `string`) continue
|
|
996
|
+
const headers = isRecord(event.headers) ? event.headers : undefined
|
|
997
|
+
const operation = headers?.operation
|
|
998
|
+
if (operation === `delete`) {
|
|
999
|
+
rows.delete(event.key)
|
|
1000
|
+
continue
|
|
1001
|
+
}
|
|
1002
|
+
if (isRecord(event.value)) {
|
|
1003
|
+
rows.set(event.key, cloneRecord(event.value))
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
return rows
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
private collectSharedStateIds(
|
|
1011
|
+
manifest: Record<string, unknown>,
|
|
1012
|
+
sharedStateIds: Set<string>
|
|
1013
|
+
): void {
|
|
1014
|
+
if (manifest.kind === `shared-state` && typeof manifest.id === `string`) {
|
|
1015
|
+
sharedStateIds.add(manifest.id)
|
|
1016
|
+
return
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
if (manifest.kind !== `source` || manifest.sourceType !== `db`) {
|
|
1020
|
+
return
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
if (typeof manifest.sourceRef === `string`) {
|
|
1024
|
+
sharedStateIds.add(manifest.sourceRef)
|
|
1025
|
+
}
|
|
1026
|
+
const config = isRecord(manifest.config) ? manifest.config : undefined
|
|
1027
|
+
if (typeof config?.id === `string`) {
|
|
1028
|
+
sharedStateIds.add(config.id)
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
private async buildForkEntityUrlMap(
|
|
1033
|
+
entitiesToFork: Array<ElectricAgentsEntity>,
|
|
1034
|
+
opts: { suffix: string; rootUrl: string; rootInstanceId?: string }
|
|
1035
|
+
): Promise<Map<string, string>> {
|
|
1036
|
+
const map = new Map<string, string>()
|
|
1037
|
+
const reserved = new Set<string>()
|
|
1038
|
+
|
|
1039
|
+
for (const entity of entitiesToFork) {
|
|
1040
|
+
const { type, instanceId } = this.parseEntityUrl(entity.url)
|
|
1041
|
+
const rootRequestedId =
|
|
1042
|
+
entity.url === opts.rootUrl ? opts.rootInstanceId : undefined
|
|
1043
|
+
const baseId = rootRequestedId ?? `${instanceId}-fork-${opts.suffix}`
|
|
1044
|
+
const forkUrl = await this.reserveForkEntityUrl(type, baseId, reserved, {
|
|
1045
|
+
exact: rootRequestedId !== undefined,
|
|
1046
|
+
})
|
|
1047
|
+
map.set(entity.url, forkUrl)
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
return map
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
private async reserveForkEntityUrl(
|
|
1054
|
+
type: string,
|
|
1055
|
+
baseId: string,
|
|
1056
|
+
reserved: Set<string>,
|
|
1057
|
+
opts?: { exact?: boolean }
|
|
1058
|
+
): Promise<string> {
|
|
1059
|
+
if (!baseId || baseId.includes(`/`)) {
|
|
1060
|
+
throw new ElectricAgentsError(
|
|
1061
|
+
ErrCodeInvalidRequest,
|
|
1062
|
+
`Fork instance_id must not be empty or contain forward slashes`,
|
|
1063
|
+
400
|
|
1064
|
+
)
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
let attempt = 0
|
|
1068
|
+
while (true) {
|
|
1069
|
+
const instanceId = attempt === 0 ? baseId : `${baseId}-${attempt}`
|
|
1070
|
+
const url = `/${type}/${instanceId}`
|
|
1071
|
+
const exists = reserved.has(url) || (await this.registry.getEntity(url))
|
|
1072
|
+
if (!exists) {
|
|
1073
|
+
reserved.add(url)
|
|
1074
|
+
return url
|
|
1075
|
+
}
|
|
1076
|
+
if (opts?.exact) {
|
|
1077
|
+
throw new ElectricAgentsError(
|
|
1078
|
+
ErrCodeDuplicateURL,
|
|
1079
|
+
`Entity already exists at URL "${url}"`,
|
|
1080
|
+
409
|
|
1081
|
+
)
|
|
1082
|
+
}
|
|
1083
|
+
attempt += 1
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
private async buildForkSharedStateIdMap(
|
|
1088
|
+
sourceIds: Set<string>,
|
|
1089
|
+
suffix: string
|
|
1090
|
+
): Promise<Map<string, string>> {
|
|
1091
|
+
const map = new Map<string, string>()
|
|
1092
|
+
const reserved = new Set<string>()
|
|
1093
|
+
|
|
1094
|
+
for (const sourceId of [...sourceIds].sort()) {
|
|
1095
|
+
const baseId = `${sourceId}-fork-${suffix}`
|
|
1096
|
+
let attempt = 0
|
|
1097
|
+
while (true) {
|
|
1098
|
+
const candidate = attempt === 0 ? baseId : `${baseId}-${attempt}`
|
|
1099
|
+
const path = getSharedStateStreamPath(candidate)
|
|
1100
|
+
if (
|
|
1101
|
+
!reserved.has(candidate) &&
|
|
1102
|
+
!(await this.streamClient.exists(path))
|
|
1103
|
+
) {
|
|
1104
|
+
reserved.add(candidate)
|
|
1105
|
+
map.set(sourceId, candidate)
|
|
1106
|
+
break
|
|
1107
|
+
}
|
|
1108
|
+
attempt += 1
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
return map
|
|
1113
|
+
}
|
|
1114
|
+
|
|
1115
|
+
private buildForkStringMap(
|
|
1116
|
+
entityUrlMap: Map<string, string>,
|
|
1117
|
+
sharedStateIdMap: Map<string, string>
|
|
1118
|
+
): Map<string, string> {
|
|
1119
|
+
const stringMap = new Map<string, string>()
|
|
1120
|
+
for (const [sourceUrl, forkUrl] of entityUrlMap) {
|
|
1121
|
+
stringMap.set(sourceUrl, forkUrl)
|
|
1122
|
+
stringMap.set(`${sourceUrl}/main`, `${forkUrl}/main`)
|
|
1123
|
+
stringMap.set(`${sourceUrl}/error`, `${forkUrl}/error`)
|
|
1124
|
+
}
|
|
1125
|
+
for (const [sourceId, forkId] of sharedStateIdMap) {
|
|
1126
|
+
stringMap.set(sourceId, forkId)
|
|
1127
|
+
stringMap.set(
|
|
1128
|
+
getSharedStateStreamPath(sourceId),
|
|
1129
|
+
getSharedStateStreamPath(forkId)
|
|
1130
|
+
)
|
|
1131
|
+
}
|
|
1132
|
+
return stringMap
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
private buildForkEntityPlans(
|
|
1136
|
+
entitiesToFork: Array<ElectricAgentsEntity>,
|
|
1137
|
+
entityUrlMap: Map<string, string>,
|
|
1138
|
+
stringMap: Map<string, string>
|
|
1139
|
+
): Array<ForkEntityPlan> {
|
|
1140
|
+
const now = Date.now()
|
|
1141
|
+
return entitiesToFork.map((source) => {
|
|
1142
|
+
const forkUrl = entityUrlMap.get(source.url)
|
|
1143
|
+
if (!forkUrl) {
|
|
1144
|
+
throw new Error(`Missing fork URL for ${source.url}`)
|
|
1145
|
+
}
|
|
1146
|
+
const { type } = this.parseEntityUrl(forkUrl)
|
|
1147
|
+
const parent = source.parent ? entityUrlMap.get(source.parent) : undefined
|
|
1148
|
+
const spawnArgs = isRecord(source.spawn_args)
|
|
1149
|
+
? (this.remapJsonValue(source.spawn_args, stringMap) as Record<
|
|
1150
|
+
string,
|
|
1151
|
+
unknown
|
|
1152
|
+
>)
|
|
1153
|
+
: source.spawn_args
|
|
1154
|
+
|
|
1155
|
+
const fork: ElectricAgentsEntity = {
|
|
1156
|
+
...source,
|
|
1157
|
+
url: forkUrl,
|
|
1158
|
+
type,
|
|
1159
|
+
status: `idle`,
|
|
1160
|
+
streams: {
|
|
1161
|
+
main: `${forkUrl}/main`,
|
|
1162
|
+
error: `${forkUrl}/error`,
|
|
1163
|
+
},
|
|
1164
|
+
subscription_id: `${type}-handler`,
|
|
1165
|
+
write_token: randomUUID(),
|
|
1166
|
+
spawn_args: spawnArgs,
|
|
1167
|
+
parent,
|
|
1168
|
+
created_at: now,
|
|
1169
|
+
updated_at: now,
|
|
1170
|
+
}
|
|
1171
|
+
if (!parent) {
|
|
1172
|
+
delete fork.parent
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
return { source, fork }
|
|
1176
|
+
})
|
|
1177
|
+
}
|
|
1178
|
+
|
|
1179
|
+
private buildForkReconciliation(
|
|
1180
|
+
plan: ForkEntityPlan,
|
|
1181
|
+
snapshot: ForkStateSnapshot,
|
|
1182
|
+
entityUrlMap: Map<string, string>,
|
|
1183
|
+
sharedStateIdMap: Map<string, string>,
|
|
1184
|
+
stringMap: Map<string, string>
|
|
1185
|
+
): {
|
|
1186
|
+
events: Array<Record<string, unknown>>
|
|
1187
|
+
manifests: Map<string, Record<string, unknown>>
|
|
1188
|
+
} {
|
|
1189
|
+
const txid = `fork-${randomUUID()}`
|
|
1190
|
+
const headers = {
|
|
1191
|
+
txid,
|
|
1192
|
+
forkedFrom: plan.source.url,
|
|
1193
|
+
}
|
|
1194
|
+
const events: Array<Record<string, unknown>> = [
|
|
1195
|
+
entityStateSchema.entityCreated.update({
|
|
1196
|
+
key: `entity-created`,
|
|
1197
|
+
value: omitUndefined({
|
|
1198
|
+
entity_type: plan.fork.type,
|
|
1199
|
+
timestamp: new Date().toISOString(),
|
|
1200
|
+
args: plan.fork.spawn_args ?? {},
|
|
1201
|
+
parent_url: plan.fork.parent,
|
|
1202
|
+
}),
|
|
1203
|
+
headers,
|
|
1204
|
+
} as any) as Record<string, unknown>,
|
|
1205
|
+
]
|
|
1206
|
+
|
|
1207
|
+
const activeManifests = new Map<string, Record<string, unknown>>()
|
|
1208
|
+
const sourceManifests =
|
|
1209
|
+
snapshot.manifestsByEntity.get(plan.source.url) ?? new Map()
|
|
1210
|
+
for (const [key, value] of sourceManifests) {
|
|
1211
|
+
const remapped = this.remapManifestEntry(
|
|
1212
|
+
key,
|
|
1213
|
+
value,
|
|
1214
|
+
entityUrlMap,
|
|
1215
|
+
sharedStateIdMap
|
|
1216
|
+
)
|
|
1217
|
+
activeManifests.set(remapped.key, remapped.value)
|
|
1218
|
+
if (!remapped.changed) {
|
|
1219
|
+
continue
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
if (remapped.key !== key) {
|
|
1223
|
+
events.push(
|
|
1224
|
+
entityStateSchema.manifests.delete({
|
|
1225
|
+
key,
|
|
1226
|
+
headers,
|
|
1227
|
+
} as any) as Record<string, unknown>
|
|
1228
|
+
)
|
|
1229
|
+
events.push(
|
|
1230
|
+
entityStateSchema.manifests.insert({
|
|
1231
|
+
key: remapped.key,
|
|
1232
|
+
value: remapped.value as any,
|
|
1233
|
+
headers,
|
|
1234
|
+
} as any) as Record<string, unknown>
|
|
1235
|
+
)
|
|
1236
|
+
} else {
|
|
1237
|
+
events.push(
|
|
1238
|
+
entityStateSchema.manifests.update({
|
|
1239
|
+
key,
|
|
1240
|
+
value: remapped.value as any,
|
|
1241
|
+
headers,
|
|
1242
|
+
} as any) as Record<string, unknown>
|
|
1243
|
+
)
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
const childStatuses =
|
|
1248
|
+
snapshot.childStatusesByEntity.get(plan.source.url) ?? new Map()
|
|
1249
|
+
for (const [key, value] of childStatuses) {
|
|
1250
|
+
const remapped = this.remapChildStatus(value, entityUrlMap)
|
|
1251
|
+
if (!remapped) continue
|
|
1252
|
+
events.push(
|
|
1253
|
+
entityStateSchema.childStatus.update({
|
|
1254
|
+
key,
|
|
1255
|
+
value: remapped as any,
|
|
1256
|
+
headers,
|
|
1257
|
+
} as any) as Record<string, unknown>
|
|
1258
|
+
)
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
const replayWatermarks =
|
|
1262
|
+
snapshot.replayWatermarksByEntity.get(plan.source.url) ?? new Map()
|
|
1263
|
+
for (const [key, value] of replayWatermarks) {
|
|
1264
|
+
const remapped = this.remapReplayWatermark(key, value, stringMap)
|
|
1265
|
+
if (!remapped) continue
|
|
1266
|
+
if (remapped.key !== key) {
|
|
1267
|
+
events.push(
|
|
1268
|
+
entityStateSchema.replayWatermarks.delete({
|
|
1269
|
+
key,
|
|
1270
|
+
headers,
|
|
1271
|
+
} as any) as Record<string, unknown>
|
|
1272
|
+
)
|
|
1273
|
+
events.push(
|
|
1274
|
+
entityStateSchema.replayWatermarks.insert({
|
|
1275
|
+
key: remapped.key,
|
|
1276
|
+
value: remapped.value as any,
|
|
1277
|
+
headers,
|
|
1278
|
+
} as any) as Record<string, unknown>
|
|
1279
|
+
)
|
|
1280
|
+
} else {
|
|
1281
|
+
events.push(
|
|
1282
|
+
entityStateSchema.replayWatermarks.update({
|
|
1283
|
+
key,
|
|
1284
|
+
value: remapped.value as any,
|
|
1285
|
+
headers,
|
|
1286
|
+
} as any) as Record<string, unknown>
|
|
1287
|
+
)
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
return { events, manifests: activeManifests }
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
private remapManifestEntry(
|
|
1295
|
+
key: string,
|
|
1296
|
+
value: Record<string, unknown>,
|
|
1297
|
+
entityUrlMap: Map<string, string>,
|
|
1298
|
+
sharedStateIdMap: Map<string, string>
|
|
1299
|
+
): {
|
|
1300
|
+
key: string
|
|
1301
|
+
value: Record<string, unknown>
|
|
1302
|
+
changed: boolean
|
|
1303
|
+
} {
|
|
1304
|
+
const next = cloneRecord(value)
|
|
1305
|
+
|
|
1306
|
+
if (next.kind === `child` && typeof next.entity_url === `string`) {
|
|
1307
|
+
const forkUrl = entityUrlMap.get(next.entity_url)
|
|
1308
|
+
if (!forkUrl) return { key, value: next, changed: false }
|
|
1309
|
+
const { instanceId } = this.parseEntityUrl(forkUrl)
|
|
1310
|
+
next.id = instanceId
|
|
1311
|
+
next.entity_url = forkUrl
|
|
1312
|
+
next.key = manifestChildKey(String(next.entity_type), instanceId)
|
|
1313
|
+
return { key: String(next.key), value: next, changed: true }
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
if (next.kind === `shared-state` && typeof next.id === `string`) {
|
|
1317
|
+
const forkId = sharedStateIdMap.get(next.id)
|
|
1318
|
+
if (!forkId) return { key, value: next, changed: false }
|
|
1319
|
+
next.id = forkId
|
|
1320
|
+
next.key = manifestSharedStateKey(forkId)
|
|
1321
|
+
return { key: String(next.key), value: next, changed: true }
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
if (next.kind === `source` && next.sourceType === `entity`) {
|
|
1325
|
+
const config = isRecord(next.config) ? next.config : {}
|
|
1326
|
+
const sourceUrl =
|
|
1327
|
+
typeof config.entityUrl === `string`
|
|
1328
|
+
? config.entityUrl
|
|
1329
|
+
: typeof next.sourceRef === `string`
|
|
1330
|
+
? next.sourceRef
|
|
1331
|
+
: undefined
|
|
1332
|
+
const forkUrl = sourceUrl ? entityUrlMap.get(sourceUrl) : undefined
|
|
1333
|
+
if (!forkUrl) return { key, value: next, changed: false }
|
|
1334
|
+
const { type } = this.parseEntityUrl(forkUrl)
|
|
1335
|
+
next.sourceRef = forkUrl
|
|
1336
|
+
next.key = manifestSourceKey(`entity`, forkUrl)
|
|
1337
|
+
next.config = {
|
|
1338
|
+
...config,
|
|
1339
|
+
entityUrl: forkUrl,
|
|
1340
|
+
streamPath: `${forkUrl}/main`,
|
|
1341
|
+
entityType: type,
|
|
1342
|
+
}
|
|
1343
|
+
return { key: String(next.key), value: next, changed: true }
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
if (next.kind === `source` && next.sourceType === `db`) {
|
|
1347
|
+
const config = isRecord(next.config) ? next.config : {}
|
|
1348
|
+
const sourceId =
|
|
1349
|
+
typeof next.sourceRef === `string`
|
|
1350
|
+
? next.sourceRef
|
|
1351
|
+
: typeof config.id === `string`
|
|
1352
|
+
? config.id
|
|
1353
|
+
: undefined
|
|
1354
|
+
const forkId = sourceId ? sharedStateIdMap.get(sourceId) : undefined
|
|
1355
|
+
if (!forkId) return { key, value: next, changed: false }
|
|
1356
|
+
next.sourceRef = forkId
|
|
1357
|
+
next.key = manifestSourceKey(`db`, forkId)
|
|
1358
|
+
next.config = {
|
|
1359
|
+
...config,
|
|
1360
|
+
id: forkId,
|
|
1361
|
+
}
|
|
1362
|
+
return { key: String(next.key), value: next, changed: true }
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
if (next.kind === `schedule` && next.scheduleType === `future_send`) {
|
|
1366
|
+
let changed = false
|
|
1367
|
+
if (typeof next.targetUrl === `string`) {
|
|
1368
|
+
const forkTarget = entityUrlMap.get(next.targetUrl)
|
|
1369
|
+
if (forkTarget) {
|
|
1370
|
+
next.targetUrl = forkTarget
|
|
1371
|
+
changed = true
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
if (typeof next.from === `string`) {
|
|
1375
|
+
const forkFrom = entityUrlMap.get(next.from)
|
|
1376
|
+
if (forkFrom) {
|
|
1377
|
+
next.from = forkFrom
|
|
1378
|
+
changed = true
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
return { key, value: next, changed }
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
return { key, value: next, changed: false }
|
|
1385
|
+
}
|
|
1386
|
+
|
|
1387
|
+
private remapChildStatus(
|
|
1388
|
+
value: Record<string, unknown>,
|
|
1389
|
+
entityUrlMap: Map<string, string>
|
|
1390
|
+
): Record<string, unknown> | null {
|
|
1391
|
+
if (typeof value.entity_url !== `string`) return null
|
|
1392
|
+
const forkUrl = entityUrlMap.get(value.entity_url)
|
|
1393
|
+
if (!forkUrl) return null
|
|
1394
|
+
const { type } = this.parseEntityUrl(forkUrl)
|
|
1395
|
+
return {
|
|
1396
|
+
...value,
|
|
1397
|
+
entity_url: forkUrl,
|
|
1398
|
+
entity_type: type,
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
private remapReplayWatermark(
|
|
1403
|
+
key: string,
|
|
1404
|
+
value: Record<string, unknown>,
|
|
1405
|
+
stringMap: Map<string, string>
|
|
1406
|
+
): { key: string; value: Record<string, unknown> } | null {
|
|
1407
|
+
if (typeof value.source_id !== `string`) return null
|
|
1408
|
+
const sourceId = value.source_id
|
|
1409
|
+
const forkSourceId = stringMap.get(sourceId)
|
|
1410
|
+
if (!forkSourceId) return null
|
|
1411
|
+
const next = { ...value, source_id: forkSourceId }
|
|
1412
|
+
return {
|
|
1413
|
+
key: key === sourceId ? forkSourceId : key,
|
|
1414
|
+
value: next,
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
private remapJsonValue(
|
|
1419
|
+
value: unknown,
|
|
1420
|
+
stringMap: Map<string, string>
|
|
1421
|
+
): unknown {
|
|
1422
|
+
if (typeof value === `string`) {
|
|
1423
|
+
return stringMap.get(value) ?? value
|
|
1424
|
+
}
|
|
1425
|
+
if (Array.isArray(value)) {
|
|
1426
|
+
return value.map((item) => this.remapJsonValue(item, stringMap))
|
|
1427
|
+
}
|
|
1428
|
+
if (isRecord(value)) {
|
|
1429
|
+
return Object.fromEntries(
|
|
1430
|
+
Object.entries(value).map(([key, item]) => [
|
|
1431
|
+
key,
|
|
1432
|
+
this.remapJsonValue(item, stringMap),
|
|
1433
|
+
])
|
|
1434
|
+
)
|
|
1435
|
+
}
|
|
1436
|
+
return value
|
|
1437
|
+
}
|
|
1438
|
+
|
|
1439
|
+
private async materializeForkManifestSideEffects(
|
|
1440
|
+
entityUrl: string,
|
|
1441
|
+
manifests: Map<string, Record<string, unknown>>
|
|
1442
|
+
): Promise<void> {
|
|
1443
|
+
for (const [manifestKey, manifest] of manifests) {
|
|
1444
|
+
await this.syncEntitiesManifestSource(
|
|
1445
|
+
entityUrl,
|
|
1446
|
+
manifestKey,
|
|
1447
|
+
`upsert`,
|
|
1448
|
+
manifest
|
|
1449
|
+
)
|
|
1450
|
+
|
|
1451
|
+
const wake = buildManifestWakeRegistration(
|
|
1452
|
+
entityUrl,
|
|
1453
|
+
manifest,
|
|
1454
|
+
manifestKey
|
|
1455
|
+
)
|
|
1456
|
+
if (wake) {
|
|
1457
|
+
await this.wakeRegistry.register({
|
|
1458
|
+
...wake,
|
|
1459
|
+
tenantId: this.tenantId,
|
|
1460
|
+
})
|
|
1461
|
+
}
|
|
1462
|
+
|
|
1463
|
+
const cronSpec = extractManifestCronSpec(manifest)
|
|
1464
|
+
if (cronSpec && this.scheduler) {
|
|
1465
|
+
await this.getOrCreateCronStream(cronSpec.expression, cronSpec.timezone)
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
await this.syncManifestFutureSendSchedule(
|
|
1469
|
+
entityUrl,
|
|
1470
|
+
manifestKey,
|
|
1471
|
+
manifest
|
|
1472
|
+
)
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
private async syncManifestFutureSendSchedule(
|
|
1477
|
+
ownerEntityUrl: string,
|
|
1478
|
+
manifestKey: string,
|
|
1479
|
+
manifest: Record<string, unknown>
|
|
1480
|
+
): Promise<void> {
|
|
1481
|
+
if (!this.scheduler) return
|
|
1482
|
+
if (
|
|
1483
|
+
manifest.kind !== `schedule` ||
|
|
1484
|
+
manifest.scheduleType !== `future_send` ||
|
|
1485
|
+
(manifest.status !== undefined && manifest.status !== `pending`)
|
|
1486
|
+
) {
|
|
1487
|
+
return
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
const fireAtRaw = manifest.fireAt
|
|
1491
|
+
const producerId = manifest.producerId
|
|
1492
|
+
const targetUrl = manifest.targetUrl
|
|
1493
|
+
if (
|
|
1494
|
+
typeof fireAtRaw !== `string` ||
|
|
1495
|
+
typeof producerId !== `string` ||
|
|
1496
|
+
typeof targetUrl !== `string`
|
|
1497
|
+
) {
|
|
1498
|
+
serverLog.warn(
|
|
1499
|
+
`[agent-server] invalid forked future_send manifest entry for ${ownerEntityUrl}/${manifestKey}`
|
|
1500
|
+
)
|
|
1501
|
+
return
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
const fireAt = new Date(fireAtRaw)
|
|
1505
|
+
if (Number.isNaN(fireAt.getTime())) {
|
|
1506
|
+
serverLog.warn(
|
|
1507
|
+
`[agent-server] invalid forked future_send fireAt for ${ownerEntityUrl}/${manifestKey}: ${fireAtRaw}`
|
|
1508
|
+
)
|
|
1509
|
+
return
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1512
|
+
await this.scheduler.syncManifestDelayedSend(
|
|
1513
|
+
ownerEntityUrl,
|
|
1514
|
+
manifestKey,
|
|
1515
|
+
{
|
|
1516
|
+
entityUrl: targetUrl,
|
|
1517
|
+
from:
|
|
1518
|
+
typeof manifest.from === `string` ? manifest.from : ownerEntityUrl,
|
|
1519
|
+
payload: manifest.payload,
|
|
1520
|
+
key: `scheduled-${producerId}`,
|
|
1521
|
+
type:
|
|
1522
|
+
typeof manifest.messageType === `string`
|
|
1523
|
+
? manifest.messageType
|
|
1524
|
+
: undefined,
|
|
1525
|
+
producerId,
|
|
1526
|
+
manifest: {
|
|
1527
|
+
ownerEntityUrl,
|
|
1528
|
+
key: manifestKey,
|
|
1529
|
+
entry: omitUndefined({
|
|
1530
|
+
...manifest,
|
|
1531
|
+
key: manifestKey,
|
|
1532
|
+
kind: `schedule`,
|
|
1533
|
+
scheduleType: `future_send`,
|
|
1534
|
+
targetUrl,
|
|
1535
|
+
fireAt: fireAt.toISOString(),
|
|
1536
|
+
producerId,
|
|
1537
|
+
status: `pending`,
|
|
1538
|
+
}),
|
|
1539
|
+
},
|
|
1540
|
+
},
|
|
1541
|
+
fireAt
|
|
1542
|
+
)
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
private parseEntityUrl(url: string): { type: string; instanceId: string } {
|
|
1546
|
+
const segments = url.split(`/`).filter(Boolean)
|
|
1547
|
+
if (segments.length !== 2 || !segments[0] || !segments[1]) {
|
|
1548
|
+
throw new ElectricAgentsError(
|
|
1549
|
+
ErrCodeInvalidRequest,
|
|
1550
|
+
`Invalid entity URL "${url}"`,
|
|
1551
|
+
400
|
|
1552
|
+
)
|
|
1553
|
+
}
|
|
1554
|
+
return { type: segments[0], instanceId: segments[1] }
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
// ==========================================================================
|
|
1558
|
+
// Send
|
|
1559
|
+
// ==========================================================================
|
|
1560
|
+
|
|
1561
|
+
/**
|
|
1562
|
+
* Deliver a message to an entity's main stream, with optional input schema
|
|
1563
|
+
* validation.
|
|
1564
|
+
*/
|
|
1565
|
+
async send(
|
|
1566
|
+
entityUrl: string,
|
|
1567
|
+
req: SendRequest,
|
|
1568
|
+
opts?: { producerId?: string }
|
|
1569
|
+
): Promise<void> {
|
|
1570
|
+
const entity = await this.validateSendRequest(entityUrl, req)
|
|
1571
|
+
if (
|
|
1572
|
+
this.isForkWorkLockedEntity(entityUrl) &&
|
|
1573
|
+
!(req.from && this.isForkWorkLockedEntity(req.from))
|
|
1574
|
+
) {
|
|
1575
|
+
this.assertEntityNotForkWorkLocked(entityUrl)
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
const now = new Date().toISOString()
|
|
1579
|
+
const key =
|
|
1580
|
+
req.key ??
|
|
1581
|
+
`msg-in-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
|
|
1582
|
+
|
|
1583
|
+
const value: Record<string, unknown> = {
|
|
1584
|
+
from: req.from,
|
|
1585
|
+
payload: req.payload,
|
|
1586
|
+
timestamp: now,
|
|
1587
|
+
}
|
|
1588
|
+
if (req.type) {
|
|
1589
|
+
value.message_type = req.type
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
const envelope = entityStateSchema.inbox.insert({
|
|
1593
|
+
key,
|
|
1594
|
+
value,
|
|
1595
|
+
} as any)
|
|
1596
|
+
|
|
1597
|
+
const encoded = this.encodeChangeEvent(envelope as Record<string, unknown>)
|
|
1598
|
+
try {
|
|
1599
|
+
if (opts?.producerId) {
|
|
1600
|
+
await this.streamClient.appendIdempotent(entity.streams.main, encoded, {
|
|
1601
|
+
producerId: opts.producerId,
|
|
1602
|
+
})
|
|
1603
|
+
return
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
await this.streamClient.append(entity.streams.main, encoded)
|
|
1607
|
+
} catch (err) {
|
|
1608
|
+
if (this.isClosedStreamError(err)) {
|
|
1609
|
+
throw new ElectricAgentsError(
|
|
1610
|
+
ErrCodeNotRunning,
|
|
1611
|
+
`Entity is stopped`,
|
|
1612
|
+
409
|
|
1613
|
+
)
|
|
1614
|
+
}
|
|
1615
|
+
throw err
|
|
1616
|
+
}
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
// ==========================================================================
|
|
1620
|
+
// Tag Updates
|
|
1621
|
+
// ==========================================================================
|
|
1622
|
+
|
|
1623
|
+
async setTag(
|
|
1624
|
+
entityUrl: string,
|
|
1625
|
+
key: string,
|
|
1626
|
+
req: SetTagRequest,
|
|
1627
|
+
token: string
|
|
1628
|
+
): Promise<ElectricAgentsEntity> {
|
|
1629
|
+
const entity = await this.registry.getEntity(entityUrl)
|
|
1630
|
+
if (!entity) {
|
|
1631
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
if (!this.isValidWriteToken(entity, token)) {
|
|
1635
|
+
throw new ElectricAgentsError(
|
|
1636
|
+
ErrCodeUnauthorized,
|
|
1637
|
+
`Invalid write token`,
|
|
1638
|
+
401
|
|
1639
|
+
)
|
|
1640
|
+
}
|
|
1641
|
+
if (entity.status === `stopped`) {
|
|
1642
|
+
throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
if (typeof req.value !== `string`) {
|
|
1646
|
+
throw new ElectricAgentsError(
|
|
1647
|
+
ErrCodeInvalidRequest,
|
|
1648
|
+
`Tag values must be strings`,
|
|
1649
|
+
400
|
|
1650
|
+
)
|
|
1651
|
+
}
|
|
1652
|
+
|
|
1653
|
+
const result = await this.registry.setEntityTag(entityUrl, key, req.value)
|
|
1654
|
+
const updated = result.entity
|
|
1655
|
+
if (!updated) {
|
|
1656
|
+
throw new ElectricAgentsError(
|
|
1657
|
+
ErrCodeEntityPersistFailed,
|
|
1658
|
+
`Entity not found after tag write`,
|
|
1659
|
+
500
|
|
1660
|
+
)
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
if (result.changed && this.entityBridgeManager) {
|
|
1664
|
+
await this.entityBridgeManager.onEntityChanged(entityUrl)
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
return updated
|
|
1668
|
+
}
|
|
1669
|
+
|
|
1670
|
+
async removeTag(
|
|
1671
|
+
entityUrl: string,
|
|
1672
|
+
key: string,
|
|
1673
|
+
token: string
|
|
1674
|
+
): Promise<ElectricAgentsEntity> {
|
|
1675
|
+
const entity = await this.registry.getEntity(entityUrl)
|
|
1676
|
+
if (!entity) {
|
|
1677
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
1678
|
+
}
|
|
1679
|
+
|
|
1680
|
+
if (!this.isValidWriteToken(entity, token)) {
|
|
1681
|
+
throw new ElectricAgentsError(
|
|
1682
|
+
ErrCodeUnauthorized,
|
|
1683
|
+
`Invalid write token`,
|
|
1684
|
+
401
|
|
1685
|
+
)
|
|
1686
|
+
}
|
|
1687
|
+
if (entity.status === `stopped`) {
|
|
1688
|
+
throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
|
|
1689
|
+
}
|
|
1690
|
+
|
|
1691
|
+
const result = await this.registry.removeEntityTag(entityUrl, key)
|
|
1692
|
+
const updated = result.entity
|
|
1693
|
+
if (!updated) {
|
|
1694
|
+
throw new ElectricAgentsError(
|
|
1695
|
+
ErrCodeEntityPersistFailed,
|
|
1696
|
+
`Entity not found after tag delete`,
|
|
1697
|
+
500
|
|
1698
|
+
)
|
|
1699
|
+
}
|
|
1700
|
+
|
|
1701
|
+
if (result.changed && this.entityBridgeManager) {
|
|
1702
|
+
await this.entityBridgeManager.onEntityChanged(entityUrl)
|
|
1703
|
+
}
|
|
1704
|
+
|
|
1705
|
+
return updated
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
async registerEntitiesSource(tags: Record<string, string>): Promise<{
|
|
1709
|
+
sourceRef: string
|
|
1710
|
+
streamUrl: string
|
|
1711
|
+
}> {
|
|
1712
|
+
if (!this.entityBridgeManager) {
|
|
1713
|
+
throw new Error(`Entity bridge manager not configured`)
|
|
1714
|
+
}
|
|
1715
|
+
return this.entityBridgeManager.register(this.validateTags(tags))
|
|
1716
|
+
}
|
|
1717
|
+
|
|
1718
|
+
async writeManifestEntry(
|
|
1719
|
+
entityUrl: string,
|
|
1720
|
+
key: string,
|
|
1721
|
+
operation: `insert` | `update` | `upsert` | `delete`,
|
|
1722
|
+
value?: Record<string, unknown>,
|
|
1723
|
+
opts?: { producerId?: string; txid?: string }
|
|
1724
|
+
): Promise<void> {
|
|
1725
|
+
const entity = await this.registry.getEntity(entityUrl)
|
|
1726
|
+
if (!entity) {
|
|
1727
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
const event: Record<string, unknown> = {
|
|
1731
|
+
type: `manifest`,
|
|
1732
|
+
key,
|
|
1733
|
+
headers: {
|
|
1734
|
+
operation,
|
|
1735
|
+
timestamp: new Date().toISOString(),
|
|
1736
|
+
...(opts?.txid ? { txid: opts.txid } : {}),
|
|
1737
|
+
},
|
|
1738
|
+
}
|
|
1739
|
+
if (value !== undefined) {
|
|
1740
|
+
event.value = value
|
|
1741
|
+
}
|
|
1742
|
+
|
|
1743
|
+
const encoded = this.encodeChangeEvent(event)
|
|
1744
|
+
if (opts?.producerId) {
|
|
1745
|
+
await this.streamClient.appendIdempotent(entity.streams.main, encoded, {
|
|
1746
|
+
producerId: opts.producerId,
|
|
1747
|
+
})
|
|
1748
|
+
await this.syncEntitiesManifestSource(entityUrl, key, operation, value)
|
|
1749
|
+
return
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
await this.streamClient.append(entity.streams.main, encoded)
|
|
1753
|
+
await this.syncEntitiesManifestSource(entityUrl, key, operation, value)
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
async upsertCronSchedule(
|
|
1757
|
+
entityUrl: string,
|
|
1758
|
+
req: {
|
|
1759
|
+
id: string
|
|
1760
|
+
expression: string
|
|
1761
|
+
timezone?: string
|
|
1762
|
+
payload?: unknown
|
|
1763
|
+
debounceMs?: number
|
|
1764
|
+
timeoutMs?: number
|
|
1765
|
+
}
|
|
1766
|
+
): Promise<{ txid: string }> {
|
|
1767
|
+
if (req.payload === undefined) {
|
|
1768
|
+
throw new ElectricAgentsError(
|
|
1769
|
+
ErrCodeInvalidRequest,
|
|
1770
|
+
`Missing required field: payload`,
|
|
1771
|
+
400
|
|
1772
|
+
)
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
const spec = resolveCronScheduleSpec(req.expression, req.timezone)
|
|
1776
|
+
|
|
1777
|
+
const manifestKey = `schedule:${req.id}`
|
|
1778
|
+
await this.wakeRegistry.unregisterByManifestKey(
|
|
1779
|
+
entityUrl,
|
|
1780
|
+
manifestKey,
|
|
1781
|
+
this.tenantId
|
|
1782
|
+
)
|
|
1783
|
+
await this.wakeRegistry.register({
|
|
1784
|
+
tenantId: this.tenantId,
|
|
1785
|
+
subscriberUrl: entityUrl,
|
|
1786
|
+
sourceUrl: getCronStreamPath(spec.expression, spec.timezone),
|
|
1787
|
+
condition: {
|
|
1788
|
+
on: `change`,
|
|
1789
|
+
},
|
|
1790
|
+
debounceMs: req.debounceMs,
|
|
1791
|
+
timeoutMs: req.timeoutMs,
|
|
1792
|
+
oneShot: false,
|
|
1793
|
+
manifestKey,
|
|
1794
|
+
})
|
|
1795
|
+
await this.getOrCreateCronStream(spec.expression, spec.timezone)
|
|
1796
|
+
|
|
1797
|
+
const txid = randomUUID()
|
|
1798
|
+
await this.writeManifestEntry(
|
|
1799
|
+
entityUrl,
|
|
1800
|
+
manifestKey,
|
|
1801
|
+
`upsert`,
|
|
1802
|
+
{
|
|
1803
|
+
key: manifestKey,
|
|
1804
|
+
kind: `schedule`,
|
|
1805
|
+
id: req.id,
|
|
1806
|
+
scheduleType: `cron`,
|
|
1807
|
+
expression: spec.expression,
|
|
1808
|
+
timezone: spec.timezone,
|
|
1809
|
+
payload: req.payload,
|
|
1810
|
+
wake: {
|
|
1811
|
+
on: `change`,
|
|
1812
|
+
...(typeof req.debounceMs === `number`
|
|
1813
|
+
? { debounceMs: req.debounceMs }
|
|
1814
|
+
: {}),
|
|
1815
|
+
...(typeof req.timeoutMs === `number`
|
|
1816
|
+
? { timeoutMs: req.timeoutMs }
|
|
1817
|
+
: {}),
|
|
1818
|
+
},
|
|
1819
|
+
},
|
|
1820
|
+
{ txid }
|
|
1821
|
+
)
|
|
1822
|
+
|
|
1823
|
+
return { txid }
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
async upsertFutureSendSchedule(
|
|
1827
|
+
ownerEntityUrl: string,
|
|
1828
|
+
req: {
|
|
1829
|
+
id: string
|
|
1830
|
+
payload: unknown
|
|
1831
|
+
targetUrl?: string
|
|
1832
|
+
fireAt: string
|
|
1833
|
+
from?: string
|
|
1834
|
+
messageType?: string
|
|
1835
|
+
}
|
|
1836
|
+
): Promise<{ txid: string }> {
|
|
1837
|
+
if (!this.scheduler) {
|
|
1838
|
+
throw new Error(`Scheduler not configured`)
|
|
1839
|
+
}
|
|
1840
|
+
|
|
1841
|
+
const targetUrl = req.targetUrl ?? ownerEntityUrl
|
|
1842
|
+
const from = req.from ?? ownerEntityUrl
|
|
1843
|
+
const fireAt = new Date(req.fireAt)
|
|
1844
|
+
if (Number.isNaN(fireAt.getTime())) {
|
|
1845
|
+
throw new ElectricAgentsError(
|
|
1846
|
+
ErrCodeInvalidRequest,
|
|
1847
|
+
`Invalid fireAt timestamp: ${req.fireAt}`,
|
|
1848
|
+
400
|
|
1849
|
+
)
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
await this.validateSendRequest(targetUrl, {
|
|
1853
|
+
from,
|
|
1854
|
+
payload: req.payload,
|
|
1855
|
+
type: req.messageType,
|
|
1856
|
+
})
|
|
1857
|
+
|
|
1858
|
+
const manifestKey = `schedule:${req.id}`
|
|
1859
|
+
const producerId = `future-send-${randomUUID()}`
|
|
1860
|
+
|
|
1861
|
+
await this.wakeRegistry.unregisterByManifestKey(
|
|
1862
|
+
ownerEntityUrl,
|
|
1863
|
+
manifestKey,
|
|
1864
|
+
this.tenantId
|
|
1865
|
+
)
|
|
1866
|
+
await this.scheduler.syncManifestDelayedSend(
|
|
1867
|
+
ownerEntityUrl,
|
|
1868
|
+
manifestKey,
|
|
1869
|
+
{
|
|
1870
|
+
entityUrl: targetUrl,
|
|
1871
|
+
from,
|
|
1872
|
+
payload: req.payload,
|
|
1873
|
+
key: `scheduled-${producerId}`,
|
|
1874
|
+
type: req.messageType,
|
|
1875
|
+
producerId,
|
|
1876
|
+
manifest: {
|
|
1877
|
+
ownerEntityUrl,
|
|
1878
|
+
key: manifestKey,
|
|
1879
|
+
entry: {
|
|
1880
|
+
key: manifestKey,
|
|
1881
|
+
kind: `schedule`,
|
|
1882
|
+
id: req.id,
|
|
1883
|
+
scheduleType: `future_send`,
|
|
1884
|
+
fireAt: fireAt.toISOString(),
|
|
1885
|
+
targetUrl,
|
|
1886
|
+
payload: req.payload,
|
|
1887
|
+
producerId,
|
|
1888
|
+
...(req.from ? { from: req.from } : {}),
|
|
1889
|
+
...(req.messageType ? { messageType: req.messageType } : {}),
|
|
1890
|
+
status: `pending`,
|
|
1891
|
+
},
|
|
1892
|
+
},
|
|
1893
|
+
},
|
|
1894
|
+
fireAt
|
|
1895
|
+
)
|
|
1896
|
+
|
|
1897
|
+
const txid = randomUUID()
|
|
1898
|
+
await this.writeManifestEntry(
|
|
1899
|
+
ownerEntityUrl,
|
|
1900
|
+
manifestKey,
|
|
1901
|
+
`upsert`,
|
|
1902
|
+
{
|
|
1903
|
+
key: manifestKey,
|
|
1904
|
+
kind: `schedule`,
|
|
1905
|
+
id: req.id,
|
|
1906
|
+
scheduleType: `future_send`,
|
|
1907
|
+
fireAt: fireAt.toISOString(),
|
|
1908
|
+
targetUrl,
|
|
1909
|
+
payload: req.payload,
|
|
1910
|
+
producerId,
|
|
1911
|
+
...(req.from ? { from: req.from } : {}),
|
|
1912
|
+
...(req.messageType ? { messageType: req.messageType } : {}),
|
|
1913
|
+
status: `pending`,
|
|
1914
|
+
},
|
|
1915
|
+
{ txid }
|
|
1916
|
+
)
|
|
1917
|
+
|
|
1918
|
+
return { txid }
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
async deleteSchedule(
|
|
1922
|
+
entityUrl: string,
|
|
1923
|
+
req: { id: string }
|
|
1924
|
+
): Promise<{ txid: string }> {
|
|
1925
|
+
const manifestKey = `schedule:${req.id}`
|
|
1926
|
+
if (this.scheduler) {
|
|
1927
|
+
await this.scheduler.cancelManifestDelayedSend(entityUrl, manifestKey)
|
|
1928
|
+
}
|
|
1929
|
+
await this.wakeRegistry.unregisterByManifestKey(
|
|
1930
|
+
entityUrl,
|
|
1931
|
+
manifestKey,
|
|
1932
|
+
this.tenantId
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
const txid = randomUUID()
|
|
1936
|
+
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, undefined, {
|
|
1937
|
+
txid,
|
|
1938
|
+
})
|
|
1939
|
+
|
|
1940
|
+
return { txid }
|
|
1941
|
+
}
|
|
1942
|
+
|
|
1943
|
+
// ==========================================================================
|
|
1944
|
+
// Wake Evaluation
|
|
1945
|
+
// ==========================================================================
|
|
1946
|
+
|
|
1947
|
+
/**
|
|
1948
|
+
* Register a wake subscription from a subscriber to a source entity.
|
|
1949
|
+
*/
|
|
1950
|
+
async registerWake(opts: {
|
|
1951
|
+
subscriberUrl: string
|
|
1952
|
+
sourceUrl: string
|
|
1953
|
+
condition: `runFinished` | { on: `change`; collections?: Array<string> }
|
|
1954
|
+
debounceMs?: number
|
|
1955
|
+
timeoutMs?: number
|
|
1956
|
+
includeResponse?: boolean
|
|
1957
|
+
manifestKey?: string
|
|
1958
|
+
}): Promise<void> {
|
|
1959
|
+
await this.wakeRegistry.register({
|
|
1960
|
+
tenantId: this.tenantId,
|
|
1961
|
+
subscriberUrl: opts.subscriberUrl,
|
|
1962
|
+
sourceUrl: opts.sourceUrl,
|
|
1963
|
+
condition: opts.condition,
|
|
1964
|
+
oneShot: false,
|
|
1965
|
+
debounceMs: opts.debounceMs,
|
|
1966
|
+
timeoutMs: opts.timeoutMs,
|
|
1967
|
+
includeResponse: opts.includeResponse,
|
|
1968
|
+
manifestKey: opts.manifestKey,
|
|
1969
|
+
})
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
async enqueueDelayedSend(
|
|
1973
|
+
entityUrl: string,
|
|
1974
|
+
req: SendRequest,
|
|
1975
|
+
fireAt: Date
|
|
1976
|
+
): Promise<void> {
|
|
1977
|
+
if (!this.scheduler) {
|
|
1978
|
+
throw new Error(`Scheduler not configured`)
|
|
1979
|
+
}
|
|
1980
|
+
|
|
1981
|
+
await this.validateSendRequest(entityUrl, req)
|
|
1982
|
+
|
|
1983
|
+
await this.scheduler.enqueueDelayedSend(
|
|
1984
|
+
{
|
|
1985
|
+
entityUrl,
|
|
1986
|
+
from: req.from,
|
|
1987
|
+
payload: req.payload,
|
|
1988
|
+
key: req.key,
|
|
1989
|
+
type: req.type,
|
|
1990
|
+
},
|
|
1991
|
+
fireAt
|
|
1992
|
+
)
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
/**
|
|
1996
|
+
* Evaluate an event against registered wake conditions and deliver results.
|
|
1997
|
+
*/
|
|
1998
|
+
async evaluateWakes(
|
|
1999
|
+
sourceUrl: string,
|
|
2000
|
+
event: Record<string, unknown>
|
|
2001
|
+
): Promise<void> {
|
|
2002
|
+
return await withSpan(`electric_agents.evaluateWakes`, async (span) => {
|
|
2003
|
+
span.setAttribute(ATTR.WAKE_SOURCE, sourceUrl)
|
|
2004
|
+
const results = this.wakeRegistry.evaluate(
|
|
2005
|
+
sourceUrl,
|
|
2006
|
+
event,
|
|
2007
|
+
this.tenantId
|
|
2008
|
+
)
|
|
2009
|
+
span.setAttribute(`electric_agents.wake.subscriber_count`, results.length)
|
|
2010
|
+
const settled = await Promise.allSettled(
|
|
2011
|
+
results.map((result) => this.deliverWakeResult(result))
|
|
2012
|
+
)
|
|
2013
|
+
for (const [index, result] of settled.entries()) {
|
|
2014
|
+
if (result.status === `rejected`) {
|
|
2015
|
+
serverLog.warn(
|
|
2016
|
+
`[agent-server] failed to deliver wake for ${results[index]!.subscriberUrl}:`,
|
|
2017
|
+
result.reason
|
|
2018
|
+
)
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
2021
|
+
})
|
|
2022
|
+
}
|
|
2023
|
+
|
|
2024
|
+
/**
|
|
2025
|
+
* Deliver a wake result: append WakeMessage to subscriber's stream and
|
|
2026
|
+
* trigger webhook notification.
|
|
2027
|
+
*/
|
|
2028
|
+
private async deliverWakeResult(result: WakeEvalResult): Promise<void> {
|
|
2029
|
+
if (result.tenantId !== this.tenantId) return
|
|
2030
|
+
|
|
2031
|
+
return await withSpan(`electric_agents.deliverWake`, async (span) => {
|
|
2032
|
+
span.setAttributes({
|
|
2033
|
+
[ATTR.WAKE_SUBSCRIBER]: result.subscriberUrl,
|
|
2034
|
+
[ATTR.WAKE_SOURCE]: result.wakeMessage.source,
|
|
2035
|
+
[ATTR.WAKE_KIND]: result.wakeMessage.timeout ? `timeout` : `change`,
|
|
2036
|
+
})
|
|
2037
|
+
// Fetch subscriber and source entity in parallel — runFinished wakes need
|
|
2038
|
+
// both, plain wakes only need subscriber but the extra read is cheap.
|
|
2039
|
+
const needsSource = result.runFinishedStatus !== undefined
|
|
2040
|
+
const [subscriber, sourceEntity] = await Promise.all([
|
|
2041
|
+
this.registry.getEntity(result.subscriberUrl),
|
|
2042
|
+
needsSource
|
|
2043
|
+
? this.registry.getEntity(result.wakeMessage.source)
|
|
2044
|
+
: Promise.resolve(null),
|
|
2045
|
+
])
|
|
2046
|
+
if (!subscriber) return
|
|
2047
|
+
const wakeMessage = await this.buildWakeMessage(
|
|
2048
|
+
subscriber,
|
|
2049
|
+
result,
|
|
2050
|
+
sourceEntity
|
|
2051
|
+
)
|
|
2052
|
+
const wakeEvent = entityStateSchema.wakes.insert({
|
|
2053
|
+
key: `wake-${result.registrationDbId}-${result.sourceEventKey}`,
|
|
2054
|
+
value: wakeMessage,
|
|
2055
|
+
} as any)
|
|
2056
|
+
await this.streamClient.appendIdempotent(
|
|
2057
|
+
subscriber.streams.main,
|
|
2058
|
+
this.encodeChangeEvent(wakeEvent as Record<string, unknown>),
|
|
2059
|
+
{
|
|
2060
|
+
producerId: `wake-reg-${result.registrationDbId}-${result.sourceEventKey}`,
|
|
2061
|
+
}
|
|
2062
|
+
)
|
|
2063
|
+
})
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
private async syncEntitiesManifestSource(
|
|
2067
|
+
entityUrl: string,
|
|
2068
|
+
manifestKey: string,
|
|
2069
|
+
operation: `insert` | `update` | `upsert` | `delete`,
|
|
2070
|
+
value?: Record<string, unknown>
|
|
2071
|
+
): Promise<void> {
|
|
2072
|
+
const sourceRef =
|
|
2073
|
+
operation === `delete` ? undefined : this.extractEntitiesSourceRef(value)
|
|
2074
|
+
await this.registry.replaceEntityManifestSource(
|
|
2075
|
+
entityUrl,
|
|
2076
|
+
manifestKey,
|
|
2077
|
+
sourceRef
|
|
2078
|
+
)
|
|
2079
|
+
}
|
|
2080
|
+
|
|
2081
|
+
private extractEntitiesSourceRef(
|
|
2082
|
+
manifest?: Record<string, unknown>
|
|
2083
|
+
): string | undefined {
|
|
2084
|
+
if (
|
|
2085
|
+
manifest?.kind === `source` &&
|
|
2086
|
+
manifest.sourceType === `entities` &&
|
|
2087
|
+
typeof manifest.sourceRef === `string`
|
|
2088
|
+
) {
|
|
2089
|
+
return manifest.sourceRef
|
|
2090
|
+
}
|
|
2091
|
+
return undefined
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
/**
|
|
2095
|
+
* Read a child entity's stream and extract concatenated text deltas
|
|
2096
|
+
* for a specific run, plus any error messages for that run.
|
|
2097
|
+
*/
|
|
2098
|
+
private async extractRunResponse(
|
|
2099
|
+
entity: ElectricAgentsEntity,
|
|
2100
|
+
runKey: string,
|
|
2101
|
+
runStatus: `completed` | `failed`
|
|
2102
|
+
): Promise<{ response?: string; error?: string }> {
|
|
2103
|
+
let events: Array<Record<string, unknown>>
|
|
2104
|
+
try {
|
|
2105
|
+
events = await this.streamClient.readJson<Record<string, unknown>>(
|
|
2106
|
+
entity.streams.main
|
|
2107
|
+
)
|
|
2108
|
+
} catch (err) {
|
|
2109
|
+
serverLog.warn(
|
|
2110
|
+
`[agent-server] failed to read child stream for ${entity.url} (${runKey}): ${err instanceof Error ? err.message : String(err)}`
|
|
2111
|
+
)
|
|
2112
|
+
return { error: `Failed to load child response` }
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
const textDeltas: Array<string> = []
|
|
2116
|
+
const errors: Array<string> = []
|
|
2117
|
+
|
|
2118
|
+
for (const parsed of events) {
|
|
2119
|
+
const value = parsed.value as Record<string, unknown> | undefined
|
|
2120
|
+
if (!value) continue
|
|
2121
|
+
|
|
2122
|
+
if (parsed.type === `text_delta`) {
|
|
2123
|
+
if ((value.run_id as string) === runKey) {
|
|
2124
|
+
textDeltas.push((value.delta as string) || ``)
|
|
2125
|
+
}
|
|
2126
|
+
} else if (parsed.type === `error` && runStatus === `failed`) {
|
|
2127
|
+
if ((value.run_id as string) === runKey) {
|
|
2128
|
+
errors.push((value.message as string) || ``)
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
|
|
2133
|
+
const result: { response?: string; error?: string } = {}
|
|
2134
|
+
|
|
2135
|
+
const runText = textDeltas.join(``)
|
|
2136
|
+
if (runText.length > 0) {
|
|
2137
|
+
result.response = runText
|
|
2138
|
+
}
|
|
2139
|
+
|
|
2140
|
+
if (errors.length > 0) {
|
|
2141
|
+
result.error = errors.join(`\n`)
|
|
2142
|
+
}
|
|
2143
|
+
|
|
2144
|
+
return result
|
|
2145
|
+
}
|
|
2146
|
+
|
|
2147
|
+
private async buildWakeMessage(
|
|
2148
|
+
subscriber: ElectricAgentsEntity,
|
|
2149
|
+
result: WakeEvalResult,
|
|
2150
|
+
sourceEntity: ElectricAgentsEntity | null
|
|
2151
|
+
): Promise<WakeMessage> {
|
|
2152
|
+
const wakeMessage: WakeMessage = {
|
|
2153
|
+
timestamp: new Date().toISOString(),
|
|
2154
|
+
...result.wakeMessage,
|
|
2155
|
+
}
|
|
2156
|
+
if (!result.runFinishedStatus) {
|
|
2157
|
+
return wakeMessage
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
if (!sourceEntity) {
|
|
2161
|
+
throw new Error(
|
|
2162
|
+
`[agent-server] runFinished wake source entity not found: ${result.wakeMessage.source}`
|
|
2163
|
+
)
|
|
2164
|
+
}
|
|
2165
|
+
|
|
2166
|
+
// `runFinished` is valid both for spawned children and explicitly observed
|
|
2167
|
+
// entities. Only child wakes get the richer sibling-status payload.
|
|
2168
|
+
if (sourceEntity.parent !== subscriber.url) {
|
|
2169
|
+
return wakeMessage
|
|
2170
|
+
}
|
|
2171
|
+
|
|
2172
|
+
const includeResponse = result.includeResponse !== false
|
|
2173
|
+
const changes = result.wakeMessage.changes
|
|
2174
|
+
const runKey = changes[changes.length - 1]?.key
|
|
2175
|
+
const { response, error } =
|
|
2176
|
+
includeResponse && runKey
|
|
2177
|
+
? await this.extractRunResponse(
|
|
2178
|
+
sourceEntity,
|
|
2179
|
+
runKey,
|
|
2180
|
+
result.runFinishedStatus
|
|
2181
|
+
)
|
|
2182
|
+
: {}
|
|
2183
|
+
|
|
2184
|
+
return {
|
|
2185
|
+
...wakeMessage,
|
|
2186
|
+
finished_child: {
|
|
2187
|
+
url: sourceEntity.url,
|
|
2188
|
+
type: sourceEntity.type,
|
|
2189
|
+
run_status: result.runFinishedStatus,
|
|
2190
|
+
...(response !== undefined ? { response } : {}),
|
|
2191
|
+
...(error !== undefined ? { error } : {}),
|
|
2192
|
+
},
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
|
|
2196
|
+
// ==========================================================================
|
|
2197
|
+
// Kill
|
|
2198
|
+
// ==========================================================================
|
|
2199
|
+
|
|
2200
|
+
async kill(entityUrl: string): Promise<{ txid: number }> {
|
|
2201
|
+
const entity = await this.registry.getEntity(entityUrl)
|
|
2202
|
+
if (!entity) {
|
|
2203
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
2204
|
+
}
|
|
2205
|
+
|
|
2206
|
+
await this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId)
|
|
2207
|
+
await this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId)
|
|
2208
|
+
|
|
2209
|
+
const txid = await this.registry.updateStatusWithTxid(entityUrl, `stopped`)
|
|
2210
|
+
if (this.entityBridgeManager) {
|
|
2211
|
+
await this.entityBridgeManager.onEntityChanged(entityUrl)
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
// Append entity_stopped to main/error streams and close them.
|
|
2215
|
+
const stoppedEvent = entityStateSchema.entityStopped.insert({
|
|
2216
|
+
key: `stopped`,
|
|
2217
|
+
value: {
|
|
2218
|
+
timestamp: new Date().toISOString(),
|
|
2219
|
+
},
|
|
2220
|
+
} as any)
|
|
2221
|
+
const eofData = this.encodeChangeEvent(
|
|
2222
|
+
stoppedEvent as Record<string, unknown>
|
|
2223
|
+
)
|
|
2224
|
+
|
|
2225
|
+
for (const streamPath of [entity.streams.main, entity.streams.error]) {
|
|
2226
|
+
try {
|
|
2227
|
+
await this.streamClient.append(streamPath, eofData, { close: true })
|
|
2228
|
+
} catch (err) {
|
|
2229
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
2230
|
+
if (
|
|
2231
|
+
/closed/i.test(message) ||
|
|
2232
|
+
/not found/i.test(message) ||
|
|
2233
|
+
/404/.test(message) ||
|
|
2234
|
+
/409/.test(message)
|
|
2235
|
+
) {
|
|
2236
|
+
continue
|
|
2237
|
+
}
|
|
2238
|
+
throw err
|
|
2239
|
+
}
|
|
2240
|
+
}
|
|
2241
|
+
|
|
2242
|
+
return { txid }
|
|
2243
|
+
}
|
|
2244
|
+
|
|
2245
|
+
// ==========================================================================
|
|
2246
|
+
// Write Validation
|
|
2247
|
+
// ==========================================================================
|
|
2248
|
+
|
|
2249
|
+
async validateWriteEvent(
|
|
2250
|
+
entity: ElectricAgentsEntity,
|
|
2251
|
+
event: Record<string, unknown>
|
|
2252
|
+
): Promise<{ code: string; message: string; status: number } | null> {
|
|
2253
|
+
if (!entity.type) return null
|
|
2254
|
+
|
|
2255
|
+
const { stateSchemas } = await this.getEffectiveSchemas(entity)
|
|
2256
|
+
if (!stateSchemas) return null
|
|
2257
|
+
|
|
2258
|
+
const eventType = event.type as string | undefined
|
|
2259
|
+
if (!eventType) return null
|
|
2260
|
+
|
|
2261
|
+
if (!(eventType in stateSchemas)) {
|
|
2262
|
+
return {
|
|
2263
|
+
code: ErrCodeUnknownEventType,
|
|
2264
|
+
message: `Unknown event type "${eventType}"`,
|
|
2265
|
+
status: 422,
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
const schema = stateSchemas[eventType]
|
|
2270
|
+
if (schema) {
|
|
2271
|
+
const headers = event.headers as Record<string, unknown> | undefined
|
|
2272
|
+
const operation = headers?.operation
|
|
2273
|
+
const rawPayload =
|
|
2274
|
+
operation === `delete` && `old_value` in event
|
|
2275
|
+
? event.old_value
|
|
2276
|
+
: event.value
|
|
2277
|
+
if (rawPayload === undefined) {
|
|
2278
|
+
return null
|
|
2279
|
+
}
|
|
2280
|
+
const payload =
|
|
2281
|
+
typeof rawPayload === `object` && rawPayload !== null
|
|
2282
|
+
? (rawPayload as Record<string, unknown>)
|
|
2283
|
+
: rawPayload
|
|
2284
|
+
const valErr = this.validator.validate(schema, payload)
|
|
2285
|
+
if (valErr) {
|
|
2286
|
+
return {
|
|
2287
|
+
code: ErrCodeSchemaValidationFailed,
|
|
2288
|
+
message: valErr.message,
|
|
2289
|
+
status: 422,
|
|
2290
|
+
}
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
return null
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
// ==========================================================================
|
|
2298
|
+
// Amend Schemas
|
|
2299
|
+
// ==========================================================================
|
|
2300
|
+
|
|
2301
|
+
/**
|
|
2302
|
+
* Add new input/output schema keys to an entity type directly in Postgres.
|
|
2303
|
+
*/
|
|
2304
|
+
async amendSchemas(
|
|
2305
|
+
typeName: string,
|
|
2306
|
+
schemas: {
|
|
2307
|
+
inbox_schemas?: Record<string, Record<string, unknown>>
|
|
2308
|
+
state_schemas?: Record<string, Record<string, unknown>>
|
|
2309
|
+
}
|
|
2310
|
+
): Promise<ElectricAgentsEntityType> {
|
|
2311
|
+
// Validate each provided schema via validateSchemaSubset.
|
|
2312
|
+
this.validateSchemaMap(schemas.inbox_schemas)
|
|
2313
|
+
this.validateSchemaMap(schemas.state_schemas)
|
|
2314
|
+
|
|
2315
|
+
// Look up current entity type.
|
|
2316
|
+
const existing = await this.registry.getEntityType(typeName)
|
|
2317
|
+
if (!existing) {
|
|
2318
|
+
throw new ElectricAgentsError(
|
|
2319
|
+
ErrCodeUnknownEntityType,
|
|
2320
|
+
`Entity type "${typeName}" not found`,
|
|
2321
|
+
404
|
|
2322
|
+
)
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
// Check for key overlap (additive only, no overwriting).
|
|
2326
|
+
if (schemas.inbox_schemas && existing.inbox_schemas) {
|
|
2327
|
+
for (const key of Object.keys(schemas.inbox_schemas)) {
|
|
2328
|
+
if (key in existing.inbox_schemas) {
|
|
2329
|
+
throw new ElectricAgentsError(
|
|
2330
|
+
ErrCodeSchemaKeyExists,
|
|
2331
|
+
`Cannot amend existing inbox schema key: ${key}`,
|
|
2332
|
+
409
|
|
2333
|
+
)
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
}
|
|
2337
|
+
if (schemas.state_schemas && existing.state_schemas) {
|
|
2338
|
+
for (const key of Object.keys(schemas.state_schemas)) {
|
|
2339
|
+
if (key in existing.state_schemas) {
|
|
2340
|
+
throw new ElectricAgentsError(
|
|
2341
|
+
ErrCodeSchemaKeyExists,
|
|
2342
|
+
`Cannot amend existing state schema key: ${key}`,
|
|
2343
|
+
409
|
|
2344
|
+
)
|
|
2345
|
+
}
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
// Merge schemas.
|
|
2350
|
+
const mergedInbox = schemas.inbox_schemas
|
|
2351
|
+
? { ...(existing.inbox_schemas ?? {}), ...schemas.inbox_schemas }
|
|
2352
|
+
: existing.inbox_schemas
|
|
2353
|
+
const mergedState = schemas.state_schemas
|
|
2354
|
+
? { ...(existing.state_schemas ?? {}), ...schemas.state_schemas }
|
|
2355
|
+
: existing.state_schemas
|
|
2356
|
+
|
|
2357
|
+
const now = new Date().toISOString()
|
|
2358
|
+
const nextRevision = existing.revision + 1
|
|
2359
|
+
|
|
2360
|
+
const updatedType: ElectricAgentsEntityType = {
|
|
2361
|
+
name: existing.name,
|
|
2362
|
+
description: existing.description,
|
|
2363
|
+
creation_schema: existing.creation_schema,
|
|
2364
|
+
inbox_schemas: mergedInbox,
|
|
2365
|
+
state_schemas: mergedState,
|
|
2366
|
+
serve_endpoint: existing.serve_endpoint,
|
|
2367
|
+
revision: nextRevision,
|
|
2368
|
+
created_at: existing.created_at,
|
|
2369
|
+
updated_at: now,
|
|
2370
|
+
}
|
|
2371
|
+
|
|
2372
|
+
await this.registry.updateEntityTypeInPlace(updatedType)
|
|
2373
|
+
|
|
2374
|
+
return (await this.registry.getEntityType(typeName)) ?? updatedType
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
// ==========================================================================
|
|
2378
|
+
// Webhook Enrichment
|
|
2379
|
+
// ==========================================================================
|
|
2380
|
+
|
|
2381
|
+
/**
|
|
2382
|
+
* Enrich webhook payload with entity context.
|
|
2383
|
+
* Called by ElectricAgentsServer during webhook forwarding to inject entity context.
|
|
2384
|
+
*/
|
|
2385
|
+
async enrichPayload(
|
|
2386
|
+
payload: Record<string, unknown>,
|
|
2387
|
+
consumer: { primary_stream: string }
|
|
2388
|
+
): Promise<Record<string, unknown>> {
|
|
2389
|
+
const entity = await this.registry.getEntityByStream(
|
|
2390
|
+
consumer.primary_stream
|
|
2391
|
+
)
|
|
2392
|
+
if (!entity) return payload
|
|
2393
|
+
|
|
2394
|
+
return {
|
|
2395
|
+
...payload,
|
|
2396
|
+
entity: {
|
|
2397
|
+
type: entity.type,
|
|
2398
|
+
status: entity.status,
|
|
2399
|
+
url: entity.url,
|
|
2400
|
+
streams: entity.streams,
|
|
2401
|
+
tags: entity.tags,
|
|
2402
|
+
spawnArgs: entity.spawn_args,
|
|
2403
|
+
},
|
|
2404
|
+
triggerEvent: `message_received`,
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
private validateSchema(schema: Record<string, unknown> | undefined): void {
|
|
2409
|
+
if (!schema) return
|
|
2410
|
+
const err = this.validator.validateSchemaSubset(schema)
|
|
2411
|
+
if (err) {
|
|
2412
|
+
throw new ElectricAgentsError(err.code, err.message, 400)
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
private validateSchemaMap(
|
|
2417
|
+
schemas: Record<string, Record<string, unknown>> | undefined
|
|
2418
|
+
): void {
|
|
2419
|
+
if (!schemas) return
|
|
2420
|
+
for (const schema of Object.values(schemas)) {
|
|
2421
|
+
this.validateSchema(schema)
|
|
2422
|
+
}
|
|
2423
|
+
}
|
|
2424
|
+
|
|
2425
|
+
private validateDispatchPolicy(
|
|
2426
|
+
input: unknown,
|
|
2427
|
+
opts: { label: string }
|
|
2428
|
+
): DispatchPolicy {
|
|
2429
|
+
try {
|
|
2430
|
+
return parseDispatchPolicy(input, opts.label)
|
|
2431
|
+
} catch (error) {
|
|
2432
|
+
throw new ElectricAgentsError(
|
|
2433
|
+
ErrCodeInvalidRequest,
|
|
2434
|
+
error instanceof Error ? error.message : `Invalid dispatch policy`,
|
|
2435
|
+
400
|
|
2436
|
+
)
|
|
2437
|
+
}
|
|
2438
|
+
}
|
|
2439
|
+
|
|
2440
|
+
private validateTags(input: unknown): Record<string, string> {
|
|
2441
|
+
try {
|
|
2442
|
+
return assertTags(input)
|
|
2443
|
+
} catch (error) {
|
|
2444
|
+
throw new ElectricAgentsError(
|
|
2445
|
+
ErrCodeInvalidRequest,
|
|
2446
|
+
error instanceof Error ? error.message : `Invalid tags`,
|
|
2447
|
+
400
|
|
2448
|
+
)
|
|
2449
|
+
}
|
|
2450
|
+
}
|
|
2451
|
+
|
|
2452
|
+
private async validateSendRequest(
|
|
2453
|
+
entityUrl: string,
|
|
2454
|
+
req: SendRequest
|
|
2455
|
+
): Promise<ElectricAgentsEntity> {
|
|
2456
|
+
const entity = await this.registry.getEntity(entityUrl)
|
|
2457
|
+
if (!entity) {
|
|
2458
|
+
throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
|
|
2459
|
+
}
|
|
2460
|
+
if (entity.status === `stopped`) {
|
|
2461
|
+
throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2464
|
+
if (req.type && entity.type) {
|
|
2465
|
+
const { inboxSchemas } = await this.getEffectiveSchemas(entity)
|
|
2466
|
+
if (inboxSchemas) {
|
|
2467
|
+
const schema = inboxSchemas[req.type]
|
|
2468
|
+
if (!schema) {
|
|
2469
|
+
throw new ElectricAgentsError(
|
|
2470
|
+
ErrCodeUnknownMessageType,
|
|
2471
|
+
`Unknown message type "${req.type}"`,
|
|
2472
|
+
422
|
|
2473
|
+
)
|
|
2474
|
+
}
|
|
2475
|
+
const valErr = this.validator.validate(schema, req.payload)
|
|
2476
|
+
if (valErr) {
|
|
2477
|
+
throw new ElectricAgentsError(
|
|
2478
|
+
valErr.code,
|
|
2479
|
+
valErr.message,
|
|
2480
|
+
422,
|
|
2481
|
+
valErr.details
|
|
2482
|
+
)
|
|
2483
|
+
}
|
|
2484
|
+
}
|
|
2485
|
+
}
|
|
2486
|
+
|
|
2487
|
+
if (!req.from) {
|
|
2488
|
+
throw new ElectricAgentsError(
|
|
2489
|
+
ErrCodeInvalidRequest,
|
|
2490
|
+
`Missing required field: from`,
|
|
2491
|
+
400
|
|
2492
|
+
)
|
|
2493
|
+
}
|
|
2494
|
+
if (req.payload === undefined) {
|
|
2495
|
+
throw new ElectricAgentsError(
|
|
2496
|
+
ErrCodeInvalidRequest,
|
|
2497
|
+
`Missing required field: payload`,
|
|
2498
|
+
400
|
|
2499
|
+
)
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
return entity
|
|
2503
|
+
}
|
|
2504
|
+
|
|
2505
|
+
private async getEffectiveSchemas(entity: ElectricAgentsEntity): Promise<{
|
|
2506
|
+
inboxSchemas?: Record<string, Record<string, unknown>>
|
|
2507
|
+
stateSchemas?: Record<string, Record<string, unknown>>
|
|
2508
|
+
}> {
|
|
2509
|
+
if (!entity.type) {
|
|
2510
|
+
return {
|
|
2511
|
+
inboxSchemas: entity.inbox_schemas,
|
|
2512
|
+
stateSchemas: entity.state_schemas,
|
|
2513
|
+
}
|
|
2514
|
+
}
|
|
2515
|
+
|
|
2516
|
+
const latestType = await this.registry.getEntityType(entity.type)
|
|
2517
|
+
|
|
2518
|
+
return {
|
|
2519
|
+
inboxSchemas: latestType?.inbox_schemas
|
|
2520
|
+
? { ...(entity.inbox_schemas ?? {}), ...latestType.inbox_schemas }
|
|
2521
|
+
: entity.inbox_schemas,
|
|
2522
|
+
stateSchemas: latestType?.state_schemas
|
|
2523
|
+
? { ...(entity.state_schemas ?? {}), ...latestType.state_schemas }
|
|
2524
|
+
: entity.state_schemas,
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
|
|
2528
|
+
private isClosedStreamError(err: unknown): boolean {
|
|
2529
|
+
if (!(err instanceof Error)) {
|
|
2530
|
+
return false
|
|
2531
|
+
}
|
|
2532
|
+
|
|
2533
|
+
const status =
|
|
2534
|
+
`status` in err ? (err as { status?: unknown }).status : undefined
|
|
2535
|
+
|
|
2536
|
+
return (
|
|
2537
|
+
(status === 409 && /Stream is closed/i.test(err.message)) ||
|
|
2538
|
+
/Stream append failed:\s*409\s+Stream is closed/i.test(err.message) ||
|
|
2539
|
+
/HTTP Error 409\b.*Stream is closed/i.test(err.message)
|
|
2540
|
+
)
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
/**
|
|
2544
|
+
* Ensure a virtual cron stream exists and schedule its next tick.
|
|
2545
|
+
* Returns the stream path (e.g. `/_cron/<base64url>`).
|
|
2546
|
+
*/
|
|
2547
|
+
async getOrCreateCronStream(
|
|
2548
|
+
expression: string,
|
|
2549
|
+
timezone?: string
|
|
2550
|
+
): Promise<string> {
|
|
2551
|
+
if (!this.scheduler) {
|
|
2552
|
+
throw new Error(`Scheduler not configured`)
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
const spec = resolveCronScheduleSpec(expression, timezone)
|
|
2556
|
+
const streamPath = getCronStreamPath(spec.expression, spec.timezone)
|
|
2557
|
+
|
|
2558
|
+
// Ensure the backing stream exists
|
|
2559
|
+
const exists = await this.streamClient.exists(streamPath)
|
|
2560
|
+
if (!exists) {
|
|
2561
|
+
await this.streamClient.create(streamPath, {
|
|
2562
|
+
contentType: `application/json`,
|
|
2563
|
+
})
|
|
2564
|
+
}
|
|
2565
|
+
|
|
2566
|
+
const fireAt = getNextCronFireAt(spec.expression, spec.timezone)
|
|
2567
|
+
await this.scheduler.enqueueCronTick(
|
|
2568
|
+
spec.expression,
|
|
2569
|
+
spec.timezone,
|
|
2570
|
+
0,
|
|
2571
|
+
streamPath,
|
|
2572
|
+
fireAt
|
|
2573
|
+
)
|
|
2574
|
+
|
|
2575
|
+
return streamPath
|
|
2576
|
+
}
|
|
2577
|
+
|
|
2578
|
+
async shutdown(): Promise<void> {
|
|
2579
|
+
if (this.stopWakeRegistryOnShutdown) {
|
|
2580
|
+
await this.wakeRegistry.stopSync()
|
|
2581
|
+
}
|
|
2582
|
+
this.registry.close()
|
|
2583
|
+
}
|
|
2584
|
+
}
|
|
2585
|
+
|
|
2586
|
+
export class ElectricAgentsError extends Error {
|
|
2587
|
+
readonly details?: unknown
|
|
2588
|
+
|
|
2589
|
+
constructor(
|
|
2590
|
+
readonly code: string,
|
|
2591
|
+
message: string,
|
|
2592
|
+
readonly status: number,
|
|
2593
|
+
details?: unknown
|
|
2594
|
+
) {
|
|
2595
|
+
super(message)
|
|
2596
|
+
this.name = `ElectricAgentsError`
|
|
2597
|
+
if (details !== undefined) {
|
|
2598
|
+
this.details = details
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
}
|