@electric-ax/agents-server 0.4.18 → 0.4.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +590 -40
- package/dist/index.cjs +576 -36
- package/dist/index.d.cts +290 -40
- package/dist/index.d.ts +290 -40
- package/dist/index.js +577 -37
- package/drizzle/0015_pg_sync_bridges.sql +14 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +6 -6
- package/src/db/schema.ts +28 -0
- package/src/entity-manager.ts +34 -29
- package/src/entity-registry.ts +144 -3
- package/src/manifest-side-effects.ts +10 -0
- package/src/pg-sync-bridge-manager.ts +552 -0
- package/src/routing/context.ts +2 -0
- package/src/routing/durable-streams-router.ts +13 -0
- package/src/routing/entities-router.ts +28 -16
- package/src/routing/global-router.ts +3 -0
- package/src/routing/hooks.ts +7 -0
- package/src/routing/internal-router.ts +2 -0
- package/src/routing/pg-sync-router.ts +113 -0
- package/src/runtime.ts +20 -1
- package/src/scheduler.ts +26 -0
- package/src/server.ts +4 -0
- package/src/standalone-runtime.ts +16 -0
- package/src/utils/server-utils.ts +97 -1
- package/src/wake-registry.ts +27 -2
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
import { DurableStream, IdempotentProducer } from '@durable-streams/client'
|
|
2
|
+
import {
|
|
3
|
+
canonicalPgSyncOptions,
|
|
4
|
+
getPgSyncStreamPath,
|
|
5
|
+
sourceRefForPgSync,
|
|
6
|
+
type CanonicalPgSyncConfig,
|
|
7
|
+
type PgSyncOptions,
|
|
8
|
+
type PgSyncRequestMetadata,
|
|
9
|
+
} from '@electric-ax/agents-runtime'
|
|
10
|
+
import {
|
|
11
|
+
ShapeStream,
|
|
12
|
+
isChangeMessage,
|
|
13
|
+
isControlMessage,
|
|
14
|
+
} from '@electric-sql/client'
|
|
15
|
+
import { serverLog } from './utils/log.js'
|
|
16
|
+
import type { StreamClient } from './stream-client.js'
|
|
17
|
+
import type { PgSyncBridgeRow, PostgresRegistry } from './entity-registry.js'
|
|
18
|
+
import type {
|
|
19
|
+
LogMode,
|
|
20
|
+
Offset,
|
|
21
|
+
ShapeStreamInterface,
|
|
22
|
+
} from '@electric-sql/client'
|
|
23
|
+
|
|
24
|
+
export const PG_SYNC_ELECTRIC_SHAPE_URL =
|
|
25
|
+
process.env.ELECTRIC_AGENTS_PG_SYNC_ELECTRIC_URL ??
|
|
26
|
+
`http://localhost:3000/v1/shape`
|
|
27
|
+
|
|
28
|
+
type PgSyncOperation = `insert` | `update` | `delete`
|
|
29
|
+
type WakeEvaluator = (
|
|
30
|
+
sourceUrl: string,
|
|
31
|
+
event: Record<string, unknown>
|
|
32
|
+
) => Promise<void> | void
|
|
33
|
+
|
|
34
|
+
export type PgSyncResolvedSource = {
|
|
35
|
+
url: string
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface PgSyncBridgeManagerOptions {
|
|
39
|
+
url?: string
|
|
40
|
+
retry?: {
|
|
41
|
+
initialDelayMs?: number
|
|
42
|
+
maxDelayMs?: number
|
|
43
|
+
random?: () => number
|
|
44
|
+
sleep?: (ms: number) => Promise<void>
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const DEFAULT_RETRY_INITIAL_DELAY_MS = 1_000
|
|
49
|
+
const DEFAULT_RETRY_MAX_DELAY_MS = 30_000
|
|
50
|
+
|
|
51
|
+
type PgSyncChangeMessage = {
|
|
52
|
+
headers: Record<string, unknown> & {
|
|
53
|
+
operation?: PgSyncOperation | string
|
|
54
|
+
offset?: unknown
|
|
55
|
+
key?: unknown
|
|
56
|
+
rowKey?: unknown
|
|
57
|
+
}
|
|
58
|
+
value?: Record<string, unknown>
|
|
59
|
+
key?: string
|
|
60
|
+
old_value?: Record<string, unknown>
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
type PgSyncCursor = {
|
|
64
|
+
handle: string
|
|
65
|
+
offset: string
|
|
66
|
+
initialSnapshotComplete: boolean
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export interface PgSyncBridgeCoordinator {
|
|
70
|
+
start?(): Promise<void>
|
|
71
|
+
register(
|
|
72
|
+
options: PgSyncOptions,
|
|
73
|
+
metadata?: PgSyncRequestMetadata
|
|
74
|
+
): Promise<{ sourceRef: string; streamUrl: string }>
|
|
75
|
+
stop(): Promise<void>
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function buildElectricShapeParams(
|
|
79
|
+
options: PgSyncOptions
|
|
80
|
+
): Record<string, unknown> {
|
|
81
|
+
return {
|
|
82
|
+
table: options.table,
|
|
83
|
+
...(options.columns !== undefined ? { columns: [...options.columns] } : {}),
|
|
84
|
+
...(options.where !== undefined ? { where: options.where } : {}),
|
|
85
|
+
...(options.params !== undefined
|
|
86
|
+
? {
|
|
87
|
+
params: Array.isArray(options.params)
|
|
88
|
+
? [...options.params]
|
|
89
|
+
: { ...options.params },
|
|
90
|
+
}
|
|
91
|
+
: {}),
|
|
92
|
+
...(options.replica !== undefined ? { replica: options.replica } : {}),
|
|
93
|
+
...(options.metadata?.tenantId
|
|
94
|
+
? { electric_agents_tenant_id: options.metadata.tenantId }
|
|
95
|
+
: {}),
|
|
96
|
+
...(options.metadata?.principalKind
|
|
97
|
+
? { electric_agents_principal_kind: options.metadata.principalKind }
|
|
98
|
+
: {}),
|
|
99
|
+
...(options.metadata?.principalId
|
|
100
|
+
? { electric_agents_principal_id: options.metadata.principalId }
|
|
101
|
+
: {}),
|
|
102
|
+
...(options.metadata?.principalKey
|
|
103
|
+
? { electric_agents_principal_key: options.metadata.principalKey }
|
|
104
|
+
: {}),
|
|
105
|
+
...(options.metadata?.principalUrl
|
|
106
|
+
? { electric_agents_principal_url: options.metadata.principalUrl }
|
|
107
|
+
: {}),
|
|
108
|
+
...(options.metadata?.entityUrl
|
|
109
|
+
? { electric_agents_entity_url: options.metadata.entityUrl }
|
|
110
|
+
: {}),
|
|
111
|
+
...(options.metadata?.entityType
|
|
112
|
+
? { electric_agents_entity_type: options.metadata.entityType }
|
|
113
|
+
: {}),
|
|
114
|
+
...(options.metadata?.streamPath
|
|
115
|
+
? { electric_agents_stream_path: options.metadata.streamPath }
|
|
116
|
+
: {}),
|
|
117
|
+
...(options.metadata?.runtimeConsumerId
|
|
118
|
+
? {
|
|
119
|
+
electric_agents_runtime_consumer_id:
|
|
120
|
+
options.metadata.runtimeConsumerId,
|
|
121
|
+
}
|
|
122
|
+
: {}),
|
|
123
|
+
...(options.metadata?.wakeId
|
|
124
|
+
? { electric_agents_wake_id: options.metadata.wakeId }
|
|
125
|
+
: {}),
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function jsonSafe(value: unknown): unknown {
|
|
130
|
+
if (typeof value === `bigint`) return value.toString()
|
|
131
|
+
if (value === null || typeof value !== `object`) return value
|
|
132
|
+
if (Array.isArray(value)) return value.map(jsonSafe)
|
|
133
|
+
return Object.fromEntries(
|
|
134
|
+
Object.entries(value as Record<string, unknown>).map(([key, item]) => [
|
|
135
|
+
key,
|
|
136
|
+
jsonSafe(item),
|
|
137
|
+
])
|
|
138
|
+
)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function stableJson(value: unknown): string {
|
|
142
|
+
if (typeof value === `bigint`) return JSON.stringify(value.toString())
|
|
143
|
+
if (value === null || typeof value !== `object`) return JSON.stringify(value)
|
|
144
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(`,`)}]`
|
|
145
|
+
return `{${Object.keys(value as Record<string, unknown>)
|
|
146
|
+
.sort()
|
|
147
|
+
.map(
|
|
148
|
+
(key) =>
|
|
149
|
+
`${JSON.stringify(key)}:${stableJson((value as Record<string, unknown>)[key])}`
|
|
150
|
+
)
|
|
151
|
+
.join(`,`)}}`
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function parseElectricOffset(offset: string): Offset | null {
|
|
155
|
+
if (offset === `-1`) return offset
|
|
156
|
+
return /^\d+_\d+$/.test(offset) ? (offset as Offset) : null
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function rowKeyForMessage(message: PgSyncChangeMessage): string | undefined {
|
|
160
|
+
const headers = message.headers as Record<string, unknown>
|
|
161
|
+
const candidate =
|
|
162
|
+
headers.key ??
|
|
163
|
+
headers.rowKey ??
|
|
164
|
+
message.value?.id ??
|
|
165
|
+
message.value?.key ??
|
|
166
|
+
message.old_value?.id ??
|
|
167
|
+
message.old_value?.key
|
|
168
|
+
return candidate === undefined ? undefined : stableJson(candidate)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function pgSyncMessageToDurableEvent(
|
|
172
|
+
message: PgSyncChangeMessage,
|
|
173
|
+
optionsOrSourceRef: PgSyncOptions | string
|
|
174
|
+
): {
|
|
175
|
+
type: `pg_sync_change`
|
|
176
|
+
key: string
|
|
177
|
+
value: Record<string, unknown>
|
|
178
|
+
headers: { operation: PgSyncOperation; timestamp: string }
|
|
179
|
+
} | null {
|
|
180
|
+
const operation = message.headers.operation
|
|
181
|
+
if (
|
|
182
|
+
operation !== `insert` &&
|
|
183
|
+
operation !== `update` &&
|
|
184
|
+
operation !== `delete`
|
|
185
|
+
)
|
|
186
|
+
return null
|
|
187
|
+
|
|
188
|
+
const sourceRef =
|
|
189
|
+
typeof optionsOrSourceRef === `string`
|
|
190
|
+
? optionsOrSourceRef
|
|
191
|
+
: sourceRefForPgSync(optionsOrSourceRef)
|
|
192
|
+
const rowKey = rowKeyForMessage(message)
|
|
193
|
+
const offset = message.headers.offset
|
|
194
|
+
if (typeof offset !== `string` || offset.length === 0) return null
|
|
195
|
+
const messageKeyPart = offset
|
|
196
|
+
const messageKey = `${sourceRef}:${operation}:${messageKeyPart}`
|
|
197
|
+
const timestamp = new Date().toISOString()
|
|
198
|
+
const oldValue = message.old_value
|
|
199
|
+
const safeValue = jsonSafe(message.value)
|
|
200
|
+
const safeOldValue = jsonSafe(oldValue)
|
|
201
|
+
const safeHeaders = jsonSafe(message.headers)
|
|
202
|
+
|
|
203
|
+
return {
|
|
204
|
+
type: `pg_sync_change`,
|
|
205
|
+
key: messageKey,
|
|
206
|
+
value: {
|
|
207
|
+
key: messageKey,
|
|
208
|
+
table:
|
|
209
|
+
typeof optionsOrSourceRef === `string`
|
|
210
|
+
? undefined
|
|
211
|
+
: optionsOrSourceRef.table,
|
|
212
|
+
operation,
|
|
213
|
+
...(rowKey !== undefined ? { rowKey } : {}),
|
|
214
|
+
...(message.value !== undefined ? { value: safeValue } : {}),
|
|
215
|
+
...(oldValue !== undefined ? { oldValue: safeOldValue } : {}),
|
|
216
|
+
headers: safeHeaders,
|
|
217
|
+
...(typeof offset === `string` ? { offset } : {}),
|
|
218
|
+
receivedAt: timestamp,
|
|
219
|
+
},
|
|
220
|
+
headers: { operation, timestamp },
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function cursorFromRow(
|
|
225
|
+
row:
|
|
226
|
+
| Pick<
|
|
227
|
+
PgSyncBridgeRow,
|
|
228
|
+
`shapeHandle` | `shapeOffset` | `initialSnapshotComplete`
|
|
229
|
+
>
|
|
230
|
+
| undefined
|
|
231
|
+
): PgSyncCursor | undefined {
|
|
232
|
+
return row?.shapeHandle && row.shapeOffset
|
|
233
|
+
? {
|
|
234
|
+
handle: row.shapeHandle,
|
|
235
|
+
offset: row.shapeOffset,
|
|
236
|
+
initialSnapshotComplete: row.initialSnapshotComplete,
|
|
237
|
+
}
|
|
238
|
+
: undefined
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
class PgSyncBridge {
|
|
242
|
+
private producer: IdempotentProducer | null = null
|
|
243
|
+
private unsubscribe: (() => void) | null = null
|
|
244
|
+
private abortController: AbortController | null = null
|
|
245
|
+
private skipChangesUntilUpToDate = false
|
|
246
|
+
private recovering = false
|
|
247
|
+
private committedCursor?: PgSyncCursor
|
|
248
|
+
private retryAttempt = 0
|
|
249
|
+
|
|
250
|
+
constructor(
|
|
251
|
+
readonly sourceRef: string,
|
|
252
|
+
readonly streamUrl: string,
|
|
253
|
+
private options: CanonicalPgSyncConfig,
|
|
254
|
+
private resolvedSource: PgSyncResolvedSource,
|
|
255
|
+
private retry: Required<NonNullable<PgSyncBridgeManagerOptions[`retry`]>>,
|
|
256
|
+
private streamClient: StreamClient,
|
|
257
|
+
private registry?: PostgresRegistry,
|
|
258
|
+
private evaluateWakes?: WakeEvaluator,
|
|
259
|
+
private initialCursor?: PgSyncCursor
|
|
260
|
+
) {
|
|
261
|
+
this.committedCursor = initialCursor
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async start(): Promise<void> {
|
|
265
|
+
if (!this.producer) {
|
|
266
|
+
this.producer = new IdempotentProducer(
|
|
267
|
+
new DurableStream({
|
|
268
|
+
url: `${this.streamClient.baseUrl}${this.streamUrl}`,
|
|
269
|
+
contentType: `application/json`,
|
|
270
|
+
}),
|
|
271
|
+
`pg-sync-bridge-${this.sourceRef}`
|
|
272
|
+
)
|
|
273
|
+
}
|
|
274
|
+
if (this.initialCursor) {
|
|
275
|
+
const offset = parseElectricOffset(this.initialCursor.offset)
|
|
276
|
+
if (offset) {
|
|
277
|
+
this.startStream(
|
|
278
|
+
offset,
|
|
279
|
+
this.initialCursor.handle,
|
|
280
|
+
!this.initialCursor.initialSnapshotComplete
|
|
281
|
+
)
|
|
282
|
+
return
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef)
|
|
286
|
+
this.startStream(`now`, undefined, true)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
async stop(): Promise<void> {
|
|
290
|
+
this.unsubscribe?.()
|
|
291
|
+
this.abortController?.abort()
|
|
292
|
+
this.unsubscribe = null
|
|
293
|
+
this.abortController = null
|
|
294
|
+
try {
|
|
295
|
+
await this.producer?.flush()
|
|
296
|
+
} finally {
|
|
297
|
+
await this.producer?.detach()
|
|
298
|
+
this.producer = null
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private startStream(
|
|
303
|
+
offset: Offset,
|
|
304
|
+
handle?: string,
|
|
305
|
+
skipChangesUntilUpToDate = false,
|
|
306
|
+
log: LogMode = offset === `now` ? `changes_only` : `full`
|
|
307
|
+
): void {
|
|
308
|
+
this.unsubscribe?.()
|
|
309
|
+
this.abortController?.abort()
|
|
310
|
+
this.skipChangesUntilUpToDate = skipChangesUntilUpToDate
|
|
311
|
+
this.abortController = new AbortController()
|
|
312
|
+
const stream: ShapeStreamInterface<Record<string, unknown>> =
|
|
313
|
+
new ShapeStream({
|
|
314
|
+
url: this.resolvedSource.url,
|
|
315
|
+
params: buildElectricShapeParams(this.options) as never,
|
|
316
|
+
offset,
|
|
317
|
+
log,
|
|
318
|
+
...(handle ? { handle } : {}),
|
|
319
|
+
signal: this.abortController.signal,
|
|
320
|
+
})
|
|
321
|
+
this.unsubscribe = stream.subscribe(
|
|
322
|
+
async (messages) => {
|
|
323
|
+
try {
|
|
324
|
+
for (const message of messages) {
|
|
325
|
+
if (isControlMessage(message)) {
|
|
326
|
+
if (message.headers.control === `must-refetch`) {
|
|
327
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef)
|
|
328
|
+
this.startStream(`now`, undefined, true)
|
|
329
|
+
return
|
|
330
|
+
}
|
|
331
|
+
if (message.headers.control === `up-to-date`) {
|
|
332
|
+
this.skipChangesUntilUpToDate = false
|
|
333
|
+
await this.persistCursor(stream, true)
|
|
334
|
+
continue
|
|
335
|
+
}
|
|
336
|
+
await this.persistCursor(stream)
|
|
337
|
+
continue
|
|
338
|
+
}
|
|
339
|
+
if (!isChangeMessage(message)) continue
|
|
340
|
+
if (!this.skipChangesUntilUpToDate) {
|
|
341
|
+
const event = pgSyncMessageToDurableEvent(message, this.options)
|
|
342
|
+
if (event) {
|
|
343
|
+
if (!this.producer)
|
|
344
|
+
throw new Error(`pg-sync producer is not started`)
|
|
345
|
+
await this.producer.append(JSON.stringify(event))
|
|
346
|
+
await this.producer.flush?.()
|
|
347
|
+
await this.evaluateWakes?.(this.streamUrl, event)
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
await this.persistCursor(stream)
|
|
351
|
+
this.retryAttempt = 0
|
|
352
|
+
}
|
|
353
|
+
} catch (error) {
|
|
354
|
+
serverLog.warn(
|
|
355
|
+
`[pg-sync-bridge] subscription callback failed for ${this.sourceRef}:`,
|
|
356
|
+
error
|
|
357
|
+
)
|
|
358
|
+
await this.recoverStream()
|
|
359
|
+
}
|
|
360
|
+
},
|
|
361
|
+
(error) => {
|
|
362
|
+
if (this.abortController?.signal.aborted) return
|
|
363
|
+
serverLog.warn(
|
|
364
|
+
`[pg-sync-bridge] subscription failed for ${this.sourceRef}:`,
|
|
365
|
+
error
|
|
366
|
+
)
|
|
367
|
+
void this.recoverStream()
|
|
368
|
+
}
|
|
369
|
+
)
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
private async recoverStream(): Promise<void> {
|
|
373
|
+
if (this.recovering) return
|
|
374
|
+
this.recovering = true
|
|
375
|
+
try {
|
|
376
|
+
const attempt = this.retryAttempt++
|
|
377
|
+
const baseDelay = Math.min(
|
|
378
|
+
this.retry.initialDelayMs * 2 ** attempt,
|
|
379
|
+
this.retry.maxDelayMs
|
|
380
|
+
)
|
|
381
|
+
const jitter = Math.floor(baseDelay * 0.2 * this.retry.random())
|
|
382
|
+
const delay = baseDelay + jitter
|
|
383
|
+
if (delay > 0) await this.retry.sleep(delay)
|
|
384
|
+
|
|
385
|
+
const offset = this.committedCursor
|
|
386
|
+
? parseElectricOffset(this.committedCursor.offset)
|
|
387
|
+
: null
|
|
388
|
+
if (offset && this.committedCursor) {
|
|
389
|
+
this.startStream(
|
|
390
|
+
offset,
|
|
391
|
+
this.committedCursor.handle,
|
|
392
|
+
!this.committedCursor.initialSnapshotComplete
|
|
393
|
+
)
|
|
394
|
+
} else {
|
|
395
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef)
|
|
396
|
+
this.startStream(`now`, undefined, true)
|
|
397
|
+
}
|
|
398
|
+
} finally {
|
|
399
|
+
this.recovering = false
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
private async persistCursor(
|
|
404
|
+
stream: ShapeStreamInterface<Record<string, unknown>>,
|
|
405
|
+
initialSnapshotComplete = !this.skipChangesUntilUpToDate
|
|
406
|
+
): Promise<void> {
|
|
407
|
+
const shapeHandle = stream.shapeHandle
|
|
408
|
+
const shapeOffset = stream.lastOffset
|
|
409
|
+
if (!shapeHandle || !shapeOffset || shapeOffset === `-1`) return
|
|
410
|
+
await this.registry?.updatePgSyncBridgeCursor(
|
|
411
|
+
this.sourceRef,
|
|
412
|
+
shapeHandle,
|
|
413
|
+
shapeOffset,
|
|
414
|
+
initialSnapshotComplete
|
|
415
|
+
)
|
|
416
|
+
this.committedCursor = {
|
|
417
|
+
handle: shapeHandle,
|
|
418
|
+
offset: shapeOffset,
|
|
419
|
+
initialSnapshotComplete,
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
export class PgSyncBridgeManager implements PgSyncBridgeCoordinator {
|
|
425
|
+
private bridges = new Map<string, PgSyncBridge>()
|
|
426
|
+
private starting = new Map<string, Promise<void>>()
|
|
427
|
+
|
|
428
|
+
private readonly url: string
|
|
429
|
+
private readonly retry: Required<
|
|
430
|
+
NonNullable<PgSyncBridgeManagerOptions[`retry`]>
|
|
431
|
+
>
|
|
432
|
+
|
|
433
|
+
constructor(
|
|
434
|
+
private streamClient: StreamClient,
|
|
435
|
+
private evaluateWakes?: WakeEvaluator,
|
|
436
|
+
private registry?: PostgresRegistry,
|
|
437
|
+
options: PgSyncBridgeManagerOptions = {}
|
|
438
|
+
) {
|
|
439
|
+
this.url = options.url ?? PG_SYNC_ELECTRIC_SHAPE_URL
|
|
440
|
+
this.retry = {
|
|
441
|
+
initialDelayMs:
|
|
442
|
+
options.retry?.initialDelayMs ?? DEFAULT_RETRY_INITIAL_DELAY_MS,
|
|
443
|
+
maxDelayMs: options.retry?.maxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS,
|
|
444
|
+
random: options.retry?.random ?? Math.random,
|
|
445
|
+
sleep:
|
|
446
|
+
options.retry?.sleep ??
|
|
447
|
+
((ms: number) =>
|
|
448
|
+
new Promise<void>((resolve) => setTimeout(resolve, ms))),
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
async start(): Promise<void> {
|
|
453
|
+
const rows = await this.registry?.listPgSyncBridges?.()
|
|
454
|
+
if (!rows) return
|
|
455
|
+
await Promise.all(
|
|
456
|
+
rows.map((row) =>
|
|
457
|
+
this.ensureBridge(row).catch((error) => {
|
|
458
|
+
serverLog.warn(
|
|
459
|
+
`[pg-sync-bridge] failed to start ${row.sourceRef}:`,
|
|
460
|
+
error
|
|
461
|
+
)
|
|
462
|
+
})
|
|
463
|
+
)
|
|
464
|
+
)
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
async register(
|
|
468
|
+
options: PgSyncOptions,
|
|
469
|
+
metadata?: PgSyncRequestMetadata
|
|
470
|
+
): Promise<{ sourceRef: string; streamUrl: string }> {
|
|
471
|
+
const mergedMetadata = { ...options.metadata, ...metadata }
|
|
472
|
+
const canonicalOptions = {
|
|
473
|
+
...canonicalPgSyncOptions(options),
|
|
474
|
+
...(Object.keys(mergedMetadata).length > 0
|
|
475
|
+
? { metadata: mergedMetadata }
|
|
476
|
+
: {}),
|
|
477
|
+
}
|
|
478
|
+
const resolvedSource = this.resolveSource(canonicalOptions)
|
|
479
|
+
const sourceRef = sourceRefForPgSync(canonicalOptions)
|
|
480
|
+
const streamUrl = getPgSyncStreamPath(sourceRef, this.registry?.tenantId)
|
|
481
|
+
const row = await this.registry?.upsertPgSyncBridge({
|
|
482
|
+
sourceRef,
|
|
483
|
+
options: canonicalOptions,
|
|
484
|
+
streamUrl,
|
|
485
|
+
})
|
|
486
|
+
await this.streamClient.ensure(streamUrl, {
|
|
487
|
+
contentType: `application/json`,
|
|
488
|
+
})
|
|
489
|
+
if (!this.bridges.has(sourceRef)) {
|
|
490
|
+
let start = this.starting.get(sourceRef)
|
|
491
|
+
if (!start) {
|
|
492
|
+
start = (async () => {
|
|
493
|
+
const bridge = new PgSyncBridge(
|
|
494
|
+
sourceRef,
|
|
495
|
+
streamUrl,
|
|
496
|
+
canonicalOptions,
|
|
497
|
+
resolvedSource,
|
|
498
|
+
this.retry,
|
|
499
|
+
this.streamClient,
|
|
500
|
+
this.registry,
|
|
501
|
+
this.evaluateWakes,
|
|
502
|
+
cursorFromRow(row)
|
|
503
|
+
)
|
|
504
|
+
await bridge.start()
|
|
505
|
+
this.bridges.set(sourceRef, bridge)
|
|
506
|
+
})().finally(() => this.starting.delete(sourceRef))
|
|
507
|
+
this.starting.set(sourceRef, start)
|
|
508
|
+
}
|
|
509
|
+
await start
|
|
510
|
+
}
|
|
511
|
+
return { sourceRef, streamUrl }
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
private async ensureBridge(row: PgSyncBridgeRow): Promise<void> {
|
|
515
|
+
if (this.bridges.has(row.sourceRef)) return
|
|
516
|
+
let start = this.starting.get(row.sourceRef)
|
|
517
|
+
if (!start) {
|
|
518
|
+
start = (async () => {
|
|
519
|
+
await this.streamClient.ensure(row.streamUrl, {
|
|
520
|
+
contentType: `application/json`,
|
|
521
|
+
})
|
|
522
|
+
const canonicalOptions = canonicalPgSyncOptions(row.options)
|
|
523
|
+
const resolvedSource = this.resolveSource(canonicalOptions)
|
|
524
|
+
const bridge = new PgSyncBridge(
|
|
525
|
+
row.sourceRef,
|
|
526
|
+
row.streamUrl,
|
|
527
|
+
canonicalOptions,
|
|
528
|
+
resolvedSource,
|
|
529
|
+
this.retry,
|
|
530
|
+
this.streamClient,
|
|
531
|
+
this.registry,
|
|
532
|
+
this.evaluateWakes,
|
|
533
|
+
cursorFromRow(row)
|
|
534
|
+
)
|
|
535
|
+
await bridge.start()
|
|
536
|
+
this.bridges.set(row.sourceRef, bridge)
|
|
537
|
+
})().finally(() => this.starting.delete(row.sourceRef))
|
|
538
|
+
this.starting.set(row.sourceRef, start)
|
|
539
|
+
}
|
|
540
|
+
await start
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
private resolveSource(options: CanonicalPgSyncConfig): PgSyncResolvedSource {
|
|
544
|
+
return { url: options.url ?? this.url }
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
async stop(): Promise<void> {
|
|
548
|
+
await Promise.allSettled(this.starting.values())
|
|
549
|
+
await Promise.all([...this.bridges.values()].map((bridge) => bridge.stop()))
|
|
550
|
+
this.bridges.clear()
|
|
551
|
+
}
|
|
552
|
+
}
|
package/src/routing/context.ts
CHANGED
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
} from '@electric-ax/agents-runtime'
|
|
6
6
|
import type { DrizzleDB } from '../db/index.js'
|
|
7
7
|
import type { EntityBridgeCoordinator } from '../entity-bridge-manager.js'
|
|
8
|
+
import type { PgSyncBridgeCoordinator } from '../pg-sync-bridge-manager.js'
|
|
8
9
|
import type { EntityManager } from '../entity-manager.js'
|
|
9
10
|
import type { ElectricAgentsTenantRuntime } from '../runtime.js'
|
|
10
11
|
import type { StreamClient } from '../stream-client.js'
|
|
@@ -54,6 +55,7 @@ export interface TenantContext {
|
|
|
54
55
|
streamClient: StreamClient
|
|
55
56
|
runtime: ElectricAgentsTenantRuntime
|
|
56
57
|
entityBridgeManager: EntityBridgeCoordinator
|
|
58
|
+
pgSyncBridgeManager?: PgSyncBridgeCoordinator
|
|
57
59
|
eventSources?: EventSourceCatalog
|
|
58
60
|
ensureEventSourceWakeSource?: (sourceUrl: string) => Promise<void> | void
|
|
59
61
|
authorizeRequest?: AuthorizeRequest
|
|
@@ -717,6 +717,19 @@ async function authorizeDurableStreamAccess(
|
|
|
717
717
|
ownerEntityUrl
|
|
718
718
|
)
|
|
719
719
|
) {
|
|
720
|
+
// Bootstrap the link synchronously so subsequent reads on this stream
|
|
721
|
+
// (e.g. the runtime's preload right after `mkdb`) can resolve the owner
|
|
722
|
+
// without waiting for the entity's manifest stream event to be
|
|
723
|
+
// processed eventually-consistently. Without this, a brand-new
|
|
724
|
+
// shared-state always 401s on the first preload because the link row
|
|
725
|
+
// is only created later via `applyManifestEntitySource`.
|
|
726
|
+
if (ownerEntityUrl) {
|
|
727
|
+
await ctx.entityManager.registry.replaceSharedStateLink(
|
|
728
|
+
ownerEntityUrl,
|
|
729
|
+
`shared-state:${sharedStateId}`,
|
|
730
|
+
sharedStateId
|
|
731
|
+
)
|
|
732
|
+
}
|
|
720
733
|
return undefined
|
|
721
734
|
}
|
|
722
735
|
return apiError(
|
|
@@ -39,6 +39,7 @@ import type {
|
|
|
39
39
|
ElectricAgentsEntity,
|
|
40
40
|
ElectricAgentsEntityType,
|
|
41
41
|
EntityPermission,
|
|
42
|
+
PublicElectricAgentsEntity,
|
|
42
43
|
SendRequest,
|
|
43
44
|
} from '../electric-agents-types.js'
|
|
44
45
|
import type { JsonRouteRequest } from './schema.js'
|
|
@@ -658,7 +659,12 @@ async function parseAttachmentForm(
|
|
|
658
659
|
}
|
|
659
660
|
|
|
660
661
|
function contentDisposition(filename: string): string {
|
|
661
|
-
|
|
662
|
+
// Header values are converted to WebIDL ByteString by undici, so every
|
|
663
|
+
// character in the raw header value must fit in a single byte. Keep the
|
|
664
|
+
// RFC 5987 filename* parameter for the full UTF-8 filename, but make the
|
|
665
|
+
// legacy filename fallback ASCII-only to avoid throwing on names containing
|
|
666
|
+
// e.g. narrow no-break spaces or emoji.
|
|
667
|
+
const fallback = filename.replace(/[^\x20-\x7e]|["\\]/g, `_`)
|
|
662
668
|
return `attachment; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(filename)}`
|
|
663
669
|
}
|
|
664
670
|
|
|
@@ -1083,6 +1089,16 @@ async function deleteEventSourceSubscription(
|
|
|
1083
1089
|
return json(result)
|
|
1084
1090
|
}
|
|
1085
1091
|
|
|
1092
|
+
function tagResponseBody(
|
|
1093
|
+
entity: ElectricAgentsEntity & { txid?: number }
|
|
1094
|
+
): PublicElectricAgentsEntity & { txid?: number } {
|
|
1095
|
+
const publicEntity = toPublicEntity(entity)
|
|
1096
|
+
if (entity.txid !== undefined) {
|
|
1097
|
+
return { ...publicEntity, txid: entity.txid }
|
|
1098
|
+
}
|
|
1099
|
+
return publicEntity
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1086
1102
|
async function setTag(
|
|
1087
1103
|
request: AgentsRouteRequest,
|
|
1088
1104
|
ctx: TenantContext
|
|
@@ -1095,14 +1111,12 @@ async function setTag(
|
|
|
1095
1111
|
|
|
1096
1112
|
const parsed = routeBody<SetTagBody>(request)
|
|
1097
1113
|
const { entityUrl } = requireExistingEntityRoute(request)
|
|
1098
|
-
const token = writeTokenFromRequest(request)
|
|
1099
1114
|
const updated = await ctx.entityManager.setTag(
|
|
1100
1115
|
entityUrl,
|
|
1101
1116
|
decodeURIComponent(request.params.tagKey),
|
|
1102
|
-
{ value: parsed.value }
|
|
1103
|
-
token
|
|
1117
|
+
{ value: parsed.value }
|
|
1104
1118
|
)
|
|
1105
|
-
return json(
|
|
1119
|
+
return json(tagResponseBody(updated))
|
|
1106
1120
|
}
|
|
1107
1121
|
|
|
1108
1122
|
async function deleteTag(
|
|
@@ -1116,13 +1130,11 @@ async function deleteTag(
|
|
|
1116
1130
|
if (principalMutationError) return principalMutationError
|
|
1117
1131
|
|
|
1118
1132
|
const { entityUrl } = requireExistingEntityRoute(request)
|
|
1119
|
-
const token = writeTokenFromRequest(request)
|
|
1120
1133
|
const updated = await ctx.entityManager.deleteTag(
|
|
1121
1134
|
entityUrl,
|
|
1122
|
-
decodeURIComponent(request.params.tagKey)
|
|
1123
|
-
token
|
|
1135
|
+
decodeURIComponent(request.params.tagKey)
|
|
1124
1136
|
)
|
|
1125
|
-
return json(
|
|
1137
|
+
return json(tagResponseBody(updated))
|
|
1126
1138
|
}
|
|
1127
1139
|
|
|
1128
1140
|
async function forkEntity(
|
|
@@ -1289,11 +1301,11 @@ async function sendEntity(
|
|
|
1289
1301
|
sendReq,
|
|
1290
1302
|
new Date(Date.now() + parsed.afterMs)
|
|
1291
1303
|
)
|
|
1292
|
-
|
|
1293
|
-
await ctx.entityManager.send(entityUrl, sendReq)
|
|
1304
|
+
return status(204)
|
|
1294
1305
|
}
|
|
1295
1306
|
|
|
1296
|
-
|
|
1307
|
+
const result = await ctx.entityManager.send(entityUrl, sendReq)
|
|
1308
|
+
return json(result)
|
|
1297
1309
|
}
|
|
1298
1310
|
|
|
1299
1311
|
async function createAttachment(
|
|
@@ -1368,12 +1380,12 @@ async function updateInboxMessage(
|
|
|
1368
1380
|
): Promise<Response> {
|
|
1369
1381
|
const parsed = routeBody<InboxMessageBody>(request)
|
|
1370
1382
|
const { entityUrl } = requireExistingEntityRoute(request)
|
|
1371
|
-
await ctx.entityManager.updateInboxMessage(
|
|
1383
|
+
const result = await ctx.entityManager.updateInboxMessage(
|
|
1372
1384
|
entityUrl,
|
|
1373
1385
|
decodeURIComponent(request.params.messageKey),
|
|
1374
1386
|
parsed
|
|
1375
1387
|
)
|
|
1376
|
-
return
|
|
1388
|
+
return json(result)
|
|
1377
1389
|
}
|
|
1378
1390
|
|
|
1379
1391
|
async function deleteInboxMessage(
|
|
@@ -1381,11 +1393,11 @@ async function deleteInboxMessage(
|
|
|
1381
1393
|
ctx: TenantContext
|
|
1382
1394
|
): Promise<Response> {
|
|
1383
1395
|
const { entityUrl } = requireExistingEntityRoute(request)
|
|
1384
|
-
await ctx.entityManager.deleteInboxMessage(
|
|
1396
|
+
const result = await ctx.entityManager.deleteInboxMessage(
|
|
1385
1397
|
entityUrl,
|
|
1386
1398
|
decodeURIComponent(request.params.messageKey)
|
|
1387
1399
|
)
|
|
1388
|
-
return
|
|
1400
|
+
return json(result)
|
|
1389
1401
|
}
|
|
1390
1402
|
|
|
1391
1403
|
async function spawnEntity(
|
|
@@ -29,5 +29,8 @@ export const globalRouter: GlobalRoutes = AutoRouter<
|
|
|
29
29
|
})
|
|
30
30
|
|
|
31
31
|
globalRouter.all(`/_electric/shared-state/*`, durableStreamsRouter.fetch)
|
|
32
|
+
globalRouter.all(`/_electric/pg-sync/register`, internalRouter.fetch)
|
|
33
|
+
globalRouter.get(`/_electric/pg-sync/*`, durableStreamsRouter.fetch)
|
|
34
|
+
globalRouter.head(`/_electric/pg-sync/*`, durableStreamsRouter.fetch)
|
|
32
35
|
globalRouter.all(`/_electric/*`, internalRouter.fetch)
|
|
33
36
|
globalRouter.all(`*`, durableStreamsRouter.fetch)
|