@electric-ax/agents-server 0.4.18 → 0.4.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +590 -40
- package/dist/index.cjs +576 -36
- package/dist/index.d.cts +290 -40
- package/dist/index.d.ts +290 -40
- package/dist/index.js +577 -37
- package/drizzle/0015_pg_sync_bridges.sql +14 -0
- package/drizzle/meta/_journal.json +7 -0
- package/package.json +6 -6
- package/src/db/schema.ts +28 -0
- package/src/entity-manager.ts +34 -29
- package/src/entity-registry.ts +144 -3
- package/src/manifest-side-effects.ts +10 -0
- package/src/pg-sync-bridge-manager.ts +552 -0
- package/src/routing/context.ts +2 -0
- package/src/routing/durable-streams-router.ts +13 -0
- package/src/routing/entities-router.ts +28 -16
- package/src/routing/global-router.ts +3 -0
- package/src/routing/hooks.ts +7 -0
- package/src/routing/internal-router.ts +2 -0
- package/src/routing/pg-sync-router.ts +113 -0
- package/src/runtime.ts +20 -1
- package/src/scheduler.ts +26 -0
- package/src/server.ts +4 -0
- package/src/standalone-runtime.ts +16 -0
- package/src/utils/server-utils.ts +97 -1
- package/src/wake-registry.ts +27 -2
package/dist/entrypoint.js
CHANGED
|
@@ -4,7 +4,7 @@ import { DurableStreamTestServer } from "@durable-streams/server";
|
|
|
4
4
|
import { createServer } from "node:http";
|
|
5
5
|
import { createServerAdapter } from "@whatwg-node/server";
|
|
6
6
|
import { Agent } from "undici";
|
|
7
|
-
import { COMPOSER_INPUT_MESSAGE_TYPE, appendPathToUrl, assertTags, buildEventSourceManifestEntry, buildTagsIndex, createEntityRegistry, createRuntimeHandler, entityStateSchema, eventSourceSubscriptionManifestKey, getCronStreamPath, getCronStreamPathFromSpec, getEntitiesStreamPath, getNextCronFireAt, getSharedStateStreamPath, getWebhookStreamPath, hashString, manifestChildKey, manifestSharedStateKey, manifestSourceKey, normalizeTags, parseCronStreamPath, resolveCronScheduleSpec, resolveEventSourceSubscription, sourceRefForTags, validateComposerInputPayload, validateSlashCommandDefinitions, verifyWebhookSignature } from "@electric-ax/agents-runtime";
|
|
7
|
+
import { COMPOSER_INPUT_MESSAGE_TYPE, appendPathToUrl, assertTags, buildEventSourceManifestEntry, buildTagsIndex, canonicalPgSyncOptions, createEntityRegistry, createRuntimeHandler, entityStateSchema, eventSourceSubscriptionManifestKey, getCronStreamPath, getCronStreamPathFromSpec, getEntitiesStreamPath, getNextCronFireAt, getPgSyncStreamPath, getSharedStateStreamPath, getWebhookStreamPath, hashString, manifestChildKey, manifestSharedStateKey, manifestSourceKey, normalizeTags, parseCronStreamPath, resolveCronScheduleSpec, resolveEventSourceSubscription, sourceRefForPgSync, sourceRefForTags, validateComposerInputPayload, validateSlashCommandDefinitions, verifyWebhookSignature } from "@electric-ax/agents-runtime";
|
|
8
8
|
import fs, { existsSync } from "node:fs";
|
|
9
9
|
import path, { dirname, resolve } from "node:path";
|
|
10
10
|
import { drizzle } from "drizzle-orm/postgres-js";
|
|
@@ -47,6 +47,7 @@ __export(schema_exports, {
|
|
|
47
47
|
entityPermissionGrants: () => entityPermissionGrants,
|
|
48
48
|
entityTypePermissionGrants: () => entityTypePermissionGrants,
|
|
49
49
|
entityTypes: () => entityTypes,
|
|
50
|
+
pgSyncBridges: () => pgSyncBridges,
|
|
50
51
|
runnerRuntimeDiagnostics: () => runnerRuntimeDiagnostics,
|
|
51
52
|
runners: () => runners,
|
|
52
53
|
scheduledTasks: () => scheduledTasks,
|
|
@@ -368,6 +369,18 @@ const scheduledTasks = pgTable(`scheduled_tasks`, {
|
|
|
368
369
|
index(`idx_scheduled_tasks_manifest_pending`).on(table.tenantId, table.ownerEntityUrl, table.manifestKey).where(sql`${table.kind} = 'delayed_send' AND ${table.completedAt} IS NULL AND ${table.manifestKey} IS NOT NULL`),
|
|
369
370
|
index(`idx_scheduled_tasks_stale_claims`).on(table.tenantId, table.claimedAt).where(sql`${table.completedAt} IS NULL AND ${table.claimedAt} IS NOT NULL`)
|
|
370
371
|
]);
|
|
372
|
+
const pgSyncBridges = pgTable(`pg_sync_bridges`, {
|
|
373
|
+
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
374
|
+
sourceRef: text(`source_ref`).notNull(),
|
|
375
|
+
options: jsonb(`options`).notNull(),
|
|
376
|
+
streamUrl: text(`stream_url`).notNull(),
|
|
377
|
+
shapeHandle: text(`shape_handle`),
|
|
378
|
+
shapeOffset: text(`shape_offset`),
|
|
379
|
+
initialSnapshotComplete: boolean(`initial_snapshot_complete`).notNull().default(false),
|
|
380
|
+
lastTouchedAt: timestamp(`last_touched_at`, { withTimezone: true }).notNull().defaultNow(),
|
|
381
|
+
createdAt: timestamp(`created_at`, { withTimezone: true }).notNull().defaultNow(),
|
|
382
|
+
updatedAt: timestamp(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
383
|
+
}, (table) => [primaryKey({ columns: [table.tenantId, table.sourceRef] }), unique(`uq_pg_sync_bridges_stream_url`).on(table.tenantId, table.streamUrl)]);
|
|
371
384
|
const entityBridges = pgTable(`entity_bridges`, {
|
|
372
385
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
373
386
|
sourceRef: text(`source_ref`).notNull(),
|
|
@@ -1134,6 +1147,22 @@ async function fileExists(filePath) {
|
|
|
1134
1147
|
return false;
|
|
1135
1148
|
}
|
|
1136
1149
|
}
|
|
1150
|
+
/**
|
|
1151
|
+
* Raised when an Electric shape proxy request must be rejected for security
|
|
1152
|
+
* reasons (an un-scoped table, or a client `where` clause that could escape the
|
|
1153
|
+
* enforced per-tenant/per-principal scoping). The global `errorMapper` hook
|
|
1154
|
+
* maps this to an HTTP error response. Defined here (rather than reusing
|
|
1155
|
+
* `ElectricAgentsError`) to keep this module free of the heavy entity-manager
|
|
1156
|
+
* import graph.
|
|
1157
|
+
*/
|
|
1158
|
+
var ElectricProxyError = class extends Error {
|
|
1159
|
+
constructor(code, message, status$1) {
|
|
1160
|
+
super(message);
|
|
1161
|
+
this.code = code;
|
|
1162
|
+
this.status = status$1;
|
|
1163
|
+
this.name = `ElectricProxyError`;
|
|
1164
|
+
}
|
|
1165
|
+
};
|
|
1137
1166
|
function buildElectricProxyTarget(options) {
|
|
1138
1167
|
const targetPath = options.incomingUrl.pathname.replace(`/_electric/electric`, ``);
|
|
1139
1168
|
const target = electricUrlWithPath(options.electricUrl, targetPath);
|
|
@@ -1143,7 +1172,12 @@ function buildElectricProxyTarget(options) {
|
|
|
1143
1172
|
applyElectricUrlQueryParams(target, options.electricUrl);
|
|
1144
1173
|
if (targetPath !== `/v1/shape`) return target;
|
|
1145
1174
|
if (options.electricSecret) target.searchParams.set(`secret`, options.electricSecret);
|
|
1146
|
-
const
|
|
1175
|
+
const clientWhere = options.incomingUrl.searchParams.get(`where`);
|
|
1176
|
+
if (clientWhere !== null && !isSelfContainedWhereClause(clientWhere)) throw new ElectricProxyError(`INVALID_WHERE`, `Invalid where clause`, 400);
|
|
1177
|
+
const tableParams = options.incomingUrl.searchParams.getAll(`table`);
|
|
1178
|
+
if (tableParams.length !== 1) throw new ElectricProxyError(`TABLE_NOT_ALLOWED`, `Table is not available through the Electric proxy`, 403);
|
|
1179
|
+
const table = tableParams[0];
|
|
1180
|
+
target.searchParams.set(`table`, table);
|
|
1147
1181
|
if (table === `entities`) {
|
|
1148
1182
|
target.searchParams.set(`columns`, `"tenant_id","url","type","status","dispatch_policy","tags","spawn_args","sandbox","parent","created_by","type_revision","inbox_schemas","state_schemas","created_at","updated_at"`);
|
|
1149
1183
|
applyShapeWhere(target, buildReadableEntitiesWhere({
|
|
@@ -1201,9 +1235,39 @@ function buildElectricProxyTarget(options) {
|
|
|
1201
1235
|
principalKind: options.principalKind ?? ``,
|
|
1202
1236
|
permissionBypass: options.permissionBypass
|
|
1203
1237
|
}));
|
|
1204
|
-
}
|
|
1238
|
+
} else throw new ElectricProxyError(`TABLE_NOT_ALLOWED`, `Table is not available through the Electric proxy`, 403);
|
|
1205
1239
|
return target;
|
|
1206
1240
|
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Returns true when a client-supplied Electric `where` clause is self-contained:
|
|
1243
|
+
* its parentheses are balanced, never close below the top level, all string
|
|
1244
|
+
* (`'`) and identifier (`"`) literals are terminated, and it contains no SQL
|
|
1245
|
+
* comment markers. Such a clause cannot break out of the `(...)` group it is
|
|
1246
|
+
* wrapped in when AND-combined with the enforced scoping predicate, nor comment
|
|
1247
|
+
* out the trailing paren the proxy appends. Characters inside string/identifier
|
|
1248
|
+
* literals are ignored. Comment markers are rejected unconditionally (even where
|
|
1249
|
+
* harmless) as a conservative defensive measure; dollar-quoted and `E''` strings
|
|
1250
|
+
* are not modeled and only ever cause fail-safe over-rejection, never a bypass.
|
|
1251
|
+
*/
|
|
1252
|
+
function isSelfContainedWhereClause(where) {
|
|
1253
|
+
let depth = 0;
|
|
1254
|
+
let quote = null;
|
|
1255
|
+
for (let i = 0; i < where.length; i++) {
|
|
1256
|
+
const ch = where[i];
|
|
1257
|
+
if (quote !== null) {
|
|
1258
|
+
if (ch === quote) if (where[i + 1] === quote) i++;
|
|
1259
|
+
else quote = null;
|
|
1260
|
+
continue;
|
|
1261
|
+
}
|
|
1262
|
+
if (ch === `'` || ch === `"`) quote = ch;
|
|
1263
|
+
else if (ch === `(`) depth++;
|
|
1264
|
+
else if (ch === `)`) {
|
|
1265
|
+
depth--;
|
|
1266
|
+
if (depth < 0) return false;
|
|
1267
|
+
} else if (ch === `-` && where[i + 1] === `-` || ch === `/` && where[i + 1] === `*`) return false;
|
|
1268
|
+
}
|
|
1269
|
+
return depth === 0 && quote === null;
|
|
1270
|
+
}
|
|
1207
1271
|
function buildReadableEntitiesWhere(options) {
|
|
1208
1272
|
const tenant = sqlStringLiteral$2(options.tenantId);
|
|
1209
1273
|
if (options.permissionBypass) return `tenant_id = ${tenant}`;
|
|
@@ -2235,7 +2299,10 @@ async function authorizeDurableStreamAccess(request, ctx) {
|
|
|
2235
2299
|
}
|
|
2236
2300
|
if (method === `PUT` || method === `POST`) {
|
|
2237
2301
|
const ownerEntityUrl = request.headers.get(SHARED_STATE_OWNER_ENTITY_HEADER)?.trim() || void 0;
|
|
2238
|
-
if (await canAccessSharedState(ctx, sharedStateId, `write`, request, ownerEntityUrl))
|
|
2302
|
+
if (await canAccessSharedState(ctx, sharedStateId, `write`, request, ownerEntityUrl)) {
|
|
2303
|
+
if (ownerEntityUrl) await ctx.entityManager.registry.replaceSharedStateLink(ownerEntityUrl, `shared-state:${sharedStateId}`, sharedStateId);
|
|
2304
|
+
return void 0;
|
|
2305
|
+
}
|
|
2239
2306
|
return apiError(401, ErrCodeUnauthorized, `Principal is not allowed to write shared state`);
|
|
2240
2307
|
}
|
|
2241
2308
|
return apiError(401, ErrCodeUnauthorized, `Principal is not allowed to access shared state`);
|
|
@@ -2718,6 +2785,9 @@ var PostgresRegistry = class {
|
|
|
2718
2785
|
entityBridgeWhere(sourceRef) {
|
|
2719
2786
|
return and(eq(entityBridges.tenantId, this.tenantId), eq(entityBridges.sourceRef, sourceRef));
|
|
2720
2787
|
}
|
|
2788
|
+
pgSyncBridgeWhere(sourceRef) {
|
|
2789
|
+
return and(eq(pgSyncBridges.tenantId, this.tenantId), eq(pgSyncBridges.sourceRef, sourceRef));
|
|
2790
|
+
}
|
|
2721
2791
|
async createEntityType(et) {
|
|
2722
2792
|
await this.db.insert(entityTypes).values({
|
|
2723
2793
|
tenantId: this.tenantId,
|
|
@@ -3187,11 +3257,12 @@ var PostgresRegistry = class {
|
|
|
3187
3257
|
};
|
|
3188
3258
|
const nextTags = normalizeTags(mutation.nextTags);
|
|
3189
3259
|
const updatedAt = Date.now();
|
|
3190
|
-
await tx.update(entities).set({
|
|
3260
|
+
const [updateResult] = await tx.update(entities).set({
|
|
3191
3261
|
tags: nextTags,
|
|
3192
3262
|
tagsIndex: buildTagsIndex(nextTags),
|
|
3193
3263
|
updatedAt
|
|
3194
|
-
}).where(this.entityWhere(url));
|
|
3264
|
+
}).where(this.entityWhere(url)).returning({ txid: sql`pg_current_xact_id()::xid::text` });
|
|
3265
|
+
const txid = updateResult ? parseInt(updateResult.txid) : void 0;
|
|
3195
3266
|
await tx.insert(tagStreamOutbox).values({
|
|
3196
3267
|
tenantId: this.tenantId,
|
|
3197
3268
|
entityUrl: url,
|
|
@@ -3209,10 +3280,63 @@ var PostgresRegistry = class {
|
|
|
3209
3280
|
return {
|
|
3210
3281
|
entity,
|
|
3211
3282
|
changed: true,
|
|
3212
|
-
...op === `insert` || op === `update` ? { op } : {}
|
|
3283
|
+
...op === `insert` || op === `update` ? { op } : {},
|
|
3284
|
+
...txid !== void 0 ? { txid } : {}
|
|
3213
3285
|
};
|
|
3214
3286
|
});
|
|
3215
3287
|
}
|
|
3288
|
+
async upsertPgSyncBridge(row) {
|
|
3289
|
+
await this.db.insert(pgSyncBridges).values({
|
|
3290
|
+
tenantId: this.tenantId,
|
|
3291
|
+
sourceRef: row.sourceRef,
|
|
3292
|
+
options: row.options,
|
|
3293
|
+
streamUrl: row.streamUrl,
|
|
3294
|
+
lastTouchedAt: new Date(),
|
|
3295
|
+
updatedAt: new Date()
|
|
3296
|
+
}).onConflictDoUpdate({
|
|
3297
|
+
target: [pgSyncBridges.tenantId, pgSyncBridges.sourceRef],
|
|
3298
|
+
set: {
|
|
3299
|
+
options: row.options,
|
|
3300
|
+
streamUrl: row.streamUrl,
|
|
3301
|
+
initialSnapshotComplete: false,
|
|
3302
|
+
lastTouchedAt: new Date(),
|
|
3303
|
+
updatedAt: new Date()
|
|
3304
|
+
}
|
|
3305
|
+
});
|
|
3306
|
+
const existing = await this.getPgSyncBridge(row.sourceRef);
|
|
3307
|
+
if (!existing) throw new Error(`Failed to load pgSync bridge ${row.sourceRef}`);
|
|
3308
|
+
return existing;
|
|
3309
|
+
}
|
|
3310
|
+
async getPgSyncBridge(sourceRef) {
|
|
3311
|
+
const rows = await this.db.select().from(pgSyncBridges).where(this.pgSyncBridgeWhere(sourceRef)).limit(1);
|
|
3312
|
+
return rows[0] ? this.rowToPgSyncBridge(rows[0]) : null;
|
|
3313
|
+
}
|
|
3314
|
+
async listPgSyncBridges(tenantId = this.tenantId) {
|
|
3315
|
+
const rows = tenantId === null ? await this.db.select().from(pgSyncBridges) : await this.db.select().from(pgSyncBridges).where(eq(pgSyncBridges.tenantId, tenantId));
|
|
3316
|
+
return rows.map((row) => this.rowToPgSyncBridge(row));
|
|
3317
|
+
}
|
|
3318
|
+
async touchPgSyncBridge(sourceRef) {
|
|
3319
|
+
await this.db.update(pgSyncBridges).set({
|
|
3320
|
+
lastTouchedAt: new Date(),
|
|
3321
|
+
updatedAt: new Date()
|
|
3322
|
+
}).where(this.pgSyncBridgeWhere(sourceRef));
|
|
3323
|
+
}
|
|
3324
|
+
async updatePgSyncBridgeCursor(sourceRef, shapeHandle, shapeOffset, initialSnapshotComplete) {
|
|
3325
|
+
await this.db.update(pgSyncBridges).set({
|
|
3326
|
+
shapeHandle,
|
|
3327
|
+
shapeOffset,
|
|
3328
|
+
...initialSnapshotComplete !== void 0 ? { initialSnapshotComplete } : {},
|
|
3329
|
+
updatedAt: new Date()
|
|
3330
|
+
}).where(this.pgSyncBridgeWhere(sourceRef));
|
|
3331
|
+
}
|
|
3332
|
+
async clearPgSyncBridgeCursor(sourceRef) {
|
|
3333
|
+
await this.db.update(pgSyncBridges).set({
|
|
3334
|
+
shapeHandle: null,
|
|
3335
|
+
shapeOffset: null,
|
|
3336
|
+
initialSnapshotComplete: false,
|
|
3337
|
+
updatedAt: new Date()
|
|
3338
|
+
}).where(this.pgSyncBridgeWhere(sourceRef));
|
|
3339
|
+
}
|
|
3216
3340
|
async upsertEntityBridge(row) {
|
|
3217
3341
|
await this.db.insert(entityBridges).values({
|
|
3218
3342
|
tenantId: this.tenantId,
|
|
@@ -3432,6 +3556,20 @@ var PostgresRegistry = class {
|
|
|
3432
3556
|
updated_at: row.updatedAt
|
|
3433
3557
|
};
|
|
3434
3558
|
}
|
|
3559
|
+
rowToPgSyncBridge(row) {
|
|
3560
|
+
return {
|
|
3561
|
+
tenantId: row.tenantId,
|
|
3562
|
+
sourceRef: row.sourceRef,
|
|
3563
|
+
options: row.options,
|
|
3564
|
+
streamUrl: row.streamUrl,
|
|
3565
|
+
shapeHandle: row.shapeHandle ?? void 0,
|
|
3566
|
+
shapeOffset: row.shapeOffset ?? void 0,
|
|
3567
|
+
initialSnapshotComplete: row.initialSnapshotComplete,
|
|
3568
|
+
lastTouchedAt: row.lastTouchedAt,
|
|
3569
|
+
createdAt: row.createdAt,
|
|
3570
|
+
updatedAt: row.updatedAt
|
|
3571
|
+
};
|
|
3572
|
+
}
|
|
3435
3573
|
rowToEntityBridge(row) {
|
|
3436
3574
|
return {
|
|
3437
3575
|
tenantId: row.tenantId,
|
|
@@ -3512,6 +3650,9 @@ var PostgresRegistry = class {
|
|
|
3512
3650
|
function isRecord$1(value) {
|
|
3513
3651
|
return typeof value === `object` && value !== null && !Array.isArray(value);
|
|
3514
3652
|
}
|
|
3653
|
+
function getPgSyncManifestStreamPath(sourceRef) {
|
|
3654
|
+
return `/_electric/pg-sync/${sourceRef}`;
|
|
3655
|
+
}
|
|
3515
3656
|
function extractManifestSourceUrl(manifest) {
|
|
3516
3657
|
if (!manifest) return void 0;
|
|
3517
3658
|
if (manifest.kind === `child` || manifest.kind === `observe`) return typeof manifest.entity_url === `string` ? manifest.entity_url : void 0;
|
|
@@ -3524,6 +3665,7 @@ function extractManifestSourceUrl(manifest) {
|
|
|
3524
3665
|
}
|
|
3525
3666
|
if (manifest.sourceType === `entities`) return typeof manifest.sourceRef === `string` ? `/_entities/${manifest.sourceRef}` : void 0;
|
|
3526
3667
|
if (manifest.sourceType === `db`) return typeof manifest.sourceRef === `string` ? getSharedStateStreamPath(manifest.sourceRef) : void 0;
|
|
3668
|
+
if (manifest.sourceType === `pgSync`) return typeof manifest.sourceRef === `string` ? getPgSyncManifestStreamPath(manifest.sourceRef) : void 0;
|
|
3527
3669
|
if (manifest.sourceType === `webhook`) {
|
|
3528
3670
|
if (typeof config?.streamUrl === `string`) return config.streamUrl;
|
|
3529
3671
|
if (typeof config?.endpointKey === `string`) return getWebhookStreamPath(config.endpointKey, typeof config.bucket === `string` ? config.bucket : void 0);
|
|
@@ -3638,6 +3780,13 @@ function isRecord(value) {
|
|
|
3638
3780
|
function cloneRecord(value) {
|
|
3639
3781
|
return JSON.parse(JSON.stringify(value));
|
|
3640
3782
|
}
|
|
3783
|
+
function withOptionalTxid(entity, txid) {
|
|
3784
|
+
if (txid === void 0) return entity;
|
|
3785
|
+
return {
|
|
3786
|
+
...entity,
|
|
3787
|
+
txid
|
|
3788
|
+
};
|
|
3789
|
+
}
|
|
3641
3790
|
/**
|
|
3642
3791
|
* Orchestrates the Electric Agents entity lifecycle: register types, spawn, send, kill.
|
|
3643
3792
|
*
|
|
@@ -4779,15 +4928,17 @@ var EntityManager = class {
|
|
|
4779
4928
|
await this.registry.updateStatus(entityUrl, `idle`);
|
|
4780
4929
|
await this.entityBridgeManager?.onEntityChanged(entityUrl);
|
|
4781
4930
|
}
|
|
4931
|
+
const txid = crypto.randomUUID();
|
|
4782
4932
|
const envelope = entityStateSchema.inbox.insert({
|
|
4783
4933
|
key,
|
|
4784
|
-
value
|
|
4934
|
+
value,
|
|
4935
|
+
headers: { txid }
|
|
4785
4936
|
});
|
|
4786
4937
|
const encoded = this.encodeChangeEvent(envelope);
|
|
4787
4938
|
try {
|
|
4788
4939
|
if (opts?.producerId) {
|
|
4789
4940
|
await this.streamClient.appendIdempotent(entity.streams.main, encoded, { producerId: opts.producerId });
|
|
4790
|
-
return;
|
|
4941
|
+
return { txid };
|
|
4791
4942
|
}
|
|
4792
4943
|
await this.streamClient.append(entity.streams.main, encoded);
|
|
4793
4944
|
if (entity.type === `principal` && req.type === `update_identity`) {
|
|
@@ -4795,9 +4946,11 @@ var EntityManager = class {
|
|
|
4795
4946
|
await this.streamClient.append(entity.streams.main, this.encodeChangeEvent({
|
|
4796
4947
|
type: `identity`,
|
|
4797
4948
|
key: `self`,
|
|
4798
|
-
value: identity
|
|
4949
|
+
value: identity,
|
|
4950
|
+
headers: { txid }
|
|
4799
4951
|
}));
|
|
4800
4952
|
}
|
|
4953
|
+
return { txid };
|
|
4801
4954
|
} catch (err) {
|
|
4802
4955
|
if (this.isClosedStreamError(err)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409);
|
|
4803
4956
|
throw err;
|
|
@@ -4818,18 +4971,26 @@ var EntityManager = class {
|
|
|
4818
4971
|
if (req.status === `cancelled`) value.cancelled_at = now;
|
|
4819
4972
|
}
|
|
4820
4973
|
if (Object.keys(value).length === 0) throw new ElectricAgentsError(ErrCodeInvalidRequest, `No inbox fields to update`, 400);
|
|
4974
|
+
const txid = crypto.randomUUID();
|
|
4821
4975
|
const envelope = entityStateSchema.inbox.update({
|
|
4822
4976
|
key,
|
|
4823
|
-
value
|
|
4977
|
+
value,
|
|
4978
|
+
headers: { txid }
|
|
4824
4979
|
});
|
|
4825
4980
|
await this.streamClient.append(entity.streams.main, this.encodeChangeEvent(envelope));
|
|
4981
|
+
return { txid };
|
|
4826
4982
|
}
|
|
4827
4983
|
async deleteInboxMessage(entityUrl, key) {
|
|
4828
4984
|
const entity = await this.registry.getEntity(entityUrl);
|
|
4829
4985
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
4830
4986
|
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
4831
|
-
const
|
|
4987
|
+
const txid = crypto.randomUUID();
|
|
4988
|
+
const envelope = entityStateSchema.inbox.delete({
|
|
4989
|
+
key,
|
|
4990
|
+
headers: { txid }
|
|
4991
|
+
});
|
|
4832
4992
|
await this.streamClient.append(entity.streams.main, this.encodeChangeEvent(envelope));
|
|
4993
|
+
return { txid };
|
|
4833
4994
|
}
|
|
4834
4995
|
isAttachmentStreamPath(path$1) {
|
|
4835
4996
|
return /^\/[^/]+\/[^/]+\/attachments\/[^/]+$/.test(path$1);
|
|
@@ -4918,28 +5079,26 @@ var EntityManager = class {
|
|
|
4918
5079
|
await this.streamClient.delete(attachment.streamPath).catch(() => void 0);
|
|
4919
5080
|
return { txid };
|
|
4920
5081
|
}
|
|
4921
|
-
async setTag(entityUrl, key, req
|
|
5082
|
+
async setTag(entityUrl, key, req) {
|
|
4922
5083
|
const entity = await this.registry.getEntity(entityUrl);
|
|
4923
5084
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
4924
|
-
if (!this.isValidWriteToken(entity, token)) throw new ElectricAgentsError(ErrCodeUnauthorized, `Invalid write token`, 401);
|
|
4925
5085
|
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
4926
5086
|
if (typeof req.value !== `string`) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Tag values must be strings`, 400);
|
|
4927
5087
|
const result = await this.registry.setEntityTag(entityUrl, key, req.value);
|
|
4928
5088
|
const updated = result.entity;
|
|
4929
5089
|
if (!updated) throw new ElectricAgentsError(ErrCodeEntityPersistFailed, `Entity not found after tag write`, 500);
|
|
4930
5090
|
if (result.changed && this.entityBridgeManager) await this.entityBridgeManager.onEntityChanged(entityUrl);
|
|
4931
|
-
return updated;
|
|
5091
|
+
return withOptionalTxid(updated, result.txid);
|
|
4932
5092
|
}
|
|
4933
|
-
async deleteTag(entityUrl, key
|
|
5093
|
+
async deleteTag(entityUrl, key) {
|
|
4934
5094
|
const entity = await this.registry.getEntity(entityUrl);
|
|
4935
5095
|
if (!entity) throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404);
|
|
4936
|
-
if (!this.isValidWriteToken(entity, token)) throw new ElectricAgentsError(ErrCodeUnauthorized, `Invalid write token`, 401);
|
|
4937
5096
|
if (rejectsNormalWrites(entity.status)) throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is not accepting writes`, 409);
|
|
4938
5097
|
const result = await this.registry.removeEntityTag(entityUrl, key);
|
|
4939
5098
|
const updated = result.entity;
|
|
4940
5099
|
if (!updated) throw new ElectricAgentsError(ErrCodeEntityPersistFailed, `Entity not found after tag delete`, 500);
|
|
4941
5100
|
if (result.changed && this.entityBridgeManager) await this.entityBridgeManager.onEntityChanged(entityUrl);
|
|
4942
|
-
return updated;
|
|
5101
|
+
return withOptionalTxid(updated, result.txid);
|
|
4943
5102
|
}
|
|
4944
5103
|
async ensureEntitiesMembershipStream(tags, principal) {
|
|
4945
5104
|
if (!this.entityBridgeManager) throw new Error(`Entity bridge manager not configured`);
|
|
@@ -6000,7 +6159,7 @@ async function parseAttachmentForm(request) {
|
|
|
6000
6159
|
};
|
|
6001
6160
|
}
|
|
6002
6161
|
function contentDisposition(filename) {
|
|
6003
|
-
const fallback = filename.replace(/["
|
|
6162
|
+
const fallback = filename.replace(/[^\x20-\x7e]|["\\]/g, `_`);
|
|
6004
6163
|
return `attachment; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(filename)}`;
|
|
6005
6164
|
}
|
|
6006
6165
|
function rejectPrincipalEntityMutation(request, action) {
|
|
@@ -6198,22 +6357,28 @@ async function deleteEventSourceSubscription(request, ctx) {
|
|
|
6198
6357
|
const result = await ctx.entityManager.deleteEventSourceSubscription(entityUrl, { id: decodeURIComponent(request.params.subscriptionId) });
|
|
6199
6358
|
return json(result);
|
|
6200
6359
|
}
|
|
6360
|
+
function tagResponseBody(entity) {
|
|
6361
|
+
const publicEntity = toPublicEntity(entity);
|
|
6362
|
+
if (entity.txid !== void 0) return {
|
|
6363
|
+
...publicEntity,
|
|
6364
|
+
txid: entity.txid
|
|
6365
|
+
};
|
|
6366
|
+
return publicEntity;
|
|
6367
|
+
}
|
|
6201
6368
|
async function setTag(request, ctx) {
|
|
6202
6369
|
const principalMutationError = rejectPrincipalEntityMutation(request, `tag updated`);
|
|
6203
6370
|
if (principalMutationError) return principalMutationError;
|
|
6204
6371
|
const parsed = routeBody(request);
|
|
6205
6372
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
6206
|
-
const
|
|
6207
|
-
|
|
6208
|
-
return json(toPublicEntity(updated));
|
|
6373
|
+
const updated = await ctx.entityManager.setTag(entityUrl, decodeURIComponent(request.params.tagKey), { value: parsed.value });
|
|
6374
|
+
return json(tagResponseBody(updated));
|
|
6209
6375
|
}
|
|
6210
6376
|
async function deleteTag(request, ctx) {
|
|
6211
6377
|
const principalMutationError = rejectPrincipalEntityMutation(request, `tag deleted`);
|
|
6212
6378
|
if (principalMutationError) return principalMutationError;
|
|
6213
6379
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
6214
|
-
const
|
|
6215
|
-
|
|
6216
|
-
return json(toPublicEntity(updated));
|
|
6380
|
+
const updated = await ctx.entityManager.deleteTag(entityUrl, decodeURIComponent(request.params.tagKey));
|
|
6381
|
+
return json(tagResponseBody(updated));
|
|
6217
6382
|
}
|
|
6218
6383
|
async function forkEntity(request, ctx) {
|
|
6219
6384
|
const principalMutationError = rejectPrincipalEntityMutation(request, `forked`);
|
|
@@ -6280,9 +6445,12 @@ async function sendEntity(request, ctx) {
|
|
|
6280
6445
|
mode: parsed.mode,
|
|
6281
6446
|
position: parsed.position
|
|
6282
6447
|
};
|
|
6283
|
-
if (parsed.afterMs && parsed.afterMs > 0)
|
|
6284
|
-
|
|
6285
|
-
|
|
6448
|
+
if (parsed.afterMs && parsed.afterMs > 0) {
|
|
6449
|
+
await ctx.entityManager.enqueueDelayedSend(entityUrl, sendReq, new Date(Date.now() + parsed.afterMs));
|
|
6450
|
+
return status(204);
|
|
6451
|
+
}
|
|
6452
|
+
const result = await ctx.entityManager.send(entityUrl, sendReq);
|
|
6453
|
+
return json(result);
|
|
6286
6454
|
}
|
|
6287
6455
|
async function createAttachment(request, ctx) {
|
|
6288
6456
|
const principalMutationError = rejectPrincipalEntityMutation(request, `given attachments`);
|
|
@@ -6325,13 +6493,13 @@ async function deleteAttachment(request, ctx) {
|
|
|
6325
6493
|
async function updateInboxMessage(request, ctx) {
|
|
6326
6494
|
const parsed = routeBody(request);
|
|
6327
6495
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
6328
|
-
await ctx.entityManager.updateInboxMessage(entityUrl, decodeURIComponent(request.params.messageKey), parsed);
|
|
6329
|
-
return
|
|
6496
|
+
const result = await ctx.entityManager.updateInboxMessage(entityUrl, decodeURIComponent(request.params.messageKey), parsed);
|
|
6497
|
+
return json(result);
|
|
6330
6498
|
}
|
|
6331
6499
|
async function deleteInboxMessage(request, ctx) {
|
|
6332
6500
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
6333
|
-
await ctx.entityManager.deleteInboxMessage(entityUrl, decodeURIComponent(request.params.messageKey));
|
|
6334
|
-
return
|
|
6501
|
+
const result = await ctx.entityManager.deleteInboxMessage(entityUrl, decodeURIComponent(request.params.messageKey));
|
|
6502
|
+
return json(result);
|
|
6335
6503
|
}
|
|
6336
6504
|
async function spawnEntity(request, ctx) {
|
|
6337
6505
|
const parsed = routeBody(request);
|
|
@@ -6602,6 +6770,49 @@ function toPublicEntityType(entityType) {
|
|
|
6602
6770
|
};
|
|
6603
6771
|
}
|
|
6604
6772
|
|
|
6773
|
+
//#endregion
|
|
6774
|
+
//#region src/routing/pg-sync-router.ts
|
|
6775
|
+
const pgSyncOptionsSchema = Type.Object({
|
|
6776
|
+
url: Type.Optional(Type.String()),
|
|
6777
|
+
table: Type.String(),
|
|
6778
|
+
columns: Type.Optional(Type.Array(Type.String())),
|
|
6779
|
+
where: Type.Optional(Type.String()),
|
|
6780
|
+
params: Type.Optional(Type.Union([Type.Array(Type.String()), Type.Record(Type.String(), Type.String())])),
|
|
6781
|
+
replica: Type.Optional(Type.Union([Type.Literal(`default`), Type.Literal(`full`)]))
|
|
6782
|
+
});
|
|
6783
|
+
const pgSyncRequestMetadataSchema = Type.Object({
|
|
6784
|
+
entityUrl: Type.Optional(Type.String()),
|
|
6785
|
+
entityType: Type.Optional(Type.String()),
|
|
6786
|
+
streamPath: Type.Optional(Type.String()),
|
|
6787
|
+
runtimeConsumerId: Type.Optional(Type.String()),
|
|
6788
|
+
wakeId: Type.Optional(Type.String())
|
|
6789
|
+
});
|
|
6790
|
+
const pgSyncRegisterBodySchema = Type.Object({
|
|
6791
|
+
options: pgSyncOptionsSchema,
|
|
6792
|
+
metadata: Type.Optional(pgSyncRequestMetadataSchema)
|
|
6793
|
+
});
|
|
6794
|
+
const pgSyncRouter = Router({ base: `/_electric/pg-sync` });
|
|
6795
|
+
pgSyncRouter.post(`/register`, withSchema(pgSyncRegisterBodySchema), registerPgSync);
|
|
6796
|
+
async function registerPgSync(request, ctx) {
|
|
6797
|
+
const { options, metadata } = routeBody(request);
|
|
6798
|
+
if (options.table.trim() === ``) return apiError(400, ErrCodeInvalidRequest, `pgSync table must be non-empty`);
|
|
6799
|
+
if (!ctx.pgSyncBridgeManager) return apiError(503, ErrCodeInvalidRequest, `pgSync bridge manager is not configured`);
|
|
6800
|
+
try {
|
|
6801
|
+
const requestMetadata$1 = {
|
|
6802
|
+
tenantId: ctx.service,
|
|
6803
|
+
principalKind: ctx.principal.kind,
|
|
6804
|
+
principalId: ctx.principal.id,
|
|
6805
|
+
principalKey: ctx.principal.key,
|
|
6806
|
+
principalUrl: ctx.principal.url,
|
|
6807
|
+
...metadata ?? {}
|
|
6808
|
+
};
|
|
6809
|
+
const result = await ctx.pgSyncBridgeManager.register(options, requestMetadata$1);
|
|
6810
|
+
return json(result);
|
|
6811
|
+
} catch (error) {
|
|
6812
|
+
return apiError(500, ErrCodeInvalidRequest, `pgSync registration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
6813
|
+
}
|
|
6814
|
+
}
|
|
6815
|
+
|
|
6605
6816
|
//#endregion
|
|
6606
6817
|
//#region src/routing/hooks.ts
|
|
6607
6818
|
const SPAN_KEY = Symbol(`agents-server.otel-span`);
|
|
@@ -6676,6 +6887,10 @@ function errorMapper(err, req) {
|
|
|
6676
6887
|
});
|
|
6677
6888
|
}
|
|
6678
6889
|
if (err instanceof ElectricAgentsError) return apiError(err.status, err.code, err.message, err.details);
|
|
6890
|
+
if (err instanceof ElectricProxyError) {
|
|
6891
|
+
serverLog.warn(`[agent-server] Electric proxy rejected request (${err.code}): ${req.url}`);
|
|
6892
|
+
return apiError(err.status, err.code, err.message);
|
|
6893
|
+
}
|
|
6679
6894
|
serverLog.error(`[agent-server] Unhandled error:`, err);
|
|
6680
6895
|
return apiError(500, `INTERNAL_SERVER_ERROR`, `Internal server error`);
|
|
6681
6896
|
}
|
|
@@ -7126,6 +7341,7 @@ internalRouter.all(`/runners`, runnersRouter.fetch);
|
|
|
7126
7341
|
internalRouter.all(`/runners/*`, runnersRouter.fetch);
|
|
7127
7342
|
internalRouter.all(`/entities/*`, entitiesRouter.fetch);
|
|
7128
7343
|
internalRouter.all(`/entity-types/*`, entityTypesRouter.fetch);
|
|
7344
|
+
internalRouter.all(`/pg-sync/*`, pgSyncRouter.fetch);
|
|
7129
7345
|
internalRouter.all(`/observations/*`, observationsRouter.fetch);
|
|
7130
7346
|
internalRouter.get(`/electric/*`, electricProxyRouter.fetch);
|
|
7131
7347
|
internalRouter.all(`*`, () => status(404));
|
|
@@ -7509,6 +7725,9 @@ const globalRouter = AutoRouter({
|
|
|
7509
7725
|
finally: [otelEndSpan, applyCors]
|
|
7510
7726
|
});
|
|
7511
7727
|
globalRouter.all(`/_electric/shared-state/*`, durableStreamsRouter.fetch);
|
|
7728
|
+
globalRouter.all(`/_electric/pg-sync/register`, internalRouter.fetch);
|
|
7729
|
+
globalRouter.get(`/_electric/pg-sync/*`, durableStreamsRouter.fetch);
|
|
7730
|
+
globalRouter.head(`/_electric/pg-sync/*`, durableStreamsRouter.fetch);
|
|
7512
7731
|
globalRouter.all(`/_electric/*`, internalRouter.fetch);
|
|
7513
7732
|
globalRouter.all(`*`, durableStreamsRouter.fetch);
|
|
7514
7733
|
|
|
@@ -7551,7 +7770,7 @@ const ENTITY_SHAPE_COLUMNS = [
|
|
|
7551
7770
|
`created_at`,
|
|
7552
7771
|
`updated_at`
|
|
7553
7772
|
];
|
|
7554
|
-
function parseElectricOffset(offset) {
|
|
7773
|
+
function parseElectricOffset$1(offset) {
|
|
7555
7774
|
if (offset === `-1`) return offset;
|
|
7556
7775
|
return /^\d+_\d+$/.test(offset) ? offset : null;
|
|
7557
7776
|
}
|
|
@@ -7643,7 +7862,7 @@ var EntityBridge = class {
|
|
|
7643
7862
|
});
|
|
7644
7863
|
await this.loadCurrentMembers();
|
|
7645
7864
|
if (this.initialShapeHandle && this.initialShapeOffset) {
|
|
7646
|
-
const initialOffset = parseElectricOffset(this.initialShapeOffset);
|
|
7865
|
+
const initialOffset = parseElectricOffset$1(this.initialShapeOffset);
|
|
7647
7866
|
if (initialOffset) {
|
|
7648
7867
|
this.startLiveStream(initialOffset, this.initialShapeHandle);
|
|
7649
7868
|
return;
|
|
@@ -8129,6 +8348,9 @@ function isPermanentElectricAgentsError(err) {
|
|
|
8129
8348
|
const name = typeof err === `object` && err !== null && `name` in err ? err.name : void 0;
|
|
8130
8349
|
return name === `ElectricAgentsError` && typeof status$1 === `number` && status$1 >= 400 && status$1 < 500;
|
|
8131
8350
|
}
|
|
8351
|
+
function cronTaskStreamPath(payload) {
|
|
8352
|
+
return typeof payload.streamPath === `string` ? payload.streamPath : null;
|
|
8353
|
+
}
|
|
8132
8354
|
function normalizeTask(row) {
|
|
8133
8355
|
return {
|
|
8134
8356
|
id: Number(row.id),
|
|
@@ -8471,6 +8693,15 @@ var Scheduler = class {
|
|
|
8471
8693
|
`;
|
|
8472
8694
|
if (completed.length === 0) return;
|
|
8473
8695
|
const nextFireAt = getNextCronFireAt(task.cronExpression, task.cronTimezone, task.fireAt);
|
|
8696
|
+
const streamPath = cronTaskStreamPath(task.payload);
|
|
8697
|
+
const subscriberRows = streamPath ? await sql$1`
|
|
8698
|
+
select 1 as exists
|
|
8699
|
+
from wake_registrations
|
|
8700
|
+
where tenant_id = ${tenantId}
|
|
8701
|
+
and source_url = ${streamPath}
|
|
8702
|
+
limit 1
|
|
8703
|
+
` : [];
|
|
8704
|
+
if (subscriberRows.length === 0) return;
|
|
8474
8705
|
await sql$1`
|
|
8475
8706
|
insert into scheduled_tasks (
|
|
8476
8707
|
tenant_id,
|
|
@@ -8570,6 +8801,308 @@ var Scheduler = class {
|
|
|
8570
8801
|
}
|
|
8571
8802
|
};
|
|
8572
8803
|
|
|
8804
|
+
//#endregion
|
|
8805
|
+
//#region src/pg-sync-bridge-manager.ts
|
|
8806
|
+
const PG_SYNC_ELECTRIC_SHAPE_URL = process.env.ELECTRIC_AGENTS_PG_SYNC_ELECTRIC_URL ?? `http://localhost:3000/v1/shape`;
|
|
8807
|
+
const DEFAULT_RETRY_INITIAL_DELAY_MS = 1e3;
|
|
8808
|
+
const DEFAULT_RETRY_MAX_DELAY_MS = 3e4;
|
|
8809
|
+
function buildElectricShapeParams(options) {
|
|
8810
|
+
return {
|
|
8811
|
+
table: options.table,
|
|
8812
|
+
...options.columns !== void 0 ? { columns: [...options.columns] } : {},
|
|
8813
|
+
...options.where !== void 0 ? { where: options.where } : {},
|
|
8814
|
+
...options.params !== void 0 ? { params: Array.isArray(options.params) ? [...options.params] : { ...options.params } } : {},
|
|
8815
|
+
...options.replica !== void 0 ? { replica: options.replica } : {},
|
|
8816
|
+
...options.metadata?.tenantId ? { electric_agents_tenant_id: options.metadata.tenantId } : {},
|
|
8817
|
+
...options.metadata?.principalKind ? { electric_agents_principal_kind: options.metadata.principalKind } : {},
|
|
8818
|
+
...options.metadata?.principalId ? { electric_agents_principal_id: options.metadata.principalId } : {},
|
|
8819
|
+
...options.metadata?.principalKey ? { electric_agents_principal_key: options.metadata.principalKey } : {},
|
|
8820
|
+
...options.metadata?.principalUrl ? { electric_agents_principal_url: options.metadata.principalUrl } : {},
|
|
8821
|
+
...options.metadata?.entityUrl ? { electric_agents_entity_url: options.metadata.entityUrl } : {},
|
|
8822
|
+
...options.metadata?.entityType ? { electric_agents_entity_type: options.metadata.entityType } : {},
|
|
8823
|
+
...options.metadata?.streamPath ? { electric_agents_stream_path: options.metadata.streamPath } : {},
|
|
8824
|
+
...options.metadata?.runtimeConsumerId ? { electric_agents_runtime_consumer_id: options.metadata.runtimeConsumerId } : {},
|
|
8825
|
+
...options.metadata?.wakeId ? { electric_agents_wake_id: options.metadata.wakeId } : {}
|
|
8826
|
+
};
|
|
8827
|
+
}
|
|
8828
|
+
function jsonSafe(value) {
|
|
8829
|
+
if (typeof value === `bigint`) return value.toString();
|
|
8830
|
+
if (value === null || typeof value !== `object`) return value;
|
|
8831
|
+
if (Array.isArray(value)) return value.map(jsonSafe);
|
|
8832
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, jsonSafe(item)]));
|
|
8833
|
+
}
|
|
8834
|
+
function stableJson(value) {
|
|
8835
|
+
if (typeof value === `bigint`) return JSON.stringify(value.toString());
|
|
8836
|
+
if (value === null || typeof value !== `object`) return JSON.stringify(value);
|
|
8837
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(`,`)}]`;
|
|
8838
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(`,`)}}`;
|
|
8839
|
+
}
|
|
8840
|
+
function parseElectricOffset(offset) {
|
|
8841
|
+
if (offset === `-1`) return offset;
|
|
8842
|
+
return /^\d+_\d+$/.test(offset) ? offset : null;
|
|
8843
|
+
}
|
|
8844
|
+
function rowKeyForMessage(message) {
|
|
8845
|
+
const headers = message.headers;
|
|
8846
|
+
const candidate = headers.key ?? headers.rowKey ?? message.value?.id ?? message.value?.key ?? message.old_value?.id ?? message.old_value?.key;
|
|
8847
|
+
return candidate === void 0 ? void 0 : stableJson(candidate);
|
|
8848
|
+
}
|
|
8849
|
+
function pgSyncMessageToDurableEvent(message, optionsOrSourceRef) {
|
|
8850
|
+
const operation = message.headers.operation;
|
|
8851
|
+
if (operation !== `insert` && operation !== `update` && operation !== `delete`) return null;
|
|
8852
|
+
const sourceRef = typeof optionsOrSourceRef === `string` ? optionsOrSourceRef : sourceRefForPgSync(optionsOrSourceRef);
|
|
8853
|
+
const rowKey = rowKeyForMessage(message);
|
|
8854
|
+
const offset = message.headers.offset;
|
|
8855
|
+
if (typeof offset !== `string` || offset.length === 0) return null;
|
|
8856
|
+
const messageKeyPart = offset;
|
|
8857
|
+
const messageKey = `${sourceRef}:${operation}:${messageKeyPart}`;
|
|
8858
|
+
const timestamp$1 = new Date().toISOString();
|
|
8859
|
+
const oldValue = message.old_value;
|
|
8860
|
+
const safeValue = jsonSafe(message.value);
|
|
8861
|
+
const safeOldValue = jsonSafe(oldValue);
|
|
8862
|
+
const safeHeaders = jsonSafe(message.headers);
|
|
8863
|
+
return {
|
|
8864
|
+
type: `pg_sync_change`,
|
|
8865
|
+
key: messageKey,
|
|
8866
|
+
value: {
|
|
8867
|
+
key: messageKey,
|
|
8868
|
+
table: typeof optionsOrSourceRef === `string` ? void 0 : optionsOrSourceRef.table,
|
|
8869
|
+
operation,
|
|
8870
|
+
...rowKey !== void 0 ? { rowKey } : {},
|
|
8871
|
+
...message.value !== void 0 ? { value: safeValue } : {},
|
|
8872
|
+
...oldValue !== void 0 ? { oldValue: safeOldValue } : {},
|
|
8873
|
+
headers: safeHeaders,
|
|
8874
|
+
...typeof offset === `string` ? { offset } : {},
|
|
8875
|
+
receivedAt: timestamp$1
|
|
8876
|
+
},
|
|
8877
|
+
headers: {
|
|
8878
|
+
operation,
|
|
8879
|
+
timestamp: timestamp$1
|
|
8880
|
+
}
|
|
8881
|
+
};
|
|
8882
|
+
}
|
|
8883
|
+
function cursorFromRow(row) {
|
|
8884
|
+
return row?.shapeHandle && row.shapeOffset ? {
|
|
8885
|
+
handle: row.shapeHandle,
|
|
8886
|
+
offset: row.shapeOffset,
|
|
8887
|
+
initialSnapshotComplete: row.initialSnapshotComplete
|
|
8888
|
+
} : void 0;
|
|
8889
|
+
}
|
|
8890
|
+
var PgSyncBridge = class {
|
|
8891
|
+
producer = null;
|
|
8892
|
+
unsubscribe = null;
|
|
8893
|
+
abortController = null;
|
|
8894
|
+
skipChangesUntilUpToDate = false;
|
|
8895
|
+
recovering = false;
|
|
8896
|
+
committedCursor;
|
|
8897
|
+
retryAttempt = 0;
|
|
8898
|
+
constructor(sourceRef, streamUrl, options, resolvedSource, retry, streamClient, registry, evaluateWakes, initialCursor) {
|
|
8899
|
+
this.sourceRef = sourceRef;
|
|
8900
|
+
this.streamUrl = streamUrl;
|
|
8901
|
+
this.options = options;
|
|
8902
|
+
this.resolvedSource = resolvedSource;
|
|
8903
|
+
this.retry = retry;
|
|
8904
|
+
this.streamClient = streamClient;
|
|
8905
|
+
this.registry = registry;
|
|
8906
|
+
this.evaluateWakes = evaluateWakes;
|
|
8907
|
+
this.initialCursor = initialCursor;
|
|
8908
|
+
this.committedCursor = initialCursor;
|
|
8909
|
+
}
|
|
8910
|
+
async start() {
|
|
8911
|
+
if (!this.producer) this.producer = new IdempotentProducer(new DurableStream({
|
|
8912
|
+
url: `${this.streamClient.baseUrl}${this.streamUrl}`,
|
|
8913
|
+
contentType: `application/json`
|
|
8914
|
+
}), `pg-sync-bridge-${this.sourceRef}`);
|
|
8915
|
+
if (this.initialCursor) {
|
|
8916
|
+
const offset = parseElectricOffset(this.initialCursor.offset);
|
|
8917
|
+
if (offset) {
|
|
8918
|
+
this.startStream(offset, this.initialCursor.handle, !this.initialCursor.initialSnapshotComplete);
|
|
8919
|
+
return;
|
|
8920
|
+
}
|
|
8921
|
+
}
|
|
8922
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef);
|
|
8923
|
+
this.startStream(`now`, void 0, true);
|
|
8924
|
+
}
|
|
8925
|
+
async stop() {
|
|
8926
|
+
this.unsubscribe?.();
|
|
8927
|
+
this.abortController?.abort();
|
|
8928
|
+
this.unsubscribe = null;
|
|
8929
|
+
this.abortController = null;
|
|
8930
|
+
try {
|
|
8931
|
+
await this.producer?.flush();
|
|
8932
|
+
} finally {
|
|
8933
|
+
await this.producer?.detach();
|
|
8934
|
+
this.producer = null;
|
|
8935
|
+
}
|
|
8936
|
+
}
|
|
8937
|
+
startStream(offset, handle, skipChangesUntilUpToDate = false, log = offset === `now` ? `changes_only` : `full`) {
|
|
8938
|
+
this.unsubscribe?.();
|
|
8939
|
+
this.abortController?.abort();
|
|
8940
|
+
this.skipChangesUntilUpToDate = skipChangesUntilUpToDate;
|
|
8941
|
+
this.abortController = new AbortController();
|
|
8942
|
+
const stream = new ShapeStream({
|
|
8943
|
+
url: this.resolvedSource.url,
|
|
8944
|
+
params: buildElectricShapeParams(this.options),
|
|
8945
|
+
offset,
|
|
8946
|
+
log,
|
|
8947
|
+
...handle ? { handle } : {},
|
|
8948
|
+
signal: this.abortController.signal
|
|
8949
|
+
});
|
|
8950
|
+
this.unsubscribe = stream.subscribe(async (messages) => {
|
|
8951
|
+
try {
|
|
8952
|
+
for (const message of messages) {
|
|
8953
|
+
if (isControlMessage(message)) {
|
|
8954
|
+
if (message.headers.control === `must-refetch`) {
|
|
8955
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef);
|
|
8956
|
+
this.startStream(`now`, void 0, true);
|
|
8957
|
+
return;
|
|
8958
|
+
}
|
|
8959
|
+
if (message.headers.control === `up-to-date`) {
|
|
8960
|
+
this.skipChangesUntilUpToDate = false;
|
|
8961
|
+
await this.persistCursor(stream, true);
|
|
8962
|
+
continue;
|
|
8963
|
+
}
|
|
8964
|
+
await this.persistCursor(stream);
|
|
8965
|
+
continue;
|
|
8966
|
+
}
|
|
8967
|
+
if (!isChangeMessage(message)) continue;
|
|
8968
|
+
if (!this.skipChangesUntilUpToDate) {
|
|
8969
|
+
const event = pgSyncMessageToDurableEvent(message, this.options);
|
|
8970
|
+
if (event) {
|
|
8971
|
+
if (!this.producer) throw new Error(`pg-sync producer is not started`);
|
|
8972
|
+
await this.producer.append(JSON.stringify(event));
|
|
8973
|
+
await this.producer.flush?.();
|
|
8974
|
+
await this.evaluateWakes?.(this.streamUrl, event);
|
|
8975
|
+
}
|
|
8976
|
+
}
|
|
8977
|
+
await this.persistCursor(stream);
|
|
8978
|
+
this.retryAttempt = 0;
|
|
8979
|
+
}
|
|
8980
|
+
} catch (error) {
|
|
8981
|
+
serverLog.warn(`[pg-sync-bridge] subscription callback failed for ${this.sourceRef}:`, error);
|
|
8982
|
+
await this.recoverStream();
|
|
8983
|
+
}
|
|
8984
|
+
}, (error) => {
|
|
8985
|
+
if (this.abortController?.signal.aborted) return;
|
|
8986
|
+
serverLog.warn(`[pg-sync-bridge] subscription failed for ${this.sourceRef}:`, error);
|
|
8987
|
+
this.recoverStream();
|
|
8988
|
+
});
|
|
8989
|
+
}
|
|
8990
|
+
async recoverStream() {
|
|
8991
|
+
if (this.recovering) return;
|
|
8992
|
+
this.recovering = true;
|
|
8993
|
+
try {
|
|
8994
|
+
const attempt = this.retryAttempt++;
|
|
8995
|
+
const baseDelay = Math.min(this.retry.initialDelayMs * 2 ** attempt, this.retry.maxDelayMs);
|
|
8996
|
+
const jitter = Math.floor(baseDelay * .2 * this.retry.random());
|
|
8997
|
+
const delay = baseDelay + jitter;
|
|
8998
|
+
if (delay > 0) await this.retry.sleep(delay);
|
|
8999
|
+
const offset = this.committedCursor ? parseElectricOffset(this.committedCursor.offset) : null;
|
|
9000
|
+
if (offset && this.committedCursor) this.startStream(offset, this.committedCursor.handle, !this.committedCursor.initialSnapshotComplete);
|
|
9001
|
+
else {
|
|
9002
|
+
await this.registry?.clearPgSyncBridgeCursor(this.sourceRef);
|
|
9003
|
+
this.startStream(`now`, void 0, true);
|
|
9004
|
+
}
|
|
9005
|
+
} finally {
|
|
9006
|
+
this.recovering = false;
|
|
9007
|
+
}
|
|
9008
|
+
}
|
|
9009
|
+
async persistCursor(stream, initialSnapshotComplete = !this.skipChangesUntilUpToDate) {
|
|
9010
|
+
const shapeHandle = stream.shapeHandle;
|
|
9011
|
+
const shapeOffset = stream.lastOffset;
|
|
9012
|
+
if (!shapeHandle || !shapeOffset || shapeOffset === `-1`) return;
|
|
9013
|
+
await this.registry?.updatePgSyncBridgeCursor(this.sourceRef, shapeHandle, shapeOffset, initialSnapshotComplete);
|
|
9014
|
+
this.committedCursor = {
|
|
9015
|
+
handle: shapeHandle,
|
|
9016
|
+
offset: shapeOffset,
|
|
9017
|
+
initialSnapshotComplete
|
|
9018
|
+
};
|
|
9019
|
+
}
|
|
9020
|
+
};
|
|
9021
|
+
var PgSyncBridgeManager = class {
|
|
9022
|
+
bridges = new Map();
|
|
9023
|
+
starting = new Map();
|
|
9024
|
+
url;
|
|
9025
|
+
retry;
|
|
9026
|
+
constructor(streamClient, evaluateWakes, registry, options = {}) {
|
|
9027
|
+
this.streamClient = streamClient;
|
|
9028
|
+
this.evaluateWakes = evaluateWakes;
|
|
9029
|
+
this.registry = registry;
|
|
9030
|
+
this.url = options.url ?? PG_SYNC_ELECTRIC_SHAPE_URL;
|
|
9031
|
+
this.retry = {
|
|
9032
|
+
initialDelayMs: options.retry?.initialDelayMs ?? DEFAULT_RETRY_INITIAL_DELAY_MS,
|
|
9033
|
+
maxDelayMs: options.retry?.maxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS,
|
|
9034
|
+
random: options.retry?.random ?? Math.random,
|
|
9035
|
+
sleep: options.retry?.sleep ?? ((ms) => new Promise((resolve$1) => setTimeout(resolve$1, ms)))
|
|
9036
|
+
};
|
|
9037
|
+
}
|
|
9038
|
+
async start() {
|
|
9039
|
+
const rows = await this.registry?.listPgSyncBridges?.();
|
|
9040
|
+
if (!rows) return;
|
|
9041
|
+
await Promise.all(rows.map((row) => this.ensureBridge(row).catch((error) => {
|
|
9042
|
+
serverLog.warn(`[pg-sync-bridge] failed to start ${row.sourceRef}:`, error);
|
|
9043
|
+
})));
|
|
9044
|
+
}
|
|
9045
|
+
async register(options, metadata) {
|
|
9046
|
+
const mergedMetadata = {
|
|
9047
|
+
...options.metadata,
|
|
9048
|
+
...metadata
|
|
9049
|
+
};
|
|
9050
|
+
const canonicalOptions = {
|
|
9051
|
+
...canonicalPgSyncOptions(options),
|
|
9052
|
+
...Object.keys(mergedMetadata).length > 0 ? { metadata: mergedMetadata } : {}
|
|
9053
|
+
};
|
|
9054
|
+
const resolvedSource = this.resolveSource(canonicalOptions);
|
|
9055
|
+
const sourceRef = sourceRefForPgSync(canonicalOptions);
|
|
9056
|
+
const streamUrl = getPgSyncStreamPath(sourceRef, this.registry?.tenantId);
|
|
9057
|
+
const row = await this.registry?.upsertPgSyncBridge({
|
|
9058
|
+
sourceRef,
|
|
9059
|
+
options: canonicalOptions,
|
|
9060
|
+
streamUrl
|
|
9061
|
+
});
|
|
9062
|
+
await this.streamClient.ensure(streamUrl, { contentType: `application/json` });
|
|
9063
|
+
if (!this.bridges.has(sourceRef)) {
|
|
9064
|
+
let start = this.starting.get(sourceRef);
|
|
9065
|
+
if (!start) {
|
|
9066
|
+
start = (async () => {
|
|
9067
|
+
const bridge = new PgSyncBridge(sourceRef, streamUrl, canonicalOptions, resolvedSource, this.retry, this.streamClient, this.registry, this.evaluateWakes, cursorFromRow(row));
|
|
9068
|
+
await bridge.start();
|
|
9069
|
+
this.bridges.set(sourceRef, bridge);
|
|
9070
|
+
})().finally(() => this.starting.delete(sourceRef));
|
|
9071
|
+
this.starting.set(sourceRef, start);
|
|
9072
|
+
}
|
|
9073
|
+
await start;
|
|
9074
|
+
}
|
|
9075
|
+
return {
|
|
9076
|
+
sourceRef,
|
|
9077
|
+
streamUrl
|
|
9078
|
+
};
|
|
9079
|
+
}
|
|
9080
|
+
async ensureBridge(row) {
|
|
9081
|
+
if (this.bridges.has(row.sourceRef)) return;
|
|
9082
|
+
let start = this.starting.get(row.sourceRef);
|
|
9083
|
+
if (!start) {
|
|
9084
|
+
start = (async () => {
|
|
9085
|
+
await this.streamClient.ensure(row.streamUrl, { contentType: `application/json` });
|
|
9086
|
+
const canonicalOptions = canonicalPgSyncOptions(row.options);
|
|
9087
|
+
const resolvedSource = this.resolveSource(canonicalOptions);
|
|
9088
|
+
const bridge = new PgSyncBridge(row.sourceRef, row.streamUrl, canonicalOptions, resolvedSource, this.retry, this.streamClient, this.registry, this.evaluateWakes, cursorFromRow(row));
|
|
9089
|
+
await bridge.start();
|
|
9090
|
+
this.bridges.set(row.sourceRef, bridge);
|
|
9091
|
+
})().finally(() => this.starting.delete(row.sourceRef));
|
|
9092
|
+
this.starting.set(row.sourceRef, start);
|
|
9093
|
+
}
|
|
9094
|
+
await start;
|
|
9095
|
+
}
|
|
9096
|
+
resolveSource(options) {
|
|
9097
|
+
return { url: options.url ?? this.url };
|
|
9098
|
+
}
|
|
9099
|
+
async stop() {
|
|
9100
|
+
await Promise.allSettled(this.starting.values());
|
|
9101
|
+
await Promise.all([...this.bridges.values()].map((bridge) => bridge.stop()));
|
|
9102
|
+
this.bridges.clear();
|
|
9103
|
+
}
|
|
9104
|
+
};
|
|
9105
|
+
|
|
8573
9106
|
//#endregion
|
|
8574
9107
|
//#region src/runtime.ts
|
|
8575
9108
|
function omitUndefined(value) {
|
|
@@ -8584,6 +9117,7 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
8584
9117
|
wakeRegistry;
|
|
8585
9118
|
scheduler;
|
|
8586
9119
|
entityBridgeManager;
|
|
9120
|
+
pgSyncBridgeManager;
|
|
8587
9121
|
claimWriteTokens;
|
|
8588
9122
|
manager;
|
|
8589
9123
|
constructor(options) {
|
|
@@ -8608,9 +9142,10 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
8608
9142
|
writeTokenValidator: (entity, token) => this.claimWriteTokens.isValid(this.serviceId, entity.streams.main, token),
|
|
8609
9143
|
stopWakeRegistryOnShutdown: options.stopWakeRegistryOnShutdown ?? false
|
|
8610
9144
|
});
|
|
9145
|
+
this.pgSyncBridgeManager = options.pgSyncBridgeManager ?? new PgSyncBridgeManager(this.streamClient, (sourceUrl, event) => this.manager.evaluateWakes(sourceUrl, event), this.registry, options.pgSync);
|
|
8611
9146
|
}
|
|
8612
9147
|
async stop() {
|
|
8613
|
-
await this.manager.shutdown();
|
|
9148
|
+
await Promise.all([this.manager.shutdown(), this.pgSyncBridgeManager.stop()]);
|
|
8614
9149
|
}
|
|
8615
9150
|
async rehydrateCronSchedules() {
|
|
8616
9151
|
const rows = await this.db.select({ sourceUrl: wakeRegistrations.sourceUrl }).from(wakeRegistrations).where(eq(wakeRegistrations.tenantId, this.serviceId));
|
|
@@ -9374,7 +9909,10 @@ var WakeRegistry = class {
|
|
|
9374
9909
|
}
|
|
9375
9910
|
if (!isChangeMessage(message)) return;
|
|
9376
9911
|
if (message.headers.operation === `delete`) {
|
|
9377
|
-
|
|
9912
|
+
const oldValue = message.old_value;
|
|
9913
|
+
const oldId = Number(oldValue?.id);
|
|
9914
|
+
if (Number.isFinite(oldId)) this.removeCachedRegistrationByDbId(oldId);
|
|
9915
|
+
else this.resetCachedRegistrations();
|
|
9378
9916
|
return;
|
|
9379
9917
|
}
|
|
9380
9918
|
this.upsertCachedRegistration(this.normalizeShapeRow(message.value));
|
|
@@ -9485,9 +10023,9 @@ var WakeRegistry = class {
|
|
|
9485
10023
|
matchCondition(reg, event) {
|
|
9486
10024
|
if (reg.condition === `runFinished`) {
|
|
9487
10025
|
if (event.type !== `run`) return null;
|
|
9488
|
-
const value = event.value;
|
|
10026
|
+
const value$1 = event.value;
|
|
9489
10027
|
const headers$1 = event.headers;
|
|
9490
|
-
const status$1 = value?.status;
|
|
10028
|
+
const status$1 = value$1?.status;
|
|
9491
10029
|
const operation$1 = headers$1?.operation;
|
|
9492
10030
|
if (operation$1 !== `update`) return null;
|
|
9493
10031
|
if (status$1 !== `completed` && status$1 !== `failed`) return null;
|
|
@@ -9510,13 +10048,15 @@ var WakeRegistry = class {
|
|
|
9510
10048
|
}
|
|
9511
10049
|
const kind = operation === `delete` ? `delete` : operation === `update` ? `update` : `insert`;
|
|
9512
10050
|
if (condition.ops && condition.ops.length > 0 && !condition.ops.includes(kind)) return null;
|
|
10051
|
+
const value = event.value;
|
|
9513
10052
|
const change = {
|
|
9514
10053
|
collection: eventType,
|
|
9515
10054
|
kind,
|
|
9516
10055
|
key: event.key || ``
|
|
9517
10056
|
};
|
|
10057
|
+
if (value && `value` in value) change.value = value.value;
|
|
10058
|
+
if (value && `oldValue` in value) change.oldValue = value.oldValue;
|
|
9518
10059
|
if (eventType === `inbox`) {
|
|
9519
|
-
const value = event.value;
|
|
9520
10060
|
if (typeof value?.from === `string`) change.from = value.from;
|
|
9521
10061
|
if (typeof value?.from_principal === `string`) change.from_principal = value.from_principal;
|
|
9522
10062
|
if (typeof value?.from_agent === `string`) change.from_agent = value.from_agent;
|
|
@@ -9560,12 +10100,15 @@ async function startStandaloneAgentsRuntime(options) {
|
|
|
9560
10100
|
wakeRegistry,
|
|
9561
10101
|
scheduler,
|
|
9562
10102
|
entityBridgeManager,
|
|
10103
|
+
pgSyncBridgeManager: options.pgSyncBridgeManager,
|
|
10104
|
+
pgSync: options.pgSync,
|
|
9563
10105
|
stopWakeRegistryOnShutdown: options.wakeRegistry ? false : true
|
|
9564
10106
|
});
|
|
9565
10107
|
const startWakeRegistry = options.startWakeRegistry ?? true;
|
|
9566
10108
|
const startScheduler = options.startScheduler ?? true;
|
|
9567
10109
|
const startTagStreamOutboxDrainer = options.startTagStreamOutboxDrainer ?? true;
|
|
9568
10110
|
const startEntityBridgeManager = options.startEntityBridgeManager ?? true;
|
|
10111
|
+
const startPgSyncBridgeManager = options.startPgSyncBridgeManager ?? true;
|
|
9569
10112
|
const rehydrateOnStart = options.rehydrateOnStart ?? true;
|
|
9570
10113
|
let entityBridgeManagerStarted = false;
|
|
9571
10114
|
let tagStreamOutboxDrainerStarted = false;
|
|
@@ -9602,6 +10145,10 @@ async function startStandaloneAgentsRuntime(options) {
|
|
|
9602
10145
|
await entityBridgeManager.start();
|
|
9603
10146
|
entityBridgeManagerStarted = true;
|
|
9604
10147
|
}
|
|
10148
|
+
if (startPgSyncBridgeManager) {
|
|
10149
|
+
serverLog.info(`[agent-server] starting pg-sync bridge manager...`);
|
|
10150
|
+
await runtime.pgSyncBridgeManager.start?.();
|
|
10151
|
+
}
|
|
9605
10152
|
if (startTagStreamOutboxDrainer) {
|
|
9606
10153
|
serverLog.info(`[agent-server] starting tag stream outbox drainer...`);
|
|
9607
10154
|
tagStreamOutboxDrainer.start();
|
|
@@ -9629,6 +10176,7 @@ async function startStandaloneAgentsRuntime(options) {
|
|
|
9629
10176
|
manager: runtime.manager,
|
|
9630
10177
|
scheduler,
|
|
9631
10178
|
entityBridgeManager,
|
|
10179
|
+
pgSyncBridgeManager: runtime.pgSyncBridgeManager,
|
|
9632
10180
|
tagStreamOutboxDrainer,
|
|
9633
10181
|
stop
|
|
9634
10182
|
};
|
|
@@ -9746,7 +10294,8 @@ var ElectricAgentsServer = class {
|
|
|
9746
10294
|
pgClient: client,
|
|
9747
10295
|
streamClient: this.streamClient,
|
|
9748
10296
|
electricUrl: this.options.electricUrl,
|
|
9749
|
-
electricSecret: this.options.electricSecret
|
|
10297
|
+
electricSecret: this.options.electricSecret,
|
|
10298
|
+
pgSync: this.options.pgSync
|
|
9750
10299
|
});
|
|
9751
10300
|
this.electricAgentsManager = this.standaloneRuntime.manager;
|
|
9752
10301
|
this.entityBridgeManager = this.standaloneRuntime.entityBridgeManager;
|
|
@@ -9868,6 +10417,7 @@ var ElectricAgentsServer = class {
|
|
|
9868
10417
|
streamClient: this.streamClient,
|
|
9869
10418
|
runtime: this.standaloneRuntime.runtime,
|
|
9870
10419
|
entityBridgeManager: this.entityBridgeManager,
|
|
10420
|
+
pgSyncBridgeManager: this.standaloneRuntime.runtime.pgSyncBridgeManager,
|
|
9871
10421
|
...this.options.eventSources ? { eventSources: this.options.eventSources } : {},
|
|
9872
10422
|
...this.options.ensureEventSourceWakeSource ? { ensureEventSourceWakeSource: this.options.ensureEventSourceWakeSource } : {},
|
|
9873
10423
|
...this.options.authorizeRequest ? { authorizeRequest: this.options.authorizeRequest } : {},
|