@electric-ax/agents-server 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +938 -869
- package/dist/index.cjs +135 -66
- package/dist/index.d.cts +26 -12
- package/dist/index.d.ts +26 -12
- package/dist/index.js +136 -67
- package/package.json +5 -5
- package/src/entity-manager.ts +31 -7
- package/src/entity-registry.ts +11 -1
- package/src/index.ts +6 -6
- package/src/manifest-side-effects.ts +2 -6
- package/src/pg-sync-bridge-manager.ts +147 -47
- package/src/routing/context.ts +11 -11
- package/src/routing/entities-router.ts +51 -28
- package/src/routing/internal-router.ts +9 -7
- package/src/routing/pg-sync-router.ts +14 -1
- package/src/server.ts +8 -8
- package/src/wake-registry.ts +2 -0
package/dist/index.js
CHANGED
|
@@ -8,7 +8,7 @@ import postgres from "postgres";
|
|
|
8
8
|
import { and, desc, eq, inArray, lt, ne, sql } from "drizzle-orm";
|
|
9
9
|
import { bigint, bigserial, boolean, check, index, integer, jsonb, pgTable, primaryKey, serial, text, timestamp, unique } from "drizzle-orm/pg-core";
|
|
10
10
|
import { createHash, createPrivateKey, createPublicKey, generateKeyPairSync, randomUUID, sign } from "node:crypto";
|
|
11
|
-
import { COMMENTS_CONTRACT, COMPOSER_INPUT_MESSAGE_TYPE, appendPathToUrl, assertTags,
|
|
11
|
+
import { COMMENTS_CONTRACT, COMPOSER_INPUT_MESSAGE_TYPE, appendPathToUrl, assertTags, buildTagsIndex, buildWebhookSourceManifestEntry, canonicalPgSyncOptions, entityStateSchema, getCronStreamPath, getCronStreamPathFromSpec, getEntitiesStreamPath, getNextCronFireAt, getPgSyncStreamPath, getSharedStateStreamPath, getWebhookStreamPath, hashString, manifestChildKey, manifestSharedStateKey, manifestSourceKey, normalizeTags, parseCronStreamPath, resolveCronScheduleSpec, resolveWebhookSourceSubscription, sourceRefForPgSync, sourceRefForTags, validateComposerInputPayload, validateSlashCommandDefinitions, verifyWebhookSignature, webhookSourceSubscriptionManifestKey } from "@electric-ax/agents-runtime";
|
|
12
12
|
import { DurableStream, DurableStreamError, FetchError, IdempotentProducer } from "@durable-streams/client";
|
|
13
13
|
import { ShapeStream, isChangeMessage, isControlMessage } from "@electric-sql/client";
|
|
14
14
|
import pino from "pino";
|
|
@@ -1344,7 +1344,6 @@ var PostgresRegistry = class {
|
|
|
1344
1344
|
set: {
|
|
1345
1345
|
options: row.options,
|
|
1346
1346
|
streamUrl: row.streamUrl,
|
|
1347
|
-
initialSnapshotComplete: false,
|
|
1348
1347
|
lastTouchedAt: new Date(),
|
|
1349
1348
|
updatedAt: new Date()
|
|
1350
1349
|
}
|
|
@@ -1383,6 +1382,9 @@ var PostgresRegistry = class {
|
|
|
1383
1382
|
updatedAt: new Date()
|
|
1384
1383
|
}).where(this.pgSyncBridgeWhere(sourceRef));
|
|
1385
1384
|
}
|
|
1385
|
+
async deletePgSyncBridge(sourceRef) {
|
|
1386
|
+
await this.db.delete(pgSyncBridges).where(this.pgSyncBridgeWhere(sourceRef));
|
|
1387
|
+
}
|
|
1386
1388
|
async upsertEntityBridge(row) {
|
|
1387
1389
|
await this.db.insert(entityBridges).values({
|
|
1388
1390
|
tenantId: this.tenantId,
|
|
@@ -3264,9 +3266,6 @@ function assertSharedSandboxColocated(key, chosenIsRemote, dispatchPolicy) {
|
|
|
3264
3266
|
function isRecord$1(value) {
|
|
3265
3267
|
return typeof value === `object` && value !== null && !Array.isArray(value);
|
|
3266
3268
|
}
|
|
3267
|
-
function getPgSyncManifestStreamPath(sourceRef) {
|
|
3268
|
-
return `/_electric/pg-sync/${sourceRef}`;
|
|
3269
|
-
}
|
|
3270
3269
|
function extractManifestSourceUrl(manifest) {
|
|
3271
3270
|
if (!manifest) return void 0;
|
|
3272
3271
|
if (manifest.kind === `child` || manifest.kind === `observe`) return typeof manifest.entity_url === `string` ? manifest.entity_url : void 0;
|
|
@@ -3279,7 +3278,7 @@ function extractManifestSourceUrl(manifest) {
|
|
|
3279
3278
|
}
|
|
3280
3279
|
if (manifest.sourceType === `entities`) return typeof manifest.sourceRef === `string` ? `/_entities/${manifest.sourceRef}` : void 0;
|
|
3281
3280
|
if (manifest.sourceType === `db`) return typeof manifest.sourceRef === `string` ? getSharedStateStreamPath(manifest.sourceRef) : void 0;
|
|
3282
|
-
if (manifest.sourceType === `pgSync`) return typeof
|
|
3281
|
+
if (manifest.sourceType === `pgSync`) return typeof config?.streamUrl === `string` ? config.streamUrl : void 0;
|
|
3283
3282
|
if (manifest.sourceType === `webhook`) {
|
|
3284
3283
|
if (typeof config?.streamUrl === `string`) return config.streamUrl;
|
|
3285
3284
|
if (typeof config?.endpointKey === `string`) return getWebhookStreamPath(config.endpointKey, typeof config.bucket === `string` ? config.bucket : void 0);
|
|
@@ -4871,7 +4870,7 @@ var EntityManager = class {
|
|
|
4871
4870
|
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, void 0, { txid });
|
|
4872
4871
|
return { txid };
|
|
4873
4872
|
}
|
|
4874
|
-
async
|
|
4873
|
+
async upsertWebhookSourceSubscription(entityUrl, req) {
|
|
4875
4874
|
const manifestKey = req.subscription.manifestKey;
|
|
4876
4875
|
const txid = randomUUID();
|
|
4877
4876
|
await this.writeManifestEntry(entityUrl, manifestKey, `upsert`, req.manifest, { txid });
|
|
@@ -4893,8 +4892,20 @@ var EntityManager = class {
|
|
|
4893
4892
|
subscription: req.subscription
|
|
4894
4893
|
};
|
|
4895
4894
|
}
|
|
4896
|
-
async
|
|
4897
|
-
const manifestKey =
|
|
4895
|
+
async deleteWebhookSourceSubscription(entityUrl, req) {
|
|
4896
|
+
const manifestKey = webhookSourceSubscriptionManifestKey(req.id);
|
|
4897
|
+
const txid = randomUUID();
|
|
4898
|
+
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, void 0, { txid });
|
|
4899
|
+
await this.wakeRegistry.unregisterByManifestKey(entityUrl, manifestKey, this.tenantId);
|
|
4900
|
+
return { txid };
|
|
4901
|
+
}
|
|
4902
|
+
/**
|
|
4903
|
+
* Stop this entity observing a pg-sync source: drop its manifest entry and
|
|
4904
|
+
* the wake it anchors. The shared pg-sync bridge (keyed by sourceRef, not by
|
|
4905
|
+
* subscriber) is intentionally left running for any other observers.
|
|
4906
|
+
*/
|
|
4907
|
+
async deletePgSyncObservation(entityUrl, req) {
|
|
4908
|
+
const manifestKey = `source:pgSync:${req.sourceRef}`;
|
|
4898
4909
|
const txid = randomUUID();
|
|
4899
4910
|
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, void 0, { txid });
|
|
4900
4911
|
await this.wakeRegistry.unregisterByManifestKey(entityUrl, manifestKey, this.tenantId);
|
|
@@ -6000,9 +6011,13 @@ var Scheduler = class {
|
|
|
6000
6011
|
|
|
6001
6012
|
//#endregion
|
|
6002
6013
|
//#region src/pg-sync-bridge-manager.ts
|
|
6003
|
-
|
|
6014
|
+
/** Registration was rejected because the source itself is invalid — map to a 4xx. */
|
|
6015
|
+
var PgSyncSourceValidationError = class extends Error {
|
|
6016
|
+
name = `PgSyncSourceValidationError`;
|
|
6017
|
+
};
|
|
6004
6018
|
const DEFAULT_RETRY_INITIAL_DELAY_MS = 1e3;
|
|
6005
6019
|
const DEFAULT_RETRY_MAX_DELAY_MS = 3e4;
|
|
6020
|
+
const DEFAULT_PROBE_TIMEOUT_MS = 1e4;
|
|
6006
6021
|
function buildElectricShapeParams(options) {
|
|
6007
6022
|
return {
|
|
6008
6023
|
table: options.table,
|
|
@@ -6022,6 +6037,31 @@ function buildElectricShapeParams(options) {
|
|
|
6022
6037
|
...options.metadata?.wakeId ? { electric_agents_wake_id: options.metadata.wakeId } : {}
|
|
6023
6038
|
};
|
|
6024
6039
|
}
|
|
6040
|
+
/**
|
|
6041
|
+
* Build the one-shot URL used to validate a shape source at registration
|
|
6042
|
+
* time. Approximates the query-param encoding of the Electric TS client
|
|
6043
|
+
* (arrays comma-joined, where-clause params as `params[n]`) — unlike the
|
|
6044
|
+
* client it does not quote column identifiers, so probe and stream encoding
|
|
6045
|
+
* can diverge for exotic column names.
|
|
6046
|
+
*/
|
|
6047
|
+
function buildShapeProbeUrl(sourceUrl, options) {
|
|
6048
|
+
let url;
|
|
6049
|
+
try {
|
|
6050
|
+
url = new URL(sourceUrl);
|
|
6051
|
+
} catch {
|
|
6052
|
+
throw new PgSyncSourceValidationError(`pgSync url "${sourceUrl}" is not a valid URL`);
|
|
6053
|
+
}
|
|
6054
|
+
if (url.protocol !== `http:` && url.protocol !== `https:`) throw new PgSyncSourceValidationError(`pgSync url "${sourceUrl}" must be an HTTP(S) Electric shape endpoint, not a database connection string`);
|
|
6055
|
+
for (const [key, value] of Object.entries(buildElectricShapeParams(options))) {
|
|
6056
|
+
if (value === void 0 || value === null) continue;
|
|
6057
|
+
if (Array.isArray(value)) if (key === `params`) value.forEach((item, index$1) => url.searchParams.set(`params[${index$1 + 1}]`, String(item)));
|
|
6058
|
+
else url.searchParams.set(key, value.join(`,`));
|
|
6059
|
+
else if (typeof value === `object`) for (const [k, v] of Object.entries(value)) url.searchParams.set(`${key}[${k}]`, String(v));
|
|
6060
|
+
else url.searchParams.set(key, String(value));
|
|
6061
|
+
}
|
|
6062
|
+
url.searchParams.set(`offset`, `now`);
|
|
6063
|
+
return url;
|
|
6064
|
+
}
|
|
6025
6065
|
function jsonSafe(value) {
|
|
6026
6066
|
if (typeof value === `bigint`) return value.toString();
|
|
6027
6067
|
if (value === null || typeof value !== `object`) return value;
|
|
@@ -6043,37 +6083,19 @@ function rowKeyForMessage(message) {
|
|
|
6043
6083
|
const candidate = headers.key ?? headers.rowKey ?? message.value?.id ?? message.value?.key ?? message.old_value?.id ?? message.old_value?.key;
|
|
6044
6084
|
return candidate === void 0 ? void 0 : stableJson(candidate);
|
|
6045
6085
|
}
|
|
6046
|
-
function pgSyncMessageToDurableEvent(message
|
|
6086
|
+
function pgSyncMessageToDurableEvent(message) {
|
|
6047
6087
|
const operation = message.headers.operation;
|
|
6048
6088
|
if (operation !== `insert` && operation !== `update` && operation !== `delete`) return null;
|
|
6049
|
-
const
|
|
6050
|
-
|
|
6051
|
-
const
|
|
6052
|
-
if (typeof offset !== `string` || offset.length === 0) return null;
|
|
6053
|
-
const messageKeyPart = offset;
|
|
6054
|
-
const messageKey = `${sourceRef}:${operation}:${messageKeyPart}`;
|
|
6055
|
-
const timestamp$1 = new Date().toISOString();
|
|
6056
|
-
const oldValue = message.old_value;
|
|
6057
|
-
const safeValue = jsonSafe(message.value);
|
|
6058
|
-
const safeOldValue = jsonSafe(oldValue);
|
|
6059
|
-
const safeHeaders = jsonSafe(message.headers);
|
|
6089
|
+
const key = message.key ?? (typeof message.headers.key === `string` ? message.headers.key : void 0) ?? rowKeyForMessage(message);
|
|
6090
|
+
if (!key) return null;
|
|
6091
|
+
const safeMessage = jsonSafe(message);
|
|
6060
6092
|
return {
|
|
6061
6093
|
type: `pg_sync_change`,
|
|
6062
|
-
key
|
|
6063
|
-
value:
|
|
6064
|
-
key: messageKey,
|
|
6065
|
-
table: typeof optionsOrSourceRef === `string` ? void 0 : optionsOrSourceRef.table,
|
|
6066
|
-
operation,
|
|
6067
|
-
...rowKey !== void 0 ? { rowKey } : {},
|
|
6068
|
-
...message.value !== void 0 ? { value: safeValue } : {},
|
|
6069
|
-
...oldValue !== void 0 ? { oldValue: safeOldValue } : {},
|
|
6070
|
-
headers: safeHeaders,
|
|
6071
|
-
...typeof offset === `string` ? { offset } : {},
|
|
6072
|
-
receivedAt: timestamp$1
|
|
6073
|
-
},
|
|
6094
|
+
key,
|
|
6095
|
+
value: safeMessage,
|
|
6074
6096
|
headers: {
|
|
6075
|
-
|
|
6076
|
-
|
|
6097
|
+
...jsonSafe(message.headers),
|
|
6098
|
+
operation
|
|
6077
6099
|
}
|
|
6078
6100
|
};
|
|
6079
6101
|
}
|
|
@@ -6163,13 +6185,13 @@ var PgSyncBridge = class {
|
|
|
6163
6185
|
}
|
|
6164
6186
|
if (!isChangeMessage(message)) continue;
|
|
6165
6187
|
if (!this.skipChangesUntilUpToDate) {
|
|
6166
|
-
const event = pgSyncMessageToDurableEvent(message
|
|
6188
|
+
const event = pgSyncMessageToDurableEvent(message);
|
|
6167
6189
|
if (event) {
|
|
6168
6190
|
if (!this.producer) throw new Error(`pg-sync producer is not started`);
|
|
6169
6191
|
await this.producer.append(JSON.stringify(event));
|
|
6170
6192
|
await this.producer.flush?.();
|
|
6171
6193
|
await this.evaluateWakes?.(this.streamUrl, event);
|
|
6172
|
-
}
|
|
6194
|
+
} else serverLog.warn(`[pg-sync-bridge] dropped change message for ${this.sourceRef} (unknown operation or missing row key):`, message.headers);
|
|
6173
6195
|
}
|
|
6174
6196
|
await this.persistCursor(stream);
|
|
6175
6197
|
this.retryAttempt = 0;
|
|
@@ -6218,13 +6240,15 @@ var PgSyncBridge = class {
|
|
|
6218
6240
|
var PgSyncBridgeManager = class {
|
|
6219
6241
|
bridges = new Map();
|
|
6220
6242
|
starting = new Map();
|
|
6221
|
-
url;
|
|
6222
6243
|
retry;
|
|
6244
|
+
fetchFn;
|
|
6245
|
+
probeTimeoutMs;
|
|
6223
6246
|
constructor(streamClient, evaluateWakes, registry, options = {}) {
|
|
6224
6247
|
this.streamClient = streamClient;
|
|
6225
6248
|
this.evaluateWakes = evaluateWakes;
|
|
6226
6249
|
this.registry = registry;
|
|
6227
|
-
this.
|
|
6250
|
+
this.fetchFn = options.fetchFn;
|
|
6251
|
+
this.probeTimeoutMs = options.probeTimeoutMs ?? DEFAULT_PROBE_TIMEOUT_MS;
|
|
6228
6252
|
this.retry = {
|
|
6229
6253
|
initialDelayMs: options.retry?.initialDelayMs ?? DEFAULT_RETRY_INITIAL_DELAY_MS,
|
|
6230
6254
|
maxDelayMs: options.retry?.maxDelayMs ?? DEFAULT_RETRY_MAX_DELAY_MS,
|
|
@@ -6235,9 +6259,16 @@ var PgSyncBridgeManager = class {
|
|
|
6235
6259
|
async start() {
|
|
6236
6260
|
const rows = await this.registry?.listPgSyncBridges?.();
|
|
6237
6261
|
if (!rows) return;
|
|
6238
|
-
await Promise.all(rows.map(
|
|
6239
|
-
|
|
6240
|
-
|
|
6262
|
+
await Promise.all(rows.map(async (row) => {
|
|
6263
|
+
if (!row.options.url) {
|
|
6264
|
+
serverLog.warn(`[pg-sync-bridge] deleting registration ${row.sourceRef}: it predates required source URLs; re-register the observation with an explicit Electric shape URL`);
|
|
6265
|
+
await this.registry?.deletePgSyncBridge?.(row.sourceRef);
|
|
6266
|
+
return;
|
|
6267
|
+
}
|
|
6268
|
+
await this.ensureBridge(row).catch((error) => {
|
|
6269
|
+
serverLog.warn(`[pg-sync-bridge] failed to start ${row.sourceRef}:`, error);
|
|
6270
|
+
});
|
|
6271
|
+
}));
|
|
6241
6272
|
}
|
|
6242
6273
|
async register(options, metadata) {
|
|
6243
6274
|
const mergedMetadata = {
|
|
@@ -6251,6 +6282,7 @@ var PgSyncBridgeManager = class {
|
|
|
6251
6282
|
const resolvedSource = this.resolveSource(canonicalOptions);
|
|
6252
6283
|
const sourceRef = sourceRefForPgSync(canonicalOptions);
|
|
6253
6284
|
const streamUrl = getPgSyncStreamPath(sourceRef, this.registry?.tenantId);
|
|
6285
|
+
if (!this.bridges.has(sourceRef) && !this.starting.has(sourceRef)) await this.probeSource(resolvedSource, canonicalOptions);
|
|
6254
6286
|
const row = await this.registry?.upsertPgSyncBridge({
|
|
6255
6287
|
sourceRef,
|
|
6256
6288
|
options: canonicalOptions,
|
|
@@ -6291,7 +6323,32 @@ var PgSyncBridgeManager = class {
|
|
|
6291
6323
|
await start;
|
|
6292
6324
|
}
|
|
6293
6325
|
resolveSource(options) {
|
|
6294
|
-
|
|
6326
|
+
if (!options.url) throw new PgSyncSourceValidationError(`pgSync source url is required; no server default is configured`);
|
|
6327
|
+
return { url: options.url };
|
|
6328
|
+
}
|
|
6329
|
+
/**
|
|
6330
|
+
* One-shot fetch of the shape log before a bridge is created, so a bad
|
|
6331
|
+
* URL or rejected shape fails the registration instead of dying silently
|
|
6332
|
+
* in the bridge's retry loop.
|
|
6333
|
+
*/
|
|
6334
|
+
async probeSource(source, options) {
|
|
6335
|
+
const probeUrl = buildShapeProbeUrl(source.url, options);
|
|
6336
|
+
const fetchFn = this.fetchFn ?? globalThis.fetch;
|
|
6337
|
+
let response;
|
|
6338
|
+
try {
|
|
6339
|
+
response = await fetchFn(probeUrl, { signal: AbortSignal.timeout(this.probeTimeoutMs) });
|
|
6340
|
+
} catch (error) {
|
|
6341
|
+
throw new PgSyncSourceValidationError(`pgSync source at ${source.url} is unreachable: ${error instanceof Error ? error.message : String(error)}`);
|
|
6342
|
+
}
|
|
6343
|
+
if (!response.ok) {
|
|
6344
|
+
const body = (await response.text().catch(() => `<failed to read body>`)).slice(0, 500);
|
|
6345
|
+
throw new PgSyncSourceValidationError(`pgSync source at ${source.url} rejected the shape request (${response.status})${body ? `: ${body}` : ``}`);
|
|
6346
|
+
}
|
|
6347
|
+
if (!response.headers.get(`electric-handle`)) {
|
|
6348
|
+
const suggestion = new URL(source.url);
|
|
6349
|
+
suggestion.pathname = `/v1/shape`;
|
|
6350
|
+
throw new PgSyncSourceValidationError(`pgSync source at ${source.url} responded but is not a shape log (missing electric-handle header) — the Electric shape API is usually served at ${suggestion.origin}/v1/shape`);
|
|
6351
|
+
}
|
|
6295
6352
|
}
|
|
6296
6353
|
async stop() {
|
|
6297
6354
|
await Promise.allSettled(this.starting.values());
|
|
@@ -7253,6 +7310,7 @@ var WakeRegistry = class {
|
|
|
7253
7310
|
};
|
|
7254
7311
|
if (value && `value` in value) change.value = value.value;
|
|
7255
7312
|
if (value && `oldValue` in value) change.oldValue = value.oldValue;
|
|
7313
|
+
else if (value && `old_value` in value) change.oldValue = value.old_value;
|
|
7256
7314
|
if (eventType === `inbox`) {
|
|
7257
7315
|
if (typeof value?.from === `string`) change.from = value.from;
|
|
7258
7316
|
if (typeof value?.from_principal === `string`) change.from_principal = value.from_principal;
|
|
@@ -8666,8 +8724,8 @@ const subscriptionLifetimeSchema = Type.Union([
|
|
|
8666
8724
|
}),
|
|
8667
8725
|
Type.Object({ kind: Type.Literal(`manual`) })
|
|
8668
8726
|
]);
|
|
8669
|
-
const
|
|
8670
|
-
|
|
8727
|
+
const webhookSourceSubscriptionBodySchema = Type.Object({
|
|
8728
|
+
webhookKey: Type.String(),
|
|
8671
8729
|
bucketKey: Type.Optional(Type.String()),
|
|
8672
8730
|
params: Type.Optional(Type.Record(Type.String(), Type.Unknown())),
|
|
8673
8731
|
filterKey: Type.Optional(Type.String()),
|
|
@@ -8700,8 +8758,9 @@ entitiesRouter.post(`/:type/:instanceId/tags/:tagKey`, withExistingEntity, withS
|
|
|
8700
8758
|
entitiesRouter.delete(`/:type/:instanceId/tags/:tagKey`, withExistingEntity, withEntityPermission(`write`), deleteTag);
|
|
8701
8759
|
entitiesRouter.put(`/:type/:instanceId/schedules/:scheduleId`, withExistingEntity, withSchema(scheduleBodySchema), withEntityPermission(`schedule`), upsertSchedule);
|
|
8702
8760
|
entitiesRouter.delete(`/:type/:instanceId/schedules/:scheduleId`, withExistingEntity, withEntityPermission(`schedule`), deleteSchedule);
|
|
8703
|
-
entitiesRouter.put(`/:type/:instanceId/
|
|
8704
|
-
entitiesRouter.delete(`/:type/:instanceId/
|
|
8761
|
+
entitiesRouter.put(`/:type/:instanceId/webhook-source-subscriptions/:subscriptionId`, withExistingEntity, withSchema(webhookSourceSubscriptionBodySchema), withEntityPermission(`write`), upsertWebhookSourceSubscription);
|
|
8762
|
+
entitiesRouter.delete(`/:type/:instanceId/webhook-source-subscriptions/:subscriptionId`, withExistingEntity, withEntityPermission(`write`), deleteWebhookSourceSubscription);
|
|
8763
|
+
entitiesRouter.delete(`/:type/:instanceId/pg-sync-observations/:sourceRef`, withExistingEntity, withEntityPermission(`write`), deletePgSyncObservation);
|
|
8705
8764
|
entitiesRouter.get(`/:type/:instanceId/grants`, withExistingEntity, withEntityPermission(`manage`), listEntityPermissionGrants);
|
|
8706
8765
|
entitiesRouter.post(`/:type/:instanceId/grants`, withExistingEntity, withSchema(entityPermissionGrantInputSchema), withEntityPermission(`manage`), createEntityPermissionGrant);
|
|
8707
8766
|
entitiesRouter.delete(`/:type/:instanceId/grants/:grantId`, withExistingEntity, withEntityPermission(`manage`), deleteEntityPermissionGrant);
|
|
@@ -8952,22 +9011,22 @@ async function deleteSchedule(request, ctx) {
|
|
|
8952
9011
|
const result = await ctx.entityManager.deleteSchedule(entityUrl, { id: decodeURIComponent(request.params.scheduleId) });
|
|
8953
9012
|
return json(result);
|
|
8954
9013
|
}
|
|
8955
|
-
async function
|
|
8956
|
-
const principalMutationError = rejectPrincipalEntityMutation(request, `subscribed to
|
|
9014
|
+
async function upsertWebhookSourceSubscription(request, ctx) {
|
|
9015
|
+
const principalMutationError = rejectPrincipalEntityMutation(request, `subscribed to webhook sources`);
|
|
8957
9016
|
if (principalMutationError) return principalMutationError;
|
|
8958
|
-
const catalog = ctx.
|
|
8959
|
-
if (!catalog) return apiError(404, ErrCodeNotFound, `No
|
|
9017
|
+
const catalog = ctx.webhookSources;
|
|
9018
|
+
if (!catalog) return apiError(404, ErrCodeNotFound, `No webhook source catalog is configured`);
|
|
8960
9019
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
8961
9020
|
const parsed = routeBody(request);
|
|
8962
|
-
const source = await catalog.
|
|
8963
|
-
if (!source) return apiError(404, ErrCodeNotFound, `
|
|
9021
|
+
const source = await catalog.getWebhookSource(parsed.webhookKey);
|
|
9022
|
+
if (!source) return apiError(404, ErrCodeNotFound, `Webhook source "${parsed.webhookKey}" not found`);
|
|
8964
9023
|
if (parsed.lifetime?.kind === `expires_at`) {
|
|
8965
9024
|
const expiresAt = new Date(parsed.lifetime.at);
|
|
8966
9025
|
if (Number.isNaN(expiresAt.getTime())) return apiError(400, ErrCodeInvalidRequest, `Invalid expires_at lifetime timestamp`);
|
|
8967
9026
|
}
|
|
8968
9027
|
let resolved;
|
|
8969
9028
|
try {
|
|
8970
|
-
resolved =
|
|
9029
|
+
resolved = resolveWebhookSourceSubscription({
|
|
8971
9030
|
contract: source,
|
|
8972
9031
|
entityUrl,
|
|
8973
9032
|
request: {
|
|
@@ -8979,18 +9038,25 @@ async function upsertEventSourceSubscription(request, ctx) {
|
|
|
8979
9038
|
} catch (error) {
|
|
8980
9039
|
return apiError(400, ErrCodeInvalidRequest, error instanceof Error ? error.message : String(error));
|
|
8981
9040
|
}
|
|
8982
|
-
await ctx.
|
|
8983
|
-
const result = await ctx.entityManager.
|
|
9041
|
+
await ctx.ensureWebhookSourceWakeSource?.(resolved.subscription.sourceUrl);
|
|
9042
|
+
const result = await ctx.entityManager.upsertWebhookSourceSubscription(entityUrl, {
|
|
8984
9043
|
subscription: resolved.subscription,
|
|
8985
|
-
manifest:
|
|
9044
|
+
manifest: buildWebhookSourceManifestEntry(resolved)
|
|
8986
9045
|
});
|
|
8987
9046
|
return json(result);
|
|
8988
9047
|
}
|
|
8989
|
-
async function
|
|
8990
|
-
const principalMutationError = rejectPrincipalEntityMutation(request, `unsubscribed from
|
|
9048
|
+
async function deleteWebhookSourceSubscription(request, ctx) {
|
|
9049
|
+
const principalMutationError = rejectPrincipalEntityMutation(request, `unsubscribed from webhook sources`);
|
|
8991
9050
|
if (principalMutationError) return principalMutationError;
|
|
8992
9051
|
const { entityUrl } = requireExistingEntityRoute(request);
|
|
8993
|
-
const result = await ctx.entityManager.
|
|
9052
|
+
const result = await ctx.entityManager.deleteWebhookSourceSubscription(entityUrl, { id: decodeURIComponent(request.params.subscriptionId) });
|
|
9053
|
+
return json(result);
|
|
9054
|
+
}
|
|
9055
|
+
async function deletePgSyncObservation(request, ctx) {
|
|
9056
|
+
const principalMutationError = rejectPrincipalEntityMutation(request, `unobserved a pg-sync source`);
|
|
9057
|
+
if (principalMutationError) return principalMutationError;
|
|
9058
|
+
const { entityUrl } = requireExistingEntityRoute(request);
|
|
9059
|
+
const result = await ctx.entityManager.deletePgSyncObservation(entityUrl, { sourceRef: decodeURIComponent(request.params.sourceRef) });
|
|
8994
9060
|
return json(result);
|
|
8995
9061
|
}
|
|
8996
9062
|
function tagResponseBody(entity) {
|
|
@@ -9456,7 +9522,7 @@ function toPublicEntityType(entityType) {
|
|
|
9456
9522
|
//#endregion
|
|
9457
9523
|
//#region src/routing/pg-sync-router.ts
|
|
9458
9524
|
const pgSyncOptionsSchema = Type.Object({
|
|
9459
|
-
url: Type.
|
|
9525
|
+
url: Type.String(),
|
|
9460
9526
|
table: Type.String(),
|
|
9461
9527
|
columns: Type.Optional(Type.Array(Type.String())),
|
|
9462
9528
|
where: Type.Optional(Type.String()),
|
|
@@ -9478,6 +9544,7 @@ const pgSyncRouter = Router({ base: `/_electric/pg-sync` });
|
|
|
9478
9544
|
pgSyncRouter.post(`/register`, withSchema(pgSyncRegisterBodySchema), registerPgSync);
|
|
9479
9545
|
async function registerPgSync(request, ctx) {
|
|
9480
9546
|
const { options, metadata } = routeBody(request);
|
|
9547
|
+
if (options.url.trim() === ``) return apiError(400, ErrCodeInvalidRequest, `pgSync url must be non-empty`);
|
|
9481
9548
|
if (options.table.trim() === ``) return apiError(400, ErrCodeInvalidRequest, `pgSync table must be non-empty`);
|
|
9482
9549
|
if (!ctx.pgSyncBridgeManager) return apiError(503, ErrCodeInvalidRequest, `pgSync bridge manager is not configured`);
|
|
9483
9550
|
try {
|
|
@@ -9492,6 +9559,8 @@ async function registerPgSync(request, ctx) {
|
|
|
9492
9559
|
const result = await ctx.pgSyncBridgeManager.register(options, requestMetadata$1);
|
|
9493
9560
|
return json(result);
|
|
9494
9561
|
} catch (error) {
|
|
9562
|
+
if (error instanceof PgSyncSourceValidationError) return apiError(400, ErrCodeInvalidRequest, error.message);
|
|
9563
|
+
serverLog.error(`[pg-sync] registration failed for table "${options.table}":`, error);
|
|
9495
9564
|
return apiError(500, ErrCodeInvalidRequest, `pgSync registration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
9496
9565
|
}
|
|
9497
9566
|
}
|
|
@@ -10016,7 +10085,7 @@ const wakeCallbackBodySchema = Type.Object({
|
|
|
10016
10085
|
const DS_SUBSCRIPTION_CALLBACK_PREFIX = `ds-subscription:`;
|
|
10017
10086
|
const internalRouter = Router({ base: `/_electric` });
|
|
10018
10087
|
internalRouter.get(`/health`, () => json({ status: `ok` }));
|
|
10019
|
-
internalRouter.get(`/
|
|
10088
|
+
internalRouter.get(`/webhook-sources`, listWebhookSources);
|
|
10020
10089
|
internalRouter.post(`/wake`, withSchema(wakeRegistrationBodySchema), registerWake);
|
|
10021
10090
|
internalRouter.post(`/subscription-webhooks/:subscriptionId`, subscriptionWebhook);
|
|
10022
10091
|
internalRouter.post(`/wake-callbacks/:consumerId`, wakeCallback);
|
|
@@ -10138,11 +10207,11 @@ async function registerWake(request, ctx) {
|
|
|
10138
10207
|
await ctx.entityManager.registerWake(opts);
|
|
10139
10208
|
return status(204);
|
|
10140
10209
|
}
|
|
10141
|
-
async function
|
|
10142
|
-
const
|
|
10143
|
-
return json({
|
|
10210
|
+
async function listWebhookSources(_request, ctx) {
|
|
10211
|
+
const webhookSources = ctx.webhookSources ? await ctx.webhookSources.listWebhookSources() : [];
|
|
10212
|
+
return json({ webhookSources: webhookSources.filter(isAgentVisibleWebhookSource) });
|
|
10144
10213
|
}
|
|
10145
|
-
function
|
|
10214
|
+
function isAgentVisibleWebhookSource(source) {
|
|
10146
10215
|
return source.agentVisible === true && source.status === `active`;
|
|
10147
10216
|
}
|
|
10148
10217
|
async function subscriptionWebhook(request, ctx) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@electric-ax/agents-server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Electric Agents entity runtime server",
|
|
5
5
|
"author": "Durable Stream contributors",
|
|
6
6
|
"bin": {
|
|
@@ -54,7 +54,7 @@
|
|
|
54
54
|
"pino-pretty": "^13.0.0",
|
|
55
55
|
"postgres": "^3.4.0",
|
|
56
56
|
"undici": "^7.24.7",
|
|
57
|
-
"@electric-ax/agents-runtime": "0.
|
|
57
|
+
"@electric-ax/agents-runtime": "0.6.0"
|
|
58
58
|
},
|
|
59
59
|
"devDependencies": {
|
|
60
60
|
"@types/node": "^22.19.15",
|
|
@@ -65,9 +65,9 @@
|
|
|
65
65
|
"tsx": "^4.19.0",
|
|
66
66
|
"typescript": "^5.0.0",
|
|
67
67
|
"vitest": "^4.1.0",
|
|
68
|
-
"@electric-ax/agents": "0.
|
|
69
|
-
"@electric-ax/agents-server-conformance-tests": "0.
|
|
70
|
-
"@electric-ax/agents-server-ui": "0.
|
|
68
|
+
"@electric-ax/agents": "0.6.0",
|
|
69
|
+
"@electric-ax/agents-server-conformance-tests": "0.6.0",
|
|
70
|
+
"@electric-ax/agents-server-ui": "0.6.0"
|
|
71
71
|
},
|
|
72
72
|
"files": [
|
|
73
73
|
"dist",
|
package/src/entity-manager.ts
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
getCronStreamPath,
|
|
8
8
|
getSharedStateStreamPath,
|
|
9
9
|
getNextCronFireAt,
|
|
10
|
-
|
|
10
|
+
webhookSourceSubscriptionManifestKey,
|
|
11
11
|
manifestChildKey,
|
|
12
12
|
manifestSharedStateKey,
|
|
13
13
|
manifestSourceKey,
|
|
@@ -56,7 +56,7 @@ import type { queueAsPromised } from 'fastq'
|
|
|
56
56
|
import type { SchedulerClient } from './scheduler.js'
|
|
57
57
|
import type { WakeEvalResult, WakeRegistry } from './wake-registry.js'
|
|
58
58
|
import type { WakeMessage } from '@electric-ax/agents-runtime'
|
|
59
|
-
import type {
|
|
59
|
+
import type { WebhookSourceSubscription } from '@electric-ax/agents-runtime'
|
|
60
60
|
import type { PostgresRegistry } from './entity-registry.js'
|
|
61
61
|
import type { SchemaValidator } from './electric-agents/schema-validator.js'
|
|
62
62
|
import type { StreamClient } from './stream-client.js'
|
|
@@ -3142,13 +3142,13 @@ export class EntityManager {
|
|
|
3142
3142
|
return { txid }
|
|
3143
3143
|
}
|
|
3144
3144
|
|
|
3145
|
-
async
|
|
3145
|
+
async upsertWebhookSourceSubscription(
|
|
3146
3146
|
entityUrl: string,
|
|
3147
3147
|
req: {
|
|
3148
|
-
subscription:
|
|
3148
|
+
subscription: WebhookSourceSubscription
|
|
3149
3149
|
manifest: Record<string, unknown>
|
|
3150
3150
|
}
|
|
3151
|
-
): Promise<{ txid: string; subscription:
|
|
3151
|
+
): Promise<{ txid: string; subscription: WebhookSourceSubscription }> {
|
|
3152
3152
|
const manifestKey = req.subscription.manifestKey
|
|
3153
3153
|
const txid = randomUUID()
|
|
3154
3154
|
await this.writeManifestEntry(
|
|
@@ -3184,11 +3184,35 @@ export class EntityManager {
|
|
|
3184
3184
|
return { txid, subscription: req.subscription }
|
|
3185
3185
|
}
|
|
3186
3186
|
|
|
3187
|
-
async
|
|
3187
|
+
async deleteWebhookSourceSubscription(
|
|
3188
3188
|
entityUrl: string,
|
|
3189
3189
|
req: { id: string }
|
|
3190
3190
|
): Promise<{ txid: string }> {
|
|
3191
|
-
const manifestKey =
|
|
3191
|
+
const manifestKey = webhookSourceSubscriptionManifestKey(req.id)
|
|
3192
|
+
const txid = randomUUID()
|
|
3193
|
+
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, undefined, {
|
|
3194
|
+
txid,
|
|
3195
|
+
})
|
|
3196
|
+
|
|
3197
|
+
await this.wakeRegistry.unregisterByManifestKey(
|
|
3198
|
+
entityUrl,
|
|
3199
|
+
manifestKey,
|
|
3200
|
+
this.tenantId
|
|
3201
|
+
)
|
|
3202
|
+
|
|
3203
|
+
return { txid }
|
|
3204
|
+
}
|
|
3205
|
+
|
|
3206
|
+
/**
|
|
3207
|
+
* Stop this entity observing a pg-sync source: drop its manifest entry and
|
|
3208
|
+
* the wake it anchors. The shared pg-sync bridge (keyed by sourceRef, not by
|
|
3209
|
+
* subscriber) is intentionally left running for any other observers.
|
|
3210
|
+
*/
|
|
3211
|
+
async deletePgSyncObservation(
|
|
3212
|
+
entityUrl: string,
|
|
3213
|
+
req: { sourceRef: string }
|
|
3214
|
+
): Promise<{ txid: string }> {
|
|
3215
|
+
const manifestKey = `source:pgSync:${req.sourceRef}`
|
|
3192
3216
|
const txid = randomUUID()
|
|
3193
3217
|
await this.writeManifestEntry(entityUrl, manifestKey, `delete`, undefined, {
|
|
3194
3218
|
txid,
|
package/src/entity-registry.ts
CHANGED
|
@@ -1575,10 +1575,16 @@ export class PostgresRegistry {
|
|
|
1575
1575
|
})
|
|
1576
1576
|
.onConflictDoUpdate({
|
|
1577
1577
|
target: [pgSyncBridges.tenantId, pgSyncBridges.sourceRef],
|
|
1578
|
+
// A conflict means the sourceRef matched, i.e. the shape-identity
|
|
1579
|
+
// options are identical, so the persisted cursor and bootstrap state
|
|
1580
|
+
// are still valid. Re-registration is the common path now that the
|
|
1581
|
+
// sourceRef ignores per-request metadata; resetting
|
|
1582
|
+
// initialSnapshotComplete here would make a later restart resume from
|
|
1583
|
+
// the saved cursor while re-skipping changes until up-to-date, dropping
|
|
1584
|
+
// real changes that arrived during downtime.
|
|
1578
1585
|
set: {
|
|
1579
1586
|
options: row.options,
|
|
1580
1587
|
streamUrl: row.streamUrl,
|
|
1581
|
-
initialSnapshotComplete: false,
|
|
1582
1588
|
lastTouchedAt: new Date(),
|
|
1583
1589
|
updatedAt: new Date(),
|
|
1584
1590
|
},
|
|
@@ -1650,6 +1656,10 @@ export class PostgresRegistry {
|
|
|
1650
1656
|
.where(this.pgSyncBridgeWhere(sourceRef))
|
|
1651
1657
|
}
|
|
1652
1658
|
|
|
1659
|
+
async deletePgSyncBridge(sourceRef: string): Promise<void> {
|
|
1660
|
+
await this.db.delete(pgSyncBridges).where(this.pgSyncBridgeWhere(sourceRef))
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1653
1663
|
async upsertEntityBridge(row: {
|
|
1654
1664
|
sourceRef: string
|
|
1655
1665
|
tags: EntityTags
|
package/src/index.ts
CHANGED
|
@@ -65,17 +65,17 @@ export type {
|
|
|
65
65
|
AuthorizeRequest,
|
|
66
66
|
} from './electric-agents-types.js'
|
|
67
67
|
export type {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
68
|
+
WebhookSourceBucket,
|
|
69
|
+
WebhookSourceContract,
|
|
70
|
+
WebhookSourceFilter,
|
|
71
|
+
WebhookSourceSubscription,
|
|
72
|
+
WebhookSourceSubscriptionInput,
|
|
73
73
|
SubscriptionLifetime,
|
|
74
74
|
} from '@electric-ax/agents-runtime'
|
|
75
75
|
export type { Principal, PrincipalKind } from './principal.js'
|
|
76
76
|
export { globalRouter } from './routing/global-router.js'
|
|
77
77
|
export type { GlobalRoutes } from './routing/global-router.js'
|
|
78
|
-
export type {
|
|
78
|
+
export type { WebhookSourceCatalog, TenantContext } from './routing/context.js'
|
|
79
79
|
export {
|
|
80
80
|
streamRootDurableStreamsRoutingAdapter,
|
|
81
81
|
pathPrefixedSingleTenantDurableStreamsRoutingAdapter,
|
|
@@ -10,10 +10,6 @@ export function isRecord(value: unknown): value is Record<string, unknown> {
|
|
|
10
10
|
return typeof value === `object` && value !== null && !Array.isArray(value)
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
function getPgSyncManifestStreamPath(sourceRef: string): string {
|
|
14
|
-
return `/_electric/pg-sync/${sourceRef}`
|
|
15
|
-
}
|
|
16
|
-
|
|
17
13
|
export function extractManifestSourceUrl(
|
|
18
14
|
manifest: Record<string, unknown> | undefined
|
|
19
15
|
): string | undefined {
|
|
@@ -62,8 +58,8 @@ export function extractManifestSourceUrl(
|
|
|
62
58
|
}
|
|
63
59
|
|
|
64
60
|
if (manifest.sourceType === `pgSync`) {
|
|
65
|
-
return typeof
|
|
66
|
-
?
|
|
61
|
+
return typeof config?.streamUrl === `string`
|
|
62
|
+
? config.streamUrl
|
|
67
63
|
: undefined
|
|
68
64
|
}
|
|
69
65
|
|