@electric-ax/agents-server 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +529 -248
- package/dist/index.cjs +1603 -1332
- package/dist/index.d.cts +274 -162
- package/dist/index.d.ts +274 -162
- package/dist/index.js +1601 -1332
- package/drizzle/0007_runner_diagnostics_and_principal.sql +22 -0
- package/drizzle/0008_runner_runtime_diagnostics.sql +50 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +6 -6
- package/src/db/schema.ts +33 -10
- package/src/electric-agents-types.ts +49 -3
- package/src/entity-registry.ts +136 -26
- package/src/host.ts +4 -5
- package/src/index.ts +5 -1
- package/src/principal.ts +23 -11
- package/src/routing/context.ts +1 -0
- package/src/routing/dispatch-policy.ts +123 -20
- package/src/routing/durable-streams-router.ts +286 -116
- package/src/routing/durable-streams-routing-adapter.ts +31 -64
- package/src/routing/electric-proxy-router.ts +1 -0
- package/src/routing/entities-router.ts +5 -5
- package/src/routing/hooks.ts +8 -1
- package/src/routing/internal-router.ts +6 -2
- package/src/routing/runners-router.ts +257 -19
- package/src/runtime.ts +4 -5
- package/src/server.ts +21 -15
- package/src/standalone-runtime.ts +4 -5
- package/src/stream-client.ts +18 -69
- package/src/utils/server-utils.ts +27 -8
package/dist/index.js
CHANGED
|
@@ -28,6 +28,7 @@ __export(schema_exports, {
|
|
|
28
28
|
entityDispatchState: () => entityDispatchState,
|
|
29
29
|
entityManifestSources: () => entityManifestSources,
|
|
30
30
|
entityTypes: () => entityTypes,
|
|
31
|
+
runnerRuntimeDiagnostics: () => runnerRuntimeDiagnostics,
|
|
31
32
|
runners: () => runners,
|
|
32
33
|
scheduledTasks: () => scheduledTasks,
|
|
33
34
|
subscriptionWebhooks: () => subscriptionWebhooks,
|
|
@@ -96,25 +97,35 @@ const users = pgTable(`users`, {
|
|
|
96
97
|
const runners = pgTable(`runners`, {
|
|
97
98
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
98
99
|
id: text(`id`).notNull(),
|
|
99
|
-
|
|
100
|
+
ownerPrincipal: text(`owner_principal`).notNull(),
|
|
100
101
|
label: text(`label`).notNull(),
|
|
101
102
|
kind: text(`kind`).notNull().default(`local`),
|
|
102
103
|
adminStatus: text(`admin_status`).notNull().default(`enabled`),
|
|
103
104
|
wakeStream: text(`wake_stream`).notNull(),
|
|
104
|
-
wakeStreamOffset: text(`wake_stream_offset`),
|
|
105
|
-
lastSeenAt: timestamp(`last_seen_at`, { withTimezone: true }),
|
|
106
|
-
livenessLeaseExpiresAt: timestamp(`liveness_lease_expires_at`, { withTimezone: true }),
|
|
107
105
|
createdAt: timestamp(`created_at`, { withTimezone: true }).notNull().defaultNow(),
|
|
108
106
|
updatedAt: timestamp(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
109
107
|
}, (table) => [
|
|
110
108
|
primaryKey({ columns: [table.tenantId, table.id] }),
|
|
111
109
|
unique(`uq_runners_wake_stream`).on(table.tenantId, table.wakeStream),
|
|
112
|
-
index(`
|
|
110
|
+
index(`idx_runners_owner_principal`).on(table.tenantId, table.ownerPrincipal),
|
|
113
111
|
index(`idx_runners_admin_status`).on(table.tenantId, table.adminStatus),
|
|
114
|
-
index(`idx_runners_liveness_lease_expires_at`).on(table.tenantId, table.livenessLeaseExpiresAt),
|
|
115
112
|
check(`chk_runners_kind`, sql`${table.kind} IN ('local', 'cloud-worker', 'sandbox', 'ci', 'server')`),
|
|
116
113
|
check(`chk_runners_admin_status`, sql`${table.adminStatus} IN ('enabled', 'disabled')`)
|
|
117
114
|
]);
|
|
115
|
+
const runnerRuntimeDiagnostics = pgTable(`runner_runtime_diagnostics`, {
|
|
116
|
+
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
117
|
+
runnerId: text(`runner_id`).notNull(),
|
|
118
|
+
ownerPrincipal: text(`owner_principal`).notNull(),
|
|
119
|
+
wakeStreamOffset: text(`wake_stream_offset`),
|
|
120
|
+
lastSeenAt: timestamp(`last_seen_at`, { withTimezone: true }).notNull(),
|
|
121
|
+
livenessLeaseExpiresAt: timestamp(`liveness_lease_expires_at`, { withTimezone: true }).notNull(),
|
|
122
|
+
diagnostics: jsonb(`diagnostics`),
|
|
123
|
+
updatedAt: timestamp(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
124
|
+
}, (table) => [
|
|
125
|
+
primaryKey({ columns: [table.tenantId, table.runnerId] }),
|
|
126
|
+
index(`idx_runner_runtime_diagnostics_owner`).on(table.tenantId, table.ownerPrincipal),
|
|
127
|
+
index(`idx_runner_runtime_diagnostics_liveness`).on(table.tenantId, table.livenessLeaseExpiresAt)
|
|
128
|
+
]);
|
|
118
129
|
const entityDispatchState = pgTable(`entity_dispatch_state`, {
|
|
119
130
|
tenantId: text(`tenant_id`).notNull().default(`default`),
|
|
120
131
|
entityUrl: text(`entity_url`).notNull(),
|
|
@@ -424,7 +435,7 @@ var PostgresRegistry = class {
|
|
|
424
435
|
await this.db.insert(runners).values({
|
|
425
436
|
tenantId: this.tenantId,
|
|
426
437
|
id: input.id,
|
|
427
|
-
|
|
438
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
428
439
|
label: input.label,
|
|
429
440
|
kind: input.kind ?? `local`,
|
|
430
441
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -433,7 +444,7 @@ var PostgresRegistry = class {
|
|
|
433
444
|
}).onConflictDoUpdate({
|
|
434
445
|
target: [runners.tenantId, runners.id],
|
|
435
446
|
set: {
|
|
436
|
-
|
|
447
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
437
448
|
label: input.label,
|
|
438
449
|
kind: input.kind ?? `local`,
|
|
439
450
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -451,20 +462,46 @@ var PostgresRegistry = class {
|
|
|
451
462
|
}
|
|
452
463
|
async listRunners(filter) {
|
|
453
464
|
const conditions = [eq(runners.tenantId, this.tenantId)];
|
|
454
|
-
if (filter?.
|
|
465
|
+
if (filter?.ownerPrincipal) conditions.push(eq(runners.ownerPrincipal, filter.ownerPrincipal));
|
|
455
466
|
const rows = await this.db.select().from(runners).where(and(...conditions)).orderBy(desc(runners.createdAt));
|
|
456
467
|
return rows.map((row) => this.rowToRunner(row));
|
|
457
468
|
}
|
|
458
469
|
async heartbeatRunner(input) {
|
|
459
470
|
const now = input.heartbeatAt ?? new Date();
|
|
460
471
|
const leaseExpiresAt = input.livenessLeaseExpiresAt ?? new Date(now.getTime() + (input.leaseMs ?? DEFAULT_RUNNER_LEASE_MS));
|
|
461
|
-
|
|
472
|
+
await this.db.insert(runnerRuntimeDiagnostics).values({
|
|
473
|
+
tenantId: this.tenantId,
|
|
474
|
+
runnerId: input.runnerId,
|
|
475
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
462
476
|
lastSeenAt: now,
|
|
463
477
|
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
464
|
-
|
|
478
|
+
wakeStreamOffset: input.wakeStreamOffset,
|
|
479
|
+
diagnostics: input.diagnostics,
|
|
465
480
|
updatedAt: now
|
|
466
|
-
}).
|
|
467
|
-
|
|
481
|
+
}).onConflictDoUpdate({
|
|
482
|
+
target: [runnerRuntimeDiagnostics.tenantId, runnerRuntimeDiagnostics.runnerId],
|
|
483
|
+
set: {
|
|
484
|
+
lastSeenAt: now,
|
|
485
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
486
|
+
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
487
|
+
...input.wakeStreamOffset !== void 0 ? { wakeStreamOffset: input.wakeStreamOffset } : {},
|
|
488
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {},
|
|
489
|
+
updatedAt: now
|
|
490
|
+
}
|
|
491
|
+
});
|
|
492
|
+
const runner = await this.getRunner(input.runnerId);
|
|
493
|
+
if (!runner) return null;
|
|
494
|
+
return {
|
|
495
|
+
...runner,
|
|
496
|
+
last_seen_at: now.toISOString(),
|
|
497
|
+
liveness_lease_expires_at: leaseExpiresAt.toISOString(),
|
|
498
|
+
...input.wakeStreamOffset !== void 0 ? { wake_stream_offset: input.wakeStreamOffset } : {},
|
|
499
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {}
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
async getRunnerDiagnostics(runnerId) {
|
|
503
|
+
const rows = await this.db.select().from(runnerRuntimeDiagnostics).where(and(eq(runnerRuntimeDiagnostics.tenantId, this.tenantId), eq(runnerRuntimeDiagnostics.runnerId, runnerId))).limit(1);
|
|
504
|
+
return rows[0] ? this.rowToRunnerRuntimeDiagnostics(rows[0]) : null;
|
|
468
505
|
}
|
|
469
506
|
async setRunnerAdminStatus(runnerId, adminStatus) {
|
|
470
507
|
const rows = await this.db.update(runners).set({
|
|
@@ -559,6 +596,27 @@ var PostgresRegistry = class {
|
|
|
559
596
|
}).where(and(eq(entityDispatchState.tenantId, this.tenantId), eq(entityDispatchState.entityUrl, claim.entity_url), eq(entityDispatchState.activeConsumerId, input.consumerId), eq(entityDispatchState.activeEpoch, input.epoch)));
|
|
560
597
|
return claim;
|
|
561
598
|
}
|
|
599
|
+
async getActiveClaimsForRunner(runnerId) {
|
|
600
|
+
const rows = await this.db.select().from(consumerClaims).where(and(eq(consumerClaims.tenantId, this.tenantId), eq(consumerClaims.runnerId, runnerId), eq(consumerClaims.status, `active`)));
|
|
601
|
+
return rows.map((row) => this.rowToConsumerClaim(row));
|
|
602
|
+
}
|
|
603
|
+
async getDispatchStatsForRunner(runnerId) {
|
|
604
|
+
const rows = await this.db.select().from(entityDispatchState).where(and(eq(entityDispatchState.tenantId, this.tenantId), eq(entityDispatchState.activeRunnerId, runnerId)));
|
|
605
|
+
let activeClaim = 0;
|
|
606
|
+
let outstandingWake = 0;
|
|
607
|
+
let pendingWork = 0;
|
|
608
|
+
for (const row of rows) {
|
|
609
|
+
if (row.activeConsumerId) activeClaim++;
|
|
610
|
+
if (row.outstandingWakeId && !row.activeConsumerId) outstandingWake++;
|
|
611
|
+
const pending = row.pendingSourceStreams;
|
|
612
|
+
if (pending && pending.length > 0) pendingWork++;
|
|
613
|
+
}
|
|
614
|
+
return {
|
|
615
|
+
entities_with_active_claim: activeClaim,
|
|
616
|
+
entities_with_outstanding_wake: outstandingWake,
|
|
617
|
+
entities_with_pending_work: pendingWork
|
|
618
|
+
};
|
|
619
|
+
}
|
|
562
620
|
entityTypeWhere(name) {
|
|
563
621
|
return and(eq(entityTypes.tenantId, this.tenantId), eq(entityTypes.name, name));
|
|
564
622
|
}
|
|
@@ -1024,23 +1082,28 @@ var PostgresRegistry = class {
|
|
|
1024
1082
|
};
|
|
1025
1083
|
}
|
|
1026
1084
|
rowToRunner(row) {
|
|
1027
|
-
const now = Date.now();
|
|
1028
|
-
const livenessExpiry = row.livenessLeaseExpiresAt?.getTime();
|
|
1029
1085
|
return {
|
|
1030
1086
|
id: row.id,
|
|
1031
|
-
|
|
1087
|
+
owner_principal: row.ownerPrincipal,
|
|
1032
1088
|
label: row.label,
|
|
1033
1089
|
kind: assertRunnerKind(row.kind),
|
|
1034
1090
|
admin_status: assertRunnerAdminStatus(row.adminStatus),
|
|
1035
|
-
liveness: livenessExpiry !== void 0 && livenessExpiry > now ? `online` : `offline`,
|
|
1036
|
-
last_seen_at: row.lastSeenAt?.toISOString(),
|
|
1037
|
-
liveness_lease_expires_at: row.livenessLeaseExpiresAt?.toISOString(),
|
|
1038
1091
|
wake_stream: row.wakeStream,
|
|
1039
|
-
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
1040
1092
|
created_at: row.createdAt.toISOString(),
|
|
1041
1093
|
updated_at: row.updatedAt.toISOString()
|
|
1042
1094
|
};
|
|
1043
1095
|
}
|
|
1096
|
+
rowToRunnerRuntimeDiagnostics(row) {
|
|
1097
|
+
return {
|
|
1098
|
+
runner_id: row.runnerId,
|
|
1099
|
+
owner_principal: row.ownerPrincipal,
|
|
1100
|
+
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
1101
|
+
last_seen_at: row.lastSeenAt.toISOString(),
|
|
1102
|
+
liveness_lease_expires_at: row.livenessLeaseExpiresAt.toISOString(),
|
|
1103
|
+
diagnostics: row.diagnostics ?? void 0,
|
|
1104
|
+
updated_at: row.updatedAt.toISOString()
|
|
1105
|
+
};
|
|
1106
|
+
}
|
|
1044
1107
|
rowToConsumerClaim(row) {
|
|
1045
1108
|
return {
|
|
1046
1109
|
consumer_id: row.consumerId,
|
|
@@ -1712,140 +1775,693 @@ function parseDispatchPolicy(value, label = `dispatch_policy`) {
|
|
|
1712
1775
|
}
|
|
1713
1776
|
|
|
1714
1777
|
//#endregion
|
|
1715
|
-
//#region src/
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1778
|
+
//#region src/tracing.ts
|
|
1779
|
+
const tracer = trace.getTracer(`agent-server`);
|
|
1780
|
+
const ATTR = {
|
|
1781
|
+
ENTITY_URL: `electric_agents.entity.url`,
|
|
1782
|
+
ENTITY_TYPE: `electric_agents.entity.type`,
|
|
1783
|
+
PARENT_URL: `electric_agents.entity.parent`,
|
|
1784
|
+
WAKE_SOURCE: `electric_agents.wake.source`,
|
|
1785
|
+
WAKE_SUBSCRIBER: `electric_agents.wake.subscriber`,
|
|
1786
|
+
WAKE_KIND: `electric_agents.wake.kind`,
|
|
1787
|
+
STREAM_PATH: `electric_agents.stream.path`,
|
|
1788
|
+
STREAM_OP: `electric_agents.stream.op`,
|
|
1789
|
+
DB_OP: `electric_agents.db.op`,
|
|
1790
|
+
HTTP_METHOD: `http.method`,
|
|
1791
|
+
HTTP_ROUTE: `http.route`,
|
|
1792
|
+
HTTP_STATUS: `http.status_code`
|
|
1793
|
+
};
|
|
1794
|
+
/**
|
|
1795
|
+
* Run `fn` inside an active span. Errors are recorded + status set to ERROR,
|
|
1796
|
+
* then re-thrown. Span ends in a finally block.
|
|
1797
|
+
*/
|
|
1798
|
+
async function withSpan(name, fn, opts) {
|
|
1799
|
+
return await tracer.startActiveSpan(name, opts ?? {}, async (span) => {
|
|
1800
|
+
try {
|
|
1801
|
+
return await fn(span);
|
|
1802
|
+
} catch (err) {
|
|
1803
|
+
span.recordException(err);
|
|
1804
|
+
span.setStatus({
|
|
1805
|
+
code: SpanStatusCode.ERROR,
|
|
1806
|
+
message: err instanceof Error ? err.message : String(err)
|
|
1807
|
+
});
|
|
1808
|
+
throw err;
|
|
1809
|
+
} finally {
|
|
1810
|
+
span.end();
|
|
1811
|
+
}
|
|
1812
|
+
});
|
|
1733
1813
|
}
|
|
1734
|
-
function
|
|
1735
|
-
|
|
1814
|
+
function injectTraceHeaders(headers, ctx = context.active()) {
|
|
1815
|
+
propagation.inject(ctx, headers);
|
|
1816
|
+
}
|
|
1817
|
+
function extractTraceContext(headers) {
|
|
1818
|
+
return propagation.extract(context.active(), headers);
|
|
1736
1819
|
}
|
|
1737
1820
|
|
|
1738
1821
|
//#endregion
|
|
1739
|
-
//#region src/
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
if (opts.dispatchPolicy) return opts.dispatchPolicy;
|
|
1754
|
-
const entityType = await ctx.entityManager.registry.getEntityType(typeName);
|
|
1755
|
-
if (opts.parent) {
|
|
1756
|
-
const parent = await ctx.entityManager.registry.getEntity(opts.parent);
|
|
1757
|
-
if (parent?.dispatch_policy) return applyTypeDefaultSubscriptionScope(parent.dispatch_policy, entityType?.default_dispatch_policy);
|
|
1822
|
+
//#region src/stream-client.ts
|
|
1823
|
+
var DurableStreamsSubscriptionError = class extends Error {
|
|
1824
|
+
code;
|
|
1825
|
+
errorMessage;
|
|
1826
|
+
constructor(message, status$1, body) {
|
|
1827
|
+
super(`${message}: ${status$1} ${body}`);
|
|
1828
|
+
this.status = status$1;
|
|
1829
|
+
this.body = body;
|
|
1830
|
+
this.name = `DurableStreamsSubscriptionError`;
|
|
1831
|
+
try {
|
|
1832
|
+
const parsed = JSON.parse(body);
|
|
1833
|
+
if (typeof parsed.error?.code === `string`) this.code = parsed.error.code;
|
|
1834
|
+
if (typeof parsed.error?.message === `string`) this.errorMessage = parsed.error.message;
|
|
1835
|
+
} catch {}
|
|
1758
1836
|
}
|
|
1759
|
-
|
|
1837
|
+
};
|
|
1838
|
+
async function resolveDurableStreamsBearer(bearer) {
|
|
1839
|
+
if (!bearer) return void 0;
|
|
1840
|
+
const value = typeof bearer === `function` ? await bearer() : bearer;
|
|
1841
|
+
const trimmed = value.trim();
|
|
1842
|
+
if (!trimmed) return void 0;
|
|
1843
|
+
return /^Bearer\s+/i.test(trimmed) ? trimmed : `Bearer ${trimmed}`;
|
|
1760
1844
|
}
|
|
1761
|
-
async function
|
|
1762
|
-
if (
|
|
1763
|
-
|
|
1764
|
-
|
|
1845
|
+
async function applyDurableStreamsBearer(headers, bearer, opts = {}) {
|
|
1846
|
+
if (!bearer) return;
|
|
1847
|
+
if (!opts.overwrite && headers.has(`authorization`)) return;
|
|
1848
|
+
const value = await resolveDurableStreamsBearer(bearer);
|
|
1849
|
+
if (value) headers.set(`authorization`, value);
|
|
1765
1850
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
const
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
dispatch_policy: dispatchPolicy
|
|
1773
|
-
};
|
|
1851
|
+
function appendPathToBaseUrl(baseUrl, path$1) {
|
|
1852
|
+
const url = new URL(baseUrl);
|
|
1853
|
+
const basePath = url.pathname.replace(/\/+$/, ``);
|
|
1854
|
+
const childPath = path$1.replace(/^\/+/, ``);
|
|
1855
|
+
url.pathname = childPath ? `${basePath === `/` ? `` : basePath}/${childPath}` : basePath || `/`;
|
|
1856
|
+
return url.toString().replace(/\/+$/, ``);
|
|
1774
1857
|
}
|
|
1775
|
-
function
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
if (!target || !defaultTarget?.subscription_id) return policy;
|
|
1779
|
-
if (!sameDispatchDestination(target, defaultTarget)) return policy;
|
|
1780
|
-
if (target.subscription_id === defaultTarget.subscription_id) return policy;
|
|
1781
|
-
return { targets: [{
|
|
1782
|
-
...target,
|
|
1783
|
-
subscription_id: defaultTarget.subscription_id
|
|
1784
|
-
}] };
|
|
1858
|
+
function durableStreamsBearerHeaders(bearer) {
|
|
1859
|
+
if (!bearer) return void 0;
|
|
1860
|
+
return { authorization: async () => await resolveDurableStreamsBearer(bearer) ?? `` };
|
|
1785
1861
|
}
|
|
1786
|
-
function
|
|
1787
|
-
|
|
1788
|
-
if (a.type === `runner` && b.type === `runner`) return a.runnerId === b.runnerId;
|
|
1789
|
-
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url;
|
|
1790
|
-
return false;
|
|
1862
|
+
function isNotFoundError(err) {
|
|
1863
|
+
return err instanceof DurableStreamError && err.code === ErrCodeNotFound || err instanceof FetchError && err.status === 404;
|
|
1791
1864
|
}
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
if (!target || target.type !== `runner`) return;
|
|
1795
|
-
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
1796
|
-
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
1797
|
-
if (ctx.principal && runner.owner_user_id !== ctx.principal.key) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
1865
|
+
function isAbortLikeError(err) {
|
|
1866
|
+
return err instanceof Error && (err.name === `AbortError` || err.message === `Stream request was aborted`);
|
|
1798
1867
|
}
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
const target = dispatchPolicy?.targets[0];
|
|
1802
|
-
if (!target) return;
|
|
1803
|
-
await linkStreamToTargetSubscription(ctx, target, entity);
|
|
1868
|
+
function normalizeSubscriptionPattern(pattern) {
|
|
1869
|
+
return pattern.replace(/^\/+/, ``);
|
|
1804
1870
|
}
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
const target = dispatchPolicy?.targets[0];
|
|
1808
|
-
if (!target) return;
|
|
1809
|
-
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
1810
|
-
await ctx.streamClient.removeSubscriptionStream(subscriptionId, entity.streams.main).catch((err) => {
|
|
1811
|
-
serverLog.warn(`[dispatch-policy] failed to remove stream from subscription`, {
|
|
1812
|
-
subscriptionId,
|
|
1813
|
-
stream: entity.streams.main
|
|
1814
|
-
}, err);
|
|
1815
|
-
});
|
|
1871
|
+
function normalizeSubscriptionStreamPath(path$1) {
|
|
1872
|
+
return path$1.replace(/^\/+/, ``);
|
|
1816
1873
|
}
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1874
|
+
function normalizeSubscriptionPath(path$1) {
|
|
1875
|
+
return path$1.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
1876
|
+
}
|
|
1877
|
+
var StreamClient = class {
|
|
1878
|
+
constructor(baseUrl, options = {}) {
|
|
1879
|
+
this.baseUrl = baseUrl;
|
|
1880
|
+
this.options = options;
|
|
1881
|
+
}
|
|
1882
|
+
streamUrl(path$1) {
|
|
1883
|
+
return appendPathToBaseUrl(this.baseUrl, path$1);
|
|
1884
|
+
}
|
|
1885
|
+
streamHeaders() {
|
|
1886
|
+
return durableStreamsBearerHeaders(this.options.bearer);
|
|
1887
|
+
}
|
|
1888
|
+
async requestHeaders(init, opts = {}) {
|
|
1889
|
+
const headers = new Headers(init);
|
|
1890
|
+
await applyDurableStreamsBearer(headers, this.options.bearer, { overwrite: opts.overwriteBearer });
|
|
1891
|
+
return headers;
|
|
1892
|
+
}
|
|
1893
|
+
backendSubscriptionPath(path$1) {
|
|
1894
|
+
return normalizeSubscriptionPath(path$1);
|
|
1895
|
+
}
|
|
1896
|
+
runtimeSubscriptionPath(path$1) {
|
|
1897
|
+
return normalizeSubscriptionPath(path$1);
|
|
1898
|
+
}
|
|
1899
|
+
subscriptionUrl(subscriptionId) {
|
|
1900
|
+
return appendPathToBaseUrl(this.baseUrl, `/__ds/subscriptions/${encodeURIComponent(subscriptionId)}`);
|
|
1901
|
+
}
|
|
1902
|
+
subscriptionChildUrl(subscriptionId, ...segments) {
|
|
1903
|
+
const url = new URL(this.subscriptionUrl(subscriptionId));
|
|
1904
|
+
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/${segments.map((segment) => encodeURIComponent(segment)).join(`/`)}`;
|
|
1905
|
+
return url.toString();
|
|
1906
|
+
}
|
|
1907
|
+
async create(path$1, opts) {
|
|
1908
|
+
return await withSpan(`stream.create`, async (span) => {
|
|
1909
|
+
span.setAttributes({
|
|
1910
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
1911
|
+
[ATTR.STREAM_OP]: `create`
|
|
1832
1912
|
});
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1913
|
+
await DurableStream.create({
|
|
1914
|
+
url: this.streamUrl(path$1),
|
|
1915
|
+
headers: this.streamHeaders(),
|
|
1916
|
+
contentType: opts.contentType,
|
|
1917
|
+
body: opts.body
|
|
1918
|
+
});
|
|
1919
|
+
});
|
|
1837
1920
|
}
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1921
|
+
async fork(path$1, sourcePath) {
|
|
1922
|
+
return await withSpan(`stream.fork`, async (span) => {
|
|
1923
|
+
span.setAttributes({
|
|
1924
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
1925
|
+
[ATTR.STREAM_OP]: `fork`
|
|
1926
|
+
});
|
|
1927
|
+
const headers = {
|
|
1928
|
+
"content-type": `application/json`,
|
|
1929
|
+
"Stream-Forked-From": new URL(this.streamUrl(sourcePath)).pathname
|
|
1930
|
+
};
|
|
1931
|
+
injectTraceHeaders(headers);
|
|
1932
|
+
const response = await fetch(this.streamUrl(path$1), {
|
|
1933
|
+
method: `PUT`,
|
|
1934
|
+
headers: await this.requestHeaders(headers)
|
|
1935
|
+
});
|
|
1936
|
+
if (response.ok) return;
|
|
1937
|
+
throw new Error(`Stream fork failed: ${response.status} ${await response.text()}`);
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
async append(path$1, data, opts) {
|
|
1941
|
+
return await withSpan(`stream.append`, async (span) => {
|
|
1942
|
+
span.setAttributes({
|
|
1943
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
1944
|
+
[ATTR.STREAM_OP]: opts?.close ? `append+close` : `append`
|
|
1945
|
+
});
|
|
1946
|
+
const handle = new DurableStream({
|
|
1947
|
+
url: this.streamUrl(path$1),
|
|
1948
|
+
headers: this.streamHeaders(),
|
|
1949
|
+
contentType: `application/json`,
|
|
1950
|
+
batching: false
|
|
1951
|
+
});
|
|
1952
|
+
if (opts?.close) {
|
|
1953
|
+
const result = await handle.close({ body: data });
|
|
1954
|
+
return { offset: result.finalOffset };
|
|
1955
|
+
}
|
|
1956
|
+
await handle.append(data);
|
|
1957
|
+
const head = await handle.head();
|
|
1958
|
+
return { offset: head.exists && head.offset || `` };
|
|
1959
|
+
});
|
|
1960
|
+
}
|
|
1961
|
+
async appendIdempotent(path$1, data, opts) {
|
|
1962
|
+
return await withSpan(`stream.appendIdempotent`, async (span) => {
|
|
1963
|
+
span.setAttributes({
|
|
1964
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
1965
|
+
[ATTR.STREAM_OP]: `appendIdempotent`
|
|
1966
|
+
});
|
|
1967
|
+
const stream = new DurableStream({
|
|
1968
|
+
url: this.streamUrl(path$1),
|
|
1969
|
+
headers: this.streamHeaders(),
|
|
1970
|
+
contentType: `application/json`
|
|
1971
|
+
});
|
|
1972
|
+
const producer = new IdempotentProducer(stream, opts.producerId, { epoch: opts.epoch ?? 0 });
|
|
1973
|
+
try {
|
|
1974
|
+
producer.append(data);
|
|
1975
|
+
await producer.flush();
|
|
1976
|
+
} finally {
|
|
1977
|
+
await producer.detach();
|
|
1978
|
+
}
|
|
1979
|
+
});
|
|
1980
|
+
}
|
|
1981
|
+
async appendWithProducerHeaders(path$1, data, opts) {
|
|
1982
|
+
return await withSpan(`stream.appendWithProducerHeaders`, async (span) => {
|
|
1983
|
+
span.setAttributes({
|
|
1984
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
1985
|
+
[ATTR.STREAM_OP]: `appendWithProducerHeaders`
|
|
1986
|
+
});
|
|
1987
|
+
const headers = {
|
|
1988
|
+
"content-type": `application/json`,
|
|
1989
|
+
"Producer-Id": opts.producerId,
|
|
1990
|
+
"Producer-Epoch": String(opts.epoch),
|
|
1991
|
+
"Producer-Seq": String(opts.seq)
|
|
1992
|
+
};
|
|
1993
|
+
injectTraceHeaders(headers);
|
|
1994
|
+
const response = await fetch(this.streamUrl(path$1), {
|
|
1995
|
+
method: `POST`,
|
|
1996
|
+
headers: await this.requestHeaders(headers),
|
|
1997
|
+
body: typeof data === `string` ? data : Buffer.from(data)
|
|
1998
|
+
});
|
|
1999
|
+
if (response.ok || response.status === 204) return;
|
|
2000
|
+
throw new Error(`Stream append failed: ${response.status} ${await response.text()}`);
|
|
2001
|
+
});
|
|
2002
|
+
}
|
|
2003
|
+
async read(path$1, fromOffset) {
|
|
2004
|
+
return await withSpan(`stream.read`, async (span) => {
|
|
2005
|
+
span.setAttributes({
|
|
2006
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
2007
|
+
[ATTR.STREAM_OP]: `read`
|
|
2008
|
+
});
|
|
2009
|
+
const handle = new DurableStream({
|
|
2010
|
+
url: this.streamUrl(path$1),
|
|
2011
|
+
headers: this.streamHeaders()
|
|
2012
|
+
});
|
|
2013
|
+
const response = await handle.stream({
|
|
2014
|
+
offset: fromOffset ?? `-1`,
|
|
2015
|
+
live: false
|
|
2016
|
+
});
|
|
2017
|
+
const messages = [];
|
|
2018
|
+
return await new Promise((resolve$1, reject) => {
|
|
2019
|
+
let settled = false;
|
|
2020
|
+
let unsub = () => {};
|
|
2021
|
+
const finish = (r) => {
|
|
2022
|
+
if (settled) return;
|
|
2023
|
+
settled = true;
|
|
2024
|
+
unsub();
|
|
2025
|
+
resolve$1(r);
|
|
2026
|
+
};
|
|
2027
|
+
unsub = response.subscribeBytes((chunk) => {
|
|
2028
|
+
messages.push({
|
|
2029
|
+
data: chunk.data,
|
|
2030
|
+
offset: chunk.offset
|
|
2031
|
+
});
|
|
2032
|
+
if (chunk.upToDate || chunk.streamClosed) finish({ messages });
|
|
2033
|
+
});
|
|
2034
|
+
response.closed.then(() => finish({ messages })).catch((err) => {
|
|
2035
|
+
if (settled) return;
|
|
2036
|
+
settled = true;
|
|
2037
|
+
unsub();
|
|
2038
|
+
reject(err);
|
|
2039
|
+
});
|
|
2040
|
+
});
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
async readJson(path$1, fromOffset) {
|
|
2044
|
+
return await withSpan(`stream.readJson`, async (span) => {
|
|
2045
|
+
span.setAttributes({
|
|
2046
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
2047
|
+
[ATTR.STREAM_OP]: `readJson`
|
|
2048
|
+
});
|
|
2049
|
+
const handle = new DurableStream({
|
|
2050
|
+
url: this.streamUrl(path$1),
|
|
2051
|
+
headers: this.streamHeaders()
|
|
2052
|
+
});
|
|
2053
|
+
const response = await handle.stream({
|
|
2054
|
+
offset: fromOffset ?? `-1`,
|
|
2055
|
+
live: false
|
|
2056
|
+
});
|
|
2057
|
+
return await response.json();
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
async waitForMessages(path$1, fromOffset, timeoutMs) {
|
|
2061
|
+
return await withSpan(`stream.waitForMessages`, async (span) => {
|
|
2062
|
+
span.setAttributes({
|
|
2063
|
+
[ATTR.STREAM_PATH]: path$1,
|
|
2064
|
+
[ATTR.STREAM_OP]: `waitForMessages`
|
|
2065
|
+
});
|
|
2066
|
+
const handle = new DurableStream({
|
|
2067
|
+
url: this.streamUrl(path$1),
|
|
2068
|
+
headers: this.streamHeaders()
|
|
2069
|
+
});
|
|
2070
|
+
const controller = new AbortController();
|
|
2071
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
2072
|
+
try {
|
|
2073
|
+
const response = await handle.stream({
|
|
2074
|
+
offset: fromOffset,
|
|
2075
|
+
live: `long-poll`,
|
|
2076
|
+
signal: controller.signal
|
|
2077
|
+
});
|
|
2078
|
+
const messages = [];
|
|
2079
|
+
return await new Promise((resolve$1, reject) => {
|
|
2080
|
+
let settled = false;
|
|
2081
|
+
let unsub = () => {};
|
|
2082
|
+
const finish = (result) => {
|
|
2083
|
+
if (settled) return;
|
|
2084
|
+
settled = true;
|
|
2085
|
+
clearTimeout(timer);
|
|
2086
|
+
unsub();
|
|
2087
|
+
resolve$1(result);
|
|
2088
|
+
};
|
|
2089
|
+
unsub = response.subscribeBytes((chunk) => {
|
|
2090
|
+
messages.push({
|
|
2091
|
+
data: chunk.data,
|
|
2092
|
+
offset: chunk.offset
|
|
2093
|
+
});
|
|
2094
|
+
if (chunk.upToDate) finish({
|
|
2095
|
+
messages,
|
|
2096
|
+
timedOut: false
|
|
2097
|
+
});
|
|
2098
|
+
});
|
|
2099
|
+
response.closed.then(() => finish({
|
|
2100
|
+
messages,
|
|
2101
|
+
timedOut: false
|
|
2102
|
+
})).catch((err) => {
|
|
2103
|
+
if (settled) return;
|
|
2104
|
+
clearTimeout(timer);
|
|
2105
|
+
if (isAbortLikeError(err)) {
|
|
2106
|
+
settled = true;
|
|
2107
|
+
unsub();
|
|
2108
|
+
resolve$1({
|
|
2109
|
+
messages: [],
|
|
2110
|
+
timedOut: true
|
|
2111
|
+
});
|
|
2112
|
+
return;
|
|
2113
|
+
}
|
|
2114
|
+
settled = true;
|
|
2115
|
+
unsub();
|
|
2116
|
+
reject(err);
|
|
2117
|
+
});
|
|
2118
|
+
});
|
|
2119
|
+
} catch (err) {
|
|
2120
|
+
clearTimeout(timer);
|
|
2121
|
+
if (isAbortLikeError(err)) return {
|
|
2122
|
+
messages: [],
|
|
2123
|
+
timedOut: true
|
|
2124
|
+
};
|
|
2125
|
+
throw err;
|
|
2126
|
+
}
|
|
2127
|
+
});
|
|
2128
|
+
}
|
|
2129
|
+
async delete(path$1) {
|
|
2130
|
+
await DurableStream.delete({
|
|
2131
|
+
url: this.streamUrl(path$1),
|
|
2132
|
+
headers: this.streamHeaders()
|
|
2133
|
+
});
|
|
2134
|
+
}
|
|
2135
|
+
async ensure(path$1, opts) {
|
|
2136
|
+
if (await this.exists(path$1)) return;
|
|
2137
|
+
try {
|
|
2138
|
+
await this.create(path$1, opts);
|
|
2139
|
+
} catch (err) {
|
|
2140
|
+
if (err && typeof err === `object` && `status` in err && err.status === 409) return;
|
|
2141
|
+
throw err;
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
async exists(path$1) {
|
|
2145
|
+
try {
|
|
2146
|
+
const result = await DurableStream.head({
|
|
2147
|
+
url: this.streamUrl(path$1),
|
|
2148
|
+
headers: this.streamHeaders()
|
|
2149
|
+
});
|
|
2150
|
+
return result.exists;
|
|
2151
|
+
} catch (err) {
|
|
2152
|
+
if (isNotFoundError(err)) return false;
|
|
2153
|
+
throw err;
|
|
2154
|
+
}
|
|
2155
|
+
}
|
|
2156
|
+
async createSubscription(pattern, subscriptionId, webhookUrl, description) {
|
|
2157
|
+
const res = await this.putSubscription(subscriptionId, {
|
|
2158
|
+
type: `webhook`,
|
|
2159
|
+
pattern: normalizeSubscriptionPattern(pattern),
|
|
2160
|
+
webhook: { url: webhookUrl },
|
|
2161
|
+
...description ? { description } : {}
|
|
2162
|
+
});
|
|
2163
|
+
return res;
|
|
2164
|
+
}
|
|
2165
|
+
async putSubscription(subscriptionId, input) {
|
|
2166
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2167
|
+
method: `PUT`,
|
|
2168
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2169
|
+
body: JSON.stringify({
|
|
2170
|
+
...input,
|
|
2171
|
+
pattern: typeof input.pattern === `string` ? this.backendSubscriptionPath(normalizeSubscriptionPattern(input.pattern)) : void 0,
|
|
2172
|
+
streams: input.streams?.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))),
|
|
2173
|
+
wake_stream: typeof input.wake_stream === `string` ? this.backendSubscriptionPath(normalizeSubscriptionStreamPath(input.wake_stream)) : void 0
|
|
2174
|
+
})
|
|
2175
|
+
});
|
|
2176
|
+
return await this.subscriptionJson(res, `Subscription creation failed`);
|
|
2177
|
+
}
|
|
2178
|
+
async getSubscription(subscriptionId) {
|
|
2179
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2180
|
+
method: `GET`,
|
|
2181
|
+
headers: await this.requestHeaders()
|
|
2182
|
+
});
|
|
2183
|
+
if (res.status === 404) return null;
|
|
2184
|
+
return await this.subscriptionJson(res, `Subscription query failed`);
|
|
2185
|
+
}
|
|
2186
|
+
async deleteSubscription(subscriptionId) {
|
|
2187
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2188
|
+
method: `DELETE`,
|
|
2189
|
+
headers: await this.requestHeaders()
|
|
2190
|
+
});
|
|
2191
|
+
if (res.status === 404 || res.status === 204) return;
|
|
2192
|
+
if (!res.ok) throw new Error(`Subscription delete failed: ${res.status} ${await res.text()}`);
|
|
2193
|
+
}
|
|
2194
|
+
async addSubscriptionStreams(subscriptionId, streams$1) {
|
|
2195
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `streams`), {
|
|
2196
|
+
method: `POST`,
|
|
2197
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2198
|
+
body: JSON.stringify({ streams: streams$1.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))) })
|
|
2199
|
+
});
|
|
2200
|
+
return await this.subscriptionJson(res, `Subscription stream add failed`);
|
|
2201
|
+
}
|
|
2202
|
+
async removeSubscriptionStream(subscriptionId, streamPath) {
|
|
2203
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `streams`, this.backendSubscriptionPath(normalizeSubscriptionStreamPath(streamPath))), {
|
|
2204
|
+
method: `DELETE`,
|
|
2205
|
+
headers: await this.requestHeaders()
|
|
2206
|
+
});
|
|
2207
|
+
if (res.status === 404 || res.status === 204) return;
|
|
2208
|
+
if (!res.ok) throw new Error(`Subscription stream remove failed: ${res.status} ${await res.text()}`);
|
|
2209
|
+
}
|
|
2210
|
+
async claimSubscription(subscriptionId, worker) {
|
|
2211
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `claim`), {
|
|
2212
|
+
method: `POST`,
|
|
2213
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2214
|
+
body: JSON.stringify({ worker })
|
|
2215
|
+
});
|
|
2216
|
+
if (res.status === 204 || res.status === 404) return null;
|
|
2217
|
+
return await this.subscriptionJson(res, `Subscription claim failed`);
|
|
2218
|
+
}
|
|
2219
|
+
async ackSubscription(subscriptionId, token, body) {
|
|
2220
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `ack`), {
|
|
2221
|
+
method: `POST`,
|
|
2222
|
+
headers: await this.requestHeaders({
|
|
2223
|
+
"content-type": `application/json`,
|
|
2224
|
+
authorization: `Bearer ${token}`
|
|
2225
|
+
}),
|
|
2226
|
+
body: JSON.stringify(this.subscriptionRequestBody(body))
|
|
2227
|
+
});
|
|
2228
|
+
return await this.subscriptionJson(res, `Subscription ack failed`);
|
|
2229
|
+
}
|
|
2230
|
+
async releaseSubscription(subscriptionId, token, body) {
|
|
2231
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `release`), {
|
|
2232
|
+
method: `POST`,
|
|
2233
|
+
headers: await this.requestHeaders({
|
|
2234
|
+
"content-type": `application/json`,
|
|
2235
|
+
authorization: `Bearer ${token}`
|
|
2236
|
+
}),
|
|
2237
|
+
body: JSON.stringify(this.subscriptionRequestBody(body))
|
|
2238
|
+
});
|
|
2239
|
+
return await this.subscriptionJson(res, `Subscription release failed`);
|
|
2240
|
+
}
|
|
2241
|
+
subscriptionRequestBody(body) {
|
|
2242
|
+
const next = { ...body };
|
|
2243
|
+
if (typeof next.stream === `string`) next.stream = this.backendSubscriptionPath(next.stream);
|
|
2244
|
+
if (typeof next.path === `string`) next.path = this.backendSubscriptionPath(next.path);
|
|
2245
|
+
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
2246
|
+
if (!ack || typeof ack !== `object`) return ack;
|
|
2247
|
+
const mapped = { ...ack };
|
|
2248
|
+
if (typeof mapped.stream === `string`) mapped.stream = this.backendSubscriptionPath(mapped.stream);
|
|
2249
|
+
if (typeof mapped.path === `string`) mapped.path = this.backendSubscriptionPath(mapped.path);
|
|
2250
|
+
return mapped;
|
|
2251
|
+
});
|
|
2252
|
+
return next;
|
|
2253
|
+
}
|
|
2254
|
+
subscriptionResponseBody(body) {
|
|
2255
|
+
const next = { ...body };
|
|
2256
|
+
if (typeof next.pattern === `string`) next.pattern = this.runtimeSubscriptionPath(next.pattern);
|
|
2257
|
+
if (typeof next.wake_stream === `string`) next.wake_stream = this.runtimeSubscriptionPath(next.wake_stream);
|
|
2258
|
+
if (Array.isArray(next.streams)) next.streams = next.streams.map((stream) => {
|
|
2259
|
+
if (typeof stream === `string`) return this.runtimeSubscriptionPath(stream);
|
|
2260
|
+
return {
|
|
2261
|
+
...stream,
|
|
2262
|
+
path: this.runtimeSubscriptionPath(stream.path)
|
|
2263
|
+
};
|
|
2264
|
+
});
|
|
2265
|
+
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
2266
|
+
if (!ack || typeof ack !== `object`) return ack;
|
|
2267
|
+
const mapped = { ...ack };
|
|
2268
|
+
if (typeof mapped.stream === `string`) mapped.stream = this.runtimeSubscriptionPath(mapped.stream);
|
|
2269
|
+
if (typeof mapped.path === `string`) mapped.path = this.runtimeSubscriptionPath(mapped.path);
|
|
2270
|
+
return mapped;
|
|
2271
|
+
});
|
|
2272
|
+
if (typeof next.stream === `string`) next.stream = this.runtimeSubscriptionPath(next.stream);
|
|
2273
|
+
return next;
|
|
2274
|
+
}
|
|
2275
|
+
async subscriptionJson(res, message) {
|
|
2276
|
+
if (!res.ok) throw new DurableStreamsSubscriptionError(message, res.status, await res.text());
|
|
2277
|
+
if (res.status === 204) return {};
|
|
2278
|
+
const text$1 = await res.text();
|
|
2279
|
+
if (!text$1.trim()) return {};
|
|
2280
|
+
return this.subscriptionResponseBody(JSON.parse(text$1));
|
|
2281
|
+
}
|
|
2282
|
+
};
|
|
2283
|
+
|
|
2284
|
+
//#endregion
|
|
2285
|
+
//#region src/utils/webhook-url.ts
|
|
2286
|
+
function rewriteLoopbackWebhookUrl(value) {
|
|
2287
|
+
if (!value) return void 0;
|
|
2288
|
+
const rewriteTarget = process.env.ELECTRIC_AGENTS_REWRITE_LOOPBACK_WEBHOOKS_TO?.trim();
|
|
2289
|
+
if (!rewriteTarget) return value;
|
|
2290
|
+
const url = new URL(value);
|
|
2291
|
+
if (!isLoopbackHostname(url.hostname)) return value;
|
|
2292
|
+
if (rewriteTarget.includes(`://`)) {
|
|
2293
|
+
const target = new URL(rewriteTarget);
|
|
2294
|
+
url.protocol = target.protocol;
|
|
2295
|
+
url.username = target.username;
|
|
2296
|
+
url.password = target.password;
|
|
2297
|
+
url.hostname = target.hostname;
|
|
2298
|
+
url.port = target.port;
|
|
2299
|
+
return url.toString();
|
|
2300
|
+
}
|
|
2301
|
+
url.host = rewriteTarget;
|
|
2302
|
+
return url.toString();
|
|
2303
|
+
}
|
|
2304
|
+
function isLoopbackHostname(hostname) {
|
|
2305
|
+
return hostname === `localhost` || hostname === `127.0.0.1` || hostname === `::1`;
|
|
2306
|
+
}
|
|
2307
|
+
|
|
2308
|
+
//#endregion
|
|
2309
|
+
//#region src/routing/dispatch-policy.ts
|
|
2310
|
+
const linkedDispatchSubscriptions = new WeakMap();
|
|
2311
|
+
function subscriptionIdForDispatchTarget(target) {
|
|
2312
|
+
if (target.subscription_id) return target.subscription_id;
|
|
2313
|
+
if (target.type === `runner`) return `runner:${target.runnerId}`;
|
|
2314
|
+
const digest = createHash(`sha256`).update(target.url).digest(`hex`);
|
|
2315
|
+
return `webhook:${digest.slice(0, 16)}`;
|
|
2316
|
+
}
|
|
2317
|
+
function subscriptionIdForEntityDispatchTarget(target, entityUrl) {
|
|
2318
|
+
const base = subscriptionIdForDispatchTarget(target);
|
|
2319
|
+
if (!target.subscription_id && target.type !== `runner`) return base;
|
|
2320
|
+
const digest = createHash(`sha256`).update(entityUrl).digest(`hex`);
|
|
2321
|
+
return `${base}:${digest.slice(0, 16)}`;
|
|
2322
|
+
}
|
|
2323
|
+
async function resolveEffectiveDispatchPolicyForSpawn(ctx, typeName, opts) {
|
|
2324
|
+
if (opts.dispatchPolicy) return opts.dispatchPolicy;
|
|
2325
|
+
const entityType = await ctx.entityManager.registry.getEntityType(typeName);
|
|
2326
|
+
if (opts.parent) {
|
|
2327
|
+
const parent = await ctx.entityManager.registry.getEntity(opts.parent);
|
|
2328
|
+
if (parent?.dispatch_policy) return applyTypeDefaultSubscriptionScope(parent.dispatch_policy, entityType?.default_dispatch_policy);
|
|
2329
|
+
}
|
|
2330
|
+
return entityType?.default_dispatch_policy;
|
|
2331
|
+
}
|
|
2332
|
+
async function resolveEffectiveDispatchPolicyForEntity(ctx, entity) {
|
|
2333
|
+
if (entity.dispatch_policy) return entity.dispatch_policy;
|
|
2334
|
+
const entityType = await ctx.entityManager.registry.getEntityType(entity.type);
|
|
2335
|
+
return entityType?.default_dispatch_policy;
|
|
2336
|
+
}
|
|
2337
|
+
async function backfillEntityDispatchPolicy(ctx, entity) {
|
|
2338
|
+
if (entity.dispatch_policy) return entity;
|
|
2339
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2340
|
+
if (!dispatchPolicy) return entity;
|
|
2341
|
+
return await ctx.entityManager.registry.updateEntityDispatchPolicy(entity.url, dispatchPolicy) ?? {
|
|
2342
|
+
...entity,
|
|
2343
|
+
dispatch_policy: dispatchPolicy
|
|
2344
|
+
};
|
|
2345
|
+
}
|
|
2346
|
+
function applyTypeDefaultSubscriptionScope(policy, typeDefault) {
|
|
2347
|
+
const target = policy.targets[0];
|
|
2348
|
+
const defaultTarget = typeDefault?.targets[0];
|
|
2349
|
+
if (!target || !defaultTarget?.subscription_id) return policy;
|
|
2350
|
+
if (!sameDispatchDestination(target, defaultTarget)) return policy;
|
|
2351
|
+
if (target.subscription_id === defaultTarget.subscription_id) return policy;
|
|
2352
|
+
return { targets: [{
|
|
2353
|
+
...target,
|
|
2354
|
+
subscription_id: defaultTarget.subscription_id
|
|
2355
|
+
}] };
|
|
2356
|
+
}
|
|
2357
|
+
function sameDispatchDestination(a, b) {
|
|
2358
|
+
if (a.type !== b.type) return false;
|
|
2359
|
+
if (a.type === `runner` && b.type === `runner`) return a.runnerId === b.runnerId;
|
|
2360
|
+
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url;
|
|
2361
|
+
return false;
|
|
2362
|
+
}
|
|
2363
|
+
function subscriptionHasStream(ctx, existing, streamPath) {
|
|
2364
|
+
const normalizedStream = streamPath.replace(/^\/+/, ``);
|
|
2365
|
+
const backendStream = `${ctx.service}/${normalizedStream}`;
|
|
2366
|
+
return existing.streams?.some((stream) => {
|
|
2367
|
+
const path$1 = typeof stream === `string` ? stream : stream.path;
|
|
2368
|
+
if (!path$1) return false;
|
|
2369
|
+
const normalized = path$1.replace(/^\/+/, ``);
|
|
2370
|
+
return normalized === normalizedStream || normalized === backendStream;
|
|
2371
|
+
}) ?? false;
|
|
2372
|
+
}
|
|
2373
|
+
function dispatchLinkCacheKey(ctx, subscriptionId, streamPath) {
|
|
2374
|
+
return `${ctx.service}:${subscriptionId}:${streamPath}`;
|
|
2375
|
+
}
|
|
2376
|
+
function getDispatchLinkCache(ctx) {
|
|
2377
|
+
let cache = linkedDispatchSubscriptions.get(ctx.streamClient);
|
|
2378
|
+
if (!cache) {
|
|
2379
|
+
cache = new Set();
|
|
2380
|
+
linkedDispatchSubscriptions.set(ctx.streamClient, cache);
|
|
2381
|
+
}
|
|
2382
|
+
return cache;
|
|
2383
|
+
}
|
|
2384
|
+
function isSubscriptionAlreadyExistsError(err) {
|
|
2385
|
+
if (!(err instanceof DurableStreamsSubscriptionError)) return false;
|
|
2386
|
+
if (err.status === 409) return true;
|
|
2387
|
+
return err.code === `SUBSCRIPTION_ALREADY_EXISTS` || err.code === `ALREADY_EXISTS` || /already exists/i.test(err.errorMessage ?? err.body ?? err.message);
|
|
2388
|
+
}
|
|
2389
|
+
async function ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, input, existing) {
|
|
2390
|
+
if (!existing) try {
|
|
2391
|
+
await ctx.streamClient.putSubscription(subscriptionId, input);
|
|
2392
|
+
return;
|
|
2393
|
+
} catch (err) {
|
|
2394
|
+
if (!isSubscriptionAlreadyExistsError(err)) throw err;
|
|
2395
|
+
existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
2396
|
+
if (!existing) {
|
|
2397
|
+
serverLog.warn(`[dispatch-policy] subscription create raced with existing subscription but it could not be read`, {
|
|
2398
|
+
subscriptionId,
|
|
2399
|
+
stream: streamPath
|
|
2400
|
+
});
|
|
2401
|
+
return;
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
if (!subscriptionHasStream(ctx, existing, streamPath)) await ctx.streamClient.addSubscriptionStreams(subscriptionId, [streamPath]);
|
|
2405
|
+
}
|
|
2406
|
+
async function assertDispatchPolicyAllowed(ctx, policy) {
|
|
2407
|
+
const target = policy?.targets[0];
|
|
2408
|
+
if (!target || target.type !== `runner`) return;
|
|
2409
|
+
if (!ctx.principal) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires an authenticated owner`, 401);
|
|
2410
|
+
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
2411
|
+
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
2412
|
+
if (runner.owner_principal !== ctx.principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
2413
|
+
}
|
|
2414
|
+
async function linkEntityDispatchSubscription(ctx, entity) {
|
|
2415
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2416
|
+
const target = dispatchPolicy?.targets[0];
|
|
2417
|
+
if (!target) return;
|
|
2418
|
+
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
2419
|
+
const cacheKey = dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main);
|
|
2420
|
+
const cache = getDispatchLinkCache(ctx);
|
|
2421
|
+
if (cache.has(cacheKey)) return;
|
|
2422
|
+
await linkStreamToTargetSubscription(ctx, target, entity, subscriptionId);
|
|
2423
|
+
cache.add(cacheKey);
|
|
2424
|
+
}
|
|
2425
|
+
async function unlinkEntityDispatchSubscription(ctx, entity) {
|
|
2426
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2427
|
+
const target = dispatchPolicy?.targets[0];
|
|
2428
|
+
if (!target) return;
|
|
2429
|
+
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
2430
|
+
getDispatchLinkCache(ctx).delete(dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main));
|
|
2431
|
+
await ctx.streamClient.removeSubscriptionStream(subscriptionId, entity.streams.main).catch((err) => {
|
|
2432
|
+
serverLog.warn(`[dispatch-policy] failed to remove stream from subscription`, {
|
|
2433
|
+
subscriptionId,
|
|
2434
|
+
stream: entity.streams.main
|
|
2435
|
+
}, err);
|
|
2436
|
+
});
|
|
2437
|
+
}
|
|
2438
|
+
async function linkStreamToTargetSubscription(ctx, target, entity, subscriptionId) {
|
|
2439
|
+
const streamPath = entity.streams.main;
|
|
2440
|
+
await ctx.streamClient.ensure(streamPath, { contentType: `application/json` });
|
|
2441
|
+
const existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
2442
|
+
if (target.type === `runner`) {
|
|
2443
|
+
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
2444
|
+
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
2445
|
+
const wakeStream = runner.wake_stream || runnerWakeStream(target.runnerId);
|
|
2446
|
+
await ctx.streamClient.ensure(wakeStream, { contentType: `application/json` });
|
|
2447
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
2448
|
+
type: `pull-wake`,
|
|
2449
|
+
streams: [streamPath],
|
|
2450
|
+
wake_stream: wakeStream,
|
|
2451
|
+
description: `Electric Agents runner ${target.runnerId}`
|
|
2452
|
+
}, existing);
|
|
2453
|
+
return;
|
|
2454
|
+
}
|
|
2455
|
+
const webhookUrl = rewriteLoopbackWebhookUrl(target.url);
|
|
2456
|
+
if (!webhookUrl) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Webhook dispatch target must include a valid URL`, 400);
|
|
2457
|
+
const forwardUrl = appendPathToUrl(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
2458
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
2459
|
+
type: `webhook`,
|
|
2460
|
+
streams: [streamPath],
|
|
2461
|
+
webhook: { url: forwardUrl },
|
|
2462
|
+
description: `Electric Agents webhook ${subscriptionId}`
|
|
2463
|
+
}, existing);
|
|
2464
|
+
await ctx.pgDb.insert(subscriptionWebhooks).values({
|
|
1849
2465
|
tenantId: ctx.service,
|
|
1850
2466
|
subscriptionId,
|
|
1851
2467
|
webhookUrl
|
|
@@ -1857,6 +2473,7 @@ async function linkStreamToTargetSubscription(ctx, target, entity) {
|
|
|
1857
2473
|
|
|
1858
2474
|
//#endregion
|
|
1859
2475
|
//#region src/principal.ts
|
|
2476
|
+
const ELECTRIC_PRINCIPAL_HEADER = `electric-principal`;
|
|
1860
2477
|
const PRINCIPAL_KINDS = new Set([
|
|
1861
2478
|
`user`,
|
|
1862
2479
|
`agent`,
|
|
@@ -1865,7 +2482,7 @@ const PRINCIPAL_KINDS = new Set([
|
|
|
1865
2482
|
]);
|
|
1866
2483
|
function parsePrincipalKey(input) {
|
|
1867
2484
|
const colon = input.indexOf(`:`);
|
|
1868
|
-
if (colon <= 0) throw new Error(`Invalid principal
|
|
2485
|
+
if (colon <= 0) throw new Error(`Invalid principal identifier`);
|
|
1869
2486
|
const kind = input.slice(0, colon);
|
|
1870
2487
|
const id = input.slice(colon + 1);
|
|
1871
2488
|
if (!PRINCIPAL_KINDS.has(kind)) throw new Error(`Invalid principal kind`);
|
|
@@ -1881,13 +2498,12 @@ function parsePrincipalKey(input) {
|
|
|
1881
2498
|
function principalUrl(key) {
|
|
1882
2499
|
return parsePrincipalKey(key).url;
|
|
1883
2500
|
}
|
|
1884
|
-
function
|
|
2501
|
+
function parsePrincipalUrl(url) {
|
|
1885
2502
|
if (!url.startsWith(`/principal/`)) return null;
|
|
1886
2503
|
const segment = url.slice(`/principal/`.length);
|
|
1887
2504
|
if (!segment || segment.includes(`/`)) return null;
|
|
1888
2505
|
try {
|
|
1889
|
-
|
|
1890
|
-
return parsePrincipalKey(key).key;
|
|
2506
|
+
return parsePrincipalKey(decodeURIComponent(segment));
|
|
1891
2507
|
} catch {
|
|
1892
2508
|
return null;
|
|
1893
2509
|
}
|
|
@@ -1900,9 +2516,8 @@ const BUILT_IN_SYSTEM_PRINCIPAL_IDS = new Set([
|
|
|
1900
2516
|
function isBuiltInSystemPrincipalUrl(url) {
|
|
1901
2517
|
if (!url?.startsWith(`/principal/`)) return false;
|
|
1902
2518
|
try {
|
|
1903
|
-
const
|
|
1904
|
-
if (!
|
|
1905
|
-
const principal = parsePrincipalKey(key);
|
|
2519
|
+
const principal = parsePrincipalUrl(url);
|
|
2520
|
+
if (!principal) return false;
|
|
1906
2521
|
return principal.kind === `system` && BUILT_IN_SYSTEM_PRINCIPAL_IDS.has(principal.id);
|
|
1907
2522
|
} catch {
|
|
1908
2523
|
return false;
|
|
@@ -1910,12 +2525,11 @@ function isBuiltInSystemPrincipalUrl(url) {
|
|
|
1910
2525
|
}
|
|
1911
2526
|
function principalFromCreatedBy(createdBy) {
|
|
1912
2527
|
if (!createdBy) return void 0;
|
|
1913
|
-
const
|
|
1914
|
-
if (!
|
|
2528
|
+
const principal = parsePrincipalUrl(createdBy);
|
|
2529
|
+
if (!principal) return {
|
|
1915
2530
|
url: createdBy,
|
|
1916
2531
|
key: null
|
|
1917
2532
|
};
|
|
1918
|
-
const principal = parsePrincipalKey(key);
|
|
1919
2533
|
return {
|
|
1920
2534
|
url: principal.url,
|
|
1921
2535
|
key: principal.key,
|
|
@@ -1993,70 +2607,26 @@ function buildManifestWakeRegistration(subscriberUrl, manifest, manifestKey) {
|
|
|
1993
2607
|
subscriberUrl,
|
|
1994
2608
|
sourceUrl,
|
|
1995
2609
|
condition: `runFinished`,
|
|
1996
|
-
oneShot: false,
|
|
1997
|
-
includeResponse: typeof wake.includeResponse === `boolean` ? wake.includeResponse : void 0,
|
|
1998
|
-
manifestKey
|
|
1999
|
-
};
|
|
2000
|
-
if (wake.on !== `change`) return null;
|
|
2001
|
-
const collections = Array.isArray(wake.collections) ? wake.collections.filter((c) => typeof c === `string`) : void 0;
|
|
2002
|
-
const ops = Array.isArray(wake.ops) ? wake.ops.filter((op) => op === `insert` || op === `update` || op === `delete`) : void 0;
|
|
2003
|
-
return {
|
|
2004
|
-
subscriberUrl,
|
|
2005
|
-
sourceUrl,
|
|
2006
|
-
condition: {
|
|
2007
|
-
on: `change`,
|
|
2008
|
-
...collections ? { collections } : {},
|
|
2009
|
-
...ops ? { ops } : {}
|
|
2010
|
-
},
|
|
2011
|
-
debounceMs: typeof wake.debounceMs === `number` ? wake.debounceMs : void 0,
|
|
2012
|
-
timeoutMs: typeof wake.timeoutMs === `number` ? wake.timeoutMs : void 0,
|
|
2013
|
-
oneShot: false,
|
|
2014
|
-
manifestKey
|
|
2015
|
-
};
|
|
2016
|
-
}
|
|
2017
|
-
|
|
2018
|
-
//#endregion
|
|
2019
|
-
//#region src/tracing.ts
|
|
2020
|
-
const tracer = trace.getTracer(`agent-server`);
|
|
2021
|
-
const ATTR = {
|
|
2022
|
-
ENTITY_URL: `electric_agents.entity.url`,
|
|
2023
|
-
ENTITY_TYPE: `electric_agents.entity.type`,
|
|
2024
|
-
PARENT_URL: `electric_agents.entity.parent`,
|
|
2025
|
-
WAKE_SOURCE: `electric_agents.wake.source`,
|
|
2026
|
-
WAKE_SUBSCRIBER: `electric_agents.wake.subscriber`,
|
|
2027
|
-
WAKE_KIND: `electric_agents.wake.kind`,
|
|
2028
|
-
STREAM_PATH: `electric_agents.stream.path`,
|
|
2029
|
-
STREAM_OP: `electric_agents.stream.op`,
|
|
2030
|
-
DB_OP: `electric_agents.db.op`,
|
|
2031
|
-
HTTP_METHOD: `http.method`,
|
|
2032
|
-
HTTP_ROUTE: `http.route`,
|
|
2033
|
-
HTTP_STATUS: `http.status_code`
|
|
2034
|
-
};
|
|
2035
|
-
/**
|
|
2036
|
-
* Run `fn` inside an active span. Errors are recorded + status set to ERROR,
|
|
2037
|
-
* then re-thrown. Span ends in a finally block.
|
|
2038
|
-
*/
|
|
2039
|
-
async function withSpan(name, fn, opts) {
|
|
2040
|
-
return await tracer.startActiveSpan(name, opts ?? {}, async (span) => {
|
|
2041
|
-
try {
|
|
2042
|
-
return await fn(span);
|
|
2043
|
-
} catch (err) {
|
|
2044
|
-
span.recordException(err);
|
|
2045
|
-
span.setStatus({
|
|
2046
|
-
code: SpanStatusCode.ERROR,
|
|
2047
|
-
message: err instanceof Error ? err.message : String(err)
|
|
2048
|
-
});
|
|
2049
|
-
throw err;
|
|
2050
|
-
} finally {
|
|
2051
|
-
span.end();
|
|
2052
|
-
}
|
|
2053
|
-
});
|
|
2054
|
-
}
|
|
2055
|
-
function injectTraceHeaders(headers, ctx = context.active()) {
|
|
2056
|
-
propagation.inject(ctx, headers);
|
|
2057
|
-
}
|
|
2058
|
-
function extractTraceContext(headers) {
|
|
2059
|
-
return propagation.extract(context.active(), headers);
|
|
2610
|
+
oneShot: false,
|
|
2611
|
+
includeResponse: typeof wake.includeResponse === `boolean` ? wake.includeResponse : void 0,
|
|
2612
|
+
manifestKey
|
|
2613
|
+
};
|
|
2614
|
+
if (wake.on !== `change`) return null;
|
|
2615
|
+
const collections = Array.isArray(wake.collections) ? wake.collections.filter((c) => typeof c === `string`) : void 0;
|
|
2616
|
+
const ops = Array.isArray(wake.ops) ? wake.ops.filter((op) => op === `insert` || op === `update` || op === `delete`) : void 0;
|
|
2617
|
+
return {
|
|
2618
|
+
subscriberUrl,
|
|
2619
|
+
sourceUrl,
|
|
2620
|
+
condition: {
|
|
2621
|
+
on: `change`,
|
|
2622
|
+
...collections ? { collections } : {},
|
|
2623
|
+
...ops ? { ops } : {}
|
|
2624
|
+
},
|
|
2625
|
+
debounceMs: typeof wake.debounceMs === `number` ? wake.debounceMs : void 0,
|
|
2626
|
+
timeoutMs: typeof wake.timeoutMs === `number` ? wake.timeoutMs : void 0,
|
|
2627
|
+
oneShot: false,
|
|
2628
|
+
manifestKey
|
|
2629
|
+
};
|
|
2060
2630
|
}
|
|
2061
2631
|
|
|
2062
2632
|
//#endregion
|
|
@@ -3602,1036 +4172,544 @@ function isPlainObject(value) {
|
|
|
3602
4172
|
//#region src/scheduler.ts
|
|
3603
4173
|
const POSTGRES_TEXT_OID = 25;
|
|
3604
4174
|
var PostgresSchedulerClient = class {
|
|
3605
|
-
constructor(pgClient, tenantId, wake) {
|
|
3606
|
-
this.pgClient = pgClient;
|
|
3607
|
-
this.tenantId = tenantId;
|
|
3608
|
-
this.wake = wake;
|
|
3609
|
-
}
|
|
3610
|
-
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3611
|
-
await this.pgClient`
|
|
3612
|
-
insert into scheduled_tasks (
|
|
3613
|
-
tenant_id,
|
|
3614
|
-
kind,
|
|
3615
|
-
payload,
|
|
3616
|
-
fire_at,
|
|
3617
|
-
owner_entity_url,
|
|
3618
|
-
manifest_key
|
|
3619
|
-
)
|
|
3620
|
-
values (
|
|
3621
|
-
${this.tenantId},
|
|
3622
|
-
'delayed_send',
|
|
3623
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3624
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3625
|
-
${opts?.ownerEntityUrl ?? null},
|
|
3626
|
-
${opts?.manifestKey ?? null}
|
|
3627
|
-
)
|
|
3628
|
-
`;
|
|
3629
|
-
this.wake?.();
|
|
3630
|
-
}
|
|
3631
|
-
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
3632
|
-
await this.pgClient.begin(async (sql$1) => {
|
|
3633
|
-
await sql$1`
|
|
3634
|
-
update scheduled_tasks
|
|
3635
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3636
|
-
where tenant_id = ${this.tenantId}
|
|
3637
|
-
and kind = 'delayed_send'
|
|
3638
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3639
|
-
and manifest_key = ${manifestKey}
|
|
3640
|
-
and completed_at is null
|
|
3641
|
-
`;
|
|
3642
|
-
await sql$1`
|
|
3643
|
-
insert into scheduled_tasks (
|
|
3644
|
-
tenant_id,
|
|
3645
|
-
kind,
|
|
3646
|
-
payload,
|
|
3647
|
-
fire_at,
|
|
3648
|
-
owner_entity_url,
|
|
3649
|
-
manifest_key
|
|
3650
|
-
)
|
|
3651
|
-
values (
|
|
3652
|
-
${this.tenantId},
|
|
3653
|
-
'delayed_send',
|
|
3654
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3655
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3656
|
-
${ownerEntityUrl},
|
|
3657
|
-
${manifestKey}
|
|
3658
|
-
)
|
|
3659
|
-
`;
|
|
3660
|
-
});
|
|
3661
|
-
this.wake?.();
|
|
3662
|
-
}
|
|
3663
|
-
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
3664
|
-
await this.pgClient`
|
|
3665
|
-
update scheduled_tasks
|
|
3666
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3667
|
-
where tenant_id = ${this.tenantId}
|
|
3668
|
-
and kind = 'delayed_send'
|
|
3669
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3670
|
-
and manifest_key = ${manifestKey}
|
|
3671
|
-
and completed_at is null
|
|
3672
|
-
`;
|
|
3673
|
-
this.wake?.();
|
|
3674
|
-
}
|
|
3675
|
-
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
3676
|
-
await this.pgClient`
|
|
3677
|
-
insert into scheduled_tasks (
|
|
3678
|
-
tenant_id,
|
|
3679
|
-
kind,
|
|
3680
|
-
payload,
|
|
3681
|
-
fire_at,
|
|
3682
|
-
cron_expression,
|
|
3683
|
-
cron_timezone,
|
|
3684
|
-
cron_tick_number
|
|
3685
|
-
)
|
|
3686
|
-
values (
|
|
3687
|
-
${this.tenantId},
|
|
3688
|
-
'cron_tick',
|
|
3689
|
-
${JSON.stringify({ streamPath })}::jsonb,
|
|
3690
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3691
|
-
${expression},
|
|
3692
|
-
${timezone},
|
|
3693
|
-
${tickNumber}
|
|
3694
|
-
)
|
|
3695
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
3696
|
-
`;
|
|
3697
|
-
this.wake?.();
|
|
3698
|
-
}
|
|
3699
|
-
};
|
|
3700
|
-
function isPermanentElectricAgentsError(err) {
|
|
3701
|
-
const status$1 = typeof err === `object` && err !== null && `status` in err ? err.status : void 0;
|
|
3702
|
-
const name = typeof err === `object` && err !== null && `name` in err ? err.name : void 0;
|
|
3703
|
-
return name === `ElectricAgentsError` && typeof status$1 === `number` && status$1 >= 400 && status$1 < 500;
|
|
3704
|
-
}
|
|
3705
|
-
function normalizeTask(row) {
|
|
3706
|
-
return {
|
|
3707
|
-
id: Number(row.id),
|
|
3708
|
-
tenantId: row.tenant_id,
|
|
3709
|
-
kind: row.kind,
|
|
3710
|
-
payload: row.payload,
|
|
3711
|
-
fireAt: row.fire_at instanceof Date ? row.fire_at : new Date(row.fire_at),
|
|
3712
|
-
cronExpression: row.cron_expression,
|
|
3713
|
-
cronTimezone: row.cron_timezone,
|
|
3714
|
-
cronTickNumber: row.cron_tick_number,
|
|
3715
|
-
ownerEntityUrl: row.owner_entity_url,
|
|
3716
|
-
manifestKey: row.manifest_key
|
|
3717
|
-
};
|
|
3718
|
-
}
|
|
3719
|
-
var Scheduler = class {
|
|
3720
|
-
claimExpiryMs;
|
|
3721
|
-
safetyPollMs;
|
|
3722
|
-
listenEnabled;
|
|
3723
|
-
pgClient;
|
|
3724
|
-
instanceId;
|
|
3725
|
-
tenantId;
|
|
3726
|
-
tenantIds;
|
|
3727
|
-
running = false;
|
|
3728
|
-
loopPromise = null;
|
|
3729
|
-
currentSleepResolve = null;
|
|
3730
|
-
currentSleepTimer = null;
|
|
3731
|
-
listenerMeta = null;
|
|
3732
|
-
constructor(options) {
|
|
3733
|
-
this.options = options;
|
|
3734
|
-
this.pgClient = options.pgClient;
|
|
3735
|
-
this.instanceId = options.instanceId;
|
|
3736
|
-
this.tenantId = options.tenantId === void 0 ? DEFAULT_TENANT_ID : options.tenantId;
|
|
3737
|
-
this.tenantIds = options.tenantIds;
|
|
3738
|
-
this.claimExpiryMs = options.claimExpiryMs ?? 3e4;
|
|
3739
|
-
this.safetyPollMs = options.safetyPollMs ?? 1e4;
|
|
3740
|
-
this.listenEnabled = options.listen !== false;
|
|
3741
|
-
}
|
|
3742
|
-
resolveTenantId(tenantId) {
|
|
3743
|
-
if (tenantId) return tenantId;
|
|
3744
|
-
if (this.tenantId) return this.tenantId;
|
|
3745
|
-
throw new Error(`Scheduler tenantId is required in shared mode`);
|
|
3746
|
-
}
|
|
3747
|
-
async start() {
|
|
3748
|
-
if (this.running) return;
|
|
3749
|
-
this.running = true;
|
|
3750
|
-
if (this.listenEnabled) this.listenerMeta = await this.pgClient.listen(`scheduled_tasks_wake`, () => {
|
|
3751
|
-
this.wakeEarly();
|
|
3752
|
-
});
|
|
3753
|
-
this.loopPromise = this.runLoop().catch((err) => {
|
|
3754
|
-
console.error(`[agent-server] scheduler loop failed:`, err);
|
|
3755
|
-
this.running = false;
|
|
3756
|
-
this.wakeEarly();
|
|
3757
|
-
});
|
|
3758
|
-
}
|
|
3759
|
-
async stop() {
|
|
3760
|
-
this.running = false;
|
|
3761
|
-
this.wakeEarly();
|
|
3762
|
-
if (this.loopPromise) {
|
|
3763
|
-
await this.loopPromise;
|
|
3764
|
-
this.loopPromise = null;
|
|
3765
|
-
}
|
|
3766
|
-
if (this.listenerMeta) {
|
|
3767
|
-
await this.listenerMeta.unlisten();
|
|
3768
|
-
this.listenerMeta = null;
|
|
3769
|
-
}
|
|
3770
|
-
}
|
|
3771
|
-
wake() {
|
|
3772
|
-
this.wakeEarly();
|
|
3773
|
-
}
|
|
3774
|
-
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3775
|
-
const tenantId = this.resolveTenantId();
|
|
3776
|
-
await this.pgClient`
|
|
3777
|
-
insert into scheduled_tasks (
|
|
3778
|
-
tenant_id,
|
|
3779
|
-
kind,
|
|
3780
|
-
payload,
|
|
3781
|
-
fire_at,
|
|
3782
|
-
owner_entity_url,
|
|
3783
|
-
manifest_key
|
|
3784
|
-
)
|
|
3785
|
-
values (
|
|
3786
|
-
${tenantId},
|
|
3787
|
-
'delayed_send',
|
|
3788
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3789
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3790
|
-
${opts?.ownerEntityUrl ?? null},
|
|
3791
|
-
${opts?.manifestKey ?? null}
|
|
3792
|
-
)
|
|
3793
|
-
`;
|
|
3794
|
-
this.wakeEarly();
|
|
3795
|
-
}
|
|
3796
|
-
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
3797
|
-
const tenantId = this.resolveTenantId();
|
|
3798
|
-
await this.pgClient.begin(async (sql$1) => {
|
|
3799
|
-
await sql$1`
|
|
3800
|
-
update scheduled_tasks
|
|
3801
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3802
|
-
where tenant_id = ${tenantId}
|
|
3803
|
-
and kind = 'delayed_send'
|
|
3804
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3805
|
-
and manifest_key = ${manifestKey}
|
|
3806
|
-
and completed_at is null
|
|
3807
|
-
`;
|
|
3808
|
-
await sql$1`
|
|
3809
|
-
insert into scheduled_tasks (
|
|
3810
|
-
tenant_id,
|
|
3811
|
-
kind,
|
|
3812
|
-
payload,
|
|
3813
|
-
fire_at,
|
|
3814
|
-
owner_entity_url,
|
|
3815
|
-
manifest_key
|
|
3816
|
-
)
|
|
3817
|
-
values (
|
|
3818
|
-
${tenantId},
|
|
3819
|
-
'delayed_send',
|
|
3820
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3821
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3822
|
-
${ownerEntityUrl},
|
|
3823
|
-
${manifestKey}
|
|
3824
|
-
)
|
|
3825
|
-
`;
|
|
3826
|
-
});
|
|
3827
|
-
this.wakeEarly();
|
|
3828
|
-
}
|
|
3829
|
-
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
3830
|
-
const tenantId = this.resolveTenantId();
|
|
3831
|
-
await this.pgClient`
|
|
3832
|
-
update scheduled_tasks
|
|
3833
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3834
|
-
where tenant_id = ${tenantId}
|
|
3835
|
-
and kind = 'delayed_send'
|
|
3836
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3837
|
-
and manifest_key = ${manifestKey}
|
|
3838
|
-
and completed_at is null
|
|
3839
|
-
`;
|
|
3840
|
-
this.wakeEarly();
|
|
4175
|
+
constructor(pgClient, tenantId, wake) {
|
|
4176
|
+
this.pgClient = pgClient;
|
|
4177
|
+
this.tenantId = tenantId;
|
|
4178
|
+
this.wake = wake;
|
|
3841
4179
|
}
|
|
3842
|
-
async
|
|
3843
|
-
const tenantId = this.resolveTenantId();
|
|
4180
|
+
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3844
4181
|
await this.pgClient`
|
|
3845
4182
|
insert into scheduled_tasks (
|
|
3846
4183
|
tenant_id,
|
|
3847
4184
|
kind,
|
|
3848
4185
|
payload,
|
|
3849
4186
|
fire_at,
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
cron_tick_number
|
|
4187
|
+
owner_entity_url,
|
|
4188
|
+
manifest_key
|
|
3853
4189
|
)
|
|
3854
4190
|
values (
|
|
3855
|
-
${tenantId},
|
|
3856
|
-
'
|
|
3857
|
-
${JSON.stringify(
|
|
4191
|
+
${this.tenantId},
|
|
4192
|
+
'delayed_send',
|
|
4193
|
+
${JSON.stringify(payload)}::jsonb,
|
|
3858
4194
|
${fireAt.toISOString()}::timestamptz,
|
|
3859
|
-
${
|
|
3860
|
-
${
|
|
3861
|
-
${tickNumber}
|
|
3862
|
-
)
|
|
3863
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
3864
|
-
`;
|
|
3865
|
-
this.wakeEarly();
|
|
3866
|
-
}
|
|
3867
|
-
async runLoop() {
|
|
3868
|
-
while (this.running) try {
|
|
3869
|
-
await this.reclaimStaleClaims();
|
|
3870
|
-
await this.fireReadyTasks();
|
|
3871
|
-
const nextFireAt = await this.getNextFireAt();
|
|
3872
|
-
const sleepTargetMs = nextFireAt ? Math.max(0, nextFireAt.getTime() - Date.now()) : this.safetyPollMs;
|
|
3873
|
-
await this.sleepOrWake(Math.min(sleepTargetMs, this.safetyPollMs));
|
|
3874
|
-
} catch (err) {
|
|
3875
|
-
console.error(`[agent-server] scheduler iteration failed:`, err);
|
|
3876
|
-
await this.sleepOrWake(this.safetyPollMs);
|
|
3877
|
-
}
|
|
3878
|
-
}
|
|
3879
|
-
async reclaimStaleClaims() {
|
|
3880
|
-
if (this.tenantId === null) {
|
|
3881
|
-
const tenantIds = this.sharedTenantIds();
|
|
3882
|
-
if (tenantIds && tenantIds.length === 0) return;
|
|
3883
|
-
if (tenantIds) {
|
|
3884
|
-
await this.pgClient`
|
|
3885
|
-
update scheduled_tasks
|
|
3886
|
-
set claimed_by = null, claimed_at = null
|
|
3887
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
3888
|
-
and completed_at is null
|
|
3889
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3890
|
-
`;
|
|
3891
|
-
return;
|
|
3892
|
-
}
|
|
3893
|
-
await this.pgClient`
|
|
3894
|
-
update scheduled_tasks
|
|
3895
|
-
set claimed_by = null, claimed_at = null
|
|
3896
|
-
where completed_at is null
|
|
3897
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3898
|
-
`;
|
|
3899
|
-
return;
|
|
3900
|
-
}
|
|
3901
|
-
await this.pgClient`
|
|
3902
|
-
update scheduled_tasks
|
|
3903
|
-
set claimed_by = null, claimed_at = null
|
|
3904
|
-
where tenant_id = ${this.tenantId}
|
|
3905
|
-
and completed_at is null
|
|
3906
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3907
|
-
`;
|
|
3908
|
-
}
|
|
3909
|
-
async fireReadyTasks() {
|
|
3910
|
-
while (this.running) {
|
|
3911
|
-
const tasks = await this.claimReadyTasks();
|
|
3912
|
-
if (tasks.length === 0) return;
|
|
3913
|
-
for (const task of tasks) await this.executeTask(task);
|
|
3914
|
-
}
|
|
3915
|
-
}
|
|
3916
|
-
async claimReadyTasks() {
|
|
3917
|
-
if (this.tenantId === null) {
|
|
3918
|
-
const tenantIds = this.sharedTenantIds();
|
|
3919
|
-
if (tenantIds && tenantIds.length === 0) return [];
|
|
3920
|
-
if (tenantIds) {
|
|
3921
|
-
const rows$2 = await this.pgClient`
|
|
3922
|
-
update scheduled_tasks
|
|
3923
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3924
|
-
where id in (
|
|
3925
|
-
select id
|
|
3926
|
-
from scheduled_tasks
|
|
3927
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
3928
|
-
and completed_at is null
|
|
3929
|
-
and claimed_at is null
|
|
3930
|
-
and fire_at <= now()
|
|
3931
|
-
order by fire_at, id
|
|
3932
|
-
for update skip locked
|
|
3933
|
-
limit 50
|
|
3934
|
-
)
|
|
3935
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
3936
|
-
, owner_entity_url, manifest_key
|
|
3937
|
-
`;
|
|
3938
|
-
return rows$2.map(normalizeTask);
|
|
3939
|
-
}
|
|
3940
|
-
const rows$1 = await this.pgClient`
|
|
3941
|
-
update scheduled_tasks
|
|
3942
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3943
|
-
where id in (
|
|
3944
|
-
select id
|
|
3945
|
-
from scheduled_tasks
|
|
3946
|
-
where completed_at is null
|
|
3947
|
-
and claimed_at is null
|
|
3948
|
-
and fire_at <= now()
|
|
3949
|
-
order by fire_at, id
|
|
3950
|
-
for update skip locked
|
|
3951
|
-
limit 50
|
|
3952
|
-
)
|
|
3953
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
3954
|
-
, owner_entity_url, manifest_key
|
|
3955
|
-
`;
|
|
3956
|
-
return rows$1.map(normalizeTask);
|
|
3957
|
-
}
|
|
3958
|
-
const rows = await this.pgClient`
|
|
3959
|
-
update scheduled_tasks
|
|
3960
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3961
|
-
where tenant_id = ${this.tenantId}
|
|
3962
|
-
and id in (
|
|
3963
|
-
select id
|
|
3964
|
-
from scheduled_tasks
|
|
3965
|
-
where tenant_id = ${this.tenantId}
|
|
3966
|
-
and completed_at is null
|
|
3967
|
-
and claimed_at is null
|
|
3968
|
-
and fire_at <= now()
|
|
3969
|
-
order by fire_at, id
|
|
3970
|
-
for update skip locked
|
|
3971
|
-
limit 50
|
|
4195
|
+
${opts?.ownerEntityUrl ?? null},
|
|
4196
|
+
${opts?.manifestKey ?? null}
|
|
3972
4197
|
)
|
|
3973
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
3974
|
-
, owner_entity_url, manifest_key
|
|
3975
|
-
`;
|
|
3976
|
-
return rows.map(normalizeTask);
|
|
3977
|
-
}
|
|
3978
|
-
async executeTask(task) {
|
|
3979
|
-
try {
|
|
3980
|
-
if (task.kind === `delayed_send`) {
|
|
3981
|
-
await this.options.executors.delayed_send(task.payload, task.id, task.tenantId);
|
|
3982
|
-
await this.markTaskComplete(task.id, task.tenantId);
|
|
3983
|
-
return;
|
|
3984
|
-
}
|
|
3985
|
-
const tickNumber = task.cronTickNumber;
|
|
3986
|
-
if (tickNumber == null || !task.cronExpression || !task.cronTimezone) throw new Error(`cron task ${task.id} is missing cron metadata`);
|
|
3987
|
-
await this.options.executors.cron_tick(task.payload, tickNumber, task.id, task.tenantId);
|
|
3988
|
-
await this.completeAndRescheduleCron(task);
|
|
3989
|
-
} catch (err) {
|
|
3990
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
3991
|
-
if (isUnregisteredTenantError(err)) {
|
|
3992
|
-
await this.releaseClaim(task.id, message, task.tenantId);
|
|
3993
|
-
serverLog.warn(`[scheduler] skipped ${task.kind} task ${task.id} for unregistered tenant "${task.tenantId}": ${message}`);
|
|
3994
|
-
return;
|
|
3995
|
-
}
|
|
3996
|
-
if (isPermanentElectricAgentsError(err)) {
|
|
3997
|
-
await this.markTaskPermanentFailure(task.id, message, task.tenantId);
|
|
3998
|
-
return;
|
|
3999
|
-
}
|
|
4000
|
-
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4001
|
-
}
|
|
4002
|
-
}
|
|
4003
|
-
async markTaskComplete(taskId, tenantId = this.resolveTenantId()) {
|
|
4004
|
-
await this.pgClient`
|
|
4005
|
-
update scheduled_tasks
|
|
4006
|
-
set completed_at = now(), last_error = null
|
|
4007
|
-
where tenant_id = ${tenantId}
|
|
4008
|
-
and id = ${taskId}
|
|
4009
|
-
and claimed_by = ${this.instanceId}
|
|
4010
|
-
and completed_at is null
|
|
4011
|
-
`;
|
|
4012
|
-
}
|
|
4013
|
-
async markTaskPermanentFailure(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4014
|
-
await this.pgClient`
|
|
4015
|
-
update scheduled_tasks
|
|
4016
|
-
set completed_at = now(), last_error = ${message}
|
|
4017
|
-
where tenant_id = ${tenantId}
|
|
4018
|
-
and id = ${taskId}
|
|
4019
|
-
and claimed_by = ${this.instanceId}
|
|
4020
|
-
and completed_at is null
|
|
4021
|
-
`;
|
|
4022
|
-
}
|
|
4023
|
-
async releaseClaim(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4024
|
-
await this.pgClient`
|
|
4025
|
-
update scheduled_tasks
|
|
4026
|
-
set claimed_at = null, claimed_by = null, last_error = ${message}
|
|
4027
|
-
where tenant_id = ${tenantId}
|
|
4028
|
-
and id = ${taskId}
|
|
4029
|
-
and claimed_by = ${this.instanceId}
|
|
4030
|
-
and completed_at is null
|
|
4031
4198
|
`;
|
|
4199
|
+
this.wake?.();
|
|
4032
4200
|
}
|
|
4033
|
-
async
|
|
4034
|
-
const tenantId = task.tenantId ?? this.resolveTenantId();
|
|
4201
|
+
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
4035
4202
|
await this.pgClient.begin(async (sql$1) => {
|
|
4036
|
-
|
|
4203
|
+
await sql$1`
|
|
4037
4204
|
update scheduled_tasks
|
|
4038
|
-
set completed_at = now(),
|
|
4039
|
-
where tenant_id = ${tenantId}
|
|
4040
|
-
and
|
|
4041
|
-
and
|
|
4205
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4206
|
+
where tenant_id = ${this.tenantId}
|
|
4207
|
+
and kind = 'delayed_send'
|
|
4208
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4209
|
+
and manifest_key = ${manifestKey}
|
|
4042
4210
|
and completed_at is null
|
|
4043
|
-
returning id
|
|
4044
4211
|
`;
|
|
4045
|
-
if (completed.length === 0) return;
|
|
4046
|
-
const nextFireAt = getNextCronFireAt(task.cronExpression, task.cronTimezone, task.fireAt);
|
|
4047
4212
|
await sql$1`
|
|
4048
|
-
insert into scheduled_tasks (
|
|
4049
|
-
tenant_id,
|
|
4050
|
-
kind,
|
|
4051
|
-
payload,
|
|
4052
|
-
fire_at,
|
|
4053
|
-
|
|
4054
|
-
|
|
4055
|
-
|
|
4056
|
-
|
|
4057
|
-
|
|
4058
|
-
|
|
4059
|
-
|
|
4060
|
-
${
|
|
4061
|
-
${
|
|
4062
|
-
${
|
|
4063
|
-
|
|
4064
|
-
${task.cronTickNumber + 1}
|
|
4065
|
-
)
|
|
4066
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4067
|
-
`;
|
|
4068
|
-
});
|
|
4069
|
-
}
|
|
4070
|
-
async getNextFireAt() {
|
|
4071
|
-
if (this.tenantId === null) {
|
|
4072
|
-
const tenantIds = this.sharedTenantIds();
|
|
4073
|
-
if (tenantIds && tenantIds.length === 0) return null;
|
|
4074
|
-
if (tenantIds) {
|
|
4075
|
-
const rows$2 = await this.pgClient`
|
|
4076
|
-
select fire_at
|
|
4077
|
-
from scheduled_tasks
|
|
4078
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4079
|
-
and completed_at is null
|
|
4080
|
-
and claimed_at is null
|
|
4081
|
-
order by fire_at, id
|
|
4082
|
-
limit 1
|
|
4083
|
-
`;
|
|
4084
|
-
if (rows$2.length === 0) return null;
|
|
4085
|
-
const fireAt$2 = rows$2[0].fire_at;
|
|
4086
|
-
return fireAt$2 instanceof Date ? fireAt$2 : new Date(fireAt$2);
|
|
4087
|
-
}
|
|
4088
|
-
const rows$1 = await this.pgClient`
|
|
4089
|
-
select fire_at
|
|
4090
|
-
from scheduled_tasks
|
|
4091
|
-
where completed_at is null
|
|
4092
|
-
and claimed_at is null
|
|
4093
|
-
order by fire_at, id
|
|
4094
|
-
limit 1
|
|
4213
|
+
insert into scheduled_tasks (
|
|
4214
|
+
tenant_id,
|
|
4215
|
+
kind,
|
|
4216
|
+
payload,
|
|
4217
|
+
fire_at,
|
|
4218
|
+
owner_entity_url,
|
|
4219
|
+
manifest_key
|
|
4220
|
+
)
|
|
4221
|
+
values (
|
|
4222
|
+
${this.tenantId},
|
|
4223
|
+
'delayed_send',
|
|
4224
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4225
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4226
|
+
${ownerEntityUrl},
|
|
4227
|
+
${manifestKey}
|
|
4228
|
+
)
|
|
4095
4229
|
`;
|
|
4096
|
-
|
|
4097
|
-
|
|
4098
|
-
|
|
4099
|
-
|
|
4100
|
-
|
|
4101
|
-
|
|
4102
|
-
|
|
4230
|
+
});
|
|
4231
|
+
this.wake?.();
|
|
4232
|
+
}
|
|
4233
|
+
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
4234
|
+
await this.pgClient`
|
|
4235
|
+
update scheduled_tasks
|
|
4236
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4103
4237
|
where tenant_id = ${this.tenantId}
|
|
4238
|
+
and kind = 'delayed_send'
|
|
4239
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4240
|
+
and manifest_key = ${manifestKey}
|
|
4104
4241
|
and completed_at is null
|
|
4105
|
-
and claimed_at is null
|
|
4106
|
-
order by fire_at, id
|
|
4107
|
-
limit 1
|
|
4108
4242
|
`;
|
|
4109
|
-
|
|
4110
|
-
const fireAt = rows[0].fire_at;
|
|
4111
|
-
return fireAt instanceof Date ? fireAt : new Date(fireAt);
|
|
4112
|
-
}
|
|
4113
|
-
async sleepOrWake(durationMs) {
|
|
4114
|
-
if (!this.running) return;
|
|
4115
|
-
await new Promise((resolve$1) => {
|
|
4116
|
-
const finish = () => {
|
|
4117
|
-
if (this.currentSleepTimer) {
|
|
4118
|
-
clearTimeout(this.currentSleepTimer);
|
|
4119
|
-
this.currentSleepTimer = null;
|
|
4120
|
-
}
|
|
4121
|
-
this.currentSleepResolve = null;
|
|
4122
|
-
resolve$1();
|
|
4123
|
-
};
|
|
4124
|
-
this.currentSleepResolve = finish;
|
|
4125
|
-
this.currentSleepTimer = setTimeout(finish, Math.max(durationMs, 0));
|
|
4126
|
-
});
|
|
4127
|
-
}
|
|
4128
|
-
wakeEarly() {
|
|
4129
|
-
const resolve$1 = this.currentSleepResolve;
|
|
4130
|
-
this.currentSleepResolve = null;
|
|
4131
|
-
if (this.currentSleepTimer) {
|
|
4132
|
-
clearTimeout(this.currentSleepTimer);
|
|
4133
|
-
this.currentSleepTimer = null;
|
|
4134
|
-
}
|
|
4135
|
-
resolve$1?.();
|
|
4136
|
-
}
|
|
4137
|
-
sharedTenantIds() {
|
|
4138
|
-
if (this.tenantId !== null || !this.tenantIds) return null;
|
|
4139
|
-
return [...new Set(this.tenantIds())];
|
|
4140
|
-
}
|
|
4141
|
-
sharedTenantIdsParameter(tenantIds) {
|
|
4142
|
-
return this.pgClient.array(tenantIds, POSTGRES_TEXT_OID);
|
|
4243
|
+
this.wake?.();
|
|
4143
4244
|
}
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
|
|
4156
|
-
|
|
4157
|
-
|
|
4158
|
-
|
|
4159
|
-
|
|
4160
|
-
|
|
4245
|
+
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
4246
|
+
await this.pgClient`
|
|
4247
|
+
insert into scheduled_tasks (
|
|
4248
|
+
tenant_id,
|
|
4249
|
+
kind,
|
|
4250
|
+
payload,
|
|
4251
|
+
fire_at,
|
|
4252
|
+
cron_expression,
|
|
4253
|
+
cron_timezone,
|
|
4254
|
+
cron_tick_number
|
|
4255
|
+
)
|
|
4256
|
+
values (
|
|
4257
|
+
${this.tenantId},
|
|
4258
|
+
'cron_tick',
|
|
4259
|
+
${JSON.stringify({ streamPath })}::jsonb,
|
|
4260
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4261
|
+
${expression},
|
|
4262
|
+
${timezone},
|
|
4263
|
+
${tickNumber}
|
|
4264
|
+
)
|
|
4265
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4266
|
+
`;
|
|
4267
|
+
this.wake?.();
|
|
4161
4268
|
}
|
|
4162
4269
|
};
|
|
4163
|
-
|
|
4164
|
-
|
|
4165
|
-
const
|
|
4166
|
-
|
|
4167
|
-
if (!trimmed) return void 0;
|
|
4168
|
-
return /^Bearer\s+/i.test(trimmed) ? trimmed : `Bearer ${trimmed}`;
|
|
4169
|
-
}
|
|
4170
|
-
async function applyDurableStreamsBearer(headers, bearer, opts = {}) {
|
|
4171
|
-
if (!bearer) return;
|
|
4172
|
-
if (!opts.overwrite && headers.has(`authorization`)) return;
|
|
4173
|
-
const value = await resolveDurableStreamsBearer(bearer);
|
|
4174
|
-
if (value) headers.set(`authorization`, value);
|
|
4175
|
-
}
|
|
4176
|
-
function durableStreamsBearerHeaders(bearer) {
|
|
4177
|
-
if (!bearer) return void 0;
|
|
4178
|
-
return { authorization: async () => await resolveDurableStreamsBearer(bearer) ?? `` };
|
|
4179
|
-
}
|
|
4180
|
-
function durableStreamsServiceUrl(baseUrl, serviceId) {
|
|
4181
|
-
const url = new URL(baseUrl);
|
|
4182
|
-
if (/^\/v1\/stream\/[^/]+\/?$/.test(url.pathname)) return baseUrl.replace(/\/+$/, ``);
|
|
4183
|
-
const base = baseUrl.replace(/\/+$/, ``);
|
|
4184
|
-
return `${base}/v1/stream/${encodeURIComponent(serviceId)}`;
|
|
4185
|
-
}
|
|
4186
|
-
function isNotFoundError(err) {
|
|
4187
|
-
return err instanceof DurableStreamError && err.code === ErrCodeNotFound || err instanceof FetchError && err.status === 404;
|
|
4188
|
-
}
|
|
4189
|
-
function isAbortLikeError(err) {
|
|
4190
|
-
return err instanceof Error && (err.name === `AbortError` || err.message === `Stream request was aborted`);
|
|
4191
|
-
}
|
|
4192
|
-
function normalizeSubscriptionPattern(pattern) {
|
|
4193
|
-
return pattern.replace(/^\/+/, ``);
|
|
4194
|
-
}
|
|
4195
|
-
function normalizeSubscriptionStreamPath(path$1) {
|
|
4196
|
-
return path$1.replace(/^\/+/, ``);
|
|
4270
|
+
function isPermanentElectricAgentsError(err) {
|
|
4271
|
+
const status$1 = typeof err === `object` && err !== null && `status` in err ? err.status : void 0;
|
|
4272
|
+
const name = typeof err === `object` && err !== null && `name` in err ? err.name : void 0;
|
|
4273
|
+
return name === `ElectricAgentsError` && typeof status$1 === `number` && status$1 >= 400 && status$1 < 500;
|
|
4197
4274
|
}
|
|
4198
|
-
function
|
|
4199
|
-
return
|
|
4275
|
+
function normalizeTask(row) {
|
|
4276
|
+
return {
|
|
4277
|
+
id: Number(row.id),
|
|
4278
|
+
tenantId: row.tenant_id,
|
|
4279
|
+
kind: row.kind,
|
|
4280
|
+
payload: row.payload,
|
|
4281
|
+
fireAt: row.fire_at instanceof Date ? row.fire_at : new Date(row.fire_at),
|
|
4282
|
+
cronExpression: row.cron_expression,
|
|
4283
|
+
cronTimezone: row.cron_timezone,
|
|
4284
|
+
cronTickNumber: row.cron_tick_number,
|
|
4285
|
+
ownerEntityUrl: row.owner_entity_url,
|
|
4286
|
+
manifestKey: row.manifest_key
|
|
4287
|
+
};
|
|
4200
4288
|
}
|
|
4201
|
-
var
|
|
4202
|
-
|
|
4203
|
-
|
|
4289
|
+
var Scheduler = class {
|
|
4290
|
+
claimExpiryMs;
|
|
4291
|
+
safetyPollMs;
|
|
4292
|
+
listenEnabled;
|
|
4293
|
+
pgClient;
|
|
4294
|
+
instanceId;
|
|
4295
|
+
tenantId;
|
|
4296
|
+
tenantIds;
|
|
4297
|
+
running = false;
|
|
4298
|
+
loopPromise = null;
|
|
4299
|
+
currentSleepResolve = null;
|
|
4300
|
+
currentSleepTimer = null;
|
|
4301
|
+
listenerMeta = null;
|
|
4302
|
+
constructor(options) {
|
|
4204
4303
|
this.options = options;
|
|
4304
|
+
this.pgClient = options.pgClient;
|
|
4305
|
+
this.instanceId = options.instanceId;
|
|
4306
|
+
this.tenantId = options.tenantId === void 0 ? DEFAULT_TENANT_ID : options.tenantId;
|
|
4307
|
+
this.tenantIds = options.tenantIds;
|
|
4308
|
+
this.claimExpiryMs = options.claimExpiryMs ?? 3e4;
|
|
4309
|
+
this.safetyPollMs = options.safetyPollMs ?? 1e4;
|
|
4310
|
+
this.listenEnabled = options.listen !== false;
|
|
4205
4311
|
}
|
|
4206
|
-
|
|
4207
|
-
return
|
|
4208
|
-
|
|
4209
|
-
|
|
4210
|
-
return durableStreamsBearerHeaders(this.options.bearer);
|
|
4211
|
-
}
|
|
4212
|
-
async requestHeaders(init, opts = {}) {
|
|
4213
|
-
const headers = new Headers(init);
|
|
4214
|
-
await applyDurableStreamsBearer(headers, this.options.bearer, { overwrite: opts.overwriteBearer });
|
|
4215
|
-
return headers;
|
|
4216
|
-
}
|
|
4217
|
-
subscriptionServiceId() {
|
|
4218
|
-
const url = new URL(this.baseUrl);
|
|
4219
|
-
const match = /^(.*)\/v1\/stream\/([^/]+)\/?$/.exec(url.pathname);
|
|
4220
|
-
return match ? decodeURIComponent(match[2]) : null;
|
|
4221
|
-
}
|
|
4222
|
-
backendSubscriptionPath(path$1) {
|
|
4223
|
-
const normalized = normalizeSubscriptionPath(path$1);
|
|
4224
|
-
const serviceId = this.subscriptionServiceId();
|
|
4225
|
-
if (!serviceId) return normalized;
|
|
4226
|
-
if (normalized === serviceId || normalized.startsWith(`${serviceId}/`)) return normalized;
|
|
4227
|
-
return `${serviceId}/${normalized}`;
|
|
4228
|
-
}
|
|
4229
|
-
runtimeSubscriptionPath(path$1) {
|
|
4230
|
-
const normalized = normalizeSubscriptionPath(path$1);
|
|
4231
|
-
const serviceId = this.subscriptionServiceId();
|
|
4232
|
-
if (!serviceId) return normalized;
|
|
4233
|
-
return normalized.startsWith(`${serviceId}/`) ? normalized.slice(serviceId.length + 1) : normalized;
|
|
4234
|
-
}
|
|
4235
|
-
subscriptionUrl(subscriptionId) {
|
|
4236
|
-
const url = new URL(this.baseUrl);
|
|
4237
|
-
const match = /^(.*)\/v1\/stream\/([^/]+)\/?$/.exec(url.pathname);
|
|
4238
|
-
if (match) {
|
|
4239
|
-
const [, prefix = ``, serviceId] = match;
|
|
4240
|
-
url.pathname = `${prefix}/v1/stream-meta/subscriptions/${encodeURIComponent(subscriptionId)}`;
|
|
4241
|
-
url.searchParams.set(`service`, decodeURIComponent(serviceId));
|
|
4242
|
-
return url.toString();
|
|
4243
|
-
}
|
|
4244
|
-
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/v1/stream-meta/subscriptions/${encodeURIComponent(subscriptionId)}`;
|
|
4245
|
-
return url.toString();
|
|
4246
|
-
}
|
|
4247
|
-
subscriptionChildUrl(subscriptionId, ...segments) {
|
|
4248
|
-
const url = new URL(this.subscriptionUrl(subscriptionId));
|
|
4249
|
-
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/${segments.map((segment) => encodeURIComponent(segment)).join(`/`)}`;
|
|
4250
|
-
return url.toString();
|
|
4251
|
-
}
|
|
4252
|
-
async create(path$1, opts) {
|
|
4253
|
-
return await withSpan(`stream.create`, async (span) => {
|
|
4254
|
-
span.setAttributes({
|
|
4255
|
-
[ATTR.STREAM_PATH]: path$1,
|
|
4256
|
-
[ATTR.STREAM_OP]: `create`
|
|
4257
|
-
});
|
|
4258
|
-
await DurableStream.create({
|
|
4259
|
-
url: this.streamUrl(path$1),
|
|
4260
|
-
headers: this.streamHeaders(),
|
|
4261
|
-
contentType: opts.contentType,
|
|
4262
|
-
body: opts.body
|
|
4263
|
-
});
|
|
4264
|
-
});
|
|
4265
|
-
}
|
|
4266
|
-
async fork(path$1, sourcePath) {
|
|
4267
|
-
return await withSpan(`stream.fork`, async (span) => {
|
|
4268
|
-
span.setAttributes({
|
|
4269
|
-
[ATTR.STREAM_PATH]: path$1,
|
|
4270
|
-
[ATTR.STREAM_OP]: `fork`
|
|
4271
|
-
});
|
|
4272
|
-
const headers = {
|
|
4273
|
-
"content-type": `application/json`,
|
|
4274
|
-
"Stream-Forked-From": sourcePath
|
|
4275
|
-
};
|
|
4276
|
-
injectTraceHeaders(headers);
|
|
4277
|
-
const response = await fetch(this.streamUrl(path$1), {
|
|
4278
|
-
method: `PUT`,
|
|
4279
|
-
headers: await this.requestHeaders(headers)
|
|
4280
|
-
});
|
|
4281
|
-
if (response.ok) return;
|
|
4282
|
-
throw new Error(`Stream fork failed: ${response.status} ${await response.text()}`);
|
|
4283
|
-
});
|
|
4284
|
-
}
|
|
4285
|
-
async append(path$1, data, opts) {
|
|
4286
|
-
return await withSpan(`stream.append`, async (span) => {
|
|
4287
|
-
span.setAttributes({
|
|
4288
|
-
[ATTR.STREAM_PATH]: path$1,
|
|
4289
|
-
[ATTR.STREAM_OP]: opts?.close ? `append+close` : `append`
|
|
4290
|
-
});
|
|
4291
|
-
const handle = new DurableStream({
|
|
4292
|
-
url: this.streamUrl(path$1),
|
|
4293
|
-
headers: this.streamHeaders(),
|
|
4294
|
-
contentType: `application/json`,
|
|
4295
|
-
batching: false
|
|
4296
|
-
});
|
|
4297
|
-
if (opts?.close) {
|
|
4298
|
-
const result = await handle.close({ body: data });
|
|
4299
|
-
return { offset: result.finalOffset };
|
|
4300
|
-
}
|
|
4301
|
-
await handle.append(data);
|
|
4302
|
-
const head = await handle.head();
|
|
4303
|
-
return { offset: head.exists && head.offset || `` };
|
|
4304
|
-
});
|
|
4305
|
-
}
|
|
4306
|
-
async appendIdempotent(path$1, data, opts) {
|
|
4307
|
-
return await withSpan(`stream.appendIdempotent`, async (span) => {
|
|
4308
|
-
span.setAttributes({
|
|
4309
|
-
[ATTR.STREAM_PATH]: path$1,
|
|
4310
|
-
[ATTR.STREAM_OP]: `appendIdempotent`
|
|
4311
|
-
});
|
|
4312
|
-
const stream = new DurableStream({
|
|
4313
|
-
url: this.streamUrl(path$1),
|
|
4314
|
-
headers: this.streamHeaders(),
|
|
4315
|
-
contentType: `application/json`
|
|
4316
|
-
});
|
|
4317
|
-
const producer = new IdempotentProducer(stream, opts.producerId, { epoch: opts.epoch ?? 0 });
|
|
4318
|
-
try {
|
|
4319
|
-
producer.append(data);
|
|
4320
|
-
await producer.flush();
|
|
4321
|
-
} finally {
|
|
4322
|
-
await producer.detach();
|
|
4323
|
-
}
|
|
4324
|
-
});
|
|
4312
|
+
resolveTenantId(tenantId) {
|
|
4313
|
+
if (tenantId) return tenantId;
|
|
4314
|
+
if (this.tenantId) return this.tenantId;
|
|
4315
|
+
throw new Error(`Scheduler tenantId is required in shared mode`);
|
|
4325
4316
|
}
|
|
4326
|
-
async
|
|
4327
|
-
|
|
4328
|
-
|
|
4329
|
-
|
|
4330
|
-
|
|
4331
|
-
});
|
|
4332
|
-
const headers = {
|
|
4333
|
-
"content-type": `application/json`,
|
|
4334
|
-
"Producer-Id": opts.producerId,
|
|
4335
|
-
"Producer-Epoch": String(opts.epoch),
|
|
4336
|
-
"Producer-Seq": String(opts.seq)
|
|
4337
|
-
};
|
|
4338
|
-
injectTraceHeaders(headers);
|
|
4339
|
-
const response = await fetch(this.streamUrl(path$1), {
|
|
4340
|
-
method: `POST`,
|
|
4341
|
-
headers: await this.requestHeaders(headers),
|
|
4342
|
-
body: typeof data === `string` ? data : Buffer.from(data)
|
|
4343
|
-
});
|
|
4344
|
-
if (response.ok || response.status === 204) return;
|
|
4345
|
-
throw new Error(`Stream append failed: ${response.status} ${await response.text()}`);
|
|
4317
|
+
async start() {
|
|
4318
|
+
if (this.running) return;
|
|
4319
|
+
this.running = true;
|
|
4320
|
+
if (this.listenEnabled) this.listenerMeta = await this.pgClient.listen(`scheduled_tasks_wake`, () => {
|
|
4321
|
+
this.wakeEarly();
|
|
4346
4322
|
});
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4350
|
-
|
|
4351
|
-
[ATTR.STREAM_PATH]: path$1,
|
|
4352
|
-
[ATTR.STREAM_OP]: `read`
|
|
4353
|
-
});
|
|
4354
|
-
const handle = new DurableStream({
|
|
4355
|
-
url: this.streamUrl(path$1),
|
|
4356
|
-
headers: this.streamHeaders()
|
|
4357
|
-
});
|
|
4358
|
-
const response = await handle.stream({
|
|
4359
|
-
offset: fromOffset ?? `-1`,
|
|
4360
|
-
live: false
|
|
4361
|
-
});
|
|
4362
|
-
const messages = [];
|
|
4363
|
-
return await new Promise((resolve$1, reject) => {
|
|
4364
|
-
let settled = false;
|
|
4365
|
-
let unsub = () => {};
|
|
4366
|
-
const finish = (r) => {
|
|
4367
|
-
if (settled) return;
|
|
4368
|
-
settled = true;
|
|
4369
|
-
unsub();
|
|
4370
|
-
resolve$1(r);
|
|
4371
|
-
};
|
|
4372
|
-
unsub = response.subscribeBytes((chunk) => {
|
|
4373
|
-
messages.push({
|
|
4374
|
-
data: chunk.data,
|
|
4375
|
-
offset: chunk.offset
|
|
4376
|
-
});
|
|
4377
|
-
if (chunk.upToDate || chunk.streamClosed) finish({ messages });
|
|
4378
|
-
});
|
|
4379
|
-
response.closed.then(() => finish({ messages })).catch((err) => {
|
|
4380
|
-
if (settled) return;
|
|
4381
|
-
settled = true;
|
|
4382
|
-
unsub();
|
|
4383
|
-
reject(err);
|
|
4384
|
-
});
|
|
4385
|
-
});
|
|
4323
|
+
this.loopPromise = this.runLoop().catch((err) => {
|
|
4324
|
+
console.error(`[agent-server] scheduler loop failed:`, err);
|
|
4325
|
+
this.running = false;
|
|
4326
|
+
this.wakeEarly();
|
|
4386
4327
|
});
|
|
4387
4328
|
}
|
|
4388
|
-
async
|
|
4389
|
-
|
|
4390
|
-
|
|
4391
|
-
|
|
4392
|
-
|
|
4393
|
-
|
|
4394
|
-
|
|
4395
|
-
|
|
4396
|
-
|
|
4397
|
-
|
|
4398
|
-
|
|
4399
|
-
offset: fromOffset ?? `-1`,
|
|
4400
|
-
live: false
|
|
4401
|
-
});
|
|
4402
|
-
return await response.json();
|
|
4403
|
-
});
|
|
4329
|
+
async stop() {
|
|
4330
|
+
this.running = false;
|
|
4331
|
+
this.wakeEarly();
|
|
4332
|
+
if (this.loopPromise) {
|
|
4333
|
+
await this.loopPromise;
|
|
4334
|
+
this.loopPromise = null;
|
|
4335
|
+
}
|
|
4336
|
+
if (this.listenerMeta) {
|
|
4337
|
+
await this.listenerMeta.unlisten();
|
|
4338
|
+
this.listenerMeta = null;
|
|
4339
|
+
}
|
|
4404
4340
|
}
|
|
4405
|
-
|
|
4406
|
-
|
|
4407
|
-
|
|
4408
|
-
|
|
4409
|
-
|
|
4410
|
-
|
|
4411
|
-
|
|
4412
|
-
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
|
|
4429
|
-
|
|
4430
|
-
|
|
4431
|
-
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
unsub();
|
|
4461
|
-
reject(err);
|
|
4462
|
-
});
|
|
4463
|
-
});
|
|
4464
|
-
} catch (err) {
|
|
4465
|
-
clearTimeout(timer);
|
|
4466
|
-
if (isAbortLikeError(err)) return {
|
|
4467
|
-
messages: [],
|
|
4468
|
-
timedOut: true
|
|
4469
|
-
};
|
|
4470
|
-
throw err;
|
|
4471
|
-
}
|
|
4341
|
+
wake() {
|
|
4342
|
+
this.wakeEarly();
|
|
4343
|
+
}
|
|
4344
|
+
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
4345
|
+
const tenantId = this.resolveTenantId();
|
|
4346
|
+
await this.pgClient`
|
|
4347
|
+
insert into scheduled_tasks (
|
|
4348
|
+
tenant_id,
|
|
4349
|
+
kind,
|
|
4350
|
+
payload,
|
|
4351
|
+
fire_at,
|
|
4352
|
+
owner_entity_url,
|
|
4353
|
+
manifest_key
|
|
4354
|
+
)
|
|
4355
|
+
values (
|
|
4356
|
+
${tenantId},
|
|
4357
|
+
'delayed_send',
|
|
4358
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4359
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4360
|
+
${opts?.ownerEntityUrl ?? null},
|
|
4361
|
+
${opts?.manifestKey ?? null}
|
|
4362
|
+
)
|
|
4363
|
+
`;
|
|
4364
|
+
this.wakeEarly();
|
|
4365
|
+
}
|
|
4366
|
+
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
4367
|
+
const tenantId = this.resolveTenantId();
|
|
4368
|
+
await this.pgClient.begin(async (sql$1) => {
|
|
4369
|
+
await sql$1`
|
|
4370
|
+
update scheduled_tasks
|
|
4371
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4372
|
+
where tenant_id = ${tenantId}
|
|
4373
|
+
and kind = 'delayed_send'
|
|
4374
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4375
|
+
and manifest_key = ${manifestKey}
|
|
4376
|
+
and completed_at is null
|
|
4377
|
+
`;
|
|
4378
|
+
await sql$1`
|
|
4379
|
+
insert into scheduled_tasks (
|
|
4380
|
+
tenant_id,
|
|
4381
|
+
kind,
|
|
4382
|
+
payload,
|
|
4383
|
+
fire_at,
|
|
4384
|
+
owner_entity_url,
|
|
4385
|
+
manifest_key
|
|
4386
|
+
)
|
|
4387
|
+
values (
|
|
4388
|
+
${tenantId},
|
|
4389
|
+
'delayed_send',
|
|
4390
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4391
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4392
|
+
${ownerEntityUrl},
|
|
4393
|
+
${manifestKey}
|
|
4394
|
+
)
|
|
4395
|
+
`;
|
|
4472
4396
|
});
|
|
4397
|
+
this.wakeEarly();
|
|
4398
|
+
}
|
|
4399
|
+
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
4400
|
+
const tenantId = this.resolveTenantId();
|
|
4401
|
+
await this.pgClient`
|
|
4402
|
+
update scheduled_tasks
|
|
4403
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4404
|
+
where tenant_id = ${tenantId}
|
|
4405
|
+
and kind = 'delayed_send'
|
|
4406
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4407
|
+
and manifest_key = ${manifestKey}
|
|
4408
|
+
and completed_at is null
|
|
4409
|
+
`;
|
|
4410
|
+
this.wakeEarly();
|
|
4473
4411
|
}
|
|
4474
|
-
async
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4412
|
+
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
4413
|
+
const tenantId = this.resolveTenantId();
|
|
4414
|
+
await this.pgClient`
|
|
4415
|
+
insert into scheduled_tasks (
|
|
4416
|
+
tenant_id,
|
|
4417
|
+
kind,
|
|
4418
|
+
payload,
|
|
4419
|
+
fire_at,
|
|
4420
|
+
cron_expression,
|
|
4421
|
+
cron_timezone,
|
|
4422
|
+
cron_tick_number
|
|
4423
|
+
)
|
|
4424
|
+
values (
|
|
4425
|
+
${tenantId},
|
|
4426
|
+
'cron_tick',
|
|
4427
|
+
${JSON.stringify({ streamPath })}::jsonb,
|
|
4428
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4429
|
+
${expression},
|
|
4430
|
+
${timezone},
|
|
4431
|
+
${tickNumber}
|
|
4432
|
+
)
|
|
4433
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4434
|
+
`;
|
|
4435
|
+
this.wakeEarly();
|
|
4479
4436
|
}
|
|
4480
|
-
async
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
await this.
|
|
4437
|
+
async runLoop() {
|
|
4438
|
+
while (this.running) try {
|
|
4439
|
+
await this.reclaimStaleClaims();
|
|
4440
|
+
await this.fireReadyTasks();
|
|
4441
|
+
const nextFireAt = await this.getNextFireAt();
|
|
4442
|
+
const sleepTargetMs = nextFireAt ? Math.max(0, nextFireAt.getTime() - Date.now()) : this.safetyPollMs;
|
|
4443
|
+
await this.sleepOrWake(Math.min(sleepTargetMs, this.safetyPollMs));
|
|
4484
4444
|
} catch (err) {
|
|
4485
|
-
|
|
4486
|
-
|
|
4445
|
+
console.error(`[agent-server] scheduler iteration failed:`, err);
|
|
4446
|
+
await this.sleepOrWake(this.safetyPollMs);
|
|
4487
4447
|
}
|
|
4488
4448
|
}
|
|
4489
|
-
async
|
|
4490
|
-
|
|
4491
|
-
const
|
|
4492
|
-
|
|
4493
|
-
|
|
4494
|
-
|
|
4495
|
-
|
|
4496
|
-
|
|
4497
|
-
|
|
4498
|
-
|
|
4449
|
+
async reclaimStaleClaims() {
|
|
4450
|
+
if (this.tenantId === null) {
|
|
4451
|
+
const tenantIds = this.sharedTenantIds();
|
|
4452
|
+
if (tenantIds && tenantIds.length === 0) return;
|
|
4453
|
+
if (tenantIds) {
|
|
4454
|
+
await this.pgClient`
|
|
4455
|
+
update scheduled_tasks
|
|
4456
|
+
set claimed_by = null, claimed_at = null
|
|
4457
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4458
|
+
and completed_at is null
|
|
4459
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4460
|
+
`;
|
|
4461
|
+
return;
|
|
4462
|
+
}
|
|
4463
|
+
await this.pgClient`
|
|
4464
|
+
update scheduled_tasks
|
|
4465
|
+
set claimed_by = null, claimed_at = null
|
|
4466
|
+
where completed_at is null
|
|
4467
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4468
|
+
`;
|
|
4469
|
+
return;
|
|
4499
4470
|
}
|
|
4471
|
+
await this.pgClient`
|
|
4472
|
+
update scheduled_tasks
|
|
4473
|
+
set claimed_by = null, claimed_at = null
|
|
4474
|
+
where tenant_id = ${this.tenantId}
|
|
4475
|
+
and completed_at is null
|
|
4476
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4477
|
+
`;
|
|
4500
4478
|
}
|
|
4501
|
-
async
|
|
4502
|
-
|
|
4503
|
-
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
});
|
|
4508
|
-
return res;
|
|
4509
|
-
}
|
|
4510
|
-
async putSubscription(subscriptionId, input) {
|
|
4511
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4512
|
-
method: `PUT`,
|
|
4513
|
-
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
4514
|
-
body: JSON.stringify({
|
|
4515
|
-
...input,
|
|
4516
|
-
pattern: typeof input.pattern === `string` ? this.backendSubscriptionPath(normalizeSubscriptionPattern(input.pattern)) : void 0,
|
|
4517
|
-
streams: input.streams?.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))),
|
|
4518
|
-
wake_stream: typeof input.wake_stream === `string` ? this.backendSubscriptionPath(normalizeSubscriptionStreamPath(input.wake_stream)) : void 0
|
|
4519
|
-
})
|
|
4520
|
-
});
|
|
4521
|
-
return await this.subscriptionJson(res, `Subscription creation failed`);
|
|
4522
|
-
}
|
|
4523
|
-
async getSubscription(subscriptionId) {
|
|
4524
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4525
|
-
method: `GET`,
|
|
4526
|
-
headers: await this.requestHeaders()
|
|
4527
|
-
});
|
|
4528
|
-
if (res.status === 404) return null;
|
|
4529
|
-
return await this.subscriptionJson(res, `Subscription query failed`);
|
|
4530
|
-
}
|
|
4531
|
-
async deleteSubscription(subscriptionId) {
|
|
4532
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4533
|
-
method: `DELETE`,
|
|
4534
|
-
headers: await this.requestHeaders()
|
|
4535
|
-
});
|
|
4536
|
-
if (res.status === 404 || res.status === 204) return;
|
|
4537
|
-
if (!res.ok) throw new Error(`Subscription delete failed: ${res.status} ${await res.text()}`);
|
|
4479
|
+
async fireReadyTasks() {
|
|
4480
|
+
while (this.running) {
|
|
4481
|
+
const tasks = await this.claimReadyTasks();
|
|
4482
|
+
if (tasks.length === 0) return;
|
|
4483
|
+
for (const task of tasks) await this.executeTask(task);
|
|
4484
|
+
}
|
|
4538
4485
|
}
|
|
4539
|
-
async
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4544
|
-
|
|
4545
|
-
|
|
4486
|
+
async claimReadyTasks() {
|
|
4487
|
+
if (this.tenantId === null) {
|
|
4488
|
+
const tenantIds = this.sharedTenantIds();
|
|
4489
|
+
if (tenantIds && tenantIds.length === 0) return [];
|
|
4490
|
+
if (tenantIds) {
|
|
4491
|
+
const rows$2 = await this.pgClient`
|
|
4492
|
+
update scheduled_tasks
|
|
4493
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4494
|
+
where id in (
|
|
4495
|
+
select id
|
|
4496
|
+
from scheduled_tasks
|
|
4497
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4498
|
+
and completed_at is null
|
|
4499
|
+
and claimed_at is null
|
|
4500
|
+
and fire_at <= now()
|
|
4501
|
+
order by fire_at, id
|
|
4502
|
+
for update skip locked
|
|
4503
|
+
limit 50
|
|
4504
|
+
)
|
|
4505
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4506
|
+
, owner_entity_url, manifest_key
|
|
4507
|
+
`;
|
|
4508
|
+
return rows$2.map(normalizeTask);
|
|
4509
|
+
}
|
|
4510
|
+
const rows$1 = await this.pgClient`
|
|
4511
|
+
update scheduled_tasks
|
|
4512
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4513
|
+
where id in (
|
|
4514
|
+
select id
|
|
4515
|
+
from scheduled_tasks
|
|
4516
|
+
where completed_at is null
|
|
4517
|
+
and claimed_at is null
|
|
4518
|
+
and fire_at <= now()
|
|
4519
|
+
order by fire_at, id
|
|
4520
|
+
for update skip locked
|
|
4521
|
+
limit 50
|
|
4522
|
+
)
|
|
4523
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4524
|
+
, owner_entity_url, manifest_key
|
|
4525
|
+
`;
|
|
4526
|
+
return rows$1.map(normalizeTask);
|
|
4527
|
+
}
|
|
4528
|
+
const rows = await this.pgClient`
|
|
4529
|
+
update scheduled_tasks
|
|
4530
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4531
|
+
where tenant_id = ${this.tenantId}
|
|
4532
|
+
and id in (
|
|
4533
|
+
select id
|
|
4534
|
+
from scheduled_tasks
|
|
4535
|
+
where tenant_id = ${this.tenantId}
|
|
4536
|
+
and completed_at is null
|
|
4537
|
+
and claimed_at is null
|
|
4538
|
+
and fire_at <= now()
|
|
4539
|
+
order by fire_at, id
|
|
4540
|
+
for update skip locked
|
|
4541
|
+
limit 50
|
|
4542
|
+
)
|
|
4543
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4544
|
+
, owner_entity_url, manifest_key
|
|
4545
|
+
`;
|
|
4546
|
+
return rows.map(normalizeTask);
|
|
4546
4547
|
}
|
|
4547
|
-
async
|
|
4548
|
-
|
|
4549
|
-
|
|
4550
|
-
|
|
4551
|
-
|
|
4552
|
-
|
|
4553
|
-
|
|
4548
|
+
async executeTask(task) {
|
|
4549
|
+
try {
|
|
4550
|
+
if (task.kind === `delayed_send`) {
|
|
4551
|
+
await this.options.executors.delayed_send(task.payload, task.id, task.tenantId);
|
|
4552
|
+
await this.markTaskComplete(task.id, task.tenantId);
|
|
4553
|
+
return;
|
|
4554
|
+
}
|
|
4555
|
+
const tickNumber = task.cronTickNumber;
|
|
4556
|
+
if (tickNumber == null || !task.cronExpression || !task.cronTimezone) throw new Error(`cron task ${task.id} is missing cron metadata`);
|
|
4557
|
+
await this.options.executors.cron_tick(task.payload, tickNumber, task.id, task.tenantId);
|
|
4558
|
+
await this.completeAndRescheduleCron(task);
|
|
4559
|
+
} catch (err) {
|
|
4560
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4561
|
+
if (isUnregisteredTenantError(err)) {
|
|
4562
|
+
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4563
|
+
serverLog.warn(`[scheduler] skipped ${task.kind} task ${task.id} for unregistered tenant "${task.tenantId}": ${message}`);
|
|
4564
|
+
return;
|
|
4565
|
+
}
|
|
4566
|
+
if (isPermanentElectricAgentsError(err)) {
|
|
4567
|
+
await this.markTaskPermanentFailure(task.id, message, task.tenantId);
|
|
4568
|
+
return;
|
|
4569
|
+
}
|
|
4570
|
+
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4571
|
+
}
|
|
4554
4572
|
}
|
|
4555
|
-
async
|
|
4556
|
-
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4573
|
+
async markTaskComplete(taskId, tenantId = this.resolveTenantId()) {
|
|
4574
|
+
await this.pgClient`
|
|
4575
|
+
update scheduled_tasks
|
|
4576
|
+
set completed_at = now(), last_error = null
|
|
4577
|
+
where tenant_id = ${tenantId}
|
|
4578
|
+
and id = ${taskId}
|
|
4579
|
+
and claimed_by = ${this.instanceId}
|
|
4580
|
+
and completed_at is null
|
|
4581
|
+
`;
|
|
4563
4582
|
}
|
|
4564
|
-
async
|
|
4565
|
-
|
|
4566
|
-
|
|
4567
|
-
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
return await this.subscriptionJson(res, `Subscription ack failed`);
|
|
4583
|
+
async markTaskPermanentFailure(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4584
|
+
await this.pgClient`
|
|
4585
|
+
update scheduled_tasks
|
|
4586
|
+
set completed_at = now(), last_error = ${message}
|
|
4587
|
+
where tenant_id = ${tenantId}
|
|
4588
|
+
and id = ${taskId}
|
|
4589
|
+
and claimed_by = ${this.instanceId}
|
|
4590
|
+
and completed_at is null
|
|
4591
|
+
`;
|
|
4574
4592
|
}
|
|
4575
|
-
async
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
return await this.subscriptionJson(res, `Subscription release failed`);
|
|
4593
|
+
async releaseClaim(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4594
|
+
await this.pgClient`
|
|
4595
|
+
update scheduled_tasks
|
|
4596
|
+
set claimed_at = null, claimed_by = null, last_error = ${message}
|
|
4597
|
+
where tenant_id = ${tenantId}
|
|
4598
|
+
and id = ${taskId}
|
|
4599
|
+
and claimed_by = ${this.instanceId}
|
|
4600
|
+
and completed_at is null
|
|
4601
|
+
`;
|
|
4585
4602
|
}
|
|
4586
|
-
|
|
4587
|
-
const
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4603
|
+
async completeAndRescheduleCron(task) {
|
|
4604
|
+
const tenantId = task.tenantId ?? this.resolveTenantId();
|
|
4605
|
+
await this.pgClient.begin(async (sql$1) => {
|
|
4606
|
+
const completed = await sql$1`
|
|
4607
|
+
update scheduled_tasks
|
|
4608
|
+
set completed_at = now(), last_error = null
|
|
4609
|
+
where tenant_id = ${tenantId}
|
|
4610
|
+
and id = ${task.id}
|
|
4611
|
+
and claimed_by = ${this.instanceId}
|
|
4612
|
+
and completed_at is null
|
|
4613
|
+
returning id
|
|
4614
|
+
`;
|
|
4615
|
+
if (completed.length === 0) return;
|
|
4616
|
+
const nextFireAt = getNextCronFireAt(task.cronExpression, task.cronTimezone, task.fireAt);
|
|
4617
|
+
await sql$1`
|
|
4618
|
+
insert into scheduled_tasks (
|
|
4619
|
+
tenant_id,
|
|
4620
|
+
kind,
|
|
4621
|
+
payload,
|
|
4622
|
+
fire_at,
|
|
4623
|
+
cron_expression,
|
|
4624
|
+
cron_timezone,
|
|
4625
|
+
cron_tick_number
|
|
4626
|
+
)
|
|
4627
|
+
values (
|
|
4628
|
+
${tenantId},
|
|
4629
|
+
'cron_tick',
|
|
4630
|
+
${JSON.stringify(task.payload)}::jsonb,
|
|
4631
|
+
${nextFireAt.toISOString()}::timestamptz,
|
|
4632
|
+
${task.cronExpression},
|
|
4633
|
+
${task.cronTimezone},
|
|
4634
|
+
${task.cronTickNumber + 1}
|
|
4635
|
+
)
|
|
4636
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4637
|
+
`;
|
|
4596
4638
|
});
|
|
4597
|
-
return next;
|
|
4598
4639
|
}
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4640
|
+
async getNextFireAt() {
|
|
4641
|
+
if (this.tenantId === null) {
|
|
4642
|
+
const tenantIds = this.sharedTenantIds();
|
|
4643
|
+
if (tenantIds && tenantIds.length === 0) return null;
|
|
4644
|
+
if (tenantIds) {
|
|
4645
|
+
const rows$2 = await this.pgClient`
|
|
4646
|
+
select fire_at
|
|
4647
|
+
from scheduled_tasks
|
|
4648
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4649
|
+
and completed_at is null
|
|
4650
|
+
and claimed_at is null
|
|
4651
|
+
order by fire_at, id
|
|
4652
|
+
limit 1
|
|
4653
|
+
`;
|
|
4654
|
+
if (rows$2.length === 0) return null;
|
|
4655
|
+
const fireAt$2 = rows$2[0].fire_at;
|
|
4656
|
+
return fireAt$2 instanceof Date ? fireAt$2 : new Date(fireAt$2);
|
|
4657
|
+
}
|
|
4658
|
+
const rows$1 = await this.pgClient`
|
|
4659
|
+
select fire_at
|
|
4660
|
+
from scheduled_tasks
|
|
4661
|
+
where completed_at is null
|
|
4662
|
+
and claimed_at is null
|
|
4663
|
+
order by fire_at, id
|
|
4664
|
+
limit 1
|
|
4665
|
+
`;
|
|
4666
|
+
if (rows$1.length === 0) return null;
|
|
4667
|
+
const fireAt$1 = rows$1[0].fire_at;
|
|
4668
|
+
return fireAt$1 instanceof Date ? fireAt$1 : new Date(fireAt$1);
|
|
4669
|
+
}
|
|
4670
|
+
const rows = await this.pgClient`
|
|
4671
|
+
select fire_at
|
|
4672
|
+
from scheduled_tasks
|
|
4673
|
+
where tenant_id = ${this.tenantId}
|
|
4674
|
+
and completed_at is null
|
|
4675
|
+
and claimed_at is null
|
|
4676
|
+
order by fire_at, id
|
|
4677
|
+
limit 1
|
|
4678
|
+
`;
|
|
4679
|
+
if (rows.length === 0) return null;
|
|
4680
|
+
const fireAt = rows[0].fire_at;
|
|
4681
|
+
return fireAt instanceof Date ? fireAt : new Date(fireAt);
|
|
4682
|
+
}
|
|
4683
|
+
async sleepOrWake(durationMs) {
|
|
4684
|
+
if (!this.running) return;
|
|
4685
|
+
await new Promise((resolve$1) => {
|
|
4686
|
+
const finish = () => {
|
|
4687
|
+
if (this.currentSleepTimer) {
|
|
4688
|
+
clearTimeout(this.currentSleepTimer);
|
|
4689
|
+
this.currentSleepTimer = null;
|
|
4690
|
+
}
|
|
4691
|
+
this.currentSleepResolve = null;
|
|
4692
|
+
resolve$1();
|
|
4608
4693
|
};
|
|
4694
|
+
this.currentSleepResolve = finish;
|
|
4695
|
+
this.currentSleepTimer = setTimeout(finish, Math.max(durationMs, 0));
|
|
4609
4696
|
});
|
|
4610
|
-
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
4611
|
-
if (!ack || typeof ack !== `object`) return ack;
|
|
4612
|
-
const mapped = { ...ack };
|
|
4613
|
-
if (typeof mapped.stream === `string`) mapped.stream = this.runtimeSubscriptionPath(mapped.stream);
|
|
4614
|
-
if (typeof mapped.path === `string`) mapped.path = this.runtimeSubscriptionPath(mapped.path);
|
|
4615
|
-
return mapped;
|
|
4616
|
-
});
|
|
4617
|
-
if (typeof next.stream === `string`) next.stream = this.runtimeSubscriptionPath(next.stream);
|
|
4618
|
-
return next;
|
|
4619
4697
|
}
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4625
|
-
|
|
4698
|
+
wakeEarly() {
|
|
4699
|
+
const resolve$1 = this.currentSleepResolve;
|
|
4700
|
+
this.currentSleepResolve = null;
|
|
4701
|
+
if (this.currentSleepTimer) {
|
|
4702
|
+
clearTimeout(this.currentSleepTimer);
|
|
4703
|
+
this.currentSleepTimer = null;
|
|
4704
|
+
}
|
|
4705
|
+
resolve$1?.();
|
|
4626
4706
|
}
|
|
4627
|
-
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
if (!res.ok) throw new Error(`Consumer query failed: ${res.status} ${await res.text()}`);
|
|
4634
|
-
return res.json();
|
|
4707
|
+
sharedTenantIds() {
|
|
4708
|
+
if (this.tenantId !== null || !this.tenantIds) return null;
|
|
4709
|
+
return [...new Set(this.tenantIds())];
|
|
4710
|
+
}
|
|
4711
|
+
sharedTenantIdsParameter(tenantIds) {
|
|
4712
|
+
return this.pgClient.array(tenantIds, POSTGRES_TEXT_OID);
|
|
4635
4713
|
}
|
|
4636
4714
|
};
|
|
4637
4715
|
|
|
@@ -4656,7 +4734,7 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
4656
4734
|
this.service = this.serviceId;
|
|
4657
4735
|
this.db = options.db;
|
|
4658
4736
|
if (options.streamClient) this.streamClient = options.streamClient;
|
|
4659
|
-
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(
|
|
4737
|
+
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer });
|
|
4660
4738
|
else throw new Error(`Either durableStreamsUrl or streamClient is required`);
|
|
4661
4739
|
this.registry = options.registry ?? new PostgresRegistry(this.db, this.serviceId);
|
|
4662
4740
|
this.wakeRegistry = options.wakeRegistry;
|
|
@@ -5747,7 +5825,7 @@ var AgentsHost = class {
|
|
|
5747
5825
|
}
|
|
5748
5826
|
createStreamClient(config) {
|
|
5749
5827
|
if (config.streamClient) return config.streamClient;
|
|
5750
|
-
if (config.durableStreamsUrl) return new StreamClient(
|
|
5828
|
+
if (config.durableStreamsUrl) return new StreamClient(config.durableStreamsUrl, { bearer: config.durableStreamsBearer });
|
|
5751
5829
|
throw new Error(`AgentsHost tenant "${config.serviceId}" must provide a streamClient or durableStreamsUrl`);
|
|
5752
5830
|
}
|
|
5753
5831
|
};
|
|
@@ -5925,74 +6003,38 @@ function validateParsedBody(schema, parsed) {
|
|
|
5925
6003
|
};
|
|
5926
6004
|
}
|
|
5927
6005
|
|
|
5928
|
-
//#endregion
|
|
5929
|
-
//#region src/routing/tenant-stream-paths.ts
|
|
5930
|
-
function withoutLeadingSlash(path$1) {
|
|
5931
|
-
return path$1.replace(/^\/+/, ``);
|
|
5932
|
-
}
|
|
5933
|
-
function withLeadingSlash(path$1) {
|
|
5934
|
-
return path$1.startsWith(`/`) ? path$1 : `/${path$1}`;
|
|
5935
|
-
}
|
|
5936
|
-
function prefixTenantStreamPath(path$1, tenantId) {
|
|
5937
|
-
const normalized = withoutLeadingSlash(path$1);
|
|
5938
|
-
if (!normalized || normalized === tenantId) return tenantId;
|
|
5939
|
-
if (normalized.startsWith(`${tenantId}/`)) return normalized;
|
|
5940
|
-
return `${tenantId}/${normalized}`;
|
|
5941
|
-
}
|
|
5942
|
-
function stripTenantStreamPrefix(path$1, tenantId) {
|
|
5943
|
-
const normalized = withoutLeadingSlash(path$1);
|
|
5944
|
-
if (normalized === tenantId) return ``;
|
|
5945
|
-
if (normalized.startsWith(`${tenantId}/`)) return normalized.slice(tenantId.length + 1);
|
|
5946
|
-
return normalized;
|
|
5947
|
-
}
|
|
5948
|
-
|
|
5949
6006
|
//#endregion
|
|
5950
6007
|
//#region src/routing/durable-streams-routing-adapter.ts
|
|
5951
6008
|
function appendSearch(target, source) {
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
}
|
|
5955
|
-
function removeServiceQuery(target) {
|
|
5956
|
-
target.searchParams.delete(`service`);
|
|
6009
|
+
source.searchParams.forEach((value, key) => {
|
|
6010
|
+
if (key !== `service`) target.searchParams.append(key, value);
|
|
6011
|
+
});
|
|
5957
6012
|
return target;
|
|
5958
6013
|
}
|
|
5959
|
-
function
|
|
5960
|
-
|
|
5961
|
-
const segments = incomingUrl.pathname.split(`/`).filter(Boolean);
|
|
5962
|
-
if (segments[0] === `v1` && segments[1] === `stream`) return {
|
|
5963
|
-
incomingUrl,
|
|
5964
|
-
streamPath: segments.length > 2 ? `/${segments.slice(3).join(`/`)}` : `/`
|
|
5965
|
-
};
|
|
5966
|
-
return {
|
|
5967
|
-
incomingUrl,
|
|
5968
|
-
streamPath: incomingUrl.pathname || `/${serviceId}`
|
|
5969
|
-
};
|
|
5970
|
-
}
|
|
5971
|
-
function backendStreamUrl(input, backendStreamPath) {
|
|
5972
|
-
const path$1 = backendStreamPath.replace(/^\/+/, ``);
|
|
5973
|
-
const target = new URL(`/v1/stream/${path$1}`, input.durableStreamsUrl);
|
|
5974
|
-
return target;
|
|
6014
|
+
function withoutTrailingSlash(pathname) {
|
|
6015
|
+
return pathname.replace(/\/+$/, ``) || `/`;
|
|
5975
6016
|
}
|
|
5976
|
-
function
|
|
6017
|
+
function appendRequestPathToStreamRoot(input) {
|
|
5977
6018
|
const incomingUrl = new URL(input.requestUrl, `http://localhost`);
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
|
|
5984
|
-
|
|
5985
|
-
|
|
5986
|
-
|
|
5987
|
-
|
|
5988
|
-
return prefixTenantStreamPath(streamPath, serviceId);
|
|
6019
|
+
const path$1 = incomingUrl.pathname.replace(/^\/+/, ``);
|
|
6020
|
+
const target = new URL(input.durableStreamsUrl);
|
|
6021
|
+
target.pathname = path$1 ? `${withoutTrailingSlash(target.pathname)}/${path$1}` : withoutTrailingSlash(target.pathname);
|
|
6022
|
+
return appendSearch(target, incomingUrl);
|
|
6023
|
+
}
|
|
6024
|
+
const streamRootDurableStreamsRoutingAdapter = {
|
|
6025
|
+
streamUrl: appendRequestPathToStreamRoot,
|
|
6026
|
+
controlUrl: appendRequestPathToStreamRoot,
|
|
6027
|
+
toBackendStreamPath(_serviceId, streamPath) {
|
|
6028
|
+
return streamPath.replace(/^\/+/, ``);
|
|
5989
6029
|
},
|
|
5990
|
-
toRuntimeStreamPath(
|
|
5991
|
-
return
|
|
6030
|
+
toRuntimeStreamPath(_serviceId, streamPath) {
|
|
6031
|
+
return streamPath.replace(/^\/+/, ``);
|
|
5992
6032
|
}
|
|
5993
6033
|
};
|
|
5994
|
-
|
|
5995
|
-
|
|
6034
|
+
const pathPrefixedSingleTenantDurableStreamsRoutingAdapter = streamRootDurableStreamsRoutingAdapter;
|
|
6035
|
+
const tenantRootDurableStreamsRoutingAdapter = streamRootDurableStreamsRoutingAdapter;
|
|
6036
|
+
function resolveDurableStreamsRoutingAdapter(adapter, _durableStreamsUrl) {
|
|
6037
|
+
return adapter ?? streamRootDurableStreamsRoutingAdapter;
|
|
5996
6038
|
}
|
|
5997
6039
|
|
|
5998
6040
|
//#endregion
|
|
@@ -6014,8 +6056,11 @@ function buildElectricProxyTarget(options) {
|
|
|
6014
6056
|
target.searchParams.set(`columns`, `"tenant_id","name","description","creation_schema","inbox_schemas","state_schemas","serve_endpoint","default_dispatch_policy","revision","created_at","updated_at"`);
|
|
6015
6057
|
applyTenantShapeWhere(target, options.tenantId);
|
|
6016
6058
|
} else if (table === `runners`) {
|
|
6017
|
-
target.searchParams.set(`columns`, `"tenant_id","id","
|
|
6018
|
-
applyTenantShapeWhere(target, options.tenantId);
|
|
6059
|
+
target.searchParams.set(`columns`, `"tenant_id","id","owner_principal","label","kind","admin_status","wake_stream","created_at","updated_at"`);
|
|
6060
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral(options.principalUrl ?? ``)}`]);
|
|
6061
|
+
} else if (table === `runner_runtime_diagnostics`) {
|
|
6062
|
+
target.searchParams.set(`columns`, `"tenant_id","runner_id","owner_principal","wake_stream_offset","last_seen_at","liveness_lease_expires_at","diagnostics","updated_at"`);
|
|
6063
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral(options.principalUrl ?? ``)}`]);
|
|
6019
6064
|
} else if (table === `entity_dispatch_state`) {
|
|
6020
6065
|
target.searchParams.set(`columns`, `"tenant_id","entity_url","pending_source_streams","pending_reason","pending_since","outstanding_wake_id","outstanding_wake_target","outstanding_wake_created_at","active_consumer_id","active_runner_id","active_epoch","active_claimed_at","active_lease_expires_at","last_wake_id","last_claimed_at","last_released_at","last_completed_at","last_error","updated_at"`);
|
|
6021
6066
|
applyTenantShapeWhere(target, options.tenantId);
|
|
@@ -6029,13 +6074,13 @@ function buildElectricProxyTarget(options) {
|
|
|
6029
6074
|
return target;
|
|
6030
6075
|
}
|
|
6031
6076
|
async function forwardFetchRequest(options) {
|
|
6032
|
-
const routingAdapter = resolveDurableStreamsRoutingAdapter(options.durableStreamsRouting);
|
|
6077
|
+
const routingAdapter = resolveDurableStreamsRoutingAdapter(options.durableStreamsRouting, options.durableStreamsUrl);
|
|
6033
6078
|
const routingInput = {
|
|
6034
6079
|
durableStreamsUrl: options.durableStreamsUrl,
|
|
6035
6080
|
serviceId: options.serviceId,
|
|
6036
6081
|
requestUrl: options.request.url
|
|
6037
6082
|
};
|
|
6038
|
-
const upstreamUrl = options.route === `
|
|
6083
|
+
const upstreamUrl = options.route === `control` ? routingAdapter.controlUrl(routingInput) : routingAdapter.streamUrl(routingInput);
|
|
6039
6084
|
const headers = new Headers(options.request.headers);
|
|
6040
6085
|
if (options.durableStreamsBearerMode !== `none`) await applyDurableStreamsBearer(headers, options.durableStreamsBearer, { overwrite: options.durableStreamsBearerMode !== `if-missing` });
|
|
6041
6086
|
const init = {
|
|
@@ -6061,8 +6106,8 @@ function decodeJsonObject(body) {
|
|
|
6061
6106
|
} catch {}
|
|
6062
6107
|
return null;
|
|
6063
6108
|
}
|
|
6064
|
-
function applyTenantShapeWhere(target, tenantId) {
|
|
6065
|
-
const tenantWhere = `tenant_id = ${sqlStringLiteral(tenantId)}
|
|
6109
|
+
function applyTenantShapeWhere(target, tenantId, extraConditions = []) {
|
|
6110
|
+
const tenantWhere = [`tenant_id = ${sqlStringLiteral(tenantId)}`, ...extraConditions].join(` AND `);
|
|
6066
6111
|
const existingWhere = target.searchParams.get(`where`);
|
|
6067
6112
|
target.searchParams.set(`where`, existingWhere ? `${tenantWhere} AND (${existingWhere})` : tenantWhere);
|
|
6068
6113
|
}
|
|
@@ -6073,8 +6118,21 @@ function sqlStringLiteral(value) {
|
|
|
6073
6118
|
//#endregion
|
|
6074
6119
|
//#region src/routing/durable-streams-router.ts
|
|
6075
6120
|
const subscriptionProxyBodySchema = Type.Object({ webhook: Type.Optional(Type.Object({ url: Type.String() }, { additionalProperties: true })) }, { additionalProperties: true });
|
|
6121
|
+
const subscriptionControlActions = [
|
|
6122
|
+
`callback`,
|
|
6123
|
+
`claim`,
|
|
6124
|
+
`ack`,
|
|
6125
|
+
`release`
|
|
6126
|
+
];
|
|
6076
6127
|
const durableStreamsRouter = Router();
|
|
6077
|
-
durableStreamsRouter.
|
|
6128
|
+
durableStreamsRouter.put(`/__ds/subscriptions/:subscriptionId`, putSubscriptionBase);
|
|
6129
|
+
durableStreamsRouter.get(`/__ds/subscriptions/:subscriptionId`, getSubscriptionBase);
|
|
6130
|
+
durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId`, deleteSubscriptionBase);
|
|
6131
|
+
durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/streams`, postSubscriptionStreams);
|
|
6132
|
+
durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId/streams/:streamPath+`, deleteSubscriptionStream);
|
|
6133
|
+
for (const action of subscriptionControlActions) durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/${action}`, subscriptionAction(action));
|
|
6134
|
+
durableStreamsRouter.all(`/__ds`, controlPassThrough);
|
|
6135
|
+
durableStreamsRouter.all(`/__ds/*`, controlPassThrough);
|
|
6078
6136
|
durableStreamsRouter.post(`*`, streamAppend);
|
|
6079
6137
|
durableStreamsRouter.all(`*`, proxyPassThrough);
|
|
6080
6138
|
function bodyFromBytes$1(body) {
|
|
@@ -6087,7 +6145,7 @@ function responseFromUpstream$1(response, body) {
|
|
|
6087
6145
|
headers: responseHeaders(response)
|
|
6088
6146
|
});
|
|
6089
6147
|
}
|
|
6090
|
-
async function forwardToDurableStreams(ctx, request, body, route = `stream`, urlOverride) {
|
|
6148
|
+
async function forwardToDurableStreams(ctx, request, body, route = `stream`, urlOverride, durableStreamsBearerMode = `overwrite`) {
|
|
6091
6149
|
const headers = new Headers(request.headers);
|
|
6092
6150
|
headers.delete(`host`);
|
|
6093
6151
|
let requestBody = body;
|
|
@@ -6101,24 +6159,13 @@ async function forwardToDurableStreams(ctx, request, body, route = `stream`, url
|
|
|
6101
6159
|
body: requestBody,
|
|
6102
6160
|
durableStreamsUrl: ctx.durableStreamsUrl,
|
|
6103
6161
|
durableStreamsBearer: ctx.durableStreamsBearer,
|
|
6104
|
-
durableStreamsBearerMode
|
|
6162
|
+
durableStreamsBearerMode,
|
|
6105
6163
|
durableStreamsRouting: ctx.durableStreamsRouting,
|
|
6106
6164
|
serviceId: ctx.service,
|
|
6107
6165
|
dispatcher: ctx.durableStreamsDispatcher,
|
|
6108
6166
|
route
|
|
6109
6167
|
});
|
|
6110
6168
|
}
|
|
6111
|
-
function subscriptionIdFromPath(pathname) {
|
|
6112
|
-
const match = /^\/v1\/stream-meta\/subscriptions\/([^/]+)(?:\/.*)?$/.exec(pathname);
|
|
6113
|
-
return match ? decodeURIComponent(match[1]) : null;
|
|
6114
|
-
}
|
|
6115
|
-
function isSubscriptionBasePath(pathname) {
|
|
6116
|
-
return /^\/v1\/stream-meta\/subscriptions\/[^/]+\/?$/.test(pathname);
|
|
6117
|
-
}
|
|
6118
|
-
function usesSubscriptionScopedBearer(requestUrl) {
|
|
6119
|
-
const pathname = new URL(requestUrl, `http://localhost`).pathname;
|
|
6120
|
-
return /^\/v1\/stream-meta\/subscriptions\/[^/]+\/(?:ack|release|callback)\/?$/.test(pathname);
|
|
6121
|
-
}
|
|
6122
6169
|
function rewriteSubscriptionBodyForBackend(payload, service, routingAdapter) {
|
|
6123
6170
|
if (typeof payload.pattern === `string`) payload.pattern = routingAdapter.toBackendStreamPath(service, payload.pattern);
|
|
6124
6171
|
if (Array.isArray(payload.streams)) payload.streams = payload.streams.map((stream) => typeof stream === `string` ? routingAdapter.toBackendStreamPath(service, stream) : stream);
|
|
@@ -6163,44 +6210,50 @@ function decodeJson(bytes) {
|
|
|
6163
6210
|
return null;
|
|
6164
6211
|
}
|
|
6165
6212
|
}
|
|
6166
|
-
function
|
|
6167
|
-
const
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
const streamPath = decodeURIComponent(encodedPath);
|
|
6171
|
-
requestUrl.pathname = `${prefix}${encodeURIComponent(routingAdapter.toBackendStreamPath(service, streamPath))}`;
|
|
6172
|
-
return requestUrl.toString();
|
|
6213
|
+
function routeParam$2(request, name) {
|
|
6214
|
+
const value = request.params[name];
|
|
6215
|
+
const raw = Array.isArray(value) ? value[0] : value;
|
|
6216
|
+
return decodeURIComponent(raw ?? ``);
|
|
6173
6217
|
}
|
|
6174
|
-
|
|
6175
|
-
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
const
|
|
6179
|
-
|
|
6218
|
+
function subscriptionRoutingAdapter(ctx) {
|
|
6219
|
+
return resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl);
|
|
6220
|
+
}
|
|
6221
|
+
async function rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter) {
|
|
6222
|
+
const body = await readRequestBody(request);
|
|
6223
|
+
if (body.length === 0) return {
|
|
6224
|
+
ok: true,
|
|
6225
|
+
body,
|
|
6226
|
+
targetWebhookUrl: null
|
|
6227
|
+
};
|
|
6228
|
+
const validation = validateBody(subscriptionProxyBodySchema, body);
|
|
6229
|
+
if (!validation.ok) return {
|
|
6230
|
+
ok: false,
|
|
6231
|
+
response: validation.response
|
|
6232
|
+
};
|
|
6233
|
+
const payload = validation.value;
|
|
6180
6234
|
let targetWebhookUrl = null;
|
|
6181
|
-
|
|
6182
|
-
|
|
6183
|
-
|
|
6184
|
-
if (requestBody.length > 0) {
|
|
6185
|
-
const validation = validateBody(subscriptionProxyBodySchema, requestBody);
|
|
6186
|
-
if (!validation.ok) return validation.response;
|
|
6187
|
-
const payload = validation.value;
|
|
6188
|
-
if (payload.webhook?.url !== void 0) {
|
|
6189
|
-
targetWebhookUrl = rewriteLoopbackWebhookUrl(payload.webhook.url) ?? null;
|
|
6190
|
-
payload.webhook.url = appendPathToUrl(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
6191
|
-
}
|
|
6192
|
-
rewriteSubscriptionBodyForBackend(payload, ctx.service, routingAdapter);
|
|
6193
|
-
requestBody = new TextEncoder().encode(JSON.stringify(payload));
|
|
6194
|
-
}
|
|
6235
|
+
if (payload.webhook?.url !== void 0) {
|
|
6236
|
+
targetWebhookUrl = rewriteLoopbackWebhookUrl(payload.webhook.url) ?? null;
|
|
6237
|
+
payload.webhook.url = appendPathToUrl(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
6195
6238
|
}
|
|
6196
|
-
|
|
6197
|
-
|
|
6239
|
+
rewriteSubscriptionBodyForBackend(payload, ctx.service, routingAdapter);
|
|
6240
|
+
return {
|
|
6241
|
+
ok: true,
|
|
6242
|
+
body: new TextEncoder().encode(JSON.stringify(payload)),
|
|
6243
|
+
targetWebhookUrl
|
|
6244
|
+
};
|
|
6245
|
+
}
|
|
6246
|
+
async function forwardSubscriptionRequest(request, ctx, routingAdapter, opts = {}) {
|
|
6247
|
+
const upstream = await forwardToDurableStreams(ctx, request, opts.body, `control`, opts.requestUrl, opts.bearerMode ?? `overwrite`);
|
|
6198
6248
|
let responseBytes = upstream.body ? new Uint8Array(await upstream.arrayBuffer()) : new Uint8Array();
|
|
6199
6249
|
responseBytes = rewriteSubscriptionResponseForClient(responseBytes, upstream, ctx.service, routingAdapter);
|
|
6200
|
-
|
|
6201
|
-
|
|
6202
|
-
|
|
6203
|
-
|
|
6250
|
+
return {
|
|
6251
|
+
upstream,
|
|
6252
|
+
response: responseFromUpstream$1(upstream, responseBytes)
|
|
6253
|
+
};
|
|
6254
|
+
}
|
|
6255
|
+
async function upsertSubscriptionWebhook(ctx, subscriptionId, targetWebhookUrl) {
|
|
6256
|
+
await ctx.pgDb.insert(subscriptionWebhooks).values({
|
|
6204
6257
|
tenantId: ctx.service,
|
|
6205
6258
|
subscriptionId,
|
|
6206
6259
|
webhookUrl: targetWebhookUrl
|
|
@@ -6208,8 +6261,64 @@ async function subscriptionProxy(request, ctx) {
|
|
|
6208
6261
|
target: [subscriptionWebhooks.tenantId, subscriptionWebhooks.subscriptionId],
|
|
6209
6262
|
set: { webhookUrl: targetWebhookUrl }
|
|
6210
6263
|
});
|
|
6264
|
+
}
|
|
6265
|
+
async function deleteSubscriptionWebhook(ctx, subscriptionId) {
|
|
6266
|
+
await ctx.pgDb.delete(subscriptionWebhooks).where(and(eq(subscriptionWebhooks.tenantId, ctx.service), eq(subscriptionWebhooks.subscriptionId, subscriptionId)));
|
|
6267
|
+
}
|
|
6268
|
+
function rewriteSubscriptionStreamPathInUrl(requestUrl, service, routingAdapter, streamPath) {
|
|
6269
|
+
const prefix = requestUrl.pathname.slice(0, requestUrl.pathname.indexOf(`/streams/`) + `/streams/`.length);
|
|
6270
|
+
requestUrl.pathname = `${prefix}${encodeURIComponent(routingAdapter.toBackendStreamPath(service, streamPath))}`;
|
|
6271
|
+
return requestUrl.toString();
|
|
6272
|
+
}
|
|
6273
|
+
async function putSubscriptionBase(request, ctx) {
|
|
6274
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6275
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6276
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6277
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6278
|
+
const { upstream, response } = await forwardSubscriptionRequest(request, ctx, routingAdapter, { body: rewrite.body });
|
|
6279
|
+
if (upstream.ok && rewrite.targetWebhookUrl) await upsertSubscriptionWebhook(ctx, subscriptionId, rewrite.targetWebhookUrl);
|
|
6280
|
+
return response;
|
|
6281
|
+
}
|
|
6282
|
+
async function getSubscriptionBase(request, ctx) {
|
|
6283
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6284
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter)).response;
|
|
6285
|
+
}
|
|
6286
|
+
async function deleteSubscriptionBase(request, ctx) {
|
|
6287
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6288
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6289
|
+
const { upstream, response } = await forwardSubscriptionRequest(request, ctx, routingAdapter);
|
|
6290
|
+
if (upstream.ok) await deleteSubscriptionWebhook(ctx, subscriptionId);
|
|
6211
6291
|
return response;
|
|
6212
6292
|
}
|
|
6293
|
+
async function postSubscriptionStreams(request, ctx) {
|
|
6294
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6295
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6296
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6297
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6298
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, { body: rewrite.body })).response;
|
|
6299
|
+
}
|
|
6300
|
+
async function deleteSubscriptionStream(request, ctx) {
|
|
6301
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6302
|
+
const requestUrl = rewriteSubscriptionStreamPathInUrl(new URL(request.url), ctx.service, routingAdapter, routeParam$2(request, `streamPath`));
|
|
6303
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, { requestUrl })).response;
|
|
6304
|
+
}
|
|
6305
|
+
function subscriptionAction(action) {
|
|
6306
|
+
return async (request, ctx) => {
|
|
6307
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6308
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6309
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6310
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6311
|
+
const bearerMode = action === `ack` || action === `release` || action === `callback` ? `if-missing` : `overwrite`;
|
|
6312
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, {
|
|
6313
|
+
body: rewrite.body,
|
|
6314
|
+
bearerMode
|
|
6315
|
+
})).response;
|
|
6316
|
+
};
|
|
6317
|
+
}
|
|
6318
|
+
async function controlPassThrough(request, ctx) {
|
|
6319
|
+
const upstream = await forwardToDurableStreams(ctx, request, void 0, `control`);
|
|
6320
|
+
return responseFromUpstream$1(upstream);
|
|
6321
|
+
}
|
|
6213
6322
|
async function streamAppend(request, ctx) {
|
|
6214
6323
|
return await electricAgentsStreamAppendRouter.fetch(createStreamAppendRouteRequest(request), ctx.runtime, (req, body) => forwardFetchRequest({
|
|
6215
6324
|
request: {
|
|
@@ -6230,10 +6339,9 @@ async function proxyPassThrough(request, ctx) {
|
|
|
6230
6339
|
const upstream = await forwardToDurableStreams(ctx, request);
|
|
6231
6340
|
const streamPath = new URL(request.url).pathname;
|
|
6232
6341
|
const method = request.method.toUpperCase();
|
|
6233
|
-
const
|
|
6234
|
-
const endTrackedRead = method === `GET` && !isControlPath ? await ctx.entityBridgeManager.beginClientRead(streamPath) : null;
|
|
6342
|
+
const endTrackedRead = method === `GET` ? await ctx.entityBridgeManager.beginClientRead(streamPath) : null;
|
|
6235
6343
|
try {
|
|
6236
|
-
if (method === `HEAD`
|
|
6344
|
+
if (method === `HEAD`) await ctx.entityBridgeManager.touchByStreamPath(streamPath);
|
|
6237
6345
|
return responseFromUpstream$1(upstream);
|
|
6238
6346
|
} finally {
|
|
6239
6347
|
await endTrackedRead?.();
|
|
@@ -6264,7 +6372,8 @@ async function proxyElectric(request, ctx) {
|
|
|
6264
6372
|
incomingUrl: new URL(request.url),
|
|
6265
6373
|
electricUrl: ctx.electricUrl,
|
|
6266
6374
|
electricSecret: ctx.electricSecret,
|
|
6267
|
-
tenantId: ctx.service
|
|
6375
|
+
tenantId: ctx.service,
|
|
6376
|
+
principalUrl: ctx.principal.url
|
|
6268
6377
|
});
|
|
6269
6378
|
const headers = new Headers(request.headers);
|
|
6270
6379
|
headers.delete(`host`);
|
|
@@ -6525,10 +6634,8 @@ async function sendEntity(request, ctx) {
|
|
|
6525
6634
|
if (parsed.from !== void 0 && parsed.from !== principal.url) return apiError(400, ErrCodeInvalidRequest, `Request from must match Electric-Principal`);
|
|
6526
6635
|
await ctx.entityManager.ensurePrincipal(principal);
|
|
6527
6636
|
const { entityUrl, entity } = requireExistingEntityRoute(request);
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
await linkEntityDispatchSubscription(ctx, updatedEntity);
|
|
6531
|
-
}
|
|
6637
|
+
const dispatchEntity = entity.dispatch_policy ? entity : await backfillEntityDispatchPolicy(ctx, entity);
|
|
6638
|
+
await linkEntityDispatchSubscription(ctx, dispatchEntity);
|
|
6532
6639
|
if (parsed.afterMs && parsed.afterMs > 0) await ctx.entityManager.enqueueDelayedSend(entityUrl, {
|
|
6533
6640
|
from: principal.url,
|
|
6534
6641
|
payload: parsed.payload,
|
|
@@ -6577,11 +6684,11 @@ async function spawnEntity(request, ctx) {
|
|
|
6577
6684
|
wake: parsed.wake,
|
|
6578
6685
|
created_by: principal.url
|
|
6579
6686
|
});
|
|
6687
|
+
await linkEntityDispatchSubscription(ctx, entity);
|
|
6580
6688
|
if (parsed.initialMessage !== void 0) await ctx.entityManager.send(entity.url, {
|
|
6581
6689
|
from: principal.url,
|
|
6582
6690
|
payload: parsed.initialMessage
|
|
6583
6691
|
});
|
|
6584
|
-
await linkEntityDispatchSubscription(ctx, entity);
|
|
6585
6692
|
return json({
|
|
6586
6693
|
...toPublicEntity(entity),
|
|
6587
6694
|
txid: entity.txid
|
|
@@ -6745,7 +6852,13 @@ function applyCors(response) {
|
|
|
6745
6852
|
const headers = new Headers(response.headers);
|
|
6746
6853
|
headers.set(`access-control-allow-origin`, `*`);
|
|
6747
6854
|
headers.set(`access-control-allow-methods`, `GET, POST, PUT, PATCH, DELETE, OPTIONS`);
|
|
6748
|
-
headers.set(`access-control-allow-headers`,
|
|
6855
|
+
headers.set(`access-control-allow-headers`, [
|
|
6856
|
+
`content-type`,
|
|
6857
|
+
`authorization`,
|
|
6858
|
+
`electric-claim-token`,
|
|
6859
|
+
ELECTRIC_PRINCIPAL_HEADER,
|
|
6860
|
+
`ngrok-skip-browser-warning`
|
|
6861
|
+
].join(`, `));
|
|
6749
6862
|
headers.set(`access-control-expose-headers`, `*`);
|
|
6750
6863
|
return new Response(response.body, {
|
|
6751
6864
|
status: response.status,
|
|
@@ -6780,11 +6893,17 @@ function getRequestSpan(req) {
|
|
|
6780
6893
|
return carrier(req)[SPAN_KEY];
|
|
6781
6894
|
}
|
|
6782
6895
|
|
|
6896
|
+
//#endregion
|
|
6897
|
+
//#region src/routing/tenant-stream-paths.ts
|
|
6898
|
+
function withLeadingSlash(path$1) {
|
|
6899
|
+
return path$1.startsWith(`/`) ? path$1 : `/${path$1}`;
|
|
6900
|
+
}
|
|
6901
|
+
|
|
6783
6902
|
//#endregion
|
|
6784
6903
|
//#region src/routing/runners-router.ts
|
|
6785
6904
|
const registerRunnerBodySchema = Type.Object({
|
|
6786
6905
|
id: Type.String(),
|
|
6787
|
-
|
|
6906
|
+
owner_principal: Type.Optional(Type.String()),
|
|
6788
6907
|
label: Type.String(),
|
|
6789
6908
|
kind: Type.Optional(Type.Union([
|
|
6790
6909
|
Type.Literal(`local`),
|
|
@@ -6800,7 +6919,8 @@ const heartbeatBodySchema = Type.Object({
|
|
|
6800
6919
|
lease_ms: Type.Optional(Type.Number()),
|
|
6801
6920
|
wake_stream_offset: Type.Optional(Type.String()),
|
|
6802
6921
|
wakeStreamOffset: Type.Optional(Type.String()),
|
|
6803
|
-
liveness_lease_expires_at: Type.Optional(Type.String())
|
|
6922
|
+
liveness_lease_expires_at: Type.Optional(Type.String()),
|
|
6923
|
+
diagnostics: Type.Optional(Type.Record(Type.String(), Type.Unknown()))
|
|
6804
6924
|
});
|
|
6805
6925
|
const claimBodySchema = Type.Object({
|
|
6806
6926
|
subscription_id: Type.Optional(Type.String()),
|
|
@@ -6808,9 +6928,39 @@ const claimBodySchema = Type.Object({
|
|
|
6808
6928
|
generation: Type.Optional(Type.Number()),
|
|
6809
6929
|
ts: Type.Optional(Type.Union([Type.String(), Type.Number()]))
|
|
6810
6930
|
}, { additionalProperties: true });
|
|
6931
|
+
const runnerClientStatuses = new Set([
|
|
6932
|
+
`stopped`,
|
|
6933
|
+
`starting`,
|
|
6934
|
+
`connecting`,
|
|
6935
|
+
`streaming`,
|
|
6936
|
+
`reconnecting`,
|
|
6937
|
+
`stopping`
|
|
6938
|
+
]);
|
|
6939
|
+
const runnerLastClaimResults = new Set([
|
|
6940
|
+
`claimed`,
|
|
6941
|
+
`no_work`,
|
|
6942
|
+
`error`
|
|
6943
|
+
]);
|
|
6944
|
+
const runnerStringOrNullDiagnostics = [
|
|
6945
|
+
`started_at`,
|
|
6946
|
+
`stream_connected_since`,
|
|
6947
|
+
`last_error`,
|
|
6948
|
+
`last_error_at`,
|
|
6949
|
+
`last_heartbeat_at`,
|
|
6950
|
+
`last_claim_at`,
|
|
6951
|
+
`last_dispatch_at`
|
|
6952
|
+
];
|
|
6953
|
+
const runnerNumberDiagnostics = [
|
|
6954
|
+
`reconnect_count`,
|
|
6955
|
+
`events_received`,
|
|
6956
|
+
`claims_succeeded`,
|
|
6957
|
+
`claims_skipped`,
|
|
6958
|
+
`claims_failed`
|
|
6959
|
+
];
|
|
6811
6960
|
const runnersRouter = Router({ base: `/_electric/runners` });
|
|
6812
6961
|
runnersRouter.post(`/`, withSchema(registerRunnerBodySchema), registerRunner);
|
|
6813
6962
|
runnersRouter.get(`/`, listRunners);
|
|
6963
|
+
runnersRouter.get(`/:id/health`, runnerHealth);
|
|
6814
6964
|
runnersRouter.get(`/:id`, getRunner);
|
|
6815
6965
|
runnersRouter.post(`/:id/heartbeat`, withSchema(heartbeatBodySchema), heartbeat);
|
|
6816
6966
|
runnersRouter.post(`/:id/enable`, setEnabled);
|
|
@@ -6823,14 +6973,41 @@ function routeParam$1(request, name) {
|
|
|
6823
6973
|
function firstQueryValue(value) {
|
|
6824
6974
|
return Array.isArray(value) ? value[0] : value;
|
|
6825
6975
|
}
|
|
6976
|
+
function requireAuthenticatedPrincipal(ctx) {
|
|
6977
|
+
if (ctx.principal) return ctx.principal;
|
|
6978
|
+
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner route requires an authenticated principal`, 401);
|
|
6979
|
+
}
|
|
6980
|
+
function canonicalOwnerPrincipal(input) {
|
|
6981
|
+
return parsePrincipalUrl(input)?.url ?? null;
|
|
6982
|
+
}
|
|
6983
|
+
function sanitizeRunnerDiagnostics(diagnostics) {
|
|
6984
|
+
if (!diagnostics) return void 0;
|
|
6985
|
+
const sanitized = {};
|
|
6986
|
+
if (typeof diagnostics.status === `string` && runnerClientStatuses.has(diagnostics.status)) sanitized.status = diagnostics.status;
|
|
6987
|
+
if (typeof diagnostics.stream_connected === `boolean`) sanitized.stream_connected = diagnostics.stream_connected;
|
|
6988
|
+
if (typeof diagnostics.last_heartbeat_ok === `boolean`) sanitized.last_heartbeat_ok = diagnostics.last_heartbeat_ok;
|
|
6989
|
+
if (diagnostics.last_claim_result === null || typeof diagnostics.last_claim_result === `string` && runnerLastClaimResults.has(diagnostics.last_claim_result)) sanitized.last_claim_result = diagnostics.last_claim_result;
|
|
6990
|
+
for (const key of runnerStringOrNullDiagnostics) {
|
|
6991
|
+
const value = diagnostics[key];
|
|
6992
|
+
if (typeof value === `string` || value === null) sanitized[key] = value;
|
|
6993
|
+
}
|
|
6994
|
+
for (const key of runnerNumberDiagnostics) {
|
|
6995
|
+
const value = diagnostics[key];
|
|
6996
|
+
if (typeof value === `number` && Number.isFinite(value) && value >= 0) sanitized[key] = value;
|
|
6997
|
+
}
|
|
6998
|
+
return Object.keys(sanitized).length > 0 ? sanitized : void 0;
|
|
6999
|
+
}
|
|
6826
7000
|
async function registerRunner(request, ctx) {
|
|
6827
7001
|
const parsed = routeBody(request);
|
|
6828
|
-
const
|
|
6829
|
-
|
|
6830
|
-
if (
|
|
7002
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
7003
|
+
const ownerPrincipal = parsed.owner_principal ?? principal.url;
|
|
7004
|
+
if (!ownerPrincipal) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal is required when no authenticated principal is present`, 400);
|
|
7005
|
+
const canonicalOwner = canonicalOwnerPrincipal(ownerPrincipal);
|
|
7006
|
+
if (!canonicalOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${ownerPrincipal}`, 400);
|
|
7007
|
+
if (canonicalOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
6831
7008
|
const runner = await ctx.entityManager.registry.createRunner({
|
|
6832
7009
|
id: parsed.id,
|
|
6833
|
-
|
|
7010
|
+
ownerPrincipal: canonicalOwner,
|
|
6834
7011
|
label: parsed.label,
|
|
6835
7012
|
kind: parsed.kind,
|
|
6836
7013
|
adminStatus: parsed.admin_status,
|
|
@@ -6840,26 +7017,32 @@ async function registerRunner(request, ctx) {
|
|
|
6840
7017
|
return json(runner, { status: 201 });
|
|
6841
7018
|
}
|
|
6842
7019
|
async function listRunners(request, ctx) {
|
|
6843
|
-
const
|
|
6844
|
-
|
|
6845
|
-
const
|
|
7020
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
7021
|
+
const requestedOwner = firstQueryValue(request.query.owner_principal);
|
|
7022
|
+
const canonicalRequestedOwner = requestedOwner ? canonicalOwnerPrincipal(requestedOwner) : void 0;
|
|
7023
|
+
if (requestedOwner && !canonicalRequestedOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${requestedOwner}`, 400);
|
|
7024
|
+
if (canonicalRequestedOwner && canonicalRequestedOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
7025
|
+
const runners$1 = await ctx.entityManager.registry.listRunners({ ownerPrincipal: principal.url });
|
|
6846
7026
|
return json(runners$1);
|
|
6847
7027
|
}
|
|
6848
7028
|
async function getRunner(request, ctx) {
|
|
6849
7029
|
const runner = await requireRunner(ctx, routeParam$1(request, `id`));
|
|
6850
|
-
assertRunnerOwnerIfAuthenticated(ctx, runner.
|
|
7030
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
6851
7031
|
return json(runner);
|
|
6852
7032
|
}
|
|
6853
7033
|
async function heartbeat(request, ctx) {
|
|
6854
7034
|
const runnerId = routeParam$1(request, `id`);
|
|
7035
|
+
requireAuthenticatedPrincipal(ctx);
|
|
6855
7036
|
const existing = await requireRunner(ctx, runnerId);
|
|
6856
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
7037
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
6857
7038
|
const parsed = routeBody(request);
|
|
6858
7039
|
const runner = await ctx.entityManager.registry.heartbeatRunner({
|
|
6859
7040
|
runnerId,
|
|
7041
|
+
ownerPrincipal: existing.owner_principal,
|
|
6860
7042
|
leaseMs: parsed.lease_ms,
|
|
6861
7043
|
wakeStreamOffset: parsed.wake_stream_offset ?? parsed.wakeStreamOffset,
|
|
6862
|
-
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0
|
|
7044
|
+
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0,
|
|
7045
|
+
diagnostics: sanitizeRunnerDiagnostics(parsed.diagnostics)
|
|
6863
7046
|
});
|
|
6864
7047
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6865
7048
|
return json(runner);
|
|
@@ -6872,16 +7055,18 @@ async function setDisabled(request, ctx) {
|
|
|
6872
7055
|
}
|
|
6873
7056
|
async function setRunnerStatus(request, ctx, adminStatus) {
|
|
6874
7057
|
const runnerId = routeParam$1(request, `id`);
|
|
7058
|
+
requireAuthenticatedPrincipal(ctx);
|
|
6875
7059
|
const existing = await requireRunner(ctx, runnerId);
|
|
6876
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
7060
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
6877
7061
|
const runner = await ctx.entityManager.registry.setRunnerAdminStatus(runnerId, adminStatus);
|
|
6878
7062
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6879
7063
|
return json(runner);
|
|
6880
7064
|
}
|
|
6881
7065
|
async function claimWake(request, ctx) {
|
|
6882
7066
|
const runnerId = routeParam$1(request, `id`);
|
|
7067
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
6883
7068
|
const runner = await requireRunner(ctx, runnerId);
|
|
6884
|
-
if (
|
|
7069
|
+
if (runner.owner_principal !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner claim requires the authenticated owner`, 403);
|
|
6885
7070
|
if (runner.admin_status !== `enabled`) throw new ElectricAgentsError(ErrCodeNotRunning, `Runner is disabled`, 409);
|
|
6886
7071
|
const parsed = routeBody(request);
|
|
6887
7072
|
const expectedSubscriptionId = subscriptionIdForDispatchTarget({
|
|
@@ -6912,11 +7097,95 @@ async function requireRunner(ctx, runnerId) {
|
|
|
6912
7097
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6913
7098
|
return runner;
|
|
6914
7099
|
}
|
|
6915
|
-
function assertRunnerOwnerIfAuthenticated(ctx,
|
|
6916
|
-
|
|
6917
|
-
if (
|
|
7100
|
+
function assertRunnerOwnerIfAuthenticated(ctx, ownerPrincipal) {
|
|
7101
|
+
requireAuthenticatedPrincipal(ctx);
|
|
7102
|
+
if (ownerPrincipal === ctx.principal.url) return;
|
|
6918
7103
|
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner access requires the authenticated owner`, 403);
|
|
6919
7104
|
}
|
|
7105
|
+
async function runnerHealth(request, ctx) {
|
|
7106
|
+
const runnerId = routeParam$1(request, `id`);
|
|
7107
|
+
const runner = await requireRunner(ctx, runnerId);
|
|
7108
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
7109
|
+
const runtimeDiagnostics = await ctx.entityManager.registry.getRunnerDiagnostics(runnerId);
|
|
7110
|
+
const now = Date.now();
|
|
7111
|
+
const parsedLeaseExpiresAt = runtimeDiagnostics?.liveness_lease_expires_at ? new Date(runtimeDiagnostics.liveness_lease_expires_at).getTime() : null;
|
|
7112
|
+
const leaseExpiresAt = parsedLeaseExpiresAt !== null && Number.isFinite(parsedLeaseExpiresAt) ? parsedLeaseExpiresAt : null;
|
|
7113
|
+
let livenessStatus;
|
|
7114
|
+
if (runner.admin_status === `disabled`) livenessStatus = `offline`;
|
|
7115
|
+
else if (leaseExpiresAt !== null && leaseExpiresAt > now) livenessStatus = `online`;
|
|
7116
|
+
else if (leaseExpiresAt !== null) livenessStatus = `expired`;
|
|
7117
|
+
else livenessStatus = `offline`;
|
|
7118
|
+
const [activeClaims, dispatchStats] = await Promise.all([ctx.entityManager.registry.getActiveClaimsForRunner(runnerId), ctx.entityManager.registry.getDispatchStatsForRunner(runnerId)]);
|
|
7119
|
+
const clientDiagnostics = sanitizeRunnerDiagnostics(runtimeDiagnostics?.diagnostics) ?? null;
|
|
7120
|
+
const issues = [];
|
|
7121
|
+
let healthStatus = `healthy`;
|
|
7122
|
+
const escalate = (floor) => {
|
|
7123
|
+
if (floor === `unhealthy`) healthStatus = `unhealthy`;
|
|
7124
|
+
else if (healthStatus === `healthy`) healthStatus = `degraded`;
|
|
7125
|
+
};
|
|
7126
|
+
if (runner.admin_status === `disabled`) {
|
|
7127
|
+
escalate(`unhealthy`);
|
|
7128
|
+
issues.push(`Runner is disabled`);
|
|
7129
|
+
}
|
|
7130
|
+
if (livenessStatus === `expired`) {
|
|
7131
|
+
escalate(`unhealthy`);
|
|
7132
|
+
const ago = leaseExpiresAt ? Math.round((now - leaseExpiresAt) / 1e3) : 0;
|
|
7133
|
+
issues.push(`Heartbeat lease expired ${ago}s ago`);
|
|
7134
|
+
}
|
|
7135
|
+
if (livenessStatus === `offline` && runner.admin_status === `enabled`) {
|
|
7136
|
+
escalate(`degraded`);
|
|
7137
|
+
issues.push(`Runner has never sent a heartbeat`);
|
|
7138
|
+
}
|
|
7139
|
+
if (clientDiagnostics) {
|
|
7140
|
+
if (clientDiagnostics.stream_connected === false) {
|
|
7141
|
+
escalate(`degraded`);
|
|
7142
|
+
issues.push(`Client reports stream disconnected`);
|
|
7143
|
+
}
|
|
7144
|
+
if (clientDiagnostics.last_heartbeat_ok === false) {
|
|
7145
|
+
escalate(`degraded`);
|
|
7146
|
+
issues.push(`Client reports last heartbeat failed`);
|
|
7147
|
+
}
|
|
7148
|
+
if (typeof clientDiagnostics.reconnect_count === `number` && clientDiagnostics.reconnect_count > 5) {
|
|
7149
|
+
escalate(`degraded`);
|
|
7150
|
+
issues.push(`Client has reconnected ${clientDiagnostics.reconnect_count} times`);
|
|
7151
|
+
}
|
|
7152
|
+
} else if (runtimeDiagnostics?.last_seen_at) {
|
|
7153
|
+
escalate(`degraded`);
|
|
7154
|
+
issues.push(`No client diagnostics available`);
|
|
7155
|
+
}
|
|
7156
|
+
const body = {
|
|
7157
|
+
runner: {
|
|
7158
|
+
id: runner.id,
|
|
7159
|
+
admin_status: runner.admin_status,
|
|
7160
|
+
liveness_status: livenessStatus,
|
|
7161
|
+
lease_expires_at: leaseExpiresAt !== null ? runtimeDiagnostics?.liveness_lease_expires_at ?? null : null,
|
|
7162
|
+
lease_remaining_ms: leaseExpiresAt !== null ? Math.max(0, leaseExpiresAt - now) : null,
|
|
7163
|
+
wake_stream: runner.wake_stream,
|
|
7164
|
+
wake_stream_offset: runtimeDiagnostics?.wake_stream_offset ?? null,
|
|
7165
|
+
last_seen_at: runtimeDiagnostics?.last_seen_at ?? null,
|
|
7166
|
+
created_at: runner.created_at
|
|
7167
|
+
},
|
|
7168
|
+
client: clientDiagnostics,
|
|
7169
|
+
claims: {
|
|
7170
|
+
active_count: activeClaims.length,
|
|
7171
|
+
active: activeClaims.map((c) => ({
|
|
7172
|
+
consumer_id: c.consumer_id,
|
|
7173
|
+
epoch: c.epoch,
|
|
7174
|
+
entity_url: c.entity_url,
|
|
7175
|
+
stream_path: c.stream_path,
|
|
7176
|
+
claimed_at: c.claimed_at,
|
|
7177
|
+
last_heartbeat_at: c.last_heartbeat_at ?? null,
|
|
7178
|
+
lease_expires_at: c.lease_expires_at ?? null
|
|
7179
|
+
}))
|
|
7180
|
+
},
|
|
7181
|
+
dispatch: dispatchStats,
|
|
7182
|
+
health: {
|
|
7183
|
+
status: healthStatus,
|
|
7184
|
+
issues
|
|
7185
|
+
}
|
|
7186
|
+
};
|
|
7187
|
+
return json(body);
|
|
7188
|
+
}
|
|
6920
7189
|
async function notificationFromClaim(ctx, input) {
|
|
6921
7190
|
const primary = input.claim.streams.find((stream) => stream.has_pending === true) ?? input.claim.streams[0];
|
|
6922
7191
|
if (!primary?.path) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Claim response did not include a stream`, 502);
|
|
@@ -7103,7 +7372,7 @@ async function webhookForward(request, ctx) {
|
|
|
7103
7372
|
let runningEntityUrl = null;
|
|
7104
7373
|
const parsedBody = parsedBodyResult.value;
|
|
7105
7374
|
const newWebhook = newWebhookPayload(parsedBody);
|
|
7106
|
-
const routingAdapter = resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting);
|
|
7375
|
+
const routingAdapter = resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl);
|
|
7107
7376
|
if (parsedBody) {
|
|
7108
7377
|
const rawPrimaryStream = newWebhook?.primaryStream ?? parsedBody.primary_stream ?? parsedBody.primaryStream ?? parsedBody.streamPath ?? null;
|
|
7109
7378
|
const primaryStream = typeof rawPrimaryStream === `string` ? toRuntimeStreamPath(rawPrimaryStream, ctx.service, routingAdapter) : null;
|
|
@@ -7232,7 +7501,7 @@ async function callbackForward(request, ctx) {
|
|
|
7232
7501
|
}
|
|
7233
7502
|
return json(responseBody);
|
|
7234
7503
|
}
|
|
7235
|
-
const upstreamBody = encodeCallbackForwardBody(ctx.service, consumerId, requestBody, resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting));
|
|
7504
|
+
const upstreamBody = encodeCallbackForwardBody(ctx.service, consumerId, requestBody, resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl));
|
|
7236
7505
|
let upstream;
|
|
7237
7506
|
try {
|
|
7238
7507
|
const subscriptionId = durableStreamsSubscriptionCallback(target.callbackUrl);
|
|
@@ -7343,4 +7612,4 @@ globalRouter.all(`/_electric/*`, internalRouter.fetch);
|
|
|
7343
7612
|
globalRouter.all(`*`, durableStreamsRouter.fetch);
|
|
7344
7613
|
|
|
7345
7614
|
//#endregion
|
|
7346
|
-
export { AgentsHost, DEFAULT_TENANT_ID, StreamClient, UnregisteredTenantError, createDb, globalRouter, isUnregisteredTenantError, pathPrefixedSingleTenantDurableStreamsRoutingAdapter, runMigrations };
|
|
7615
|
+
export { AgentsHost, DEFAULT_TENANT_ID, StreamClient, UnregisteredTenantError, createDb, globalRouter, isUnregisteredTenantError, pathPrefixedSingleTenantDurableStreamsRoutingAdapter, runMigrations, streamRootDurableStreamsRoutingAdapter, tenantRootDurableStreamsRoutingAdapter };
|