@electric-ax/agents-server 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/entrypoint.js +529 -248
- package/dist/index.cjs +1603 -1332
- package/dist/index.d.cts +274 -162
- package/dist/index.d.ts +274 -162
- package/dist/index.js +1601 -1332
- package/drizzle/0007_runner_diagnostics_and_principal.sql +22 -0
- package/drizzle/0008_runner_runtime_diagnostics.sql +50 -0
- package/drizzle/meta/_journal.json +14 -0
- package/package.json +6 -6
- package/src/db/schema.ts +33 -10
- package/src/electric-agents-types.ts +49 -3
- package/src/entity-registry.ts +136 -26
- package/src/host.ts +4 -5
- package/src/index.ts +5 -1
- package/src/principal.ts +23 -11
- package/src/routing/context.ts +1 -0
- package/src/routing/dispatch-policy.ts +123 -20
- package/src/routing/durable-streams-router.ts +286 -116
- package/src/routing/durable-streams-routing-adapter.ts +31 -64
- package/src/routing/electric-proxy-router.ts +1 -0
- package/src/routing/entities-router.ts +5 -5
- package/src/routing/hooks.ts +8 -1
- package/src/routing/internal-router.ts +6 -2
- package/src/routing/runners-router.ts +257 -19
- package/src/runtime.ts +4 -5
- package/src/server.ts +21 -15
- package/src/standalone-runtime.ts +4 -5
- package/src/stream-client.ts +18 -69
- package/src/utils/server-utils.ts +27 -8
package/dist/index.cjs
CHANGED
|
@@ -57,6 +57,7 @@ __export(schema_exports, {
|
|
|
57
57
|
entityDispatchState: () => entityDispatchState,
|
|
58
58
|
entityManifestSources: () => entityManifestSources,
|
|
59
59
|
entityTypes: () => entityTypes,
|
|
60
|
+
runnerRuntimeDiagnostics: () => runnerRuntimeDiagnostics,
|
|
60
61
|
runners: () => runners,
|
|
61
62
|
scheduledTasks: () => scheduledTasks,
|
|
62
63
|
subscriptionWebhooks: () => subscriptionWebhooks,
|
|
@@ -125,25 +126,35 @@ const users = (0, drizzle_orm_pg_core.pgTable)(`users`, {
|
|
|
125
126
|
const runners = (0, drizzle_orm_pg_core.pgTable)(`runners`, {
|
|
126
127
|
tenantId: (0, drizzle_orm_pg_core.text)(`tenant_id`).notNull().default(`default`),
|
|
127
128
|
id: (0, drizzle_orm_pg_core.text)(`id`).notNull(),
|
|
128
|
-
|
|
129
|
+
ownerPrincipal: (0, drizzle_orm_pg_core.text)(`owner_principal`).notNull(),
|
|
129
130
|
label: (0, drizzle_orm_pg_core.text)(`label`).notNull(),
|
|
130
131
|
kind: (0, drizzle_orm_pg_core.text)(`kind`).notNull().default(`local`),
|
|
131
132
|
adminStatus: (0, drizzle_orm_pg_core.text)(`admin_status`).notNull().default(`enabled`),
|
|
132
133
|
wakeStream: (0, drizzle_orm_pg_core.text)(`wake_stream`).notNull(),
|
|
133
|
-
wakeStreamOffset: (0, drizzle_orm_pg_core.text)(`wake_stream_offset`),
|
|
134
|
-
lastSeenAt: (0, drizzle_orm_pg_core.timestamp)(`last_seen_at`, { withTimezone: true }),
|
|
135
|
-
livenessLeaseExpiresAt: (0, drizzle_orm_pg_core.timestamp)(`liveness_lease_expires_at`, { withTimezone: true }),
|
|
136
134
|
createdAt: (0, drizzle_orm_pg_core.timestamp)(`created_at`, { withTimezone: true }).notNull().defaultNow(),
|
|
137
135
|
updatedAt: (0, drizzle_orm_pg_core.timestamp)(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
138
136
|
}, (table) => [
|
|
139
137
|
(0, drizzle_orm_pg_core.primaryKey)({ columns: [table.tenantId, table.id] }),
|
|
140
138
|
(0, drizzle_orm_pg_core.unique)(`uq_runners_wake_stream`).on(table.tenantId, table.wakeStream),
|
|
141
|
-
(0, drizzle_orm_pg_core.index)(`
|
|
139
|
+
(0, drizzle_orm_pg_core.index)(`idx_runners_owner_principal`).on(table.tenantId, table.ownerPrincipal),
|
|
142
140
|
(0, drizzle_orm_pg_core.index)(`idx_runners_admin_status`).on(table.tenantId, table.adminStatus),
|
|
143
|
-
(0, drizzle_orm_pg_core.index)(`idx_runners_liveness_lease_expires_at`).on(table.tenantId, table.livenessLeaseExpiresAt),
|
|
144
141
|
(0, drizzle_orm_pg_core.check)(`chk_runners_kind`, drizzle_orm.sql`${table.kind} IN ('local', 'cloud-worker', 'sandbox', 'ci', 'server')`),
|
|
145
142
|
(0, drizzle_orm_pg_core.check)(`chk_runners_admin_status`, drizzle_orm.sql`${table.adminStatus} IN ('enabled', 'disabled')`)
|
|
146
143
|
]);
|
|
144
|
+
const runnerRuntimeDiagnostics = (0, drizzle_orm_pg_core.pgTable)(`runner_runtime_diagnostics`, {
|
|
145
|
+
tenantId: (0, drizzle_orm_pg_core.text)(`tenant_id`).notNull().default(`default`),
|
|
146
|
+
runnerId: (0, drizzle_orm_pg_core.text)(`runner_id`).notNull(),
|
|
147
|
+
ownerPrincipal: (0, drizzle_orm_pg_core.text)(`owner_principal`).notNull(),
|
|
148
|
+
wakeStreamOffset: (0, drizzle_orm_pg_core.text)(`wake_stream_offset`),
|
|
149
|
+
lastSeenAt: (0, drizzle_orm_pg_core.timestamp)(`last_seen_at`, { withTimezone: true }).notNull(),
|
|
150
|
+
livenessLeaseExpiresAt: (0, drizzle_orm_pg_core.timestamp)(`liveness_lease_expires_at`, { withTimezone: true }).notNull(),
|
|
151
|
+
diagnostics: (0, drizzle_orm_pg_core.jsonb)(`diagnostics`),
|
|
152
|
+
updatedAt: (0, drizzle_orm_pg_core.timestamp)(`updated_at`, { withTimezone: true }).notNull().defaultNow()
|
|
153
|
+
}, (table) => [
|
|
154
|
+
(0, drizzle_orm_pg_core.primaryKey)({ columns: [table.tenantId, table.runnerId] }),
|
|
155
|
+
(0, drizzle_orm_pg_core.index)(`idx_runner_runtime_diagnostics_owner`).on(table.tenantId, table.ownerPrincipal),
|
|
156
|
+
(0, drizzle_orm_pg_core.index)(`idx_runner_runtime_diagnostics_liveness`).on(table.tenantId, table.livenessLeaseExpiresAt)
|
|
157
|
+
]);
|
|
147
158
|
const entityDispatchState = (0, drizzle_orm_pg_core.pgTable)(`entity_dispatch_state`, {
|
|
148
159
|
tenantId: (0, drizzle_orm_pg_core.text)(`tenant_id`).notNull().default(`default`),
|
|
149
160
|
entityUrl: (0, drizzle_orm_pg_core.text)(`entity_url`).notNull(),
|
|
@@ -453,7 +464,7 @@ var PostgresRegistry = class {
|
|
|
453
464
|
await this.db.insert(runners).values({
|
|
454
465
|
tenantId: this.tenantId,
|
|
455
466
|
id: input.id,
|
|
456
|
-
|
|
467
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
457
468
|
label: input.label,
|
|
458
469
|
kind: input.kind ?? `local`,
|
|
459
470
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -462,7 +473,7 @@ var PostgresRegistry = class {
|
|
|
462
473
|
}).onConflictDoUpdate({
|
|
463
474
|
target: [runners.tenantId, runners.id],
|
|
464
475
|
set: {
|
|
465
|
-
|
|
476
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
466
477
|
label: input.label,
|
|
467
478
|
kind: input.kind ?? `local`,
|
|
468
479
|
adminStatus: input.adminStatus ?? `enabled`,
|
|
@@ -480,20 +491,46 @@ var PostgresRegistry = class {
|
|
|
480
491
|
}
|
|
481
492
|
async listRunners(filter) {
|
|
482
493
|
const conditions = [(0, drizzle_orm.eq)(runners.tenantId, this.tenantId)];
|
|
483
|
-
if (filter?.
|
|
494
|
+
if (filter?.ownerPrincipal) conditions.push((0, drizzle_orm.eq)(runners.ownerPrincipal, filter.ownerPrincipal));
|
|
484
495
|
const rows = await this.db.select().from(runners).where((0, drizzle_orm.and)(...conditions)).orderBy((0, drizzle_orm.desc)(runners.createdAt));
|
|
485
496
|
return rows.map((row) => this.rowToRunner(row));
|
|
486
497
|
}
|
|
487
498
|
async heartbeatRunner(input) {
|
|
488
499
|
const now = input.heartbeatAt ?? new Date();
|
|
489
500
|
const leaseExpiresAt = input.livenessLeaseExpiresAt ?? new Date(now.getTime() + (input.leaseMs ?? DEFAULT_RUNNER_LEASE_MS));
|
|
490
|
-
|
|
501
|
+
await this.db.insert(runnerRuntimeDiagnostics).values({
|
|
502
|
+
tenantId: this.tenantId,
|
|
503
|
+
runnerId: input.runnerId,
|
|
504
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
491
505
|
lastSeenAt: now,
|
|
492
506
|
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
493
|
-
|
|
507
|
+
wakeStreamOffset: input.wakeStreamOffset,
|
|
508
|
+
diagnostics: input.diagnostics,
|
|
494
509
|
updatedAt: now
|
|
495
|
-
}).
|
|
496
|
-
|
|
510
|
+
}).onConflictDoUpdate({
|
|
511
|
+
target: [runnerRuntimeDiagnostics.tenantId, runnerRuntimeDiagnostics.runnerId],
|
|
512
|
+
set: {
|
|
513
|
+
lastSeenAt: now,
|
|
514
|
+
ownerPrincipal: input.ownerPrincipal,
|
|
515
|
+
livenessLeaseExpiresAt: leaseExpiresAt,
|
|
516
|
+
...input.wakeStreamOffset !== void 0 ? { wakeStreamOffset: input.wakeStreamOffset } : {},
|
|
517
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {},
|
|
518
|
+
updatedAt: now
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
const runner = await this.getRunner(input.runnerId);
|
|
522
|
+
if (!runner) return null;
|
|
523
|
+
return {
|
|
524
|
+
...runner,
|
|
525
|
+
last_seen_at: now.toISOString(),
|
|
526
|
+
liveness_lease_expires_at: leaseExpiresAt.toISOString(),
|
|
527
|
+
...input.wakeStreamOffset !== void 0 ? { wake_stream_offset: input.wakeStreamOffset } : {},
|
|
528
|
+
...input.diagnostics !== void 0 ? { diagnostics: input.diagnostics } : {}
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
async getRunnerDiagnostics(runnerId) {
|
|
532
|
+
const rows = await this.db.select().from(runnerRuntimeDiagnostics).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(runnerRuntimeDiagnostics.tenantId, this.tenantId), (0, drizzle_orm.eq)(runnerRuntimeDiagnostics.runnerId, runnerId))).limit(1);
|
|
533
|
+
return rows[0] ? this.rowToRunnerRuntimeDiagnostics(rows[0]) : null;
|
|
497
534
|
}
|
|
498
535
|
async setRunnerAdminStatus(runnerId, adminStatus) {
|
|
499
536
|
const rows = await this.db.update(runners).set({
|
|
@@ -588,6 +625,27 @@ var PostgresRegistry = class {
|
|
|
588
625
|
}).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(entityDispatchState.tenantId, this.tenantId), (0, drizzle_orm.eq)(entityDispatchState.entityUrl, claim.entity_url), (0, drizzle_orm.eq)(entityDispatchState.activeConsumerId, input.consumerId), (0, drizzle_orm.eq)(entityDispatchState.activeEpoch, input.epoch)));
|
|
589
626
|
return claim;
|
|
590
627
|
}
|
|
628
|
+
async getActiveClaimsForRunner(runnerId) {
|
|
629
|
+
const rows = await this.db.select().from(consumerClaims).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(consumerClaims.tenantId, this.tenantId), (0, drizzle_orm.eq)(consumerClaims.runnerId, runnerId), (0, drizzle_orm.eq)(consumerClaims.status, `active`)));
|
|
630
|
+
return rows.map((row) => this.rowToConsumerClaim(row));
|
|
631
|
+
}
|
|
632
|
+
async getDispatchStatsForRunner(runnerId) {
|
|
633
|
+
const rows = await this.db.select().from(entityDispatchState).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(entityDispatchState.tenantId, this.tenantId), (0, drizzle_orm.eq)(entityDispatchState.activeRunnerId, runnerId)));
|
|
634
|
+
let activeClaim = 0;
|
|
635
|
+
let outstandingWake = 0;
|
|
636
|
+
let pendingWork = 0;
|
|
637
|
+
for (const row of rows) {
|
|
638
|
+
if (row.activeConsumerId) activeClaim++;
|
|
639
|
+
if (row.outstandingWakeId && !row.activeConsumerId) outstandingWake++;
|
|
640
|
+
const pending = row.pendingSourceStreams;
|
|
641
|
+
if (pending && pending.length > 0) pendingWork++;
|
|
642
|
+
}
|
|
643
|
+
return {
|
|
644
|
+
entities_with_active_claim: activeClaim,
|
|
645
|
+
entities_with_outstanding_wake: outstandingWake,
|
|
646
|
+
entities_with_pending_work: pendingWork
|
|
647
|
+
};
|
|
648
|
+
}
|
|
591
649
|
entityTypeWhere(name) {
|
|
592
650
|
return (0, drizzle_orm.and)((0, drizzle_orm.eq)(entityTypes.tenantId, this.tenantId), (0, drizzle_orm.eq)(entityTypes.name, name));
|
|
593
651
|
}
|
|
@@ -1053,23 +1111,28 @@ var PostgresRegistry = class {
|
|
|
1053
1111
|
};
|
|
1054
1112
|
}
|
|
1055
1113
|
rowToRunner(row) {
|
|
1056
|
-
const now = Date.now();
|
|
1057
|
-
const livenessExpiry = row.livenessLeaseExpiresAt?.getTime();
|
|
1058
1114
|
return {
|
|
1059
1115
|
id: row.id,
|
|
1060
|
-
|
|
1116
|
+
owner_principal: row.ownerPrincipal,
|
|
1061
1117
|
label: row.label,
|
|
1062
1118
|
kind: assertRunnerKind(row.kind),
|
|
1063
1119
|
admin_status: assertRunnerAdminStatus(row.adminStatus),
|
|
1064
|
-
liveness: livenessExpiry !== void 0 && livenessExpiry > now ? `online` : `offline`,
|
|
1065
|
-
last_seen_at: row.lastSeenAt?.toISOString(),
|
|
1066
|
-
liveness_lease_expires_at: row.livenessLeaseExpiresAt?.toISOString(),
|
|
1067
1120
|
wake_stream: row.wakeStream,
|
|
1068
|
-
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
1069
1121
|
created_at: row.createdAt.toISOString(),
|
|
1070
1122
|
updated_at: row.updatedAt.toISOString()
|
|
1071
1123
|
};
|
|
1072
1124
|
}
|
|
1125
|
+
rowToRunnerRuntimeDiagnostics(row) {
|
|
1126
|
+
return {
|
|
1127
|
+
runner_id: row.runnerId,
|
|
1128
|
+
owner_principal: row.ownerPrincipal,
|
|
1129
|
+
wake_stream_offset: row.wakeStreamOffset ?? void 0,
|
|
1130
|
+
last_seen_at: row.lastSeenAt.toISOString(),
|
|
1131
|
+
liveness_lease_expires_at: row.livenessLeaseExpiresAt.toISOString(),
|
|
1132
|
+
diagnostics: row.diagnostics ?? void 0,
|
|
1133
|
+
updated_at: row.updatedAt.toISOString()
|
|
1134
|
+
};
|
|
1135
|
+
}
|
|
1073
1136
|
rowToConsumerClaim(row) {
|
|
1074
1137
|
return {
|
|
1075
1138
|
consumer_id: row.consumerId,
|
|
@@ -1741,140 +1804,693 @@ function parseDispatchPolicy(value, label = `dispatch_policy`) {
|
|
|
1741
1804
|
}
|
|
1742
1805
|
|
|
1743
1806
|
//#endregion
|
|
1744
|
-
//#region src/
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1807
|
+
//#region src/tracing.ts
|
|
1808
|
+
const tracer = __opentelemetry_api.trace.getTracer(`agent-server`);
|
|
1809
|
+
const ATTR = {
|
|
1810
|
+
ENTITY_URL: `electric_agents.entity.url`,
|
|
1811
|
+
ENTITY_TYPE: `electric_agents.entity.type`,
|
|
1812
|
+
PARENT_URL: `electric_agents.entity.parent`,
|
|
1813
|
+
WAKE_SOURCE: `electric_agents.wake.source`,
|
|
1814
|
+
WAKE_SUBSCRIBER: `electric_agents.wake.subscriber`,
|
|
1815
|
+
WAKE_KIND: `electric_agents.wake.kind`,
|
|
1816
|
+
STREAM_PATH: `electric_agents.stream.path`,
|
|
1817
|
+
STREAM_OP: `electric_agents.stream.op`,
|
|
1818
|
+
DB_OP: `electric_agents.db.op`,
|
|
1819
|
+
HTTP_METHOD: `http.method`,
|
|
1820
|
+
HTTP_ROUTE: `http.route`,
|
|
1821
|
+
HTTP_STATUS: `http.status_code`
|
|
1822
|
+
};
|
|
1823
|
+
/**
|
|
1824
|
+
* Run `fn` inside an active span. Errors are recorded + status set to ERROR,
|
|
1825
|
+
* then re-thrown. Span ends in a finally block.
|
|
1826
|
+
*/
|
|
1827
|
+
async function withSpan(name, fn, opts) {
|
|
1828
|
+
return await tracer.startActiveSpan(name, opts ?? {}, async (span) => {
|
|
1829
|
+
try {
|
|
1830
|
+
return await fn(span);
|
|
1831
|
+
} catch (err) {
|
|
1832
|
+
span.recordException(err);
|
|
1833
|
+
span.setStatus({
|
|
1834
|
+
code: __opentelemetry_api.SpanStatusCode.ERROR,
|
|
1835
|
+
message: err instanceof Error ? err.message : String(err)
|
|
1836
|
+
});
|
|
1837
|
+
throw err;
|
|
1838
|
+
} finally {
|
|
1839
|
+
span.end();
|
|
1840
|
+
}
|
|
1841
|
+
});
|
|
1762
1842
|
}
|
|
1763
|
-
function
|
|
1764
|
-
|
|
1843
|
+
function injectTraceHeaders(headers, ctx = __opentelemetry_api.context.active()) {
|
|
1844
|
+
__opentelemetry_api.propagation.inject(ctx, headers);
|
|
1845
|
+
}
|
|
1846
|
+
function extractTraceContext(headers) {
|
|
1847
|
+
return __opentelemetry_api.propagation.extract(__opentelemetry_api.context.active(), headers);
|
|
1765
1848
|
}
|
|
1766
1849
|
|
|
1767
1850
|
//#endregion
|
|
1768
|
-
//#region src/
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
if (opts.dispatchPolicy) return opts.dispatchPolicy;
|
|
1783
|
-
const entityType = await ctx.entityManager.registry.getEntityType(typeName);
|
|
1784
|
-
if (opts.parent) {
|
|
1785
|
-
const parent = await ctx.entityManager.registry.getEntity(opts.parent);
|
|
1786
|
-
if (parent?.dispatch_policy) return applyTypeDefaultSubscriptionScope(parent.dispatch_policy, entityType?.default_dispatch_policy);
|
|
1851
|
+
//#region src/stream-client.ts
|
|
1852
|
+
var DurableStreamsSubscriptionError = class extends Error {
|
|
1853
|
+
code;
|
|
1854
|
+
errorMessage;
|
|
1855
|
+
constructor(message, status$4, body) {
|
|
1856
|
+
super(`${message}: ${status$4} ${body}`);
|
|
1857
|
+
this.status = status$4;
|
|
1858
|
+
this.body = body;
|
|
1859
|
+
this.name = `DurableStreamsSubscriptionError`;
|
|
1860
|
+
try {
|
|
1861
|
+
const parsed = JSON.parse(body);
|
|
1862
|
+
if (typeof parsed.error?.code === `string`) this.code = parsed.error.code;
|
|
1863
|
+
if (typeof parsed.error?.message === `string`) this.errorMessage = parsed.error.message;
|
|
1864
|
+
} catch {}
|
|
1787
1865
|
}
|
|
1788
|
-
|
|
1866
|
+
};
|
|
1867
|
+
async function resolveDurableStreamsBearer(bearer) {
|
|
1868
|
+
if (!bearer) return void 0;
|
|
1869
|
+
const value = typeof bearer === `function` ? await bearer() : bearer;
|
|
1870
|
+
const trimmed = value.trim();
|
|
1871
|
+
if (!trimmed) return void 0;
|
|
1872
|
+
return /^Bearer\s+/i.test(trimmed) ? trimmed : `Bearer ${trimmed}`;
|
|
1789
1873
|
}
|
|
1790
|
-
async function
|
|
1791
|
-
if (
|
|
1792
|
-
|
|
1793
|
-
|
|
1874
|
+
async function applyDurableStreamsBearer(headers, bearer, opts = {}) {
|
|
1875
|
+
if (!bearer) return;
|
|
1876
|
+
if (!opts.overwrite && headers.has(`authorization`)) return;
|
|
1877
|
+
const value = await resolveDurableStreamsBearer(bearer);
|
|
1878
|
+
if (value) headers.set(`authorization`, value);
|
|
1794
1879
|
}
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
const
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
dispatch_policy: dispatchPolicy
|
|
1802
|
-
};
|
|
1880
|
+
function appendPathToBaseUrl(baseUrl, path$2) {
|
|
1881
|
+
const url = new URL(baseUrl);
|
|
1882
|
+
const basePath = url.pathname.replace(/\/+$/, ``);
|
|
1883
|
+
const childPath = path$2.replace(/^\/+/, ``);
|
|
1884
|
+
url.pathname = childPath ? `${basePath === `/` ? `` : basePath}/${childPath}` : basePath || `/`;
|
|
1885
|
+
return url.toString().replace(/\/+$/, ``);
|
|
1803
1886
|
}
|
|
1804
|
-
function
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
if (!target || !defaultTarget?.subscription_id) return policy;
|
|
1808
|
-
if (!sameDispatchDestination(target, defaultTarget)) return policy;
|
|
1809
|
-
if (target.subscription_id === defaultTarget.subscription_id) return policy;
|
|
1810
|
-
return { targets: [{
|
|
1811
|
-
...target,
|
|
1812
|
-
subscription_id: defaultTarget.subscription_id
|
|
1813
|
-
}] };
|
|
1887
|
+
function durableStreamsBearerHeaders(bearer) {
|
|
1888
|
+
if (!bearer) return void 0;
|
|
1889
|
+
return { authorization: async () => await resolveDurableStreamsBearer(bearer) ?? `` };
|
|
1814
1890
|
}
|
|
1815
|
-
function
|
|
1816
|
-
|
|
1817
|
-
if (a.type === `runner` && b.type === `runner`) return a.runnerId === b.runnerId;
|
|
1818
|
-
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url;
|
|
1819
|
-
return false;
|
|
1891
|
+
function isNotFoundError(err) {
|
|
1892
|
+
return err instanceof __durable_streams_client.DurableStreamError && err.code === ErrCodeNotFound || err instanceof __durable_streams_client.FetchError && err.status === 404;
|
|
1820
1893
|
}
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
if (!target || target.type !== `runner`) return;
|
|
1824
|
-
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
1825
|
-
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
1826
|
-
if (ctx.principal && runner.owner_user_id !== ctx.principal.key) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
1894
|
+
function isAbortLikeError(err) {
|
|
1895
|
+
return err instanceof Error && (err.name === `AbortError` || err.message === `Stream request was aborted`);
|
|
1827
1896
|
}
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
const target = dispatchPolicy?.targets[0];
|
|
1831
|
-
if (!target) return;
|
|
1832
|
-
await linkStreamToTargetSubscription(ctx, target, entity);
|
|
1897
|
+
function normalizeSubscriptionPattern(pattern) {
|
|
1898
|
+
return pattern.replace(/^\/+/, ``);
|
|
1833
1899
|
}
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
const target = dispatchPolicy?.targets[0];
|
|
1837
|
-
if (!target) return;
|
|
1838
|
-
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
1839
|
-
await ctx.streamClient.removeSubscriptionStream(subscriptionId, entity.streams.main).catch((err) => {
|
|
1840
|
-
serverLog.warn(`[dispatch-policy] failed to remove stream from subscription`, {
|
|
1841
|
-
subscriptionId,
|
|
1842
|
-
stream: entity.streams.main
|
|
1843
|
-
}, err);
|
|
1844
|
-
});
|
|
1900
|
+
function normalizeSubscriptionStreamPath(path$2) {
|
|
1901
|
+
return path$2.replace(/^\/+/, ``);
|
|
1845
1902
|
}
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1903
|
+
function normalizeSubscriptionPath(path$2) {
|
|
1904
|
+
return path$2.replace(/^\/+/, ``).replace(/\/+$/, ``);
|
|
1905
|
+
}
|
|
1906
|
+
var StreamClient = class {
|
|
1907
|
+
constructor(baseUrl, options = {}) {
|
|
1908
|
+
this.baseUrl = baseUrl;
|
|
1909
|
+
this.options = options;
|
|
1910
|
+
}
|
|
1911
|
+
streamUrl(path$2) {
|
|
1912
|
+
return appendPathToBaseUrl(this.baseUrl, path$2);
|
|
1913
|
+
}
|
|
1914
|
+
streamHeaders() {
|
|
1915
|
+
return durableStreamsBearerHeaders(this.options.bearer);
|
|
1916
|
+
}
|
|
1917
|
+
async requestHeaders(init, opts = {}) {
|
|
1918
|
+
const headers = new Headers(init);
|
|
1919
|
+
await applyDurableStreamsBearer(headers, this.options.bearer, { overwrite: opts.overwriteBearer });
|
|
1920
|
+
return headers;
|
|
1921
|
+
}
|
|
1922
|
+
backendSubscriptionPath(path$2) {
|
|
1923
|
+
return normalizeSubscriptionPath(path$2);
|
|
1924
|
+
}
|
|
1925
|
+
runtimeSubscriptionPath(path$2) {
|
|
1926
|
+
return normalizeSubscriptionPath(path$2);
|
|
1927
|
+
}
|
|
1928
|
+
subscriptionUrl(subscriptionId) {
|
|
1929
|
+
return appendPathToBaseUrl(this.baseUrl, `/__ds/subscriptions/${encodeURIComponent(subscriptionId)}`);
|
|
1930
|
+
}
|
|
1931
|
+
subscriptionChildUrl(subscriptionId, ...segments) {
|
|
1932
|
+
const url = new URL(this.subscriptionUrl(subscriptionId));
|
|
1933
|
+
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/${segments.map((segment) => encodeURIComponent(segment)).join(`/`)}`;
|
|
1934
|
+
return url.toString();
|
|
1935
|
+
}
|
|
1936
|
+
async create(path$2, opts) {
|
|
1937
|
+
return await withSpan(`stream.create`, async (span) => {
|
|
1938
|
+
span.setAttributes({
|
|
1939
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
1940
|
+
[ATTR.STREAM_OP]: `create`
|
|
1861
1941
|
});
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1942
|
+
await __durable_streams_client.DurableStream.create({
|
|
1943
|
+
url: this.streamUrl(path$2),
|
|
1944
|
+
headers: this.streamHeaders(),
|
|
1945
|
+
contentType: opts.contentType,
|
|
1946
|
+
body: opts.body
|
|
1947
|
+
});
|
|
1948
|
+
});
|
|
1866
1949
|
}
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1950
|
+
async fork(path$2, sourcePath) {
|
|
1951
|
+
return await withSpan(`stream.fork`, async (span) => {
|
|
1952
|
+
span.setAttributes({
|
|
1953
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
1954
|
+
[ATTR.STREAM_OP]: `fork`
|
|
1955
|
+
});
|
|
1956
|
+
const headers = {
|
|
1957
|
+
"content-type": `application/json`,
|
|
1958
|
+
"Stream-Forked-From": new URL(this.streamUrl(sourcePath)).pathname
|
|
1959
|
+
};
|
|
1960
|
+
injectTraceHeaders(headers);
|
|
1961
|
+
const response = await fetch(this.streamUrl(path$2), {
|
|
1962
|
+
method: `PUT`,
|
|
1963
|
+
headers: await this.requestHeaders(headers)
|
|
1964
|
+
});
|
|
1965
|
+
if (response.ok) return;
|
|
1966
|
+
throw new Error(`Stream fork failed: ${response.status} ${await response.text()}`);
|
|
1967
|
+
});
|
|
1968
|
+
}
|
|
1969
|
+
async append(path$2, data, opts) {
|
|
1970
|
+
return await withSpan(`stream.append`, async (span) => {
|
|
1971
|
+
span.setAttributes({
|
|
1972
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
1973
|
+
[ATTR.STREAM_OP]: opts?.close ? `append+close` : `append`
|
|
1974
|
+
});
|
|
1975
|
+
const handle = new __durable_streams_client.DurableStream({
|
|
1976
|
+
url: this.streamUrl(path$2),
|
|
1977
|
+
headers: this.streamHeaders(),
|
|
1978
|
+
contentType: `application/json`,
|
|
1979
|
+
batching: false
|
|
1980
|
+
});
|
|
1981
|
+
if (opts?.close) {
|
|
1982
|
+
const result = await handle.close({ body: data });
|
|
1983
|
+
return { offset: result.finalOffset };
|
|
1984
|
+
}
|
|
1985
|
+
await handle.append(data);
|
|
1986
|
+
const head = await handle.head();
|
|
1987
|
+
return { offset: head.exists && head.offset || `` };
|
|
1988
|
+
});
|
|
1989
|
+
}
|
|
1990
|
+
async appendIdempotent(path$2, data, opts) {
|
|
1991
|
+
return await withSpan(`stream.appendIdempotent`, async (span) => {
|
|
1992
|
+
span.setAttributes({
|
|
1993
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
1994
|
+
[ATTR.STREAM_OP]: `appendIdempotent`
|
|
1995
|
+
});
|
|
1996
|
+
const stream = new __durable_streams_client.DurableStream({
|
|
1997
|
+
url: this.streamUrl(path$2),
|
|
1998
|
+
headers: this.streamHeaders(),
|
|
1999
|
+
contentType: `application/json`
|
|
2000
|
+
});
|
|
2001
|
+
const producer = new __durable_streams_client.IdempotentProducer(stream, opts.producerId, { epoch: opts.epoch ?? 0 });
|
|
2002
|
+
try {
|
|
2003
|
+
producer.append(data);
|
|
2004
|
+
await producer.flush();
|
|
2005
|
+
} finally {
|
|
2006
|
+
await producer.detach();
|
|
2007
|
+
}
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
async appendWithProducerHeaders(path$2, data, opts) {
|
|
2011
|
+
return await withSpan(`stream.appendWithProducerHeaders`, async (span) => {
|
|
2012
|
+
span.setAttributes({
|
|
2013
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
2014
|
+
[ATTR.STREAM_OP]: `appendWithProducerHeaders`
|
|
2015
|
+
});
|
|
2016
|
+
const headers = {
|
|
2017
|
+
"content-type": `application/json`,
|
|
2018
|
+
"Producer-Id": opts.producerId,
|
|
2019
|
+
"Producer-Epoch": String(opts.epoch),
|
|
2020
|
+
"Producer-Seq": String(opts.seq)
|
|
2021
|
+
};
|
|
2022
|
+
injectTraceHeaders(headers);
|
|
2023
|
+
const response = await fetch(this.streamUrl(path$2), {
|
|
2024
|
+
method: `POST`,
|
|
2025
|
+
headers: await this.requestHeaders(headers),
|
|
2026
|
+
body: typeof data === `string` ? data : Buffer.from(data)
|
|
2027
|
+
});
|
|
2028
|
+
if (response.ok || response.status === 204) return;
|
|
2029
|
+
throw new Error(`Stream append failed: ${response.status} ${await response.text()}`);
|
|
2030
|
+
});
|
|
2031
|
+
}
|
|
2032
|
+
async read(path$2, fromOffset) {
|
|
2033
|
+
return await withSpan(`stream.read`, async (span) => {
|
|
2034
|
+
span.setAttributes({
|
|
2035
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
2036
|
+
[ATTR.STREAM_OP]: `read`
|
|
2037
|
+
});
|
|
2038
|
+
const handle = new __durable_streams_client.DurableStream({
|
|
2039
|
+
url: this.streamUrl(path$2),
|
|
2040
|
+
headers: this.streamHeaders()
|
|
2041
|
+
});
|
|
2042
|
+
const response = await handle.stream({
|
|
2043
|
+
offset: fromOffset ?? `-1`,
|
|
2044
|
+
live: false
|
|
2045
|
+
});
|
|
2046
|
+
const messages = [];
|
|
2047
|
+
return await new Promise((resolve$1, reject) => {
|
|
2048
|
+
let settled = false;
|
|
2049
|
+
let unsub = () => {};
|
|
2050
|
+
const finish = (r) => {
|
|
2051
|
+
if (settled) return;
|
|
2052
|
+
settled = true;
|
|
2053
|
+
unsub();
|
|
2054
|
+
resolve$1(r);
|
|
2055
|
+
};
|
|
2056
|
+
unsub = response.subscribeBytes((chunk) => {
|
|
2057
|
+
messages.push({
|
|
2058
|
+
data: chunk.data,
|
|
2059
|
+
offset: chunk.offset
|
|
2060
|
+
});
|
|
2061
|
+
if (chunk.upToDate || chunk.streamClosed) finish({ messages });
|
|
2062
|
+
});
|
|
2063
|
+
response.closed.then(() => finish({ messages })).catch((err) => {
|
|
2064
|
+
if (settled) return;
|
|
2065
|
+
settled = true;
|
|
2066
|
+
unsub();
|
|
2067
|
+
reject(err);
|
|
2068
|
+
});
|
|
2069
|
+
});
|
|
2070
|
+
});
|
|
2071
|
+
}
|
|
2072
|
+
async readJson(path$2, fromOffset) {
|
|
2073
|
+
return await withSpan(`stream.readJson`, async (span) => {
|
|
2074
|
+
span.setAttributes({
|
|
2075
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
2076
|
+
[ATTR.STREAM_OP]: `readJson`
|
|
2077
|
+
});
|
|
2078
|
+
const handle = new __durable_streams_client.DurableStream({
|
|
2079
|
+
url: this.streamUrl(path$2),
|
|
2080
|
+
headers: this.streamHeaders()
|
|
2081
|
+
});
|
|
2082
|
+
const response = await handle.stream({
|
|
2083
|
+
offset: fromOffset ?? `-1`,
|
|
2084
|
+
live: false
|
|
2085
|
+
});
|
|
2086
|
+
return await response.json();
|
|
2087
|
+
});
|
|
2088
|
+
}
|
|
2089
|
+
async waitForMessages(path$2, fromOffset, timeoutMs) {
|
|
2090
|
+
return await withSpan(`stream.waitForMessages`, async (span) => {
|
|
2091
|
+
span.setAttributes({
|
|
2092
|
+
[ATTR.STREAM_PATH]: path$2,
|
|
2093
|
+
[ATTR.STREAM_OP]: `waitForMessages`
|
|
2094
|
+
});
|
|
2095
|
+
const handle = new __durable_streams_client.DurableStream({
|
|
2096
|
+
url: this.streamUrl(path$2),
|
|
2097
|
+
headers: this.streamHeaders()
|
|
2098
|
+
});
|
|
2099
|
+
const controller = new AbortController();
|
|
2100
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
2101
|
+
try {
|
|
2102
|
+
const response = await handle.stream({
|
|
2103
|
+
offset: fromOffset,
|
|
2104
|
+
live: `long-poll`,
|
|
2105
|
+
signal: controller.signal
|
|
2106
|
+
});
|
|
2107
|
+
const messages = [];
|
|
2108
|
+
return await new Promise((resolve$1, reject) => {
|
|
2109
|
+
let settled = false;
|
|
2110
|
+
let unsub = () => {};
|
|
2111
|
+
const finish = (result) => {
|
|
2112
|
+
if (settled) return;
|
|
2113
|
+
settled = true;
|
|
2114
|
+
clearTimeout(timer);
|
|
2115
|
+
unsub();
|
|
2116
|
+
resolve$1(result);
|
|
2117
|
+
};
|
|
2118
|
+
unsub = response.subscribeBytes((chunk) => {
|
|
2119
|
+
messages.push({
|
|
2120
|
+
data: chunk.data,
|
|
2121
|
+
offset: chunk.offset
|
|
2122
|
+
});
|
|
2123
|
+
if (chunk.upToDate) finish({
|
|
2124
|
+
messages,
|
|
2125
|
+
timedOut: false
|
|
2126
|
+
});
|
|
2127
|
+
});
|
|
2128
|
+
response.closed.then(() => finish({
|
|
2129
|
+
messages,
|
|
2130
|
+
timedOut: false
|
|
2131
|
+
})).catch((err) => {
|
|
2132
|
+
if (settled) return;
|
|
2133
|
+
clearTimeout(timer);
|
|
2134
|
+
if (isAbortLikeError(err)) {
|
|
2135
|
+
settled = true;
|
|
2136
|
+
unsub();
|
|
2137
|
+
resolve$1({
|
|
2138
|
+
messages: [],
|
|
2139
|
+
timedOut: true
|
|
2140
|
+
});
|
|
2141
|
+
return;
|
|
2142
|
+
}
|
|
2143
|
+
settled = true;
|
|
2144
|
+
unsub();
|
|
2145
|
+
reject(err);
|
|
2146
|
+
});
|
|
2147
|
+
});
|
|
2148
|
+
} catch (err) {
|
|
2149
|
+
clearTimeout(timer);
|
|
2150
|
+
if (isAbortLikeError(err)) return {
|
|
2151
|
+
messages: [],
|
|
2152
|
+
timedOut: true
|
|
2153
|
+
};
|
|
2154
|
+
throw err;
|
|
2155
|
+
}
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2158
|
+
async delete(path$2) {
|
|
2159
|
+
await __durable_streams_client.DurableStream.delete({
|
|
2160
|
+
url: this.streamUrl(path$2),
|
|
2161
|
+
headers: this.streamHeaders()
|
|
2162
|
+
});
|
|
2163
|
+
}
|
|
2164
|
+
async ensure(path$2, opts) {
|
|
2165
|
+
if (await this.exists(path$2)) return;
|
|
2166
|
+
try {
|
|
2167
|
+
await this.create(path$2, opts);
|
|
2168
|
+
} catch (err) {
|
|
2169
|
+
if (err && typeof err === `object` && `status` in err && err.status === 409) return;
|
|
2170
|
+
throw err;
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
async exists(path$2) {
|
|
2174
|
+
try {
|
|
2175
|
+
const result = await __durable_streams_client.DurableStream.head({
|
|
2176
|
+
url: this.streamUrl(path$2),
|
|
2177
|
+
headers: this.streamHeaders()
|
|
2178
|
+
});
|
|
2179
|
+
return result.exists;
|
|
2180
|
+
} catch (err) {
|
|
2181
|
+
if (isNotFoundError(err)) return false;
|
|
2182
|
+
throw err;
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
async createSubscription(pattern, subscriptionId, webhookUrl, description) {
|
|
2186
|
+
const res = await this.putSubscription(subscriptionId, {
|
|
2187
|
+
type: `webhook`,
|
|
2188
|
+
pattern: normalizeSubscriptionPattern(pattern),
|
|
2189
|
+
webhook: { url: webhookUrl },
|
|
2190
|
+
...description ? { description } : {}
|
|
2191
|
+
});
|
|
2192
|
+
return res;
|
|
2193
|
+
}
|
|
2194
|
+
async putSubscription(subscriptionId, input) {
|
|
2195
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2196
|
+
method: `PUT`,
|
|
2197
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2198
|
+
body: JSON.stringify({
|
|
2199
|
+
...input,
|
|
2200
|
+
pattern: typeof input.pattern === `string` ? this.backendSubscriptionPath(normalizeSubscriptionPattern(input.pattern)) : void 0,
|
|
2201
|
+
streams: input.streams?.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))),
|
|
2202
|
+
wake_stream: typeof input.wake_stream === `string` ? this.backendSubscriptionPath(normalizeSubscriptionStreamPath(input.wake_stream)) : void 0
|
|
2203
|
+
})
|
|
2204
|
+
});
|
|
2205
|
+
return await this.subscriptionJson(res, `Subscription creation failed`);
|
|
2206
|
+
}
|
|
2207
|
+
async getSubscription(subscriptionId) {
|
|
2208
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2209
|
+
method: `GET`,
|
|
2210
|
+
headers: await this.requestHeaders()
|
|
2211
|
+
});
|
|
2212
|
+
if (res.status === 404) return null;
|
|
2213
|
+
return await this.subscriptionJson(res, `Subscription query failed`);
|
|
2214
|
+
}
|
|
2215
|
+
async deleteSubscription(subscriptionId) {
|
|
2216
|
+
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
2217
|
+
method: `DELETE`,
|
|
2218
|
+
headers: await this.requestHeaders()
|
|
2219
|
+
});
|
|
2220
|
+
if (res.status === 404 || res.status === 204) return;
|
|
2221
|
+
if (!res.ok) throw new Error(`Subscription delete failed: ${res.status} ${await res.text()}`);
|
|
2222
|
+
}
|
|
2223
|
+
async addSubscriptionStreams(subscriptionId, streams$1) {
|
|
2224
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `streams`), {
|
|
2225
|
+
method: `POST`,
|
|
2226
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2227
|
+
body: JSON.stringify({ streams: streams$1.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))) })
|
|
2228
|
+
});
|
|
2229
|
+
return await this.subscriptionJson(res, `Subscription stream add failed`);
|
|
2230
|
+
}
|
|
2231
|
+
async removeSubscriptionStream(subscriptionId, streamPath) {
|
|
2232
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `streams`, this.backendSubscriptionPath(normalizeSubscriptionStreamPath(streamPath))), {
|
|
2233
|
+
method: `DELETE`,
|
|
2234
|
+
headers: await this.requestHeaders()
|
|
2235
|
+
});
|
|
2236
|
+
if (res.status === 404 || res.status === 204) return;
|
|
2237
|
+
if (!res.ok) throw new Error(`Subscription stream remove failed: ${res.status} ${await res.text()}`);
|
|
2238
|
+
}
|
|
2239
|
+
async claimSubscription(subscriptionId, worker) {
|
|
2240
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `claim`), {
|
|
2241
|
+
method: `POST`,
|
|
2242
|
+
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
2243
|
+
body: JSON.stringify({ worker })
|
|
2244
|
+
});
|
|
2245
|
+
if (res.status === 204 || res.status === 404) return null;
|
|
2246
|
+
return await this.subscriptionJson(res, `Subscription claim failed`);
|
|
2247
|
+
}
|
|
2248
|
+
async ackSubscription(subscriptionId, token, body) {
|
|
2249
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `ack`), {
|
|
2250
|
+
method: `POST`,
|
|
2251
|
+
headers: await this.requestHeaders({
|
|
2252
|
+
"content-type": `application/json`,
|
|
2253
|
+
authorization: `Bearer ${token}`
|
|
2254
|
+
}),
|
|
2255
|
+
body: JSON.stringify(this.subscriptionRequestBody(body))
|
|
2256
|
+
});
|
|
2257
|
+
return await this.subscriptionJson(res, `Subscription ack failed`);
|
|
2258
|
+
}
|
|
2259
|
+
async releaseSubscription(subscriptionId, token, body) {
|
|
2260
|
+
const res = await fetch(this.subscriptionChildUrl(subscriptionId, `release`), {
|
|
2261
|
+
method: `POST`,
|
|
2262
|
+
headers: await this.requestHeaders({
|
|
2263
|
+
"content-type": `application/json`,
|
|
2264
|
+
authorization: `Bearer ${token}`
|
|
2265
|
+
}),
|
|
2266
|
+
body: JSON.stringify(this.subscriptionRequestBody(body))
|
|
2267
|
+
});
|
|
2268
|
+
return await this.subscriptionJson(res, `Subscription release failed`);
|
|
2269
|
+
}
|
|
2270
|
+
subscriptionRequestBody(body) {
|
|
2271
|
+
const next = { ...body };
|
|
2272
|
+
if (typeof next.stream === `string`) next.stream = this.backendSubscriptionPath(next.stream);
|
|
2273
|
+
if (typeof next.path === `string`) next.path = this.backendSubscriptionPath(next.path);
|
|
2274
|
+
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
2275
|
+
if (!ack || typeof ack !== `object`) return ack;
|
|
2276
|
+
const mapped = { ...ack };
|
|
2277
|
+
if (typeof mapped.stream === `string`) mapped.stream = this.backendSubscriptionPath(mapped.stream);
|
|
2278
|
+
if (typeof mapped.path === `string`) mapped.path = this.backendSubscriptionPath(mapped.path);
|
|
2279
|
+
return mapped;
|
|
2280
|
+
});
|
|
2281
|
+
return next;
|
|
2282
|
+
}
|
|
2283
|
+
subscriptionResponseBody(body) {
|
|
2284
|
+
const next = { ...body };
|
|
2285
|
+
if (typeof next.pattern === `string`) next.pattern = this.runtimeSubscriptionPath(next.pattern);
|
|
2286
|
+
if (typeof next.wake_stream === `string`) next.wake_stream = this.runtimeSubscriptionPath(next.wake_stream);
|
|
2287
|
+
if (Array.isArray(next.streams)) next.streams = next.streams.map((stream) => {
|
|
2288
|
+
if (typeof stream === `string`) return this.runtimeSubscriptionPath(stream);
|
|
2289
|
+
return {
|
|
2290
|
+
...stream,
|
|
2291
|
+
path: this.runtimeSubscriptionPath(stream.path)
|
|
2292
|
+
};
|
|
2293
|
+
});
|
|
2294
|
+
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
2295
|
+
if (!ack || typeof ack !== `object`) return ack;
|
|
2296
|
+
const mapped = { ...ack };
|
|
2297
|
+
if (typeof mapped.stream === `string`) mapped.stream = this.runtimeSubscriptionPath(mapped.stream);
|
|
2298
|
+
if (typeof mapped.path === `string`) mapped.path = this.runtimeSubscriptionPath(mapped.path);
|
|
2299
|
+
return mapped;
|
|
2300
|
+
});
|
|
2301
|
+
if (typeof next.stream === `string`) next.stream = this.runtimeSubscriptionPath(next.stream);
|
|
2302
|
+
return next;
|
|
2303
|
+
}
|
|
2304
|
+
async subscriptionJson(res, message) {
|
|
2305
|
+
if (!res.ok) throw new DurableStreamsSubscriptionError(message, res.status, await res.text());
|
|
2306
|
+
if (res.status === 204) return {};
|
|
2307
|
+
const text$1 = await res.text();
|
|
2308
|
+
if (!text$1.trim()) return {};
|
|
2309
|
+
return this.subscriptionResponseBody(JSON.parse(text$1));
|
|
2310
|
+
}
|
|
2311
|
+
};
|
|
2312
|
+
|
|
2313
|
+
//#endregion
|
|
2314
|
+
//#region src/utils/webhook-url.ts
|
|
2315
|
+
function rewriteLoopbackWebhookUrl(value) {
|
|
2316
|
+
if (!value) return void 0;
|
|
2317
|
+
const rewriteTarget = process.env.ELECTRIC_AGENTS_REWRITE_LOOPBACK_WEBHOOKS_TO?.trim();
|
|
2318
|
+
if (!rewriteTarget) return value;
|
|
2319
|
+
const url = new URL(value);
|
|
2320
|
+
if (!isLoopbackHostname(url.hostname)) return value;
|
|
2321
|
+
if (rewriteTarget.includes(`://`)) {
|
|
2322
|
+
const target = new URL(rewriteTarget);
|
|
2323
|
+
url.protocol = target.protocol;
|
|
2324
|
+
url.username = target.username;
|
|
2325
|
+
url.password = target.password;
|
|
2326
|
+
url.hostname = target.hostname;
|
|
2327
|
+
url.port = target.port;
|
|
2328
|
+
return url.toString();
|
|
2329
|
+
}
|
|
2330
|
+
url.host = rewriteTarget;
|
|
2331
|
+
return url.toString();
|
|
2332
|
+
}
|
|
2333
|
+
function isLoopbackHostname(hostname) {
|
|
2334
|
+
return hostname === `localhost` || hostname === `127.0.0.1` || hostname === `::1`;
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2337
|
+
//#endregion
|
|
2338
|
+
//#region src/routing/dispatch-policy.ts
|
|
2339
|
+
const linkedDispatchSubscriptions = new WeakMap();
|
|
2340
|
+
function subscriptionIdForDispatchTarget(target) {
|
|
2341
|
+
if (target.subscription_id) return target.subscription_id;
|
|
2342
|
+
if (target.type === `runner`) return `runner:${target.runnerId}`;
|
|
2343
|
+
const digest = (0, node_crypto.createHash)(`sha256`).update(target.url).digest(`hex`);
|
|
2344
|
+
return `webhook:${digest.slice(0, 16)}`;
|
|
2345
|
+
}
|
|
2346
|
+
function subscriptionIdForEntityDispatchTarget(target, entityUrl) {
|
|
2347
|
+
const base = subscriptionIdForDispatchTarget(target);
|
|
2348
|
+
if (!target.subscription_id && target.type !== `runner`) return base;
|
|
2349
|
+
const digest = (0, node_crypto.createHash)(`sha256`).update(entityUrl).digest(`hex`);
|
|
2350
|
+
return `${base}:${digest.slice(0, 16)}`;
|
|
2351
|
+
}
|
|
2352
|
+
async function resolveEffectiveDispatchPolicyForSpawn(ctx, typeName, opts) {
|
|
2353
|
+
if (opts.dispatchPolicy) return opts.dispatchPolicy;
|
|
2354
|
+
const entityType = await ctx.entityManager.registry.getEntityType(typeName);
|
|
2355
|
+
if (opts.parent) {
|
|
2356
|
+
const parent = await ctx.entityManager.registry.getEntity(opts.parent);
|
|
2357
|
+
if (parent?.dispatch_policy) return applyTypeDefaultSubscriptionScope(parent.dispatch_policy, entityType?.default_dispatch_policy);
|
|
2358
|
+
}
|
|
2359
|
+
return entityType?.default_dispatch_policy;
|
|
2360
|
+
}
|
|
2361
|
+
async function resolveEffectiveDispatchPolicyForEntity(ctx, entity) {
|
|
2362
|
+
if (entity.dispatch_policy) return entity.dispatch_policy;
|
|
2363
|
+
const entityType = await ctx.entityManager.registry.getEntityType(entity.type);
|
|
2364
|
+
return entityType?.default_dispatch_policy;
|
|
2365
|
+
}
|
|
2366
|
+
async function backfillEntityDispatchPolicy(ctx, entity) {
|
|
2367
|
+
if (entity.dispatch_policy) return entity;
|
|
2368
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2369
|
+
if (!dispatchPolicy) return entity;
|
|
2370
|
+
return await ctx.entityManager.registry.updateEntityDispatchPolicy(entity.url, dispatchPolicy) ?? {
|
|
2371
|
+
...entity,
|
|
2372
|
+
dispatch_policy: dispatchPolicy
|
|
2373
|
+
};
|
|
2374
|
+
}
|
|
2375
|
+
function applyTypeDefaultSubscriptionScope(policy, typeDefault) {
|
|
2376
|
+
const target = policy.targets[0];
|
|
2377
|
+
const defaultTarget = typeDefault?.targets[0];
|
|
2378
|
+
if (!target || !defaultTarget?.subscription_id) return policy;
|
|
2379
|
+
if (!sameDispatchDestination(target, defaultTarget)) return policy;
|
|
2380
|
+
if (target.subscription_id === defaultTarget.subscription_id) return policy;
|
|
2381
|
+
return { targets: [{
|
|
2382
|
+
...target,
|
|
2383
|
+
subscription_id: defaultTarget.subscription_id
|
|
2384
|
+
}] };
|
|
2385
|
+
}
|
|
2386
|
+
function sameDispatchDestination(a, b) {
|
|
2387
|
+
if (a.type !== b.type) return false;
|
|
2388
|
+
if (a.type === `runner` && b.type === `runner`) return a.runnerId === b.runnerId;
|
|
2389
|
+
if (a.type === `webhook` && b.type === `webhook`) return a.url === b.url;
|
|
2390
|
+
return false;
|
|
2391
|
+
}
|
|
2392
|
+
function subscriptionHasStream(ctx, existing, streamPath) {
|
|
2393
|
+
const normalizedStream = streamPath.replace(/^\/+/, ``);
|
|
2394
|
+
const backendStream = `${ctx.service}/${normalizedStream}`;
|
|
2395
|
+
return existing.streams?.some((stream) => {
|
|
2396
|
+
const path$2 = typeof stream === `string` ? stream : stream.path;
|
|
2397
|
+
if (!path$2) return false;
|
|
2398
|
+
const normalized = path$2.replace(/^\/+/, ``);
|
|
2399
|
+
return normalized === normalizedStream || normalized === backendStream;
|
|
2400
|
+
}) ?? false;
|
|
2401
|
+
}
|
|
2402
|
+
function dispatchLinkCacheKey(ctx, subscriptionId, streamPath) {
|
|
2403
|
+
return `${ctx.service}:${subscriptionId}:${streamPath}`;
|
|
2404
|
+
}
|
|
2405
|
+
function getDispatchLinkCache(ctx) {
|
|
2406
|
+
let cache = linkedDispatchSubscriptions.get(ctx.streamClient);
|
|
2407
|
+
if (!cache) {
|
|
2408
|
+
cache = new Set();
|
|
2409
|
+
linkedDispatchSubscriptions.set(ctx.streamClient, cache);
|
|
2410
|
+
}
|
|
2411
|
+
return cache;
|
|
2412
|
+
}
|
|
2413
|
+
function isSubscriptionAlreadyExistsError(err) {
|
|
2414
|
+
if (!(err instanceof DurableStreamsSubscriptionError)) return false;
|
|
2415
|
+
if (err.status === 409) return true;
|
|
2416
|
+
return err.code === `SUBSCRIPTION_ALREADY_EXISTS` || err.code === `ALREADY_EXISTS` || /already exists/i.test(err.errorMessage ?? err.body ?? err.message);
|
|
2417
|
+
}
|
|
2418
|
+
async function ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, input, existing) {
|
|
2419
|
+
if (!existing) try {
|
|
2420
|
+
await ctx.streamClient.putSubscription(subscriptionId, input);
|
|
2421
|
+
return;
|
|
2422
|
+
} catch (err) {
|
|
2423
|
+
if (!isSubscriptionAlreadyExistsError(err)) throw err;
|
|
2424
|
+
existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
2425
|
+
if (!existing) {
|
|
2426
|
+
serverLog.warn(`[dispatch-policy] subscription create raced with existing subscription but it could not be read`, {
|
|
2427
|
+
subscriptionId,
|
|
2428
|
+
stream: streamPath
|
|
2429
|
+
});
|
|
2430
|
+
return;
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
if (!subscriptionHasStream(ctx, existing, streamPath)) await ctx.streamClient.addSubscriptionStreams(subscriptionId, [streamPath]);
|
|
2434
|
+
}
|
|
2435
|
+
async function assertDispatchPolicyAllowed(ctx, policy) {
|
|
2436
|
+
const target = policy?.targets[0];
|
|
2437
|
+
if (!target || target.type !== `runner`) return;
|
|
2438
|
+
if (!ctx.principal) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires an authenticated owner`, 401);
|
|
2439
|
+
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
2440
|
+
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
2441
|
+
if (runner.owner_principal !== ctx.principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner dispatch requires the authenticated owner`, 403);
|
|
2442
|
+
}
|
|
2443
|
+
async function linkEntityDispatchSubscription(ctx, entity) {
|
|
2444
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2445
|
+
const target = dispatchPolicy?.targets[0];
|
|
2446
|
+
if (!target) return;
|
|
2447
|
+
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
2448
|
+
const cacheKey = dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main);
|
|
2449
|
+
const cache = getDispatchLinkCache(ctx);
|
|
2450
|
+
if (cache.has(cacheKey)) return;
|
|
2451
|
+
await linkStreamToTargetSubscription(ctx, target, entity, subscriptionId);
|
|
2452
|
+
cache.add(cacheKey);
|
|
2453
|
+
}
|
|
2454
|
+
async function unlinkEntityDispatchSubscription(ctx, entity) {
|
|
2455
|
+
const dispatchPolicy = await resolveEffectiveDispatchPolicyForEntity(ctx, entity);
|
|
2456
|
+
const target = dispatchPolicy?.targets[0];
|
|
2457
|
+
if (!target) return;
|
|
2458
|
+
const subscriptionId = subscriptionIdForEntityDispatchTarget(target, entity.url);
|
|
2459
|
+
getDispatchLinkCache(ctx).delete(dispatchLinkCacheKey(ctx, subscriptionId, entity.streams.main));
|
|
2460
|
+
await ctx.streamClient.removeSubscriptionStream(subscriptionId, entity.streams.main).catch((err) => {
|
|
2461
|
+
serverLog.warn(`[dispatch-policy] failed to remove stream from subscription`, {
|
|
2462
|
+
subscriptionId,
|
|
2463
|
+
stream: entity.streams.main
|
|
2464
|
+
}, err);
|
|
2465
|
+
});
|
|
2466
|
+
}
|
|
2467
|
+
async function linkStreamToTargetSubscription(ctx, target, entity, subscriptionId) {
|
|
2468
|
+
const streamPath = entity.streams.main;
|
|
2469
|
+
await ctx.streamClient.ensure(streamPath, { contentType: `application/json` });
|
|
2470
|
+
const existing = await ctx.streamClient.getSubscription(subscriptionId);
|
|
2471
|
+
if (target.type === `runner`) {
|
|
2472
|
+
const runner = await ctx.entityManager.registry.getRunner(target.runnerId);
|
|
2473
|
+
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner "${target.runnerId}" not found`, 404);
|
|
2474
|
+
const wakeStream = runner.wake_stream || runnerWakeStream(target.runnerId);
|
|
2475
|
+
await ctx.streamClient.ensure(wakeStream, { contentType: `application/json` });
|
|
2476
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
2477
|
+
type: `pull-wake`,
|
|
2478
|
+
streams: [streamPath],
|
|
2479
|
+
wake_stream: wakeStream,
|
|
2480
|
+
description: `Electric Agents runner ${target.runnerId}`
|
|
2481
|
+
}, existing);
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
2484
|
+
const webhookUrl = rewriteLoopbackWebhookUrl(target.url);
|
|
2485
|
+
if (!webhookUrl) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Webhook dispatch target must include a valid URL`, 400);
|
|
2486
|
+
const forwardUrl = (0, __electric_ax_agents_runtime.appendPathToUrl)(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
2487
|
+
await ensureSubscriptionIncludesStream(ctx, subscriptionId, streamPath, {
|
|
2488
|
+
type: `webhook`,
|
|
2489
|
+
streams: [streamPath],
|
|
2490
|
+
webhook: { url: forwardUrl },
|
|
2491
|
+
description: `Electric Agents webhook ${subscriptionId}`
|
|
2492
|
+
}, existing);
|
|
2493
|
+
await ctx.pgDb.insert(subscriptionWebhooks).values({
|
|
1878
2494
|
tenantId: ctx.service,
|
|
1879
2495
|
subscriptionId,
|
|
1880
2496
|
webhookUrl
|
|
@@ -1886,6 +2502,7 @@ async function linkStreamToTargetSubscription(ctx, target, entity) {
|
|
|
1886
2502
|
|
|
1887
2503
|
//#endregion
|
|
1888
2504
|
//#region src/principal.ts
|
|
2505
|
+
const ELECTRIC_PRINCIPAL_HEADER = `electric-principal`;
|
|
1889
2506
|
const PRINCIPAL_KINDS = new Set([
|
|
1890
2507
|
`user`,
|
|
1891
2508
|
`agent`,
|
|
@@ -1894,7 +2511,7 @@ const PRINCIPAL_KINDS = new Set([
|
|
|
1894
2511
|
]);
|
|
1895
2512
|
function parsePrincipalKey(input) {
|
|
1896
2513
|
const colon = input.indexOf(`:`);
|
|
1897
|
-
if (colon <= 0) throw new Error(`Invalid principal
|
|
2514
|
+
if (colon <= 0) throw new Error(`Invalid principal identifier`);
|
|
1898
2515
|
const kind = input.slice(0, colon);
|
|
1899
2516
|
const id = input.slice(colon + 1);
|
|
1900
2517
|
if (!PRINCIPAL_KINDS.has(kind)) throw new Error(`Invalid principal kind`);
|
|
@@ -1910,13 +2527,12 @@ function parsePrincipalKey(input) {
|
|
|
1910
2527
|
function principalUrl(key) {
|
|
1911
2528
|
return parsePrincipalKey(key).url;
|
|
1912
2529
|
}
|
|
1913
|
-
function
|
|
2530
|
+
function parsePrincipalUrl(url) {
|
|
1914
2531
|
if (!url.startsWith(`/principal/`)) return null;
|
|
1915
2532
|
const segment = url.slice(`/principal/`.length);
|
|
1916
2533
|
if (!segment || segment.includes(`/`)) return null;
|
|
1917
2534
|
try {
|
|
1918
|
-
|
|
1919
|
-
return parsePrincipalKey(key).key;
|
|
2535
|
+
return parsePrincipalKey(decodeURIComponent(segment));
|
|
1920
2536
|
} catch {
|
|
1921
2537
|
return null;
|
|
1922
2538
|
}
|
|
@@ -1929,9 +2545,8 @@ const BUILT_IN_SYSTEM_PRINCIPAL_IDS = new Set([
|
|
|
1929
2545
|
function isBuiltInSystemPrincipalUrl(url) {
|
|
1930
2546
|
if (!url?.startsWith(`/principal/`)) return false;
|
|
1931
2547
|
try {
|
|
1932
|
-
const
|
|
1933
|
-
if (!
|
|
1934
|
-
const principal = parsePrincipalKey(key);
|
|
2548
|
+
const principal = parsePrincipalUrl(url);
|
|
2549
|
+
if (!principal) return false;
|
|
1935
2550
|
return principal.kind === `system` && BUILT_IN_SYSTEM_PRINCIPAL_IDS.has(principal.id);
|
|
1936
2551
|
} catch {
|
|
1937
2552
|
return false;
|
|
@@ -1939,12 +2554,11 @@ function isBuiltInSystemPrincipalUrl(url) {
|
|
|
1939
2554
|
}
|
|
1940
2555
|
function principalFromCreatedBy(createdBy) {
|
|
1941
2556
|
if (!createdBy) return void 0;
|
|
1942
|
-
const
|
|
1943
|
-
if (!
|
|
2557
|
+
const principal = parsePrincipalUrl(createdBy);
|
|
2558
|
+
if (!principal) return {
|
|
1944
2559
|
url: createdBy,
|
|
1945
2560
|
key: null
|
|
1946
2561
|
};
|
|
1947
|
-
const principal = parsePrincipalKey(key);
|
|
1948
2562
|
return {
|
|
1949
2563
|
url: principal.url,
|
|
1950
2564
|
key: principal.key,
|
|
@@ -2022,70 +2636,26 @@ function buildManifestWakeRegistration(subscriberUrl, manifest, manifestKey) {
|
|
|
2022
2636
|
subscriberUrl,
|
|
2023
2637
|
sourceUrl,
|
|
2024
2638
|
condition: `runFinished`,
|
|
2025
|
-
oneShot: false,
|
|
2026
|
-
includeResponse: typeof wake.includeResponse === `boolean` ? wake.includeResponse : void 0,
|
|
2027
|
-
manifestKey
|
|
2028
|
-
};
|
|
2029
|
-
if (wake.on !== `change`) return null;
|
|
2030
|
-
const collections = Array.isArray(wake.collections) ? wake.collections.filter((c) => typeof c === `string`) : void 0;
|
|
2031
|
-
const ops = Array.isArray(wake.ops) ? wake.ops.filter((op) => op === `insert` || op === `update` || op === `delete`) : void 0;
|
|
2032
|
-
return {
|
|
2033
|
-
subscriberUrl,
|
|
2034
|
-
sourceUrl,
|
|
2035
|
-
condition: {
|
|
2036
|
-
on: `change`,
|
|
2037
|
-
...collections ? { collections } : {},
|
|
2038
|
-
...ops ? { ops } : {}
|
|
2039
|
-
},
|
|
2040
|
-
debounceMs: typeof wake.debounceMs === `number` ? wake.debounceMs : void 0,
|
|
2041
|
-
timeoutMs: typeof wake.timeoutMs === `number` ? wake.timeoutMs : void 0,
|
|
2042
|
-
oneShot: false,
|
|
2043
|
-
manifestKey
|
|
2044
|
-
};
|
|
2045
|
-
}
|
|
2046
|
-
|
|
2047
|
-
//#endregion
|
|
2048
|
-
//#region src/tracing.ts
|
|
2049
|
-
const tracer = __opentelemetry_api.trace.getTracer(`agent-server`);
|
|
2050
|
-
const ATTR = {
|
|
2051
|
-
ENTITY_URL: `electric_agents.entity.url`,
|
|
2052
|
-
ENTITY_TYPE: `electric_agents.entity.type`,
|
|
2053
|
-
PARENT_URL: `electric_agents.entity.parent`,
|
|
2054
|
-
WAKE_SOURCE: `electric_agents.wake.source`,
|
|
2055
|
-
WAKE_SUBSCRIBER: `electric_agents.wake.subscriber`,
|
|
2056
|
-
WAKE_KIND: `electric_agents.wake.kind`,
|
|
2057
|
-
STREAM_PATH: `electric_agents.stream.path`,
|
|
2058
|
-
STREAM_OP: `electric_agents.stream.op`,
|
|
2059
|
-
DB_OP: `electric_agents.db.op`,
|
|
2060
|
-
HTTP_METHOD: `http.method`,
|
|
2061
|
-
HTTP_ROUTE: `http.route`,
|
|
2062
|
-
HTTP_STATUS: `http.status_code`
|
|
2063
|
-
};
|
|
2064
|
-
/**
|
|
2065
|
-
* Run `fn` inside an active span. Errors are recorded + status set to ERROR,
|
|
2066
|
-
* then re-thrown. Span ends in a finally block.
|
|
2067
|
-
*/
|
|
2068
|
-
async function withSpan(name, fn, opts) {
|
|
2069
|
-
return await tracer.startActiveSpan(name, opts ?? {}, async (span) => {
|
|
2070
|
-
try {
|
|
2071
|
-
return await fn(span);
|
|
2072
|
-
} catch (err) {
|
|
2073
|
-
span.recordException(err);
|
|
2074
|
-
span.setStatus({
|
|
2075
|
-
code: __opentelemetry_api.SpanStatusCode.ERROR,
|
|
2076
|
-
message: err instanceof Error ? err.message : String(err)
|
|
2077
|
-
});
|
|
2078
|
-
throw err;
|
|
2079
|
-
} finally {
|
|
2080
|
-
span.end();
|
|
2081
|
-
}
|
|
2082
|
-
});
|
|
2083
|
-
}
|
|
2084
|
-
function injectTraceHeaders(headers, ctx = __opentelemetry_api.context.active()) {
|
|
2085
|
-
__opentelemetry_api.propagation.inject(ctx, headers);
|
|
2086
|
-
}
|
|
2087
|
-
function extractTraceContext(headers) {
|
|
2088
|
-
return __opentelemetry_api.propagation.extract(__opentelemetry_api.context.active(), headers);
|
|
2639
|
+
oneShot: false,
|
|
2640
|
+
includeResponse: typeof wake.includeResponse === `boolean` ? wake.includeResponse : void 0,
|
|
2641
|
+
manifestKey
|
|
2642
|
+
};
|
|
2643
|
+
if (wake.on !== `change`) return null;
|
|
2644
|
+
const collections = Array.isArray(wake.collections) ? wake.collections.filter((c) => typeof c === `string`) : void 0;
|
|
2645
|
+
const ops = Array.isArray(wake.ops) ? wake.ops.filter((op) => op === `insert` || op === `update` || op === `delete`) : void 0;
|
|
2646
|
+
return {
|
|
2647
|
+
subscriberUrl,
|
|
2648
|
+
sourceUrl,
|
|
2649
|
+
condition: {
|
|
2650
|
+
on: `change`,
|
|
2651
|
+
...collections ? { collections } : {},
|
|
2652
|
+
...ops ? { ops } : {}
|
|
2653
|
+
},
|
|
2654
|
+
debounceMs: typeof wake.debounceMs === `number` ? wake.debounceMs : void 0,
|
|
2655
|
+
timeoutMs: typeof wake.timeoutMs === `number` ? wake.timeoutMs : void 0,
|
|
2656
|
+
oneShot: false,
|
|
2657
|
+
manifestKey
|
|
2658
|
+
};
|
|
2089
2659
|
}
|
|
2090
2660
|
|
|
2091
2661
|
//#endregion
|
|
@@ -3631,1036 +4201,544 @@ function isPlainObject(value) {
|
|
|
3631
4201
|
//#region src/scheduler.ts
|
|
3632
4202
|
const POSTGRES_TEXT_OID = 25;
|
|
3633
4203
|
var PostgresSchedulerClient = class {
|
|
3634
|
-
constructor(pgClient, tenantId, wake) {
|
|
3635
|
-
this.pgClient = pgClient;
|
|
3636
|
-
this.tenantId = tenantId;
|
|
3637
|
-
this.wake = wake;
|
|
3638
|
-
}
|
|
3639
|
-
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3640
|
-
await this.pgClient`
|
|
3641
|
-
insert into scheduled_tasks (
|
|
3642
|
-
tenant_id,
|
|
3643
|
-
kind,
|
|
3644
|
-
payload,
|
|
3645
|
-
fire_at,
|
|
3646
|
-
owner_entity_url,
|
|
3647
|
-
manifest_key
|
|
3648
|
-
)
|
|
3649
|
-
values (
|
|
3650
|
-
${this.tenantId},
|
|
3651
|
-
'delayed_send',
|
|
3652
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3653
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3654
|
-
${opts?.ownerEntityUrl ?? null},
|
|
3655
|
-
${opts?.manifestKey ?? null}
|
|
3656
|
-
)
|
|
3657
|
-
`;
|
|
3658
|
-
this.wake?.();
|
|
3659
|
-
}
|
|
3660
|
-
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
3661
|
-
await this.pgClient.begin(async (sql$2) => {
|
|
3662
|
-
await sql$2`
|
|
3663
|
-
update scheduled_tasks
|
|
3664
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3665
|
-
where tenant_id = ${this.tenantId}
|
|
3666
|
-
and kind = 'delayed_send'
|
|
3667
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3668
|
-
and manifest_key = ${manifestKey}
|
|
3669
|
-
and completed_at is null
|
|
3670
|
-
`;
|
|
3671
|
-
await sql$2`
|
|
3672
|
-
insert into scheduled_tasks (
|
|
3673
|
-
tenant_id,
|
|
3674
|
-
kind,
|
|
3675
|
-
payload,
|
|
3676
|
-
fire_at,
|
|
3677
|
-
owner_entity_url,
|
|
3678
|
-
manifest_key
|
|
3679
|
-
)
|
|
3680
|
-
values (
|
|
3681
|
-
${this.tenantId},
|
|
3682
|
-
'delayed_send',
|
|
3683
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3684
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3685
|
-
${ownerEntityUrl},
|
|
3686
|
-
${manifestKey}
|
|
3687
|
-
)
|
|
3688
|
-
`;
|
|
3689
|
-
});
|
|
3690
|
-
this.wake?.();
|
|
3691
|
-
}
|
|
3692
|
-
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
3693
|
-
await this.pgClient`
|
|
3694
|
-
update scheduled_tasks
|
|
3695
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3696
|
-
where tenant_id = ${this.tenantId}
|
|
3697
|
-
and kind = 'delayed_send'
|
|
3698
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3699
|
-
and manifest_key = ${manifestKey}
|
|
3700
|
-
and completed_at is null
|
|
3701
|
-
`;
|
|
3702
|
-
this.wake?.();
|
|
3703
|
-
}
|
|
3704
|
-
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
3705
|
-
await this.pgClient`
|
|
3706
|
-
insert into scheduled_tasks (
|
|
3707
|
-
tenant_id,
|
|
3708
|
-
kind,
|
|
3709
|
-
payload,
|
|
3710
|
-
fire_at,
|
|
3711
|
-
cron_expression,
|
|
3712
|
-
cron_timezone,
|
|
3713
|
-
cron_tick_number
|
|
3714
|
-
)
|
|
3715
|
-
values (
|
|
3716
|
-
${this.tenantId},
|
|
3717
|
-
'cron_tick',
|
|
3718
|
-
${JSON.stringify({ streamPath })}::jsonb,
|
|
3719
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3720
|
-
${expression},
|
|
3721
|
-
${timezone},
|
|
3722
|
-
${tickNumber}
|
|
3723
|
-
)
|
|
3724
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
3725
|
-
`;
|
|
3726
|
-
this.wake?.();
|
|
3727
|
-
}
|
|
3728
|
-
};
|
|
3729
|
-
function isPermanentElectricAgentsError(err) {
|
|
3730
|
-
const status$4 = typeof err === `object` && err !== null && `status` in err ? err.status : void 0;
|
|
3731
|
-
const name = typeof err === `object` && err !== null && `name` in err ? err.name : void 0;
|
|
3732
|
-
return name === `ElectricAgentsError` && typeof status$4 === `number` && status$4 >= 400 && status$4 < 500;
|
|
3733
|
-
}
|
|
3734
|
-
function normalizeTask(row) {
|
|
3735
|
-
return {
|
|
3736
|
-
id: Number(row.id),
|
|
3737
|
-
tenantId: row.tenant_id,
|
|
3738
|
-
kind: row.kind,
|
|
3739
|
-
payload: row.payload,
|
|
3740
|
-
fireAt: row.fire_at instanceof Date ? row.fire_at : new Date(row.fire_at),
|
|
3741
|
-
cronExpression: row.cron_expression,
|
|
3742
|
-
cronTimezone: row.cron_timezone,
|
|
3743
|
-
cronTickNumber: row.cron_tick_number,
|
|
3744
|
-
ownerEntityUrl: row.owner_entity_url,
|
|
3745
|
-
manifestKey: row.manifest_key
|
|
3746
|
-
};
|
|
3747
|
-
}
|
|
3748
|
-
var Scheduler = class {
|
|
3749
|
-
claimExpiryMs;
|
|
3750
|
-
safetyPollMs;
|
|
3751
|
-
listenEnabled;
|
|
3752
|
-
pgClient;
|
|
3753
|
-
instanceId;
|
|
3754
|
-
tenantId;
|
|
3755
|
-
tenantIds;
|
|
3756
|
-
running = false;
|
|
3757
|
-
loopPromise = null;
|
|
3758
|
-
currentSleepResolve = null;
|
|
3759
|
-
currentSleepTimer = null;
|
|
3760
|
-
listenerMeta = null;
|
|
3761
|
-
constructor(options) {
|
|
3762
|
-
this.options = options;
|
|
3763
|
-
this.pgClient = options.pgClient;
|
|
3764
|
-
this.instanceId = options.instanceId;
|
|
3765
|
-
this.tenantId = options.tenantId === void 0 ? DEFAULT_TENANT_ID : options.tenantId;
|
|
3766
|
-
this.tenantIds = options.tenantIds;
|
|
3767
|
-
this.claimExpiryMs = options.claimExpiryMs ?? 3e4;
|
|
3768
|
-
this.safetyPollMs = options.safetyPollMs ?? 1e4;
|
|
3769
|
-
this.listenEnabled = options.listen !== false;
|
|
3770
|
-
}
|
|
3771
|
-
resolveTenantId(tenantId) {
|
|
3772
|
-
if (tenantId) return tenantId;
|
|
3773
|
-
if (this.tenantId) return this.tenantId;
|
|
3774
|
-
throw new Error(`Scheduler tenantId is required in shared mode`);
|
|
3775
|
-
}
|
|
3776
|
-
async start() {
|
|
3777
|
-
if (this.running) return;
|
|
3778
|
-
this.running = true;
|
|
3779
|
-
if (this.listenEnabled) this.listenerMeta = await this.pgClient.listen(`scheduled_tasks_wake`, () => {
|
|
3780
|
-
this.wakeEarly();
|
|
3781
|
-
});
|
|
3782
|
-
this.loopPromise = this.runLoop().catch((err) => {
|
|
3783
|
-
console.error(`[agent-server] scheduler loop failed:`, err);
|
|
3784
|
-
this.running = false;
|
|
3785
|
-
this.wakeEarly();
|
|
3786
|
-
});
|
|
3787
|
-
}
|
|
3788
|
-
async stop() {
|
|
3789
|
-
this.running = false;
|
|
3790
|
-
this.wakeEarly();
|
|
3791
|
-
if (this.loopPromise) {
|
|
3792
|
-
await this.loopPromise;
|
|
3793
|
-
this.loopPromise = null;
|
|
3794
|
-
}
|
|
3795
|
-
if (this.listenerMeta) {
|
|
3796
|
-
await this.listenerMeta.unlisten();
|
|
3797
|
-
this.listenerMeta = null;
|
|
3798
|
-
}
|
|
3799
|
-
}
|
|
3800
|
-
wake() {
|
|
3801
|
-
this.wakeEarly();
|
|
3802
|
-
}
|
|
3803
|
-
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3804
|
-
const tenantId = this.resolveTenantId();
|
|
3805
|
-
await this.pgClient`
|
|
3806
|
-
insert into scheduled_tasks (
|
|
3807
|
-
tenant_id,
|
|
3808
|
-
kind,
|
|
3809
|
-
payload,
|
|
3810
|
-
fire_at,
|
|
3811
|
-
owner_entity_url,
|
|
3812
|
-
manifest_key
|
|
3813
|
-
)
|
|
3814
|
-
values (
|
|
3815
|
-
${tenantId},
|
|
3816
|
-
'delayed_send',
|
|
3817
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3818
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3819
|
-
${opts?.ownerEntityUrl ?? null},
|
|
3820
|
-
${opts?.manifestKey ?? null}
|
|
3821
|
-
)
|
|
3822
|
-
`;
|
|
3823
|
-
this.wakeEarly();
|
|
3824
|
-
}
|
|
3825
|
-
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
3826
|
-
const tenantId = this.resolveTenantId();
|
|
3827
|
-
await this.pgClient.begin(async (sql$2) => {
|
|
3828
|
-
await sql$2`
|
|
3829
|
-
update scheduled_tasks
|
|
3830
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3831
|
-
where tenant_id = ${tenantId}
|
|
3832
|
-
and kind = 'delayed_send'
|
|
3833
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3834
|
-
and manifest_key = ${manifestKey}
|
|
3835
|
-
and completed_at is null
|
|
3836
|
-
`;
|
|
3837
|
-
await sql$2`
|
|
3838
|
-
insert into scheduled_tasks (
|
|
3839
|
-
tenant_id,
|
|
3840
|
-
kind,
|
|
3841
|
-
payload,
|
|
3842
|
-
fire_at,
|
|
3843
|
-
owner_entity_url,
|
|
3844
|
-
manifest_key
|
|
3845
|
-
)
|
|
3846
|
-
values (
|
|
3847
|
-
${tenantId},
|
|
3848
|
-
'delayed_send',
|
|
3849
|
-
${JSON.stringify(payload)}::jsonb,
|
|
3850
|
-
${fireAt.toISOString()}::timestamptz,
|
|
3851
|
-
${ownerEntityUrl},
|
|
3852
|
-
${manifestKey}
|
|
3853
|
-
)
|
|
3854
|
-
`;
|
|
3855
|
-
});
|
|
3856
|
-
this.wakeEarly();
|
|
3857
|
-
}
|
|
3858
|
-
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
3859
|
-
const tenantId = this.resolveTenantId();
|
|
3860
|
-
await this.pgClient`
|
|
3861
|
-
update scheduled_tasks
|
|
3862
|
-
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
3863
|
-
where tenant_id = ${tenantId}
|
|
3864
|
-
and kind = 'delayed_send'
|
|
3865
|
-
and owner_entity_url = ${ownerEntityUrl}
|
|
3866
|
-
and manifest_key = ${manifestKey}
|
|
3867
|
-
and completed_at is null
|
|
3868
|
-
`;
|
|
3869
|
-
this.wakeEarly();
|
|
4204
|
+
constructor(pgClient, tenantId, wake) {
|
|
4205
|
+
this.pgClient = pgClient;
|
|
4206
|
+
this.tenantId = tenantId;
|
|
4207
|
+
this.wake = wake;
|
|
3870
4208
|
}
|
|
3871
|
-
async
|
|
3872
|
-
const tenantId = this.resolveTenantId();
|
|
4209
|
+
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
3873
4210
|
await this.pgClient`
|
|
3874
4211
|
insert into scheduled_tasks (
|
|
3875
4212
|
tenant_id,
|
|
3876
4213
|
kind,
|
|
3877
4214
|
payload,
|
|
3878
4215
|
fire_at,
|
|
3879
|
-
|
|
3880
|
-
|
|
3881
|
-
cron_tick_number
|
|
4216
|
+
owner_entity_url,
|
|
4217
|
+
manifest_key
|
|
3882
4218
|
)
|
|
3883
4219
|
values (
|
|
3884
|
-
${tenantId},
|
|
3885
|
-
'
|
|
3886
|
-
${JSON.stringify(
|
|
4220
|
+
${this.tenantId},
|
|
4221
|
+
'delayed_send',
|
|
4222
|
+
${JSON.stringify(payload)}::jsonb,
|
|
3887
4223
|
${fireAt.toISOString()}::timestamptz,
|
|
3888
|
-
${
|
|
3889
|
-
${
|
|
3890
|
-
${tickNumber}
|
|
3891
|
-
)
|
|
3892
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
3893
|
-
`;
|
|
3894
|
-
this.wakeEarly();
|
|
3895
|
-
}
|
|
3896
|
-
async runLoop() {
|
|
3897
|
-
while (this.running) try {
|
|
3898
|
-
await this.reclaimStaleClaims();
|
|
3899
|
-
await this.fireReadyTasks();
|
|
3900
|
-
const nextFireAt = await this.getNextFireAt();
|
|
3901
|
-
const sleepTargetMs = nextFireAt ? Math.max(0, nextFireAt.getTime() - Date.now()) : this.safetyPollMs;
|
|
3902
|
-
await this.sleepOrWake(Math.min(sleepTargetMs, this.safetyPollMs));
|
|
3903
|
-
} catch (err) {
|
|
3904
|
-
console.error(`[agent-server] scheduler iteration failed:`, err);
|
|
3905
|
-
await this.sleepOrWake(this.safetyPollMs);
|
|
3906
|
-
}
|
|
3907
|
-
}
|
|
3908
|
-
async reclaimStaleClaims() {
|
|
3909
|
-
if (this.tenantId === null) {
|
|
3910
|
-
const tenantIds = this.sharedTenantIds();
|
|
3911
|
-
if (tenantIds && tenantIds.length === 0) return;
|
|
3912
|
-
if (tenantIds) {
|
|
3913
|
-
await this.pgClient`
|
|
3914
|
-
update scheduled_tasks
|
|
3915
|
-
set claimed_by = null, claimed_at = null
|
|
3916
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
3917
|
-
and completed_at is null
|
|
3918
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3919
|
-
`;
|
|
3920
|
-
return;
|
|
3921
|
-
}
|
|
3922
|
-
await this.pgClient`
|
|
3923
|
-
update scheduled_tasks
|
|
3924
|
-
set claimed_by = null, claimed_at = null
|
|
3925
|
-
where completed_at is null
|
|
3926
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3927
|
-
`;
|
|
3928
|
-
return;
|
|
3929
|
-
}
|
|
3930
|
-
await this.pgClient`
|
|
3931
|
-
update scheduled_tasks
|
|
3932
|
-
set claimed_by = null, claimed_at = null
|
|
3933
|
-
where tenant_id = ${this.tenantId}
|
|
3934
|
-
and completed_at is null
|
|
3935
|
-
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
3936
|
-
`;
|
|
3937
|
-
}
|
|
3938
|
-
async fireReadyTasks() {
|
|
3939
|
-
while (this.running) {
|
|
3940
|
-
const tasks = await this.claimReadyTasks();
|
|
3941
|
-
if (tasks.length === 0) return;
|
|
3942
|
-
for (const task of tasks) await this.executeTask(task);
|
|
3943
|
-
}
|
|
3944
|
-
}
|
|
3945
|
-
async claimReadyTasks() {
|
|
3946
|
-
if (this.tenantId === null) {
|
|
3947
|
-
const tenantIds = this.sharedTenantIds();
|
|
3948
|
-
if (tenantIds && tenantIds.length === 0) return [];
|
|
3949
|
-
if (tenantIds) {
|
|
3950
|
-
const rows$2 = await this.pgClient`
|
|
3951
|
-
update scheduled_tasks
|
|
3952
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3953
|
-
where id in (
|
|
3954
|
-
select id
|
|
3955
|
-
from scheduled_tasks
|
|
3956
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
3957
|
-
and completed_at is null
|
|
3958
|
-
and claimed_at is null
|
|
3959
|
-
and fire_at <= now()
|
|
3960
|
-
order by fire_at, id
|
|
3961
|
-
for update skip locked
|
|
3962
|
-
limit 50
|
|
3963
|
-
)
|
|
3964
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
3965
|
-
, owner_entity_url, manifest_key
|
|
3966
|
-
`;
|
|
3967
|
-
return rows$2.map(normalizeTask);
|
|
3968
|
-
}
|
|
3969
|
-
const rows$1 = await this.pgClient`
|
|
3970
|
-
update scheduled_tasks
|
|
3971
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3972
|
-
where id in (
|
|
3973
|
-
select id
|
|
3974
|
-
from scheduled_tasks
|
|
3975
|
-
where completed_at is null
|
|
3976
|
-
and claimed_at is null
|
|
3977
|
-
and fire_at <= now()
|
|
3978
|
-
order by fire_at, id
|
|
3979
|
-
for update skip locked
|
|
3980
|
-
limit 50
|
|
3981
|
-
)
|
|
3982
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
3983
|
-
, owner_entity_url, manifest_key
|
|
3984
|
-
`;
|
|
3985
|
-
return rows$1.map(normalizeTask);
|
|
3986
|
-
}
|
|
3987
|
-
const rows = await this.pgClient`
|
|
3988
|
-
update scheduled_tasks
|
|
3989
|
-
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
3990
|
-
where tenant_id = ${this.tenantId}
|
|
3991
|
-
and id in (
|
|
3992
|
-
select id
|
|
3993
|
-
from scheduled_tasks
|
|
3994
|
-
where tenant_id = ${this.tenantId}
|
|
3995
|
-
and completed_at is null
|
|
3996
|
-
and claimed_at is null
|
|
3997
|
-
and fire_at <= now()
|
|
3998
|
-
order by fire_at, id
|
|
3999
|
-
for update skip locked
|
|
4000
|
-
limit 50
|
|
4224
|
+
${opts?.ownerEntityUrl ?? null},
|
|
4225
|
+
${opts?.manifestKey ?? null}
|
|
4001
4226
|
)
|
|
4002
|
-
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4003
|
-
, owner_entity_url, manifest_key
|
|
4004
|
-
`;
|
|
4005
|
-
return rows.map(normalizeTask);
|
|
4006
|
-
}
|
|
4007
|
-
async executeTask(task) {
|
|
4008
|
-
try {
|
|
4009
|
-
if (task.kind === `delayed_send`) {
|
|
4010
|
-
await this.options.executors.delayed_send(task.payload, task.id, task.tenantId);
|
|
4011
|
-
await this.markTaskComplete(task.id, task.tenantId);
|
|
4012
|
-
return;
|
|
4013
|
-
}
|
|
4014
|
-
const tickNumber = task.cronTickNumber;
|
|
4015
|
-
if (tickNumber == null || !task.cronExpression || !task.cronTimezone) throw new Error(`cron task ${task.id} is missing cron metadata`);
|
|
4016
|
-
await this.options.executors.cron_tick(task.payload, tickNumber, task.id, task.tenantId);
|
|
4017
|
-
await this.completeAndRescheduleCron(task);
|
|
4018
|
-
} catch (err) {
|
|
4019
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
4020
|
-
if (isUnregisteredTenantError(err)) {
|
|
4021
|
-
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4022
|
-
serverLog.warn(`[scheduler] skipped ${task.kind} task ${task.id} for unregistered tenant "${task.tenantId}": ${message}`);
|
|
4023
|
-
return;
|
|
4024
|
-
}
|
|
4025
|
-
if (isPermanentElectricAgentsError(err)) {
|
|
4026
|
-
await this.markTaskPermanentFailure(task.id, message, task.tenantId);
|
|
4027
|
-
return;
|
|
4028
|
-
}
|
|
4029
|
-
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4030
|
-
}
|
|
4031
|
-
}
|
|
4032
|
-
async markTaskComplete(taskId, tenantId = this.resolveTenantId()) {
|
|
4033
|
-
await this.pgClient`
|
|
4034
|
-
update scheduled_tasks
|
|
4035
|
-
set completed_at = now(), last_error = null
|
|
4036
|
-
where tenant_id = ${tenantId}
|
|
4037
|
-
and id = ${taskId}
|
|
4038
|
-
and claimed_by = ${this.instanceId}
|
|
4039
|
-
and completed_at is null
|
|
4040
|
-
`;
|
|
4041
|
-
}
|
|
4042
|
-
async markTaskPermanentFailure(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4043
|
-
await this.pgClient`
|
|
4044
|
-
update scheduled_tasks
|
|
4045
|
-
set completed_at = now(), last_error = ${message}
|
|
4046
|
-
where tenant_id = ${tenantId}
|
|
4047
|
-
and id = ${taskId}
|
|
4048
|
-
and claimed_by = ${this.instanceId}
|
|
4049
|
-
and completed_at is null
|
|
4050
|
-
`;
|
|
4051
|
-
}
|
|
4052
|
-
async releaseClaim(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4053
|
-
await this.pgClient`
|
|
4054
|
-
update scheduled_tasks
|
|
4055
|
-
set claimed_at = null, claimed_by = null, last_error = ${message}
|
|
4056
|
-
where tenant_id = ${tenantId}
|
|
4057
|
-
and id = ${taskId}
|
|
4058
|
-
and claimed_by = ${this.instanceId}
|
|
4059
|
-
and completed_at is null
|
|
4060
4227
|
`;
|
|
4228
|
+
this.wake?.();
|
|
4061
4229
|
}
|
|
4062
|
-
async
|
|
4063
|
-
const tenantId = task.tenantId ?? this.resolveTenantId();
|
|
4230
|
+
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
4064
4231
|
await this.pgClient.begin(async (sql$2) => {
|
|
4065
|
-
|
|
4232
|
+
await sql$2`
|
|
4066
4233
|
update scheduled_tasks
|
|
4067
|
-
set completed_at = now(),
|
|
4068
|
-
where tenant_id = ${tenantId}
|
|
4069
|
-
and
|
|
4070
|
-
and
|
|
4234
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4235
|
+
where tenant_id = ${this.tenantId}
|
|
4236
|
+
and kind = 'delayed_send'
|
|
4237
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4238
|
+
and manifest_key = ${manifestKey}
|
|
4071
4239
|
and completed_at is null
|
|
4072
|
-
returning id
|
|
4073
4240
|
`;
|
|
4074
|
-
if (completed.length === 0) return;
|
|
4075
|
-
const nextFireAt = (0, __electric_ax_agents_runtime.getNextCronFireAt)(task.cronExpression, task.cronTimezone, task.fireAt);
|
|
4076
4241
|
await sql$2`
|
|
4077
|
-
insert into scheduled_tasks (
|
|
4078
|
-
tenant_id,
|
|
4079
|
-
kind,
|
|
4080
|
-
payload,
|
|
4081
|
-
fire_at,
|
|
4082
|
-
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
${
|
|
4090
|
-
${
|
|
4091
|
-
${
|
|
4092
|
-
|
|
4093
|
-
${task.cronTickNumber + 1}
|
|
4094
|
-
)
|
|
4095
|
-
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4096
|
-
`;
|
|
4097
|
-
});
|
|
4098
|
-
}
|
|
4099
|
-
async getNextFireAt() {
|
|
4100
|
-
if (this.tenantId === null) {
|
|
4101
|
-
const tenantIds = this.sharedTenantIds();
|
|
4102
|
-
if (tenantIds && tenantIds.length === 0) return null;
|
|
4103
|
-
if (tenantIds) {
|
|
4104
|
-
const rows$2 = await this.pgClient`
|
|
4105
|
-
select fire_at
|
|
4106
|
-
from scheduled_tasks
|
|
4107
|
-
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4108
|
-
and completed_at is null
|
|
4109
|
-
and claimed_at is null
|
|
4110
|
-
order by fire_at, id
|
|
4111
|
-
limit 1
|
|
4112
|
-
`;
|
|
4113
|
-
if (rows$2.length === 0) return null;
|
|
4114
|
-
const fireAt$2 = rows$2[0].fire_at;
|
|
4115
|
-
return fireAt$2 instanceof Date ? fireAt$2 : new Date(fireAt$2);
|
|
4116
|
-
}
|
|
4117
|
-
const rows$1 = await this.pgClient`
|
|
4118
|
-
select fire_at
|
|
4119
|
-
from scheduled_tasks
|
|
4120
|
-
where completed_at is null
|
|
4121
|
-
and claimed_at is null
|
|
4122
|
-
order by fire_at, id
|
|
4123
|
-
limit 1
|
|
4242
|
+
insert into scheduled_tasks (
|
|
4243
|
+
tenant_id,
|
|
4244
|
+
kind,
|
|
4245
|
+
payload,
|
|
4246
|
+
fire_at,
|
|
4247
|
+
owner_entity_url,
|
|
4248
|
+
manifest_key
|
|
4249
|
+
)
|
|
4250
|
+
values (
|
|
4251
|
+
${this.tenantId},
|
|
4252
|
+
'delayed_send',
|
|
4253
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4254
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4255
|
+
${ownerEntityUrl},
|
|
4256
|
+
${manifestKey}
|
|
4257
|
+
)
|
|
4124
4258
|
`;
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4259
|
+
});
|
|
4260
|
+
this.wake?.();
|
|
4261
|
+
}
|
|
4262
|
+
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
4263
|
+
await this.pgClient`
|
|
4264
|
+
update scheduled_tasks
|
|
4265
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4132
4266
|
where tenant_id = ${this.tenantId}
|
|
4267
|
+
and kind = 'delayed_send'
|
|
4268
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4269
|
+
and manifest_key = ${manifestKey}
|
|
4133
4270
|
and completed_at is null
|
|
4134
|
-
and claimed_at is null
|
|
4135
|
-
order by fire_at, id
|
|
4136
|
-
limit 1
|
|
4137
4271
|
`;
|
|
4138
|
-
|
|
4139
|
-
const fireAt = rows[0].fire_at;
|
|
4140
|
-
return fireAt instanceof Date ? fireAt : new Date(fireAt);
|
|
4141
|
-
}
|
|
4142
|
-
async sleepOrWake(durationMs) {
|
|
4143
|
-
if (!this.running) return;
|
|
4144
|
-
await new Promise((resolve$1) => {
|
|
4145
|
-
const finish = () => {
|
|
4146
|
-
if (this.currentSleepTimer) {
|
|
4147
|
-
clearTimeout(this.currentSleepTimer);
|
|
4148
|
-
this.currentSleepTimer = null;
|
|
4149
|
-
}
|
|
4150
|
-
this.currentSleepResolve = null;
|
|
4151
|
-
resolve$1();
|
|
4152
|
-
};
|
|
4153
|
-
this.currentSleepResolve = finish;
|
|
4154
|
-
this.currentSleepTimer = setTimeout(finish, Math.max(durationMs, 0));
|
|
4155
|
-
});
|
|
4156
|
-
}
|
|
4157
|
-
wakeEarly() {
|
|
4158
|
-
const resolve$1 = this.currentSleepResolve;
|
|
4159
|
-
this.currentSleepResolve = null;
|
|
4160
|
-
if (this.currentSleepTimer) {
|
|
4161
|
-
clearTimeout(this.currentSleepTimer);
|
|
4162
|
-
this.currentSleepTimer = null;
|
|
4163
|
-
}
|
|
4164
|
-
resolve$1?.();
|
|
4165
|
-
}
|
|
4166
|
-
sharedTenantIds() {
|
|
4167
|
-
if (this.tenantId !== null || !this.tenantIds) return null;
|
|
4168
|
-
return [...new Set(this.tenantIds())];
|
|
4169
|
-
}
|
|
4170
|
-
sharedTenantIdsParameter(tenantIds) {
|
|
4171
|
-
return this.pgClient.array(tenantIds, POSTGRES_TEXT_OID);
|
|
4272
|
+
this.wake?.();
|
|
4172
4273
|
}
|
|
4173
|
-
|
|
4174
|
-
|
|
4175
|
-
|
|
4176
|
-
|
|
4177
|
-
|
|
4178
|
-
|
|
4179
|
-
|
|
4180
|
-
|
|
4181
|
-
|
|
4182
|
-
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
|
|
4187
|
-
|
|
4188
|
-
|
|
4189
|
-
|
|
4274
|
+
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
4275
|
+
await this.pgClient`
|
|
4276
|
+
insert into scheduled_tasks (
|
|
4277
|
+
tenant_id,
|
|
4278
|
+
kind,
|
|
4279
|
+
payload,
|
|
4280
|
+
fire_at,
|
|
4281
|
+
cron_expression,
|
|
4282
|
+
cron_timezone,
|
|
4283
|
+
cron_tick_number
|
|
4284
|
+
)
|
|
4285
|
+
values (
|
|
4286
|
+
${this.tenantId},
|
|
4287
|
+
'cron_tick',
|
|
4288
|
+
${JSON.stringify({ streamPath })}::jsonb,
|
|
4289
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4290
|
+
${expression},
|
|
4291
|
+
${timezone},
|
|
4292
|
+
${tickNumber}
|
|
4293
|
+
)
|
|
4294
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4295
|
+
`;
|
|
4296
|
+
this.wake?.();
|
|
4190
4297
|
}
|
|
4191
4298
|
};
|
|
4192
|
-
|
|
4193
|
-
|
|
4194
|
-
const
|
|
4195
|
-
|
|
4196
|
-
if (!trimmed) return void 0;
|
|
4197
|
-
return /^Bearer\s+/i.test(trimmed) ? trimmed : `Bearer ${trimmed}`;
|
|
4198
|
-
}
|
|
4199
|
-
async function applyDurableStreamsBearer(headers, bearer, opts = {}) {
|
|
4200
|
-
if (!bearer) return;
|
|
4201
|
-
if (!opts.overwrite && headers.has(`authorization`)) return;
|
|
4202
|
-
const value = await resolveDurableStreamsBearer(bearer);
|
|
4203
|
-
if (value) headers.set(`authorization`, value);
|
|
4204
|
-
}
|
|
4205
|
-
function durableStreamsBearerHeaders(bearer) {
|
|
4206
|
-
if (!bearer) return void 0;
|
|
4207
|
-
return { authorization: async () => await resolveDurableStreamsBearer(bearer) ?? `` };
|
|
4208
|
-
}
|
|
4209
|
-
function durableStreamsServiceUrl(baseUrl, serviceId) {
|
|
4210
|
-
const url = new URL(baseUrl);
|
|
4211
|
-
if (/^\/v1\/stream\/[^/]+\/?$/.test(url.pathname)) return baseUrl.replace(/\/+$/, ``);
|
|
4212
|
-
const base = baseUrl.replace(/\/+$/, ``);
|
|
4213
|
-
return `${base}/v1/stream/${encodeURIComponent(serviceId)}`;
|
|
4214
|
-
}
|
|
4215
|
-
function isNotFoundError(err) {
|
|
4216
|
-
return err instanceof __durable_streams_client.DurableStreamError && err.code === ErrCodeNotFound || err instanceof __durable_streams_client.FetchError && err.status === 404;
|
|
4217
|
-
}
|
|
4218
|
-
function isAbortLikeError(err) {
|
|
4219
|
-
return err instanceof Error && (err.name === `AbortError` || err.message === `Stream request was aborted`);
|
|
4220
|
-
}
|
|
4221
|
-
function normalizeSubscriptionPattern(pattern) {
|
|
4222
|
-
return pattern.replace(/^\/+/, ``);
|
|
4223
|
-
}
|
|
4224
|
-
function normalizeSubscriptionStreamPath(path$2) {
|
|
4225
|
-
return path$2.replace(/^\/+/, ``);
|
|
4299
|
+
function isPermanentElectricAgentsError(err) {
|
|
4300
|
+
const status$4 = typeof err === `object` && err !== null && `status` in err ? err.status : void 0;
|
|
4301
|
+
const name = typeof err === `object` && err !== null && `name` in err ? err.name : void 0;
|
|
4302
|
+
return name === `ElectricAgentsError` && typeof status$4 === `number` && status$4 >= 400 && status$4 < 500;
|
|
4226
4303
|
}
|
|
4227
|
-
function
|
|
4228
|
-
return
|
|
4304
|
+
function normalizeTask(row) {
|
|
4305
|
+
return {
|
|
4306
|
+
id: Number(row.id),
|
|
4307
|
+
tenantId: row.tenant_id,
|
|
4308
|
+
kind: row.kind,
|
|
4309
|
+
payload: row.payload,
|
|
4310
|
+
fireAt: row.fire_at instanceof Date ? row.fire_at : new Date(row.fire_at),
|
|
4311
|
+
cronExpression: row.cron_expression,
|
|
4312
|
+
cronTimezone: row.cron_timezone,
|
|
4313
|
+
cronTickNumber: row.cron_tick_number,
|
|
4314
|
+
ownerEntityUrl: row.owner_entity_url,
|
|
4315
|
+
manifestKey: row.manifest_key
|
|
4316
|
+
};
|
|
4229
4317
|
}
|
|
4230
|
-
var
|
|
4231
|
-
|
|
4232
|
-
|
|
4318
|
+
var Scheduler = class {
|
|
4319
|
+
claimExpiryMs;
|
|
4320
|
+
safetyPollMs;
|
|
4321
|
+
listenEnabled;
|
|
4322
|
+
pgClient;
|
|
4323
|
+
instanceId;
|
|
4324
|
+
tenantId;
|
|
4325
|
+
tenantIds;
|
|
4326
|
+
running = false;
|
|
4327
|
+
loopPromise = null;
|
|
4328
|
+
currentSleepResolve = null;
|
|
4329
|
+
currentSleepTimer = null;
|
|
4330
|
+
listenerMeta = null;
|
|
4331
|
+
constructor(options) {
|
|
4233
4332
|
this.options = options;
|
|
4333
|
+
this.pgClient = options.pgClient;
|
|
4334
|
+
this.instanceId = options.instanceId;
|
|
4335
|
+
this.tenantId = options.tenantId === void 0 ? DEFAULT_TENANT_ID : options.tenantId;
|
|
4336
|
+
this.tenantIds = options.tenantIds;
|
|
4337
|
+
this.claimExpiryMs = options.claimExpiryMs ?? 3e4;
|
|
4338
|
+
this.safetyPollMs = options.safetyPollMs ?? 1e4;
|
|
4339
|
+
this.listenEnabled = options.listen !== false;
|
|
4234
4340
|
}
|
|
4235
|
-
|
|
4236
|
-
return
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
return durableStreamsBearerHeaders(this.options.bearer);
|
|
4240
|
-
}
|
|
4241
|
-
async requestHeaders(init, opts = {}) {
|
|
4242
|
-
const headers = new Headers(init);
|
|
4243
|
-
await applyDurableStreamsBearer(headers, this.options.bearer, { overwrite: opts.overwriteBearer });
|
|
4244
|
-
return headers;
|
|
4245
|
-
}
|
|
4246
|
-
subscriptionServiceId() {
|
|
4247
|
-
const url = new URL(this.baseUrl);
|
|
4248
|
-
const match = /^(.*)\/v1\/stream\/([^/]+)\/?$/.exec(url.pathname);
|
|
4249
|
-
return match ? decodeURIComponent(match[2]) : null;
|
|
4250
|
-
}
|
|
4251
|
-
backendSubscriptionPath(path$2) {
|
|
4252
|
-
const normalized = normalizeSubscriptionPath(path$2);
|
|
4253
|
-
const serviceId = this.subscriptionServiceId();
|
|
4254
|
-
if (!serviceId) return normalized;
|
|
4255
|
-
if (normalized === serviceId || normalized.startsWith(`${serviceId}/`)) return normalized;
|
|
4256
|
-
return `${serviceId}/${normalized}`;
|
|
4257
|
-
}
|
|
4258
|
-
runtimeSubscriptionPath(path$2) {
|
|
4259
|
-
const normalized = normalizeSubscriptionPath(path$2);
|
|
4260
|
-
const serviceId = this.subscriptionServiceId();
|
|
4261
|
-
if (!serviceId) return normalized;
|
|
4262
|
-
return normalized.startsWith(`${serviceId}/`) ? normalized.slice(serviceId.length + 1) : normalized;
|
|
4263
|
-
}
|
|
4264
|
-
subscriptionUrl(subscriptionId) {
|
|
4265
|
-
const url = new URL(this.baseUrl);
|
|
4266
|
-
const match = /^(.*)\/v1\/stream\/([^/]+)\/?$/.exec(url.pathname);
|
|
4267
|
-
if (match) {
|
|
4268
|
-
const [, prefix = ``, serviceId] = match;
|
|
4269
|
-
url.pathname = `${prefix}/v1/stream-meta/subscriptions/${encodeURIComponent(subscriptionId)}`;
|
|
4270
|
-
url.searchParams.set(`service`, decodeURIComponent(serviceId));
|
|
4271
|
-
return url.toString();
|
|
4272
|
-
}
|
|
4273
|
-
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/v1/stream-meta/subscriptions/${encodeURIComponent(subscriptionId)}`;
|
|
4274
|
-
return url.toString();
|
|
4275
|
-
}
|
|
4276
|
-
subscriptionChildUrl(subscriptionId, ...segments) {
|
|
4277
|
-
const url = new URL(this.subscriptionUrl(subscriptionId));
|
|
4278
|
-
url.pathname = `${url.pathname.replace(/\/+$/, ``)}/${segments.map((segment) => encodeURIComponent(segment)).join(`/`)}`;
|
|
4279
|
-
return url.toString();
|
|
4280
|
-
}
|
|
4281
|
-
async create(path$2, opts) {
|
|
4282
|
-
return await withSpan(`stream.create`, async (span) => {
|
|
4283
|
-
span.setAttributes({
|
|
4284
|
-
[ATTR.STREAM_PATH]: path$2,
|
|
4285
|
-
[ATTR.STREAM_OP]: `create`
|
|
4286
|
-
});
|
|
4287
|
-
await __durable_streams_client.DurableStream.create({
|
|
4288
|
-
url: this.streamUrl(path$2),
|
|
4289
|
-
headers: this.streamHeaders(),
|
|
4290
|
-
contentType: opts.contentType,
|
|
4291
|
-
body: opts.body
|
|
4292
|
-
});
|
|
4293
|
-
});
|
|
4294
|
-
}
|
|
4295
|
-
async fork(path$2, sourcePath) {
|
|
4296
|
-
return await withSpan(`stream.fork`, async (span) => {
|
|
4297
|
-
span.setAttributes({
|
|
4298
|
-
[ATTR.STREAM_PATH]: path$2,
|
|
4299
|
-
[ATTR.STREAM_OP]: `fork`
|
|
4300
|
-
});
|
|
4301
|
-
const headers = {
|
|
4302
|
-
"content-type": `application/json`,
|
|
4303
|
-
"Stream-Forked-From": sourcePath
|
|
4304
|
-
};
|
|
4305
|
-
injectTraceHeaders(headers);
|
|
4306
|
-
const response = await fetch(this.streamUrl(path$2), {
|
|
4307
|
-
method: `PUT`,
|
|
4308
|
-
headers: await this.requestHeaders(headers)
|
|
4309
|
-
});
|
|
4310
|
-
if (response.ok) return;
|
|
4311
|
-
throw new Error(`Stream fork failed: ${response.status} ${await response.text()}`);
|
|
4312
|
-
});
|
|
4313
|
-
}
|
|
4314
|
-
async append(path$2, data, opts) {
|
|
4315
|
-
return await withSpan(`stream.append`, async (span) => {
|
|
4316
|
-
span.setAttributes({
|
|
4317
|
-
[ATTR.STREAM_PATH]: path$2,
|
|
4318
|
-
[ATTR.STREAM_OP]: opts?.close ? `append+close` : `append`
|
|
4319
|
-
});
|
|
4320
|
-
const handle = new __durable_streams_client.DurableStream({
|
|
4321
|
-
url: this.streamUrl(path$2),
|
|
4322
|
-
headers: this.streamHeaders(),
|
|
4323
|
-
contentType: `application/json`,
|
|
4324
|
-
batching: false
|
|
4325
|
-
});
|
|
4326
|
-
if (opts?.close) {
|
|
4327
|
-
const result = await handle.close({ body: data });
|
|
4328
|
-
return { offset: result.finalOffset };
|
|
4329
|
-
}
|
|
4330
|
-
await handle.append(data);
|
|
4331
|
-
const head = await handle.head();
|
|
4332
|
-
return { offset: head.exists && head.offset || `` };
|
|
4333
|
-
});
|
|
4334
|
-
}
|
|
4335
|
-
async appendIdempotent(path$2, data, opts) {
|
|
4336
|
-
return await withSpan(`stream.appendIdempotent`, async (span) => {
|
|
4337
|
-
span.setAttributes({
|
|
4338
|
-
[ATTR.STREAM_PATH]: path$2,
|
|
4339
|
-
[ATTR.STREAM_OP]: `appendIdempotent`
|
|
4340
|
-
});
|
|
4341
|
-
const stream = new __durable_streams_client.DurableStream({
|
|
4342
|
-
url: this.streamUrl(path$2),
|
|
4343
|
-
headers: this.streamHeaders(),
|
|
4344
|
-
contentType: `application/json`
|
|
4345
|
-
});
|
|
4346
|
-
const producer = new __durable_streams_client.IdempotentProducer(stream, opts.producerId, { epoch: opts.epoch ?? 0 });
|
|
4347
|
-
try {
|
|
4348
|
-
producer.append(data);
|
|
4349
|
-
await producer.flush();
|
|
4350
|
-
} finally {
|
|
4351
|
-
await producer.detach();
|
|
4352
|
-
}
|
|
4353
|
-
});
|
|
4341
|
+
resolveTenantId(tenantId) {
|
|
4342
|
+
if (tenantId) return tenantId;
|
|
4343
|
+
if (this.tenantId) return this.tenantId;
|
|
4344
|
+
throw new Error(`Scheduler tenantId is required in shared mode`);
|
|
4354
4345
|
}
|
|
4355
|
-
async
|
|
4356
|
-
|
|
4357
|
-
|
|
4358
|
-
|
|
4359
|
-
|
|
4360
|
-
});
|
|
4361
|
-
const headers = {
|
|
4362
|
-
"content-type": `application/json`,
|
|
4363
|
-
"Producer-Id": opts.producerId,
|
|
4364
|
-
"Producer-Epoch": String(opts.epoch),
|
|
4365
|
-
"Producer-Seq": String(opts.seq)
|
|
4366
|
-
};
|
|
4367
|
-
injectTraceHeaders(headers);
|
|
4368
|
-
const response = await fetch(this.streamUrl(path$2), {
|
|
4369
|
-
method: `POST`,
|
|
4370
|
-
headers: await this.requestHeaders(headers),
|
|
4371
|
-
body: typeof data === `string` ? data : Buffer.from(data)
|
|
4372
|
-
});
|
|
4373
|
-
if (response.ok || response.status === 204) return;
|
|
4374
|
-
throw new Error(`Stream append failed: ${response.status} ${await response.text()}`);
|
|
4346
|
+
async start() {
|
|
4347
|
+
if (this.running) return;
|
|
4348
|
+
this.running = true;
|
|
4349
|
+
if (this.listenEnabled) this.listenerMeta = await this.pgClient.listen(`scheduled_tasks_wake`, () => {
|
|
4350
|
+
this.wakeEarly();
|
|
4375
4351
|
});
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
[ATTR.STREAM_PATH]: path$2,
|
|
4381
|
-
[ATTR.STREAM_OP]: `read`
|
|
4382
|
-
});
|
|
4383
|
-
const handle = new __durable_streams_client.DurableStream({
|
|
4384
|
-
url: this.streamUrl(path$2),
|
|
4385
|
-
headers: this.streamHeaders()
|
|
4386
|
-
});
|
|
4387
|
-
const response = await handle.stream({
|
|
4388
|
-
offset: fromOffset ?? `-1`,
|
|
4389
|
-
live: false
|
|
4390
|
-
});
|
|
4391
|
-
const messages = [];
|
|
4392
|
-
return await new Promise((resolve$1, reject) => {
|
|
4393
|
-
let settled = false;
|
|
4394
|
-
let unsub = () => {};
|
|
4395
|
-
const finish = (r) => {
|
|
4396
|
-
if (settled) return;
|
|
4397
|
-
settled = true;
|
|
4398
|
-
unsub();
|
|
4399
|
-
resolve$1(r);
|
|
4400
|
-
};
|
|
4401
|
-
unsub = response.subscribeBytes((chunk) => {
|
|
4402
|
-
messages.push({
|
|
4403
|
-
data: chunk.data,
|
|
4404
|
-
offset: chunk.offset
|
|
4405
|
-
});
|
|
4406
|
-
if (chunk.upToDate || chunk.streamClosed) finish({ messages });
|
|
4407
|
-
});
|
|
4408
|
-
response.closed.then(() => finish({ messages })).catch((err) => {
|
|
4409
|
-
if (settled) return;
|
|
4410
|
-
settled = true;
|
|
4411
|
-
unsub();
|
|
4412
|
-
reject(err);
|
|
4413
|
-
});
|
|
4414
|
-
});
|
|
4352
|
+
this.loopPromise = this.runLoop().catch((err) => {
|
|
4353
|
+
console.error(`[agent-server] scheduler loop failed:`, err);
|
|
4354
|
+
this.running = false;
|
|
4355
|
+
this.wakeEarly();
|
|
4415
4356
|
});
|
|
4416
4357
|
}
|
|
4417
|
-
async
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
|
|
4421
|
-
|
|
4422
|
-
|
|
4423
|
-
|
|
4424
|
-
|
|
4425
|
-
|
|
4426
|
-
|
|
4427
|
-
|
|
4428
|
-
offset: fromOffset ?? `-1`,
|
|
4429
|
-
live: false
|
|
4430
|
-
});
|
|
4431
|
-
return await response.json();
|
|
4432
|
-
});
|
|
4358
|
+
async stop() {
|
|
4359
|
+
this.running = false;
|
|
4360
|
+
this.wakeEarly();
|
|
4361
|
+
if (this.loopPromise) {
|
|
4362
|
+
await this.loopPromise;
|
|
4363
|
+
this.loopPromise = null;
|
|
4364
|
+
}
|
|
4365
|
+
if (this.listenerMeta) {
|
|
4366
|
+
await this.listenerMeta.unlisten();
|
|
4367
|
+
this.listenerMeta = null;
|
|
4368
|
+
}
|
|
4433
4369
|
}
|
|
4434
|
-
|
|
4435
|
-
|
|
4436
|
-
|
|
4437
|
-
|
|
4438
|
-
|
|
4439
|
-
|
|
4440
|
-
|
|
4441
|
-
|
|
4442
|
-
|
|
4443
|
-
|
|
4444
|
-
|
|
4445
|
-
|
|
4446
|
-
|
|
4447
|
-
|
|
4448
|
-
|
|
4449
|
-
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
|
|
4477
|
-
|
|
4478
|
-
|
|
4479
|
-
|
|
4480
|
-
|
|
4481
|
-
|
|
4482
|
-
|
|
4483
|
-
|
|
4484
|
-
|
|
4485
|
-
|
|
4486
|
-
|
|
4487
|
-
|
|
4488
|
-
|
|
4489
|
-
unsub();
|
|
4490
|
-
reject(err);
|
|
4491
|
-
});
|
|
4492
|
-
});
|
|
4493
|
-
} catch (err) {
|
|
4494
|
-
clearTimeout(timer);
|
|
4495
|
-
if (isAbortLikeError(err)) return {
|
|
4496
|
-
messages: [],
|
|
4497
|
-
timedOut: true
|
|
4498
|
-
};
|
|
4499
|
-
throw err;
|
|
4500
|
-
}
|
|
4370
|
+
wake() {
|
|
4371
|
+
this.wakeEarly();
|
|
4372
|
+
}
|
|
4373
|
+
async enqueueDelayedSend(payload, fireAt, opts) {
|
|
4374
|
+
const tenantId = this.resolveTenantId();
|
|
4375
|
+
await this.pgClient`
|
|
4376
|
+
insert into scheduled_tasks (
|
|
4377
|
+
tenant_id,
|
|
4378
|
+
kind,
|
|
4379
|
+
payload,
|
|
4380
|
+
fire_at,
|
|
4381
|
+
owner_entity_url,
|
|
4382
|
+
manifest_key
|
|
4383
|
+
)
|
|
4384
|
+
values (
|
|
4385
|
+
${tenantId},
|
|
4386
|
+
'delayed_send',
|
|
4387
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4388
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4389
|
+
${opts?.ownerEntityUrl ?? null},
|
|
4390
|
+
${opts?.manifestKey ?? null}
|
|
4391
|
+
)
|
|
4392
|
+
`;
|
|
4393
|
+
this.wakeEarly();
|
|
4394
|
+
}
|
|
4395
|
+
async syncManifestDelayedSend(ownerEntityUrl, manifestKey, payload, fireAt) {
|
|
4396
|
+
const tenantId = this.resolveTenantId();
|
|
4397
|
+
await this.pgClient.begin(async (sql$2) => {
|
|
4398
|
+
await sql$2`
|
|
4399
|
+
update scheduled_tasks
|
|
4400
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4401
|
+
where tenant_id = ${tenantId}
|
|
4402
|
+
and kind = 'delayed_send'
|
|
4403
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4404
|
+
and manifest_key = ${manifestKey}
|
|
4405
|
+
and completed_at is null
|
|
4406
|
+
`;
|
|
4407
|
+
await sql$2`
|
|
4408
|
+
insert into scheduled_tasks (
|
|
4409
|
+
tenant_id,
|
|
4410
|
+
kind,
|
|
4411
|
+
payload,
|
|
4412
|
+
fire_at,
|
|
4413
|
+
owner_entity_url,
|
|
4414
|
+
manifest_key
|
|
4415
|
+
)
|
|
4416
|
+
values (
|
|
4417
|
+
${tenantId},
|
|
4418
|
+
'delayed_send',
|
|
4419
|
+
${JSON.stringify(payload)}::jsonb,
|
|
4420
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4421
|
+
${ownerEntityUrl},
|
|
4422
|
+
${manifestKey}
|
|
4423
|
+
)
|
|
4424
|
+
`;
|
|
4501
4425
|
});
|
|
4426
|
+
this.wakeEarly();
|
|
4427
|
+
}
|
|
4428
|
+
async cancelManifestDelayedSend(ownerEntityUrl, manifestKey) {
|
|
4429
|
+
const tenantId = this.resolveTenantId();
|
|
4430
|
+
await this.pgClient`
|
|
4431
|
+
update scheduled_tasks
|
|
4432
|
+
set completed_at = now(), claimed_at = null, claimed_by = null
|
|
4433
|
+
where tenant_id = ${tenantId}
|
|
4434
|
+
and kind = 'delayed_send'
|
|
4435
|
+
and owner_entity_url = ${ownerEntityUrl}
|
|
4436
|
+
and manifest_key = ${manifestKey}
|
|
4437
|
+
and completed_at is null
|
|
4438
|
+
`;
|
|
4439
|
+
this.wakeEarly();
|
|
4502
4440
|
}
|
|
4503
|
-
async
|
|
4504
|
-
|
|
4505
|
-
|
|
4506
|
-
|
|
4507
|
-
|
|
4441
|
+
async enqueueCronTick(expression, timezone, tickNumber, streamPath, fireAt) {
|
|
4442
|
+
const tenantId = this.resolveTenantId();
|
|
4443
|
+
await this.pgClient`
|
|
4444
|
+
insert into scheduled_tasks (
|
|
4445
|
+
tenant_id,
|
|
4446
|
+
kind,
|
|
4447
|
+
payload,
|
|
4448
|
+
fire_at,
|
|
4449
|
+
cron_expression,
|
|
4450
|
+
cron_timezone,
|
|
4451
|
+
cron_tick_number
|
|
4452
|
+
)
|
|
4453
|
+
values (
|
|
4454
|
+
${tenantId},
|
|
4455
|
+
'cron_tick',
|
|
4456
|
+
${JSON.stringify({ streamPath })}::jsonb,
|
|
4457
|
+
${fireAt.toISOString()}::timestamptz,
|
|
4458
|
+
${expression},
|
|
4459
|
+
${timezone},
|
|
4460
|
+
${tickNumber}
|
|
4461
|
+
)
|
|
4462
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4463
|
+
`;
|
|
4464
|
+
this.wakeEarly();
|
|
4508
4465
|
}
|
|
4509
|
-
async
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
await this.
|
|
4466
|
+
async runLoop() {
|
|
4467
|
+
while (this.running) try {
|
|
4468
|
+
await this.reclaimStaleClaims();
|
|
4469
|
+
await this.fireReadyTasks();
|
|
4470
|
+
const nextFireAt = await this.getNextFireAt();
|
|
4471
|
+
const sleepTargetMs = nextFireAt ? Math.max(0, nextFireAt.getTime() - Date.now()) : this.safetyPollMs;
|
|
4472
|
+
await this.sleepOrWake(Math.min(sleepTargetMs, this.safetyPollMs));
|
|
4513
4473
|
} catch (err) {
|
|
4514
|
-
|
|
4515
|
-
|
|
4474
|
+
console.error(`[agent-server] scheduler iteration failed:`, err);
|
|
4475
|
+
await this.sleepOrWake(this.safetyPollMs);
|
|
4516
4476
|
}
|
|
4517
4477
|
}
|
|
4518
|
-
async
|
|
4519
|
-
|
|
4520
|
-
const
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
|
|
4478
|
+
async reclaimStaleClaims() {
|
|
4479
|
+
if (this.tenantId === null) {
|
|
4480
|
+
const tenantIds = this.sharedTenantIds();
|
|
4481
|
+
if (tenantIds && tenantIds.length === 0) return;
|
|
4482
|
+
if (tenantIds) {
|
|
4483
|
+
await this.pgClient`
|
|
4484
|
+
update scheduled_tasks
|
|
4485
|
+
set claimed_by = null, claimed_at = null
|
|
4486
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4487
|
+
and completed_at is null
|
|
4488
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4489
|
+
`;
|
|
4490
|
+
return;
|
|
4491
|
+
}
|
|
4492
|
+
await this.pgClient`
|
|
4493
|
+
update scheduled_tasks
|
|
4494
|
+
set claimed_by = null, claimed_at = null
|
|
4495
|
+
where completed_at is null
|
|
4496
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4497
|
+
`;
|
|
4498
|
+
return;
|
|
4528
4499
|
}
|
|
4500
|
+
await this.pgClient`
|
|
4501
|
+
update scheduled_tasks
|
|
4502
|
+
set claimed_by = null, claimed_at = null
|
|
4503
|
+
where tenant_id = ${this.tenantId}
|
|
4504
|
+
and completed_at is null
|
|
4505
|
+
and claimed_at < now() - (${this.claimExpiryMs} * interval '1 millisecond')
|
|
4506
|
+
`;
|
|
4529
4507
|
}
|
|
4530
|
-
async
|
|
4531
|
-
|
|
4532
|
-
|
|
4533
|
-
|
|
4534
|
-
|
|
4535
|
-
|
|
4536
|
-
});
|
|
4537
|
-
return res;
|
|
4538
|
-
}
|
|
4539
|
-
async putSubscription(subscriptionId, input) {
|
|
4540
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4541
|
-
method: `PUT`,
|
|
4542
|
-
headers: await this.requestHeaders({ "content-type": `application/json` }),
|
|
4543
|
-
body: JSON.stringify({
|
|
4544
|
-
...input,
|
|
4545
|
-
pattern: typeof input.pattern === `string` ? this.backendSubscriptionPath(normalizeSubscriptionPattern(input.pattern)) : void 0,
|
|
4546
|
-
streams: input.streams?.map((stream) => this.backendSubscriptionPath(normalizeSubscriptionStreamPath(stream))),
|
|
4547
|
-
wake_stream: typeof input.wake_stream === `string` ? this.backendSubscriptionPath(normalizeSubscriptionStreamPath(input.wake_stream)) : void 0
|
|
4548
|
-
})
|
|
4549
|
-
});
|
|
4550
|
-
return await this.subscriptionJson(res, `Subscription creation failed`);
|
|
4551
|
-
}
|
|
4552
|
-
async getSubscription(subscriptionId) {
|
|
4553
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4554
|
-
method: `GET`,
|
|
4555
|
-
headers: await this.requestHeaders()
|
|
4556
|
-
});
|
|
4557
|
-
if (res.status === 404) return null;
|
|
4558
|
-
return await this.subscriptionJson(res, `Subscription query failed`);
|
|
4559
|
-
}
|
|
4560
|
-
async deleteSubscription(subscriptionId) {
|
|
4561
|
-
const res = await fetch(this.subscriptionUrl(subscriptionId), {
|
|
4562
|
-
method: `DELETE`,
|
|
4563
|
-
headers: await this.requestHeaders()
|
|
4564
|
-
});
|
|
4565
|
-
if (res.status === 404 || res.status === 204) return;
|
|
4566
|
-
if (!res.ok) throw new Error(`Subscription delete failed: ${res.status} ${await res.text()}`);
|
|
4508
|
+
async fireReadyTasks() {
|
|
4509
|
+
while (this.running) {
|
|
4510
|
+
const tasks = await this.claimReadyTasks();
|
|
4511
|
+
if (tasks.length === 0) return;
|
|
4512
|
+
for (const task of tasks) await this.executeTask(task);
|
|
4513
|
+
}
|
|
4567
4514
|
}
|
|
4568
|
-
async
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
|
|
4515
|
+
async claimReadyTasks() {
|
|
4516
|
+
if (this.tenantId === null) {
|
|
4517
|
+
const tenantIds = this.sharedTenantIds();
|
|
4518
|
+
if (tenantIds && tenantIds.length === 0) return [];
|
|
4519
|
+
if (tenantIds) {
|
|
4520
|
+
const rows$2 = await this.pgClient`
|
|
4521
|
+
update scheduled_tasks
|
|
4522
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4523
|
+
where id in (
|
|
4524
|
+
select id
|
|
4525
|
+
from scheduled_tasks
|
|
4526
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4527
|
+
and completed_at is null
|
|
4528
|
+
and claimed_at is null
|
|
4529
|
+
and fire_at <= now()
|
|
4530
|
+
order by fire_at, id
|
|
4531
|
+
for update skip locked
|
|
4532
|
+
limit 50
|
|
4533
|
+
)
|
|
4534
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4535
|
+
, owner_entity_url, manifest_key
|
|
4536
|
+
`;
|
|
4537
|
+
return rows$2.map(normalizeTask);
|
|
4538
|
+
}
|
|
4539
|
+
const rows$1 = await this.pgClient`
|
|
4540
|
+
update scheduled_tasks
|
|
4541
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4542
|
+
where id in (
|
|
4543
|
+
select id
|
|
4544
|
+
from scheduled_tasks
|
|
4545
|
+
where completed_at is null
|
|
4546
|
+
and claimed_at is null
|
|
4547
|
+
and fire_at <= now()
|
|
4548
|
+
order by fire_at, id
|
|
4549
|
+
for update skip locked
|
|
4550
|
+
limit 50
|
|
4551
|
+
)
|
|
4552
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4553
|
+
, owner_entity_url, manifest_key
|
|
4554
|
+
`;
|
|
4555
|
+
return rows$1.map(normalizeTask);
|
|
4556
|
+
}
|
|
4557
|
+
const rows = await this.pgClient`
|
|
4558
|
+
update scheduled_tasks
|
|
4559
|
+
set claimed_by = ${this.instanceId}, claimed_at = now()
|
|
4560
|
+
where tenant_id = ${this.tenantId}
|
|
4561
|
+
and id in (
|
|
4562
|
+
select id
|
|
4563
|
+
from scheduled_tasks
|
|
4564
|
+
where tenant_id = ${this.tenantId}
|
|
4565
|
+
and completed_at is null
|
|
4566
|
+
and claimed_at is null
|
|
4567
|
+
and fire_at <= now()
|
|
4568
|
+
order by fire_at, id
|
|
4569
|
+
for update skip locked
|
|
4570
|
+
limit 50
|
|
4571
|
+
)
|
|
4572
|
+
returning tenant_id, id, kind, payload, fire_at, cron_expression, cron_timezone, cron_tick_number
|
|
4573
|
+
, owner_entity_url, manifest_key
|
|
4574
|
+
`;
|
|
4575
|
+
return rows.map(normalizeTask);
|
|
4575
4576
|
}
|
|
4576
|
-
async
|
|
4577
|
-
|
|
4578
|
-
|
|
4579
|
-
|
|
4580
|
-
|
|
4581
|
-
|
|
4582
|
-
|
|
4577
|
+
async executeTask(task) {
|
|
4578
|
+
try {
|
|
4579
|
+
if (task.kind === `delayed_send`) {
|
|
4580
|
+
await this.options.executors.delayed_send(task.payload, task.id, task.tenantId);
|
|
4581
|
+
await this.markTaskComplete(task.id, task.tenantId);
|
|
4582
|
+
return;
|
|
4583
|
+
}
|
|
4584
|
+
const tickNumber = task.cronTickNumber;
|
|
4585
|
+
if (tickNumber == null || !task.cronExpression || !task.cronTimezone) throw new Error(`cron task ${task.id} is missing cron metadata`);
|
|
4586
|
+
await this.options.executors.cron_tick(task.payload, tickNumber, task.id, task.tenantId);
|
|
4587
|
+
await this.completeAndRescheduleCron(task);
|
|
4588
|
+
} catch (err) {
|
|
4589
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
4590
|
+
if (isUnregisteredTenantError(err)) {
|
|
4591
|
+
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4592
|
+
serverLog.warn(`[scheduler] skipped ${task.kind} task ${task.id} for unregistered tenant "${task.tenantId}": ${message}`);
|
|
4593
|
+
return;
|
|
4594
|
+
}
|
|
4595
|
+
if (isPermanentElectricAgentsError(err)) {
|
|
4596
|
+
await this.markTaskPermanentFailure(task.id, message, task.tenantId);
|
|
4597
|
+
return;
|
|
4598
|
+
}
|
|
4599
|
+
await this.releaseClaim(task.id, message, task.tenantId);
|
|
4600
|
+
}
|
|
4583
4601
|
}
|
|
4584
|
-
async
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
|
|
4588
|
-
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4602
|
+
async markTaskComplete(taskId, tenantId = this.resolveTenantId()) {
|
|
4603
|
+
await this.pgClient`
|
|
4604
|
+
update scheduled_tasks
|
|
4605
|
+
set completed_at = now(), last_error = null
|
|
4606
|
+
where tenant_id = ${tenantId}
|
|
4607
|
+
and id = ${taskId}
|
|
4608
|
+
and claimed_by = ${this.instanceId}
|
|
4609
|
+
and completed_at is null
|
|
4610
|
+
`;
|
|
4592
4611
|
}
|
|
4593
|
-
async
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
return await this.subscriptionJson(res, `Subscription ack failed`);
|
|
4612
|
+
async markTaskPermanentFailure(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4613
|
+
await this.pgClient`
|
|
4614
|
+
update scheduled_tasks
|
|
4615
|
+
set completed_at = now(), last_error = ${message}
|
|
4616
|
+
where tenant_id = ${tenantId}
|
|
4617
|
+
and id = ${taskId}
|
|
4618
|
+
and claimed_by = ${this.instanceId}
|
|
4619
|
+
and completed_at is null
|
|
4620
|
+
`;
|
|
4603
4621
|
}
|
|
4604
|
-
async
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
return await this.subscriptionJson(res, `Subscription release failed`);
|
|
4622
|
+
async releaseClaim(taskId, message, tenantId = this.resolveTenantId()) {
|
|
4623
|
+
await this.pgClient`
|
|
4624
|
+
update scheduled_tasks
|
|
4625
|
+
set claimed_at = null, claimed_by = null, last_error = ${message}
|
|
4626
|
+
where tenant_id = ${tenantId}
|
|
4627
|
+
and id = ${taskId}
|
|
4628
|
+
and claimed_by = ${this.instanceId}
|
|
4629
|
+
and completed_at is null
|
|
4630
|
+
`;
|
|
4614
4631
|
}
|
|
4615
|
-
|
|
4616
|
-
const
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
|
|
4624
|
-
|
|
4632
|
+
async completeAndRescheduleCron(task) {
|
|
4633
|
+
const tenantId = task.tenantId ?? this.resolveTenantId();
|
|
4634
|
+
await this.pgClient.begin(async (sql$2) => {
|
|
4635
|
+
const completed = await sql$2`
|
|
4636
|
+
update scheduled_tasks
|
|
4637
|
+
set completed_at = now(), last_error = null
|
|
4638
|
+
where tenant_id = ${tenantId}
|
|
4639
|
+
and id = ${task.id}
|
|
4640
|
+
and claimed_by = ${this.instanceId}
|
|
4641
|
+
and completed_at is null
|
|
4642
|
+
returning id
|
|
4643
|
+
`;
|
|
4644
|
+
if (completed.length === 0) return;
|
|
4645
|
+
const nextFireAt = (0, __electric_ax_agents_runtime.getNextCronFireAt)(task.cronExpression, task.cronTimezone, task.fireAt);
|
|
4646
|
+
await sql$2`
|
|
4647
|
+
insert into scheduled_tasks (
|
|
4648
|
+
tenant_id,
|
|
4649
|
+
kind,
|
|
4650
|
+
payload,
|
|
4651
|
+
fire_at,
|
|
4652
|
+
cron_expression,
|
|
4653
|
+
cron_timezone,
|
|
4654
|
+
cron_tick_number
|
|
4655
|
+
)
|
|
4656
|
+
values (
|
|
4657
|
+
${tenantId},
|
|
4658
|
+
'cron_tick',
|
|
4659
|
+
${JSON.stringify(task.payload)}::jsonb,
|
|
4660
|
+
${nextFireAt.toISOString()}::timestamptz,
|
|
4661
|
+
${task.cronExpression},
|
|
4662
|
+
${task.cronTimezone},
|
|
4663
|
+
${task.cronTickNumber + 1}
|
|
4664
|
+
)
|
|
4665
|
+
on conflict (tenant_id, cron_expression, cron_timezone, cron_tick_number) do nothing
|
|
4666
|
+
`;
|
|
4625
4667
|
});
|
|
4626
|
-
return next;
|
|
4627
4668
|
}
|
|
4628
|
-
|
|
4629
|
-
|
|
4630
|
-
|
|
4631
|
-
|
|
4632
|
-
|
|
4633
|
-
|
|
4634
|
-
|
|
4635
|
-
|
|
4636
|
-
|
|
4669
|
+
async getNextFireAt() {
|
|
4670
|
+
if (this.tenantId === null) {
|
|
4671
|
+
const tenantIds = this.sharedTenantIds();
|
|
4672
|
+
if (tenantIds && tenantIds.length === 0) return null;
|
|
4673
|
+
if (tenantIds) {
|
|
4674
|
+
const rows$2 = await this.pgClient`
|
|
4675
|
+
select fire_at
|
|
4676
|
+
from scheduled_tasks
|
|
4677
|
+
where tenant_id = any(${this.sharedTenantIdsParameter(tenantIds)})
|
|
4678
|
+
and completed_at is null
|
|
4679
|
+
and claimed_at is null
|
|
4680
|
+
order by fire_at, id
|
|
4681
|
+
limit 1
|
|
4682
|
+
`;
|
|
4683
|
+
if (rows$2.length === 0) return null;
|
|
4684
|
+
const fireAt$2 = rows$2[0].fire_at;
|
|
4685
|
+
return fireAt$2 instanceof Date ? fireAt$2 : new Date(fireAt$2);
|
|
4686
|
+
}
|
|
4687
|
+
const rows$1 = await this.pgClient`
|
|
4688
|
+
select fire_at
|
|
4689
|
+
from scheduled_tasks
|
|
4690
|
+
where completed_at is null
|
|
4691
|
+
and claimed_at is null
|
|
4692
|
+
order by fire_at, id
|
|
4693
|
+
limit 1
|
|
4694
|
+
`;
|
|
4695
|
+
if (rows$1.length === 0) return null;
|
|
4696
|
+
const fireAt$1 = rows$1[0].fire_at;
|
|
4697
|
+
return fireAt$1 instanceof Date ? fireAt$1 : new Date(fireAt$1);
|
|
4698
|
+
}
|
|
4699
|
+
const rows = await this.pgClient`
|
|
4700
|
+
select fire_at
|
|
4701
|
+
from scheduled_tasks
|
|
4702
|
+
where tenant_id = ${this.tenantId}
|
|
4703
|
+
and completed_at is null
|
|
4704
|
+
and claimed_at is null
|
|
4705
|
+
order by fire_at, id
|
|
4706
|
+
limit 1
|
|
4707
|
+
`;
|
|
4708
|
+
if (rows.length === 0) return null;
|
|
4709
|
+
const fireAt = rows[0].fire_at;
|
|
4710
|
+
return fireAt instanceof Date ? fireAt : new Date(fireAt);
|
|
4711
|
+
}
|
|
4712
|
+
async sleepOrWake(durationMs) {
|
|
4713
|
+
if (!this.running) return;
|
|
4714
|
+
await new Promise((resolve$1) => {
|
|
4715
|
+
const finish = () => {
|
|
4716
|
+
if (this.currentSleepTimer) {
|
|
4717
|
+
clearTimeout(this.currentSleepTimer);
|
|
4718
|
+
this.currentSleepTimer = null;
|
|
4719
|
+
}
|
|
4720
|
+
this.currentSleepResolve = null;
|
|
4721
|
+
resolve$1();
|
|
4637
4722
|
};
|
|
4723
|
+
this.currentSleepResolve = finish;
|
|
4724
|
+
this.currentSleepTimer = setTimeout(finish, Math.max(durationMs, 0));
|
|
4638
4725
|
});
|
|
4639
|
-
if (Array.isArray(next.acks)) next.acks = next.acks.map((ack) => {
|
|
4640
|
-
if (!ack || typeof ack !== `object`) return ack;
|
|
4641
|
-
const mapped = { ...ack };
|
|
4642
|
-
if (typeof mapped.stream === `string`) mapped.stream = this.runtimeSubscriptionPath(mapped.stream);
|
|
4643
|
-
if (typeof mapped.path === `string`) mapped.path = this.runtimeSubscriptionPath(mapped.path);
|
|
4644
|
-
return mapped;
|
|
4645
|
-
});
|
|
4646
|
-
if (typeof next.stream === `string`) next.stream = this.runtimeSubscriptionPath(next.stream);
|
|
4647
|
-
return next;
|
|
4648
4726
|
}
|
|
4649
|
-
|
|
4650
|
-
|
|
4651
|
-
|
|
4652
|
-
|
|
4653
|
-
|
|
4654
|
-
|
|
4727
|
+
wakeEarly() {
|
|
4728
|
+
const resolve$1 = this.currentSleepResolve;
|
|
4729
|
+
this.currentSleepResolve = null;
|
|
4730
|
+
if (this.currentSleepTimer) {
|
|
4731
|
+
clearTimeout(this.currentSleepTimer);
|
|
4732
|
+
this.currentSleepTimer = null;
|
|
4733
|
+
}
|
|
4734
|
+
resolve$1?.();
|
|
4655
4735
|
}
|
|
4656
|
-
|
|
4657
|
-
|
|
4658
|
-
|
|
4659
|
-
|
|
4660
|
-
|
|
4661
|
-
|
|
4662
|
-
if (!res.ok) throw new Error(`Consumer query failed: ${res.status} ${await res.text()}`);
|
|
4663
|
-
return res.json();
|
|
4736
|
+
sharedTenantIds() {
|
|
4737
|
+
if (this.tenantId !== null || !this.tenantIds) return null;
|
|
4738
|
+
return [...new Set(this.tenantIds())];
|
|
4739
|
+
}
|
|
4740
|
+
sharedTenantIdsParameter(tenantIds) {
|
|
4741
|
+
return this.pgClient.array(tenantIds, POSTGRES_TEXT_OID);
|
|
4664
4742
|
}
|
|
4665
4743
|
};
|
|
4666
4744
|
|
|
@@ -4685,7 +4763,7 @@ var ElectricAgentsTenantRuntime = class {
|
|
|
4685
4763
|
this.service = this.serviceId;
|
|
4686
4764
|
this.db = options.db;
|
|
4687
4765
|
if (options.streamClient) this.streamClient = options.streamClient;
|
|
4688
|
-
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(
|
|
4766
|
+
else if (options.durableStreamsUrl) this.streamClient = new StreamClient(options.durableStreamsUrl, { bearer: options.durableStreamsBearer });
|
|
4689
4767
|
else throw new Error(`Either durableStreamsUrl or streamClient is required`);
|
|
4690
4768
|
this.registry = options.registry ?? new PostgresRegistry(this.db, this.serviceId);
|
|
4691
4769
|
this.wakeRegistry = options.wakeRegistry;
|
|
@@ -5776,7 +5854,7 @@ var AgentsHost = class {
|
|
|
5776
5854
|
}
|
|
5777
5855
|
createStreamClient(config) {
|
|
5778
5856
|
if (config.streamClient) return config.streamClient;
|
|
5779
|
-
if (config.durableStreamsUrl) return new StreamClient(
|
|
5857
|
+
if (config.durableStreamsUrl) return new StreamClient(config.durableStreamsUrl, { bearer: config.durableStreamsBearer });
|
|
5780
5858
|
throw new Error(`AgentsHost tenant "${config.serviceId}" must provide a streamClient or durableStreamsUrl`);
|
|
5781
5859
|
}
|
|
5782
5860
|
};
|
|
@@ -5954,74 +6032,38 @@ function validateParsedBody(schema, parsed) {
|
|
|
5954
6032
|
};
|
|
5955
6033
|
}
|
|
5956
6034
|
|
|
5957
|
-
//#endregion
|
|
5958
|
-
//#region src/routing/tenant-stream-paths.ts
|
|
5959
|
-
function withoutLeadingSlash(path$2) {
|
|
5960
|
-
return path$2.replace(/^\/+/, ``);
|
|
5961
|
-
}
|
|
5962
|
-
function withLeadingSlash(path$2) {
|
|
5963
|
-
return path$2.startsWith(`/`) ? path$2 : `/${path$2}`;
|
|
5964
|
-
}
|
|
5965
|
-
function prefixTenantStreamPath(path$2, tenantId) {
|
|
5966
|
-
const normalized = withoutLeadingSlash(path$2);
|
|
5967
|
-
if (!normalized || normalized === tenantId) return tenantId;
|
|
5968
|
-
if (normalized.startsWith(`${tenantId}/`)) return normalized;
|
|
5969
|
-
return `${tenantId}/${normalized}`;
|
|
5970
|
-
}
|
|
5971
|
-
function stripTenantStreamPrefix(path$2, tenantId) {
|
|
5972
|
-
const normalized = withoutLeadingSlash(path$2);
|
|
5973
|
-
if (normalized === tenantId) return ``;
|
|
5974
|
-
if (normalized.startsWith(`${tenantId}/`)) return normalized.slice(tenantId.length + 1);
|
|
5975
|
-
return normalized;
|
|
5976
|
-
}
|
|
5977
|
-
|
|
5978
6035
|
//#endregion
|
|
5979
6036
|
//#region src/routing/durable-streams-routing-adapter.ts
|
|
5980
6037
|
function appendSearch(target, source) {
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
}
|
|
5984
|
-
function removeServiceQuery(target) {
|
|
5985
|
-
target.searchParams.delete(`service`);
|
|
6038
|
+
source.searchParams.forEach((value, key) => {
|
|
6039
|
+
if (key !== `service`) target.searchParams.append(key, value);
|
|
6040
|
+
});
|
|
5986
6041
|
return target;
|
|
5987
6042
|
}
|
|
5988
|
-
function
|
|
5989
|
-
|
|
5990
|
-
const segments = incomingUrl.pathname.split(`/`).filter(Boolean);
|
|
5991
|
-
if (segments[0] === `v1` && segments[1] === `stream`) return {
|
|
5992
|
-
incomingUrl,
|
|
5993
|
-
streamPath: segments.length > 2 ? `/${segments.slice(3).join(`/`)}` : `/`
|
|
5994
|
-
};
|
|
5995
|
-
return {
|
|
5996
|
-
incomingUrl,
|
|
5997
|
-
streamPath: incomingUrl.pathname || `/${serviceId}`
|
|
5998
|
-
};
|
|
5999
|
-
}
|
|
6000
|
-
function backendStreamUrl(input, backendStreamPath) {
|
|
6001
|
-
const path$2 = backendStreamPath.replace(/^\/+/, ``);
|
|
6002
|
-
const target = new URL(`/v1/stream/${path$2}`, input.durableStreamsUrl);
|
|
6003
|
-
return target;
|
|
6043
|
+
function withoutTrailingSlash(pathname) {
|
|
6044
|
+
return pathname.replace(/\/+$/, ``) || `/`;
|
|
6004
6045
|
}
|
|
6005
|
-
function
|
|
6046
|
+
function appendRequestPathToStreamRoot(input) {
|
|
6006
6047
|
const incomingUrl = new URL(input.requestUrl, `http://localhost`);
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
return prefixTenantStreamPath(streamPath, serviceId);
|
|
6048
|
+
const path$2 = incomingUrl.pathname.replace(/^\/+/, ``);
|
|
6049
|
+
const target = new URL(input.durableStreamsUrl);
|
|
6050
|
+
target.pathname = path$2 ? `${withoutTrailingSlash(target.pathname)}/${path$2}` : withoutTrailingSlash(target.pathname);
|
|
6051
|
+
return appendSearch(target, incomingUrl);
|
|
6052
|
+
}
|
|
6053
|
+
const streamRootDurableStreamsRoutingAdapter = {
|
|
6054
|
+
streamUrl: appendRequestPathToStreamRoot,
|
|
6055
|
+
controlUrl: appendRequestPathToStreamRoot,
|
|
6056
|
+
toBackendStreamPath(_serviceId, streamPath) {
|
|
6057
|
+
return streamPath.replace(/^\/+/, ``);
|
|
6018
6058
|
},
|
|
6019
|
-
toRuntimeStreamPath(
|
|
6020
|
-
return
|
|
6059
|
+
toRuntimeStreamPath(_serviceId, streamPath) {
|
|
6060
|
+
return streamPath.replace(/^\/+/, ``);
|
|
6021
6061
|
}
|
|
6022
6062
|
};
|
|
6023
|
-
|
|
6024
|
-
|
|
6063
|
+
const pathPrefixedSingleTenantDurableStreamsRoutingAdapter = streamRootDurableStreamsRoutingAdapter;
|
|
6064
|
+
const tenantRootDurableStreamsRoutingAdapter = streamRootDurableStreamsRoutingAdapter;
|
|
6065
|
+
function resolveDurableStreamsRoutingAdapter(adapter, _durableStreamsUrl) {
|
|
6066
|
+
return adapter ?? streamRootDurableStreamsRoutingAdapter;
|
|
6025
6067
|
}
|
|
6026
6068
|
|
|
6027
6069
|
//#endregion
|
|
@@ -6043,8 +6085,11 @@ function buildElectricProxyTarget(options) {
|
|
|
6043
6085
|
target.searchParams.set(`columns`, `"tenant_id","name","description","creation_schema","inbox_schemas","state_schemas","serve_endpoint","default_dispatch_policy","revision","created_at","updated_at"`);
|
|
6044
6086
|
applyTenantShapeWhere(target, options.tenantId);
|
|
6045
6087
|
} else if (table === `runners`) {
|
|
6046
|
-
target.searchParams.set(`columns`, `"tenant_id","id","
|
|
6047
|
-
applyTenantShapeWhere(target, options.tenantId);
|
|
6088
|
+
target.searchParams.set(`columns`, `"tenant_id","id","owner_principal","label","kind","admin_status","wake_stream","created_at","updated_at"`);
|
|
6089
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral(options.principalUrl ?? ``)}`]);
|
|
6090
|
+
} else if (table === `runner_runtime_diagnostics`) {
|
|
6091
|
+
target.searchParams.set(`columns`, `"tenant_id","runner_id","owner_principal","wake_stream_offset","last_seen_at","liveness_lease_expires_at","diagnostics","updated_at"`);
|
|
6092
|
+
applyTenantShapeWhere(target, options.tenantId, [`owner_principal = ${sqlStringLiteral(options.principalUrl ?? ``)}`]);
|
|
6048
6093
|
} else if (table === `entity_dispatch_state`) {
|
|
6049
6094
|
target.searchParams.set(`columns`, `"tenant_id","entity_url","pending_source_streams","pending_reason","pending_since","outstanding_wake_id","outstanding_wake_target","outstanding_wake_created_at","active_consumer_id","active_runner_id","active_epoch","active_claimed_at","active_lease_expires_at","last_wake_id","last_claimed_at","last_released_at","last_completed_at","last_error","updated_at"`);
|
|
6050
6095
|
applyTenantShapeWhere(target, options.tenantId);
|
|
@@ -6058,13 +6103,13 @@ function buildElectricProxyTarget(options) {
|
|
|
6058
6103
|
return target;
|
|
6059
6104
|
}
|
|
6060
6105
|
async function forwardFetchRequest(options) {
|
|
6061
|
-
const routingAdapter = resolveDurableStreamsRoutingAdapter(options.durableStreamsRouting);
|
|
6106
|
+
const routingAdapter = resolveDurableStreamsRoutingAdapter(options.durableStreamsRouting, options.durableStreamsUrl);
|
|
6062
6107
|
const routingInput = {
|
|
6063
6108
|
durableStreamsUrl: options.durableStreamsUrl,
|
|
6064
6109
|
serviceId: options.serviceId,
|
|
6065
6110
|
requestUrl: options.request.url
|
|
6066
6111
|
};
|
|
6067
|
-
const upstreamUrl = options.route === `
|
|
6112
|
+
const upstreamUrl = options.route === `control` ? routingAdapter.controlUrl(routingInput) : routingAdapter.streamUrl(routingInput);
|
|
6068
6113
|
const headers = new Headers(options.request.headers);
|
|
6069
6114
|
if (options.durableStreamsBearerMode !== `none`) await applyDurableStreamsBearer(headers, options.durableStreamsBearer, { overwrite: options.durableStreamsBearerMode !== `if-missing` });
|
|
6070
6115
|
const init = {
|
|
@@ -6090,8 +6135,8 @@ function decodeJsonObject(body) {
|
|
|
6090
6135
|
} catch {}
|
|
6091
6136
|
return null;
|
|
6092
6137
|
}
|
|
6093
|
-
function applyTenantShapeWhere(target, tenantId) {
|
|
6094
|
-
const tenantWhere = `tenant_id = ${sqlStringLiteral(tenantId)}
|
|
6138
|
+
function applyTenantShapeWhere(target, tenantId, extraConditions = []) {
|
|
6139
|
+
const tenantWhere = [`tenant_id = ${sqlStringLiteral(tenantId)}`, ...extraConditions].join(` AND `);
|
|
6095
6140
|
const existingWhere = target.searchParams.get(`where`);
|
|
6096
6141
|
target.searchParams.set(`where`, existingWhere ? `${tenantWhere} AND (${existingWhere})` : tenantWhere);
|
|
6097
6142
|
}
|
|
@@ -6102,8 +6147,21 @@ function sqlStringLiteral(value) {
|
|
|
6102
6147
|
//#endregion
|
|
6103
6148
|
//#region src/routing/durable-streams-router.ts
|
|
6104
6149
|
const subscriptionProxyBodySchema = __sinclair_typebox.Type.Object({ webhook: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Object({ url: __sinclair_typebox.Type.String() }, { additionalProperties: true })) }, { additionalProperties: true });
|
|
6150
|
+
const subscriptionControlActions = [
|
|
6151
|
+
`callback`,
|
|
6152
|
+
`claim`,
|
|
6153
|
+
`ack`,
|
|
6154
|
+
`release`
|
|
6155
|
+
];
|
|
6105
6156
|
const durableStreamsRouter = (0, itty_router.Router)();
|
|
6106
|
-
durableStreamsRouter.
|
|
6157
|
+
durableStreamsRouter.put(`/__ds/subscriptions/:subscriptionId`, putSubscriptionBase);
|
|
6158
|
+
durableStreamsRouter.get(`/__ds/subscriptions/:subscriptionId`, getSubscriptionBase);
|
|
6159
|
+
durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId`, deleteSubscriptionBase);
|
|
6160
|
+
durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/streams`, postSubscriptionStreams);
|
|
6161
|
+
durableStreamsRouter.delete(`/__ds/subscriptions/:subscriptionId/streams/:streamPath+`, deleteSubscriptionStream);
|
|
6162
|
+
for (const action of subscriptionControlActions) durableStreamsRouter.post(`/__ds/subscriptions/:subscriptionId/${action}`, subscriptionAction(action));
|
|
6163
|
+
durableStreamsRouter.all(`/__ds`, controlPassThrough);
|
|
6164
|
+
durableStreamsRouter.all(`/__ds/*`, controlPassThrough);
|
|
6107
6165
|
durableStreamsRouter.post(`*`, streamAppend);
|
|
6108
6166
|
durableStreamsRouter.all(`*`, proxyPassThrough);
|
|
6109
6167
|
function bodyFromBytes$1(body) {
|
|
@@ -6116,7 +6174,7 @@ function responseFromUpstream$1(response, body) {
|
|
|
6116
6174
|
headers: responseHeaders(response)
|
|
6117
6175
|
});
|
|
6118
6176
|
}
|
|
6119
|
-
async function forwardToDurableStreams(ctx, request, body, route = `stream`, urlOverride) {
|
|
6177
|
+
async function forwardToDurableStreams(ctx, request, body, route = `stream`, urlOverride, durableStreamsBearerMode = `overwrite`) {
|
|
6120
6178
|
const headers = new Headers(request.headers);
|
|
6121
6179
|
headers.delete(`host`);
|
|
6122
6180
|
let requestBody = body;
|
|
@@ -6130,24 +6188,13 @@ async function forwardToDurableStreams(ctx, request, body, route = `stream`, url
|
|
|
6130
6188
|
body: requestBody,
|
|
6131
6189
|
durableStreamsUrl: ctx.durableStreamsUrl,
|
|
6132
6190
|
durableStreamsBearer: ctx.durableStreamsBearer,
|
|
6133
|
-
durableStreamsBearerMode
|
|
6191
|
+
durableStreamsBearerMode,
|
|
6134
6192
|
durableStreamsRouting: ctx.durableStreamsRouting,
|
|
6135
6193
|
serviceId: ctx.service,
|
|
6136
6194
|
dispatcher: ctx.durableStreamsDispatcher,
|
|
6137
6195
|
route
|
|
6138
6196
|
});
|
|
6139
6197
|
}
|
|
6140
|
-
function subscriptionIdFromPath(pathname) {
|
|
6141
|
-
const match = /^\/v1\/stream-meta\/subscriptions\/([^/]+)(?:\/.*)?$/.exec(pathname);
|
|
6142
|
-
return match ? decodeURIComponent(match[1]) : null;
|
|
6143
|
-
}
|
|
6144
|
-
function isSubscriptionBasePath(pathname) {
|
|
6145
|
-
return /^\/v1\/stream-meta\/subscriptions\/[^/]+\/?$/.test(pathname);
|
|
6146
|
-
}
|
|
6147
|
-
function usesSubscriptionScopedBearer(requestUrl) {
|
|
6148
|
-
const pathname = new URL(requestUrl, `http://localhost`).pathname;
|
|
6149
|
-
return /^\/v1\/stream-meta\/subscriptions\/[^/]+\/(?:ack|release|callback)\/?$/.test(pathname);
|
|
6150
|
-
}
|
|
6151
6198
|
function rewriteSubscriptionBodyForBackend(payload, service, routingAdapter) {
|
|
6152
6199
|
if (typeof payload.pattern === `string`) payload.pattern = routingAdapter.toBackendStreamPath(service, payload.pattern);
|
|
6153
6200
|
if (Array.isArray(payload.streams)) payload.streams = payload.streams.map((stream) => typeof stream === `string` ? routingAdapter.toBackendStreamPath(service, stream) : stream);
|
|
@@ -6192,44 +6239,50 @@ function decodeJson(bytes) {
|
|
|
6192
6239
|
return null;
|
|
6193
6240
|
}
|
|
6194
6241
|
}
|
|
6195
|
-
function
|
|
6196
|
-
const
|
|
6197
|
-
|
|
6198
|
-
|
|
6199
|
-
const streamPath = decodeURIComponent(encodedPath);
|
|
6200
|
-
requestUrl.pathname = `${prefix}${encodeURIComponent(routingAdapter.toBackendStreamPath(service, streamPath))}`;
|
|
6201
|
-
return requestUrl.toString();
|
|
6242
|
+
function routeParam$2(request, name) {
|
|
6243
|
+
const value = request.params[name];
|
|
6244
|
+
const raw = Array.isArray(value) ? value[0] : value;
|
|
6245
|
+
return decodeURIComponent(raw ?? ``);
|
|
6202
6246
|
}
|
|
6203
|
-
|
|
6204
|
-
|
|
6205
|
-
|
|
6206
|
-
|
|
6207
|
-
const
|
|
6208
|
-
|
|
6247
|
+
function subscriptionRoutingAdapter(ctx) {
|
|
6248
|
+
return resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl);
|
|
6249
|
+
}
|
|
6250
|
+
async function rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter) {
|
|
6251
|
+
const body = await readRequestBody(request);
|
|
6252
|
+
if (body.length === 0) return {
|
|
6253
|
+
ok: true,
|
|
6254
|
+
body,
|
|
6255
|
+
targetWebhookUrl: null
|
|
6256
|
+
};
|
|
6257
|
+
const validation = validateBody(subscriptionProxyBodySchema, body);
|
|
6258
|
+
if (!validation.ok) return {
|
|
6259
|
+
ok: false,
|
|
6260
|
+
response: validation.response
|
|
6261
|
+
};
|
|
6262
|
+
const payload = validation.value;
|
|
6209
6263
|
let targetWebhookUrl = null;
|
|
6210
|
-
|
|
6211
|
-
|
|
6212
|
-
|
|
6213
|
-
if (requestBody.length > 0) {
|
|
6214
|
-
const validation = validateBody(subscriptionProxyBodySchema, requestBody);
|
|
6215
|
-
if (!validation.ok) return validation.response;
|
|
6216
|
-
const payload = validation.value;
|
|
6217
|
-
if (payload.webhook?.url !== void 0) {
|
|
6218
|
-
targetWebhookUrl = rewriteLoopbackWebhookUrl(payload.webhook.url) ?? null;
|
|
6219
|
-
payload.webhook.url = (0, __electric_ax_agents_runtime.appendPathToUrl)(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
6220
|
-
}
|
|
6221
|
-
rewriteSubscriptionBodyForBackend(payload, ctx.service, routingAdapter);
|
|
6222
|
-
requestBody = new TextEncoder().encode(JSON.stringify(payload));
|
|
6223
|
-
}
|
|
6264
|
+
if (payload.webhook?.url !== void 0) {
|
|
6265
|
+
targetWebhookUrl = rewriteLoopbackWebhookUrl(payload.webhook.url) ?? null;
|
|
6266
|
+
payload.webhook.url = (0, __electric_ax_agents_runtime.appendPathToUrl)(ctx.publicUrl, `/_electric/webhook-forward/${encodeURIComponent(subscriptionId)}`);
|
|
6224
6267
|
}
|
|
6225
|
-
|
|
6226
|
-
|
|
6268
|
+
rewriteSubscriptionBodyForBackend(payload, ctx.service, routingAdapter);
|
|
6269
|
+
return {
|
|
6270
|
+
ok: true,
|
|
6271
|
+
body: new TextEncoder().encode(JSON.stringify(payload)),
|
|
6272
|
+
targetWebhookUrl
|
|
6273
|
+
};
|
|
6274
|
+
}
|
|
6275
|
+
async function forwardSubscriptionRequest(request, ctx, routingAdapter, opts = {}) {
|
|
6276
|
+
const upstream = await forwardToDurableStreams(ctx, request, opts.body, `control`, opts.requestUrl, opts.bearerMode ?? `overwrite`);
|
|
6227
6277
|
let responseBytes = upstream.body ? new Uint8Array(await upstream.arrayBuffer()) : new Uint8Array();
|
|
6228
6278
|
responseBytes = rewriteSubscriptionResponseForClient(responseBytes, upstream, ctx.service, routingAdapter);
|
|
6229
|
-
|
|
6230
|
-
|
|
6231
|
-
|
|
6232
|
-
|
|
6279
|
+
return {
|
|
6280
|
+
upstream,
|
|
6281
|
+
response: responseFromUpstream$1(upstream, responseBytes)
|
|
6282
|
+
};
|
|
6283
|
+
}
|
|
6284
|
+
async function upsertSubscriptionWebhook(ctx, subscriptionId, targetWebhookUrl) {
|
|
6285
|
+
await ctx.pgDb.insert(subscriptionWebhooks).values({
|
|
6233
6286
|
tenantId: ctx.service,
|
|
6234
6287
|
subscriptionId,
|
|
6235
6288
|
webhookUrl: targetWebhookUrl
|
|
@@ -6237,8 +6290,64 @@ async function subscriptionProxy(request, ctx) {
|
|
|
6237
6290
|
target: [subscriptionWebhooks.tenantId, subscriptionWebhooks.subscriptionId],
|
|
6238
6291
|
set: { webhookUrl: targetWebhookUrl }
|
|
6239
6292
|
});
|
|
6293
|
+
}
|
|
6294
|
+
async function deleteSubscriptionWebhook(ctx, subscriptionId) {
|
|
6295
|
+
await ctx.pgDb.delete(subscriptionWebhooks).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(subscriptionWebhooks.tenantId, ctx.service), (0, drizzle_orm.eq)(subscriptionWebhooks.subscriptionId, subscriptionId)));
|
|
6296
|
+
}
|
|
6297
|
+
function rewriteSubscriptionStreamPathInUrl(requestUrl, service, routingAdapter, streamPath) {
|
|
6298
|
+
const prefix = requestUrl.pathname.slice(0, requestUrl.pathname.indexOf(`/streams/`) + `/streams/`.length);
|
|
6299
|
+
requestUrl.pathname = `${prefix}${encodeURIComponent(routingAdapter.toBackendStreamPath(service, streamPath))}`;
|
|
6300
|
+
return requestUrl.toString();
|
|
6301
|
+
}
|
|
6302
|
+
async function putSubscriptionBase(request, ctx) {
|
|
6303
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6304
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6305
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6306
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6307
|
+
const { upstream, response } = await forwardSubscriptionRequest(request, ctx, routingAdapter, { body: rewrite.body });
|
|
6308
|
+
if (upstream.ok && rewrite.targetWebhookUrl) await upsertSubscriptionWebhook(ctx, subscriptionId, rewrite.targetWebhookUrl);
|
|
6309
|
+
return response;
|
|
6310
|
+
}
|
|
6311
|
+
async function getSubscriptionBase(request, ctx) {
|
|
6312
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6313
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter)).response;
|
|
6314
|
+
}
|
|
6315
|
+
async function deleteSubscriptionBase(request, ctx) {
|
|
6316
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6317
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6318
|
+
const { upstream, response } = await forwardSubscriptionRequest(request, ctx, routingAdapter);
|
|
6319
|
+
if (upstream.ok) await deleteSubscriptionWebhook(ctx, subscriptionId);
|
|
6240
6320
|
return response;
|
|
6241
6321
|
}
|
|
6322
|
+
async function postSubscriptionStreams(request, ctx) {
|
|
6323
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6324
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6325
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6326
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6327
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, { body: rewrite.body })).response;
|
|
6328
|
+
}
|
|
6329
|
+
async function deleteSubscriptionStream(request, ctx) {
|
|
6330
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6331
|
+
const requestUrl = rewriteSubscriptionStreamPathInUrl(new URL(request.url), ctx.service, routingAdapter, routeParam$2(request, `streamPath`));
|
|
6332
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, { requestUrl })).response;
|
|
6333
|
+
}
|
|
6334
|
+
function subscriptionAction(action) {
|
|
6335
|
+
return async (request, ctx) => {
|
|
6336
|
+
const subscriptionId = routeParam$2(request, `subscriptionId`);
|
|
6337
|
+
const routingAdapter = subscriptionRoutingAdapter(ctx);
|
|
6338
|
+
const rewrite = await rewriteSubscriptionRequestBody(request, ctx, subscriptionId, routingAdapter);
|
|
6339
|
+
if (!rewrite.ok) return rewrite.response;
|
|
6340
|
+
const bearerMode = action === `ack` || action === `release` || action === `callback` ? `if-missing` : `overwrite`;
|
|
6341
|
+
return (await forwardSubscriptionRequest(request, ctx, routingAdapter, {
|
|
6342
|
+
body: rewrite.body,
|
|
6343
|
+
bearerMode
|
|
6344
|
+
})).response;
|
|
6345
|
+
};
|
|
6346
|
+
}
|
|
6347
|
+
async function controlPassThrough(request, ctx) {
|
|
6348
|
+
const upstream = await forwardToDurableStreams(ctx, request, void 0, `control`);
|
|
6349
|
+
return responseFromUpstream$1(upstream);
|
|
6350
|
+
}
|
|
6242
6351
|
async function streamAppend(request, ctx) {
|
|
6243
6352
|
return await electricAgentsStreamAppendRouter.fetch(createStreamAppendRouteRequest(request), ctx.runtime, (req, body) => forwardFetchRequest({
|
|
6244
6353
|
request: {
|
|
@@ -6259,10 +6368,9 @@ async function proxyPassThrough(request, ctx) {
|
|
|
6259
6368
|
const upstream = await forwardToDurableStreams(ctx, request);
|
|
6260
6369
|
const streamPath = new URL(request.url).pathname;
|
|
6261
6370
|
const method = request.method.toUpperCase();
|
|
6262
|
-
const
|
|
6263
|
-
const endTrackedRead = method === `GET` && !isControlPath ? await ctx.entityBridgeManager.beginClientRead(streamPath) : null;
|
|
6371
|
+
const endTrackedRead = method === `GET` ? await ctx.entityBridgeManager.beginClientRead(streamPath) : null;
|
|
6264
6372
|
try {
|
|
6265
|
-
if (method === `HEAD`
|
|
6373
|
+
if (method === `HEAD`) await ctx.entityBridgeManager.touchByStreamPath(streamPath);
|
|
6266
6374
|
return responseFromUpstream$1(upstream);
|
|
6267
6375
|
} finally {
|
|
6268
6376
|
await endTrackedRead?.();
|
|
@@ -6293,7 +6401,8 @@ async function proxyElectric(request, ctx) {
|
|
|
6293
6401
|
incomingUrl: new URL(request.url),
|
|
6294
6402
|
electricUrl: ctx.electricUrl,
|
|
6295
6403
|
electricSecret: ctx.electricSecret,
|
|
6296
|
-
tenantId: ctx.service
|
|
6404
|
+
tenantId: ctx.service,
|
|
6405
|
+
principalUrl: ctx.principal.url
|
|
6297
6406
|
});
|
|
6298
6407
|
const headers = new Headers(request.headers);
|
|
6299
6408
|
headers.delete(`host`);
|
|
@@ -6554,10 +6663,8 @@ async function sendEntity(request, ctx) {
|
|
|
6554
6663
|
if (parsed.from !== void 0 && parsed.from !== principal.url) return apiError(400, ErrCodeInvalidRequest, `Request from must match Electric-Principal`);
|
|
6555
6664
|
await ctx.entityManager.ensurePrincipal(principal);
|
|
6556
6665
|
const { entityUrl, entity } = requireExistingEntityRoute(request);
|
|
6557
|
-
|
|
6558
|
-
|
|
6559
|
-
await linkEntityDispatchSubscription(ctx, updatedEntity);
|
|
6560
|
-
}
|
|
6666
|
+
const dispatchEntity = entity.dispatch_policy ? entity : await backfillEntityDispatchPolicy(ctx, entity);
|
|
6667
|
+
await linkEntityDispatchSubscription(ctx, dispatchEntity);
|
|
6561
6668
|
if (parsed.afterMs && parsed.afterMs > 0) await ctx.entityManager.enqueueDelayedSend(entityUrl, {
|
|
6562
6669
|
from: principal.url,
|
|
6563
6670
|
payload: parsed.payload,
|
|
@@ -6606,11 +6713,11 @@ async function spawnEntity(request, ctx) {
|
|
|
6606
6713
|
wake: parsed.wake,
|
|
6607
6714
|
created_by: principal.url
|
|
6608
6715
|
});
|
|
6716
|
+
await linkEntityDispatchSubscription(ctx, entity);
|
|
6609
6717
|
if (parsed.initialMessage !== void 0) await ctx.entityManager.send(entity.url, {
|
|
6610
6718
|
from: principal.url,
|
|
6611
6719
|
payload: parsed.initialMessage
|
|
6612
6720
|
});
|
|
6613
|
-
await linkEntityDispatchSubscription(ctx, entity);
|
|
6614
6721
|
return (0, itty_router.json)({
|
|
6615
6722
|
...toPublicEntity(entity),
|
|
6616
6723
|
txid: entity.txid
|
|
@@ -6774,7 +6881,13 @@ function applyCors(response) {
|
|
|
6774
6881
|
const headers = new Headers(response.headers);
|
|
6775
6882
|
headers.set(`access-control-allow-origin`, `*`);
|
|
6776
6883
|
headers.set(`access-control-allow-methods`, `GET, POST, PUT, PATCH, DELETE, OPTIONS`);
|
|
6777
|
-
headers.set(`access-control-allow-headers`,
|
|
6884
|
+
headers.set(`access-control-allow-headers`, [
|
|
6885
|
+
`content-type`,
|
|
6886
|
+
`authorization`,
|
|
6887
|
+
`electric-claim-token`,
|
|
6888
|
+
ELECTRIC_PRINCIPAL_HEADER,
|
|
6889
|
+
`ngrok-skip-browser-warning`
|
|
6890
|
+
].join(`, `));
|
|
6778
6891
|
headers.set(`access-control-expose-headers`, `*`);
|
|
6779
6892
|
return new Response(response.body, {
|
|
6780
6893
|
status: response.status,
|
|
@@ -6809,11 +6922,17 @@ function getRequestSpan(req) {
|
|
|
6809
6922
|
return carrier(req)[SPAN_KEY];
|
|
6810
6923
|
}
|
|
6811
6924
|
|
|
6925
|
+
//#endregion
|
|
6926
|
+
//#region src/routing/tenant-stream-paths.ts
|
|
6927
|
+
function withLeadingSlash(path$2) {
|
|
6928
|
+
return path$2.startsWith(`/`) ? path$2 : `/${path$2}`;
|
|
6929
|
+
}
|
|
6930
|
+
|
|
6812
6931
|
//#endregion
|
|
6813
6932
|
//#region src/routing/runners-router.ts
|
|
6814
6933
|
const registerRunnerBodySchema = __sinclair_typebox.Type.Object({
|
|
6815
6934
|
id: __sinclair_typebox.Type.String(),
|
|
6816
|
-
|
|
6935
|
+
owner_principal: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String()),
|
|
6817
6936
|
label: __sinclair_typebox.Type.String(),
|
|
6818
6937
|
kind: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Union([
|
|
6819
6938
|
__sinclair_typebox.Type.Literal(`local`),
|
|
@@ -6829,7 +6948,8 @@ const heartbeatBodySchema = __sinclair_typebox.Type.Object({
|
|
|
6829
6948
|
lease_ms: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Number()),
|
|
6830
6949
|
wake_stream_offset: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String()),
|
|
6831
6950
|
wakeStreamOffset: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String()),
|
|
6832
|
-
liveness_lease_expires_at: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String())
|
|
6951
|
+
liveness_lease_expires_at: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String()),
|
|
6952
|
+
diagnostics: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Record(__sinclair_typebox.Type.String(), __sinclair_typebox.Type.Unknown()))
|
|
6833
6953
|
});
|
|
6834
6954
|
const claimBodySchema = __sinclair_typebox.Type.Object({
|
|
6835
6955
|
subscription_id: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.String()),
|
|
@@ -6837,9 +6957,39 @@ const claimBodySchema = __sinclair_typebox.Type.Object({
|
|
|
6837
6957
|
generation: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Number()),
|
|
6838
6958
|
ts: __sinclair_typebox.Type.Optional(__sinclair_typebox.Type.Union([__sinclair_typebox.Type.String(), __sinclair_typebox.Type.Number()]))
|
|
6839
6959
|
}, { additionalProperties: true });
|
|
6960
|
+
const runnerClientStatuses = new Set([
|
|
6961
|
+
`stopped`,
|
|
6962
|
+
`starting`,
|
|
6963
|
+
`connecting`,
|
|
6964
|
+
`streaming`,
|
|
6965
|
+
`reconnecting`,
|
|
6966
|
+
`stopping`
|
|
6967
|
+
]);
|
|
6968
|
+
const runnerLastClaimResults = new Set([
|
|
6969
|
+
`claimed`,
|
|
6970
|
+
`no_work`,
|
|
6971
|
+
`error`
|
|
6972
|
+
]);
|
|
6973
|
+
const runnerStringOrNullDiagnostics = [
|
|
6974
|
+
`started_at`,
|
|
6975
|
+
`stream_connected_since`,
|
|
6976
|
+
`last_error`,
|
|
6977
|
+
`last_error_at`,
|
|
6978
|
+
`last_heartbeat_at`,
|
|
6979
|
+
`last_claim_at`,
|
|
6980
|
+
`last_dispatch_at`
|
|
6981
|
+
];
|
|
6982
|
+
const runnerNumberDiagnostics = [
|
|
6983
|
+
`reconnect_count`,
|
|
6984
|
+
`events_received`,
|
|
6985
|
+
`claims_succeeded`,
|
|
6986
|
+
`claims_skipped`,
|
|
6987
|
+
`claims_failed`
|
|
6988
|
+
];
|
|
6840
6989
|
const runnersRouter = (0, itty_router.Router)({ base: `/_electric/runners` });
|
|
6841
6990
|
runnersRouter.post(`/`, withSchema(registerRunnerBodySchema), registerRunner);
|
|
6842
6991
|
runnersRouter.get(`/`, listRunners);
|
|
6992
|
+
runnersRouter.get(`/:id/health`, runnerHealth);
|
|
6843
6993
|
runnersRouter.get(`/:id`, getRunner);
|
|
6844
6994
|
runnersRouter.post(`/:id/heartbeat`, withSchema(heartbeatBodySchema), heartbeat);
|
|
6845
6995
|
runnersRouter.post(`/:id/enable`, setEnabled);
|
|
@@ -6852,14 +7002,41 @@ function routeParam$1(request, name) {
|
|
|
6852
7002
|
function firstQueryValue(value) {
|
|
6853
7003
|
return Array.isArray(value) ? value[0] : value;
|
|
6854
7004
|
}
|
|
7005
|
+
function requireAuthenticatedPrincipal(ctx) {
|
|
7006
|
+
if (ctx.principal) return ctx.principal;
|
|
7007
|
+
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner route requires an authenticated principal`, 401);
|
|
7008
|
+
}
|
|
7009
|
+
function canonicalOwnerPrincipal(input) {
|
|
7010
|
+
return parsePrincipalUrl(input)?.url ?? null;
|
|
7011
|
+
}
|
|
7012
|
+
function sanitizeRunnerDiagnostics(diagnostics) {
|
|
7013
|
+
if (!diagnostics) return void 0;
|
|
7014
|
+
const sanitized = {};
|
|
7015
|
+
if (typeof diagnostics.status === `string` && runnerClientStatuses.has(diagnostics.status)) sanitized.status = diagnostics.status;
|
|
7016
|
+
if (typeof diagnostics.stream_connected === `boolean`) sanitized.stream_connected = diagnostics.stream_connected;
|
|
7017
|
+
if (typeof diagnostics.last_heartbeat_ok === `boolean`) sanitized.last_heartbeat_ok = diagnostics.last_heartbeat_ok;
|
|
7018
|
+
if (diagnostics.last_claim_result === null || typeof diagnostics.last_claim_result === `string` && runnerLastClaimResults.has(diagnostics.last_claim_result)) sanitized.last_claim_result = diagnostics.last_claim_result;
|
|
7019
|
+
for (const key of runnerStringOrNullDiagnostics) {
|
|
7020
|
+
const value = diagnostics[key];
|
|
7021
|
+
if (typeof value === `string` || value === null) sanitized[key] = value;
|
|
7022
|
+
}
|
|
7023
|
+
for (const key of runnerNumberDiagnostics) {
|
|
7024
|
+
const value = diagnostics[key];
|
|
7025
|
+
if (typeof value === `number` && Number.isFinite(value) && value >= 0) sanitized[key] = value;
|
|
7026
|
+
}
|
|
7027
|
+
return Object.keys(sanitized).length > 0 ? sanitized : void 0;
|
|
7028
|
+
}
|
|
6855
7029
|
async function registerRunner(request, ctx) {
|
|
6856
7030
|
const parsed = routeBody(request);
|
|
6857
|
-
const
|
|
6858
|
-
|
|
6859
|
-
if (
|
|
7031
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
7032
|
+
const ownerPrincipal = parsed.owner_principal ?? principal.url;
|
|
7033
|
+
if (!ownerPrincipal) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal is required when no authenticated principal is present`, 400);
|
|
7034
|
+
const canonicalOwner = canonicalOwnerPrincipal(ownerPrincipal);
|
|
7035
|
+
if (!canonicalOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${ownerPrincipal}`, 400);
|
|
7036
|
+
if (canonicalOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
6860
7037
|
const runner = await ctx.entityManager.registry.createRunner({
|
|
6861
7038
|
id: parsed.id,
|
|
6862
|
-
|
|
7039
|
+
ownerPrincipal: canonicalOwner,
|
|
6863
7040
|
label: parsed.label,
|
|
6864
7041
|
kind: parsed.kind,
|
|
6865
7042
|
adminStatus: parsed.admin_status,
|
|
@@ -6869,26 +7046,32 @@ async function registerRunner(request, ctx) {
|
|
|
6869
7046
|
return (0, itty_router.json)(runner, { status: 201 });
|
|
6870
7047
|
}
|
|
6871
7048
|
async function listRunners(request, ctx) {
|
|
6872
|
-
const
|
|
6873
|
-
|
|
6874
|
-
const
|
|
7049
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
7050
|
+
const requestedOwner = firstQueryValue(request.query.owner_principal);
|
|
7051
|
+
const canonicalRequestedOwner = requestedOwner ? canonicalOwnerPrincipal(requestedOwner) : void 0;
|
|
7052
|
+
if (requestedOwner && !canonicalRequestedOwner) throw new ElectricAgentsError(ErrCodeInvalidRequest, `owner_principal must be a valid principal URL (e.g. /principal/user%3Aalice), got: ${requestedOwner}`, 400);
|
|
7053
|
+
if (canonicalRequestedOwner && canonicalRequestedOwner !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `owner_principal must match the authenticated principal`, 403);
|
|
7054
|
+
const runners$1 = await ctx.entityManager.registry.listRunners({ ownerPrincipal: principal.url });
|
|
6875
7055
|
return (0, itty_router.json)(runners$1);
|
|
6876
7056
|
}
|
|
6877
7057
|
async function getRunner(request, ctx) {
|
|
6878
7058
|
const runner = await requireRunner(ctx, routeParam$1(request, `id`));
|
|
6879
|
-
assertRunnerOwnerIfAuthenticated(ctx, runner.
|
|
7059
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
6880
7060
|
return (0, itty_router.json)(runner);
|
|
6881
7061
|
}
|
|
6882
7062
|
async function heartbeat(request, ctx) {
|
|
6883
7063
|
const runnerId = routeParam$1(request, `id`);
|
|
7064
|
+
requireAuthenticatedPrincipal(ctx);
|
|
6884
7065
|
const existing = await requireRunner(ctx, runnerId);
|
|
6885
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
7066
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
6886
7067
|
const parsed = routeBody(request);
|
|
6887
7068
|
const runner = await ctx.entityManager.registry.heartbeatRunner({
|
|
6888
7069
|
runnerId,
|
|
7070
|
+
ownerPrincipal: existing.owner_principal,
|
|
6889
7071
|
leaseMs: parsed.lease_ms,
|
|
6890
7072
|
wakeStreamOffset: parsed.wake_stream_offset ?? parsed.wakeStreamOffset,
|
|
6891
|
-
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0
|
|
7073
|
+
livenessLeaseExpiresAt: parsed.liveness_lease_expires_at ? new Date(parsed.liveness_lease_expires_at) : void 0,
|
|
7074
|
+
diagnostics: sanitizeRunnerDiagnostics(parsed.diagnostics)
|
|
6892
7075
|
});
|
|
6893
7076
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6894
7077
|
return (0, itty_router.json)(runner);
|
|
@@ -6901,16 +7084,18 @@ async function setDisabled(request, ctx) {
|
|
|
6901
7084
|
}
|
|
6902
7085
|
async function setRunnerStatus(request, ctx, adminStatus) {
|
|
6903
7086
|
const runnerId = routeParam$1(request, `id`);
|
|
7087
|
+
requireAuthenticatedPrincipal(ctx);
|
|
6904
7088
|
const existing = await requireRunner(ctx, runnerId);
|
|
6905
|
-
assertRunnerOwnerIfAuthenticated(ctx, existing.
|
|
7089
|
+
assertRunnerOwnerIfAuthenticated(ctx, existing.owner_principal);
|
|
6906
7090
|
const runner = await ctx.entityManager.registry.setRunnerAdminStatus(runnerId, adminStatus);
|
|
6907
7091
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6908
7092
|
return (0, itty_router.json)(runner);
|
|
6909
7093
|
}
|
|
6910
7094
|
async function claimWake(request, ctx) {
|
|
6911
7095
|
const runnerId = routeParam$1(request, `id`);
|
|
7096
|
+
const principal = requireAuthenticatedPrincipal(ctx);
|
|
6912
7097
|
const runner = await requireRunner(ctx, runnerId);
|
|
6913
|
-
if (
|
|
7098
|
+
if (runner.owner_principal !== principal.url) throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner claim requires the authenticated owner`, 403);
|
|
6914
7099
|
if (runner.admin_status !== `enabled`) throw new ElectricAgentsError(ErrCodeNotRunning, `Runner is disabled`, 409);
|
|
6915
7100
|
const parsed = routeBody(request);
|
|
6916
7101
|
const expectedSubscriptionId = subscriptionIdForDispatchTarget({
|
|
@@ -6941,11 +7126,95 @@ async function requireRunner(ctx, runnerId) {
|
|
|
6941
7126
|
if (!runner) throw new ElectricAgentsError(ErrCodeNotFound, `Runner not found`, 404);
|
|
6942
7127
|
return runner;
|
|
6943
7128
|
}
|
|
6944
|
-
function assertRunnerOwnerIfAuthenticated(ctx,
|
|
6945
|
-
|
|
6946
|
-
if (
|
|
7129
|
+
function assertRunnerOwnerIfAuthenticated(ctx, ownerPrincipal) {
|
|
7130
|
+
requireAuthenticatedPrincipal(ctx);
|
|
7131
|
+
if (ownerPrincipal === ctx.principal.url) return;
|
|
6947
7132
|
throw new ElectricAgentsError(ErrCodeUnauthorized, `Runner access requires the authenticated owner`, 403);
|
|
6948
7133
|
}
|
|
7134
|
+
async function runnerHealth(request, ctx) {
|
|
7135
|
+
const runnerId = routeParam$1(request, `id`);
|
|
7136
|
+
const runner = await requireRunner(ctx, runnerId);
|
|
7137
|
+
assertRunnerOwnerIfAuthenticated(ctx, runner.owner_principal);
|
|
7138
|
+
const runtimeDiagnostics = await ctx.entityManager.registry.getRunnerDiagnostics(runnerId);
|
|
7139
|
+
const now = Date.now();
|
|
7140
|
+
const parsedLeaseExpiresAt = runtimeDiagnostics?.liveness_lease_expires_at ? new Date(runtimeDiagnostics.liveness_lease_expires_at).getTime() : null;
|
|
7141
|
+
const leaseExpiresAt = parsedLeaseExpiresAt !== null && Number.isFinite(parsedLeaseExpiresAt) ? parsedLeaseExpiresAt : null;
|
|
7142
|
+
let livenessStatus;
|
|
7143
|
+
if (runner.admin_status === `disabled`) livenessStatus = `offline`;
|
|
7144
|
+
else if (leaseExpiresAt !== null && leaseExpiresAt > now) livenessStatus = `online`;
|
|
7145
|
+
else if (leaseExpiresAt !== null) livenessStatus = `expired`;
|
|
7146
|
+
else livenessStatus = `offline`;
|
|
7147
|
+
const [activeClaims, dispatchStats] = await Promise.all([ctx.entityManager.registry.getActiveClaimsForRunner(runnerId), ctx.entityManager.registry.getDispatchStatsForRunner(runnerId)]);
|
|
7148
|
+
const clientDiagnostics = sanitizeRunnerDiagnostics(runtimeDiagnostics?.diagnostics) ?? null;
|
|
7149
|
+
const issues = [];
|
|
7150
|
+
let healthStatus = `healthy`;
|
|
7151
|
+
const escalate = (floor) => {
|
|
7152
|
+
if (floor === `unhealthy`) healthStatus = `unhealthy`;
|
|
7153
|
+
else if (healthStatus === `healthy`) healthStatus = `degraded`;
|
|
7154
|
+
};
|
|
7155
|
+
if (runner.admin_status === `disabled`) {
|
|
7156
|
+
escalate(`unhealthy`);
|
|
7157
|
+
issues.push(`Runner is disabled`);
|
|
7158
|
+
}
|
|
7159
|
+
if (livenessStatus === `expired`) {
|
|
7160
|
+
escalate(`unhealthy`);
|
|
7161
|
+
const ago = leaseExpiresAt ? Math.round((now - leaseExpiresAt) / 1e3) : 0;
|
|
7162
|
+
issues.push(`Heartbeat lease expired ${ago}s ago`);
|
|
7163
|
+
}
|
|
7164
|
+
if (livenessStatus === `offline` && runner.admin_status === `enabled`) {
|
|
7165
|
+
escalate(`degraded`);
|
|
7166
|
+
issues.push(`Runner has never sent a heartbeat`);
|
|
7167
|
+
}
|
|
7168
|
+
if (clientDiagnostics) {
|
|
7169
|
+
if (clientDiagnostics.stream_connected === false) {
|
|
7170
|
+
escalate(`degraded`);
|
|
7171
|
+
issues.push(`Client reports stream disconnected`);
|
|
7172
|
+
}
|
|
7173
|
+
if (clientDiagnostics.last_heartbeat_ok === false) {
|
|
7174
|
+
escalate(`degraded`);
|
|
7175
|
+
issues.push(`Client reports last heartbeat failed`);
|
|
7176
|
+
}
|
|
7177
|
+
if (typeof clientDiagnostics.reconnect_count === `number` && clientDiagnostics.reconnect_count > 5) {
|
|
7178
|
+
escalate(`degraded`);
|
|
7179
|
+
issues.push(`Client has reconnected ${clientDiagnostics.reconnect_count} times`);
|
|
7180
|
+
}
|
|
7181
|
+
} else if (runtimeDiagnostics?.last_seen_at) {
|
|
7182
|
+
escalate(`degraded`);
|
|
7183
|
+
issues.push(`No client diagnostics available`);
|
|
7184
|
+
}
|
|
7185
|
+
const body = {
|
|
7186
|
+
runner: {
|
|
7187
|
+
id: runner.id,
|
|
7188
|
+
admin_status: runner.admin_status,
|
|
7189
|
+
liveness_status: livenessStatus,
|
|
7190
|
+
lease_expires_at: leaseExpiresAt !== null ? runtimeDiagnostics?.liveness_lease_expires_at ?? null : null,
|
|
7191
|
+
lease_remaining_ms: leaseExpiresAt !== null ? Math.max(0, leaseExpiresAt - now) : null,
|
|
7192
|
+
wake_stream: runner.wake_stream,
|
|
7193
|
+
wake_stream_offset: runtimeDiagnostics?.wake_stream_offset ?? null,
|
|
7194
|
+
last_seen_at: runtimeDiagnostics?.last_seen_at ?? null,
|
|
7195
|
+
created_at: runner.created_at
|
|
7196
|
+
},
|
|
7197
|
+
client: clientDiagnostics,
|
|
7198
|
+
claims: {
|
|
7199
|
+
active_count: activeClaims.length,
|
|
7200
|
+
active: activeClaims.map((c) => ({
|
|
7201
|
+
consumer_id: c.consumer_id,
|
|
7202
|
+
epoch: c.epoch,
|
|
7203
|
+
entity_url: c.entity_url,
|
|
7204
|
+
stream_path: c.stream_path,
|
|
7205
|
+
claimed_at: c.claimed_at,
|
|
7206
|
+
last_heartbeat_at: c.last_heartbeat_at ?? null,
|
|
7207
|
+
lease_expires_at: c.lease_expires_at ?? null
|
|
7208
|
+
}))
|
|
7209
|
+
},
|
|
7210
|
+
dispatch: dispatchStats,
|
|
7211
|
+
health: {
|
|
7212
|
+
status: healthStatus,
|
|
7213
|
+
issues
|
|
7214
|
+
}
|
|
7215
|
+
};
|
|
7216
|
+
return (0, itty_router.json)(body);
|
|
7217
|
+
}
|
|
6949
7218
|
async function notificationFromClaim(ctx, input) {
|
|
6950
7219
|
const primary = input.claim.streams.find((stream) => stream.has_pending === true) ?? input.claim.streams[0];
|
|
6951
7220
|
if (!primary?.path) throw new ElectricAgentsError(ErrCodeInvalidRequest, `Claim response did not include a stream`, 502);
|
|
@@ -7132,7 +7401,7 @@ async function webhookForward(request, ctx) {
|
|
|
7132
7401
|
let runningEntityUrl = null;
|
|
7133
7402
|
const parsedBody = parsedBodyResult.value;
|
|
7134
7403
|
const newWebhook = newWebhookPayload(parsedBody);
|
|
7135
|
-
const routingAdapter = resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting);
|
|
7404
|
+
const routingAdapter = resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl);
|
|
7136
7405
|
if (parsedBody) {
|
|
7137
7406
|
const rawPrimaryStream = newWebhook?.primaryStream ?? parsedBody.primary_stream ?? parsedBody.primaryStream ?? parsedBody.streamPath ?? null;
|
|
7138
7407
|
const primaryStream = typeof rawPrimaryStream === `string` ? toRuntimeStreamPath(rawPrimaryStream, ctx.service, routingAdapter) : null;
|
|
@@ -7261,7 +7530,7 @@ async function callbackForward(request, ctx) {
|
|
|
7261
7530
|
}
|
|
7262
7531
|
return (0, itty_router.json)(responseBody);
|
|
7263
7532
|
}
|
|
7264
|
-
const upstreamBody = encodeCallbackForwardBody(ctx.service, consumerId, requestBody, resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting));
|
|
7533
|
+
const upstreamBody = encodeCallbackForwardBody(ctx.service, consumerId, requestBody, resolveDurableStreamsRoutingAdapter(ctx.durableStreamsRouting, ctx.durableStreamsUrl));
|
|
7265
7534
|
let upstream;
|
|
7266
7535
|
try {
|
|
7267
7536
|
const subscriptionId = durableStreamsSubscriptionCallback(target.callbackUrl);
|
|
@@ -7380,4 +7649,6 @@ exports.createDb = createDb
|
|
|
7380
7649
|
exports.globalRouter = globalRouter
|
|
7381
7650
|
exports.isUnregisteredTenantError = isUnregisteredTenantError
|
|
7382
7651
|
exports.pathPrefixedSingleTenantDurableStreamsRoutingAdapter = pathPrefixedSingleTenantDurableStreamsRoutingAdapter
|
|
7383
|
-
exports.runMigrations = runMigrations
|
|
7652
|
+
exports.runMigrations = runMigrations
|
|
7653
|
+
exports.streamRootDurableStreamsRoutingAdapter = streamRootDurableStreamsRoutingAdapter
|
|
7654
|
+
exports.tenantRootDurableStreamsRoutingAdapter = tenantRootDurableStreamsRoutingAdapter
|