@electric-ax/agents-server 0.4.7 → 0.4.11

package/src/routing/internal-router.ts

@@ -16,7 +16,7 @@ import {
 } from '../electric-agents-http.js'
 import { consumerCallbacks, subscriptionWebhooks } from '../db/schema.js'
 import {
-  ErrCodeCallbackNotFound,
+  ErrCodeWakeCallbackNotFound,
   ErrCodeForkInProgress,
   ErrCodeSubscriptionNotFound,
   ErrCodeUnauthorized,
@@ -26,17 +26,20 @@ import { decodeJsonObject } from '../utils/server-utils.js'
 import { serverLog } from '../utils/log.js'
 import { applyDurableStreamsBearer } from '../stream-client.js'
 import { getDefaultWebhookSigner } from '../webhook-signing.js'
-import { cronRouter } from './cron-router.js'
 import { resolveDurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js'
 import { electricProxyRouter } from './electric-proxy-router.js'
 import { entitiesRouter } from './entities-router.js'
 import { entityTypesRouter } from './entity-types-router.js'
 import { getRequestSpan } from './hooks.js'
+import { observationsRouter } from './observations-router.js'
 import { runnersRouter } from './runners-router.js'
 import { routeBody, validateOptionalJsonBody, withSchema } from './schema.js'
 import { withLeadingSlash } from './tenant-stream-paths.js'
 import type { IRequest, RouterType } from 'itty-router'
-import type { WebhookSignatureVerifierConfig } from '@electric-ax/agents-runtime'
+import type {
+  EventSourceContract,
+  WebhookSignatureVerifierConfig,
+} from '@electric-ax/agents-runtime'
 import type { TenantContext } from './context.js'
 import type { DurableStreamsRoutingAdapter } from './durable-streams-routing-adapter.js'
 import type { WebhookSigner } from '../webhook-signing.js'
@@ -66,7 +69,7 @@ const wakeRegistrationBodySchema = Type.Object({
   manifestKey: Type.Optional(Type.String()),
 })
 
-const webhookForwardBodySchema = Type.Object(
+const subscriptionWebhookBodySchema = Type.Object(
   {
     subscription_id: Type.Optional(Type.String()),
     wake_id: Type.Optional(Type.String()),
@@ -84,7 +87,7 @@ const webhookForwardBodySchema = Type.Object(
   { additionalProperties: true }
 )
 
-const callbackForwardBodySchema = Type.Object(
+const wakeCallbackBodySchema = Type.Object(
   {
     epoch: Type.Optional(Type.Number()),
     generation: Type.Optional(Type.Number()),
@@ -97,8 +100,8 @@ const callbackForwardBodySchema = Type.Object(
 )
 
 type WakeRegistrationBody = Static<typeof wakeRegistrationBodySchema>
-type WebhookForwardBody = Static<typeof webhookForwardBodySchema>
-type CallbackForwardBody = Static<typeof callbackForwardBodySchema>
+type SubscriptionWebhookBody = Static<typeof subscriptionWebhookBodySchema>
+type WakeCallbackBody = Static<typeof wakeCallbackBodySchema>
 
 const DS_SUBSCRIPTION_CALLBACK_PREFIX = `ds-subscription:`
 
@@ -117,18 +120,22 @@ export const internalRouter: InternalRoutes = Router<
 })
 
 internalRouter.get(`/health`, () => json({ status: `ok` }))
+internalRouter.get(`/event-sources`, listEventSources)
 internalRouter.post(
   `/wake`,
   withSchema(wakeRegistrationBodySchema),
   registerWake
 )
-internalRouter.post(`/webhook-forward/:subscriptionId`, webhookForward)
-internalRouter.post(`/callback-forward/:consumerId`, callbackForward)
+internalRouter.post(
+  `/subscription-webhooks/:subscriptionId`,
+  subscriptionWebhook
+)
+internalRouter.post(`/wake-callbacks/:consumerId`, wakeCallback)
 internalRouter.all(`/runners`, runnersRouter.fetch)
 internalRouter.all(`/runners/*`, runnersRouter.fetch)
 internalRouter.all(`/entities/*`, entitiesRouter.fetch)
 internalRouter.all(`/entity-types/*`, entityTypesRouter.fetch)
-internalRouter.all(`/cron/*`, cronRouter.fetch)
+internalRouter.all(`/observations/*`, observationsRouter.fetch)
 internalRouter.get(`/electric/*`, electricProxyRouter.fetch)
 internalRouter.all(`*`, () => status(404))
 
@@ -179,7 +186,7 @@ function resolveWebhookSigner(ctx: TenantContext): WebhookSigner {
 
 function durableStreamsWebhookJwksUrl(ctx: TenantContext): string {
   if (!ctx.durableStreamsRouting) {
-    return appendPathToUrl(ctx.durableStreamsUrl, `/__ds/jwks.json`)
+    return appendPathToBackendUrl(ctx.durableStreamsUrl, `/__ds/jwks.json`)
   }
 
   return resolveDurableStreamsRoutingAdapter(
@@ -194,6 +201,28 @@ function durableStreamsWebhookJwksUrl(ctx: TenantContext): string {
     .toString()
 }
 
+function appendPathToBackendUrl(baseUrl: string, path: string): string {
+  const base = new URL(baseUrl)
+  const pathUrl = new URL(path, `http://electric-agents.local`)
+  const basePath =
+    base.pathname === `/` ? `` : base.pathname.replace(/\/+$/, ``)
+  const suffix = pathUrl.pathname.startsWith(`/`)
+    ? pathUrl.pathname
+    : `/${pathUrl.pathname}`
+  const target = new URL(base)
+
+  target.pathname = `${basePath}${suffix}`
+  target.search = ``
+  target.hash = pathUrl.hash
+  base.searchParams.forEach((value, key) => {
+    target.searchParams.append(key, value)
+  })
+  pathUrl.searchParams.forEach((value, key) => {
+    target.searchParams.append(key, value)
+  })
+  return target.toString()
+}
+
 function durableStreamsJwksFetchClient(ctx: TenantContext): typeof fetch {
   return async (input, init) => {
     const headers = new Headers(init?.headers)
@@ -265,7 +294,7 @@ function claimTokenFromRequest(request: IRequest): string | undefined {
   )
 }
 
-function newWebhookPayload(body: WebhookForwardBody | undefined): {
+function newWebhookPayload(body: SubscriptionWebhookBody | undefined): {
   wakeId: string
   generation: number
   primaryStream: string
@@ -335,13 +364,27 @@ async function registerWake(
   return status(204)
 }
 
-async function webhookForward(
+async function listEventSources(
+  _request: IRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const eventSources = ctx.eventSources
+    ? await ctx.eventSources.listEventSources()
+    : []
+  return json({ eventSources: eventSources.filter(isAgentVisibleEventSource) })
+}
+
+function isAgentVisibleEventSource(source: EventSourceContract): boolean {
+  return source.agentVisible === true && source.status === `active`
+}
+
+async function subscriptionWebhook(
   request: IRequest,
   ctx: TenantContext
 ): Promise<Response> {
   const subscriptionId = routeParam(request, `subscriptionId`)
   const rootSpan = getRequestSpan(request)
-  rootSpan?.updateName(`webhook-forward`)
+  rootSpan?.updateName(`subscription-webhook`)
   rootSpan?.setAttribute(
     `electric_agents.webhook.subscription_id`,
     subscriptionId
@@ -380,7 +423,7 @@ async function webhookForward(
     )
   }
   const parsedBodyResult = validateOptionalJsonBody(
-    webhookForwardBodySchema,
+    subscriptionWebhookBodySchema,
     body,
     request.headers.get(`content-type`)
   )
@@ -388,7 +431,9 @@ async function webhookForward(
 
   let forwardBody = body
   let runningEntityUrl: string | null = null
-  const parsedBody = parsedBodyResult.value as WebhookForwardBody | undefined
+  const parsedBody = parsedBodyResult.value as
+    | SubscriptionWebhookBody
+    | undefined
   const newWebhook = newWebhookPayload(parsedBody)
   const routingAdapter = resolveDurableStreamsRoutingAdapter(
     ctx.durableStreamsRouting,
@@ -467,7 +512,7 @@ async function webhookForward(
               })
               .catch((err) => {
                 serverLog.warn(
-                  `[webhook-forward] consumerCallbacks upsert failed (non-fatal): ${
+                  `[subscription-webhook] consumerCallbacks upsert failed (non-fatal): ${
                     err instanceof Error ? err.message : String(err)
                   }`
                 )
@@ -515,7 +560,7 @@ async function webhookForward(
       if (consumerId && callbackUrl) {
         const callback = appendPathToUrl(
           ctx.publicUrl,
-          `/_electric/callback-forward/${encodeURIComponent(consumerId)}`
+          `/_electric/wake-callbacks/${encodeURIComponent(consumerId)}`
         )
         enriched.callback = callback
         if (newWebhook) {
@@ -564,7 +609,7 @@ async function webhookForward(
     }
     return apiError(
       502,
-      `WEBHOOK_FORWARD_FAILED`,
+      `SUBSCRIPTION_WEBHOOK_FAILED`,
       err instanceof Error ? err.message : String(err)
     )
   }
@@ -575,7 +620,7 @@ async function webhookForward(
   return responseFromUpstream(upstream, responseBytes)
 }
 
-async function callbackForward(
+async function wakeCallback(
   request: IRequest,
   ctx: TenantContext
 ): Promise<Response> {
@@ -600,19 +645,19 @@ async function callbackForward(
   if (!target) {
     return apiError(
       404,
-      ErrCodeCallbackNotFound,
-      `Unknown callback-forward consumer`
+      ErrCodeWakeCallbackNotFound,
+      `Unknown wake-callback consumer`
     )
   }
 
   const body = await readRequestBody(request as Request)
   const parsedBodyResult = validateOptionalJsonBody(
-    callbackForwardBodySchema,
+    wakeCallbackBodySchema,
     body,
     request.headers.get(`content-type`)
   )
   if (!parsedBodyResult.ok) return parsedBodyResult.response
-  const requestBody = parsedBodyResult.value as CallbackForwardBody | undefined
+  const requestBody = parsedBodyResult.value as WakeCallbackBody | undefined
   const isClaimRequest =
     requestBody?.wakeId !== undefined || requestBody?.wake_id !== undefined
   const isDoneRequest = requestBody?.done === true
@@ -635,7 +680,7 @@ async function callbackForward(
     return json(responseBody)
   }
 
-  const upstreamBody = encodeCallbackForwardBody(
+  const upstreamBody = encodeWakeCallbackBody(
     ctx.service,
     consumerId,
     requestBody,
@@ -655,7 +700,7 @@ async function callbackForward(
       if (!token) {
         return apiError(401, `UNAUTHORIZED`, `Missing claim token`)
       }
-      const upstreamPayload = encodeCallbackForwardPayload(
+      const upstreamPayload = encodeWakeCallbackPayload(
         consumerId,
         requestBody,
         (stream) => stream.replace(/^\/+/, ``)
@@ -676,7 +721,7 @@ async function callbackForward(
   } catch (err) {
     return apiError(
       502,
-      `CALLBACK_FORWARD_FAILED`,
+      `WAKE_CALLBACK_FAILED`,
       err instanceof Error ? err.message : String(err)
     )
   }
@@ -715,7 +760,7 @@ async function callbackForward(
     }
     if (upstream.ok && isDoneRequest && target.primaryStream) {
       serverLog.info(
-        `[callback-forward] done received for stream=${target.primaryStream} consumer=${consumerId}`
+        `[wake-callback] done received for stream=${target.primaryStream} consumer=${consumerId}`
       )
       const stillOwnsClaim = ctx.runtime.claimWriteTokens.owns(
         ctx.service,
@@ -770,11 +815,11 @@ async function callbackForward(
         )
         await ctx.entityBridgeManager.onEntityChanged(entity.url)
         serverLog.info(
-          `[callback-forward] status updated after done for ${entity.url}`
+          `[wake-callback] status updated after done for ${entity.url}`
         )
       } else if (!entity) {
         serverLog.warn(
-          `[callback-forward] done received but no entity found for stream=${target.primaryStream}`
+          `[wake-callback] done received but no entity found for stream=${target.primaryStream}`
         )
       }
 
@@ -788,19 +833,19 @@ async function callbackForward(
         )
       } else if (entity) {
         serverLog.info(
-          `[callback-forward] done arrived after in-memory token evicted (stream=${target.primaryStream} consumer=${consumerId})`
+          `[wake-callback] done arrived after in-memory token evicted (stream=${target.primaryStream} consumer=${consumerId})`
         )
       }
     } else if (requestBody?.done === true) {
       serverLog.warn(
-        `[callback-forward] done received but skipped: upstream.ok=${upstream.ok} primaryStream=${
+        `[wake-callback] done received but skipped: upstream.ok=${upstream.ok} primaryStream=${
           target.primaryStream ?? `null`
         } consumer=${consumerId}`
       )
     }
   } catch (err) {
     serverLog.error(
-      `[callback-forward] error processing done for consumer=${consumerId}: ${
+      `[wake-callback] error processing done for consumer=${consumerId}: ${
         err instanceof Error ? err.message : String(err)
       }`
     )
@@ -820,21 +865,21 @@ async function mintClaimWriteToken(
   return ctx.runtime.claimWriteTokens.mint(ctx.service, streamPath, consumerId)
 }
 
-function encodeCallbackForwardBody(
+function encodeWakeCallbackBody(
   service: string,
   consumerId: string,
-  body: CallbackForwardBody | undefined,
+  body: WakeCallbackBody | undefined,
   routingAdapter: DurableStreamsRoutingAdapter
 ): Uint8Array {
-  const payload = encodeCallbackForwardPayload(consumerId, body, (stream) =>
+  const payload = encodeWakeCallbackPayload(consumerId, body, (stream) =>
     routingAdapter.toBackendStreamPath(service, stream)
   )
   return new TextEncoder().encode(JSON.stringify(payload))
 }
 
-function encodeCallbackForwardPayload(
+function encodeWakeCallbackPayload(
   consumerId: string,
-  body: CallbackForwardBody | undefined,
+  body: WakeCallbackBody | undefined,
   mapStream: (stream: string) => string
 ): Record<string, unknown> {
   if (!body) return {}

package/src/routing/observations-router.ts

@@ -0,0 +1,74 @@
+/**
+ * HTTP routes for ensuring observation backing streams.
+ */
+
+import { Type, type Static } from '@sinclair/typebox'
+import { Router, json } from 'itty-router'
+import { routeBody, withSchema } from './schema.js'
+import type { JsonRouteRequest } from './schema.js'
+import type { RouterType } from 'itty-router'
+import type { TenantContext } from './context.js'
+
+const stringRecordSchema = Type.Record(Type.String(), Type.String())
+
+const ensureEntitiesMembershipStreamBodySchema = Type.Object({
+  tags: Type.Optional(stringRecordSchema),
+})
+
+const ensureCronStreamBodySchema = Type.Object({
+  expression: Type.String(),
+  timezone: Type.Optional(Type.String()),
+})
+
+type EnsureEntitiesMembershipStreamBody = Static<
+  typeof ensureEntitiesMembershipStreamBodySchema
+>
+type EnsureCronStreamBody = Static<typeof ensureCronStreamBodySchema>
+
+export type ObservationsRoutes = RouterType<
+  JsonRouteRequest,
+  [TenantContext],
+  Response | undefined
+>
+
+export const observationsRouter: ObservationsRoutes = Router<
+  JsonRouteRequest,
+  [TenantContext],
+  Response | undefined
+>({
+  base: `/_electric/observations`,
+})
+
+observationsRouter.post(
+  `/entities/ensure-stream`,
+  withSchema(ensureEntitiesMembershipStreamBodySchema),
+  ensureEntitiesMembershipStream
+)
+observationsRouter.post(
+  `/cron/ensure-stream`,
+  withSchema(ensureCronStreamBodySchema),
+  ensureCronStream
+)
+
+async function ensureEntitiesMembershipStream(
+  request: JsonRouteRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const parsed = routeBody<EnsureEntitiesMembershipStreamBody>(request)
+  const result = await ctx.entityManager.ensureEntitiesMembershipStream(
+    parsed.tags ?? {}
+  )
+  return json(result)
+}
+
+async function ensureCronStream(
+  request: JsonRouteRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const parsed = routeBody<EnsureCronStreamBody>(request)
+  const streamPath = await ctx.entityManager.getOrCreateCronStream(
+    parsed.expression,
+    parsed.timezone
+  )
+  return json({ streamUrl: streamPath })
+}

package/src/routing/runners-router.ts

@@ -628,7 +628,7 @@ async function notificationFromClaim(
     streams,
     callback: appendPathToUrl(
       ctx.publicUrl,
-      `/_electric/callback-forward/${encodeURIComponent(input.claim.wake_id)}`
+      `/_electric/wake-callbacks/${encodeURIComponent(input.claim.wake_id)}`
     ),
     claimToken: input.claim.token,
     triggerEvent: `message_received`,

package/src/server.ts

@@ -34,6 +34,7 @@ import type { Principal } from './principal.js'
 import type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
 import type { DurableStreamsRoutingAdapter } from './routing/durable-streams-routing-adapter.js'
 import type { OssServerContext } from './routing/oss-server-router.js'
+import type { EventSourceCatalog } from './routing/context.js'
 import type { StartedStandaloneAgentsRuntime } from './standalone-runtime.js'
 import type { DurableStreamsBearerProvider } from './stream-client.js'
 import type {
@@ -67,6 +68,8 @@ export interface ElectricAgentsServerOptions {
     request: Request
   ) => Promise<Principal | null> | Principal | null
   allowDevPrincipalFallback?: boolean
+  eventSources?: EventSourceCatalog
+  ensureEventSourceWakeSource?: (sourceUrl: string) => Promise<void> | void
   /**
    * Disabled by default. When set to a positive interval, periodically
    * recovers expired dispatch claims and stale outstanding wakes.
@@ -441,6 +444,15 @@ export class ElectricAgentsServer {
       streamClient: this.streamClient,
       runtime: this.standaloneRuntime.runtime,
       entityBridgeManager: this.entityBridgeManager,
+      ...(this.options.eventSources
+        ? { eventSources: this.options.eventSources }
+        : {}),
+      ...(this.options.ensureEventSourceWakeSource
+        ? {
+            ensureEventSourceWakeSource:
+              this.options.ensureEventSourceWakeSource,
+          }
+        : {}),
       isShuttingDown: () => this.shuttingDown,
       mockAgent: this.mockAgentBootstrap
         ? { runtime: this.mockAgentBootstrap.runtime }

package/src/wake-registry.ts

@@ -229,7 +229,7 @@ export class WakeRegistry {
               await this.applyShapeMessage(message)
               if (
                 !settled &&
-                `control` in message.headers &&
+                isControlMessage(message) &&
                 message.headers.control === `up-to-date`
               ) {
                 settled = true

package/src/routing/cron-router.ts

@@ -1,45 +0,0 @@
-/**
- * HTTP routes under /_electric/cron.
- */
-
-import { Type, type Static } from '@sinclair/typebox'
-import { Router, json } from 'itty-router'
-import { routeBody, withSchema } from './schema.js'
-import type { JsonRouteRequest } from './schema.js'
-import type { RouterType } from 'itty-router'
-import type { TenantContext } from './context.js'
-
-const cronRegisterBodySchema = Type.Object({
-  expression: Type.String(),
-  timezone: Type.Optional(Type.String()),
-})
-
-type CronRegisterBody = Static<typeof cronRegisterBodySchema>
-
-export type CronRoutes = RouterType<
-  JsonRouteRequest,
-  [TenantContext],
-  Response | undefined
->
-
-export const cronRouter: CronRoutes = Router<
-  JsonRouteRequest,
-  [TenantContext],
-  Response | undefined
->({
-  base: `/_electric/cron`,
-})
-
-cronRouter.post(`/register`, withSchema(cronRegisterBodySchema), registerCron)
-
-async function registerCron(
-  request: JsonRouteRequest,
-  ctx: TenantContext
-): Promise<Response> {
-  const parsed = routeBody<CronRegisterBody>(request)
-  const streamPath = await ctx.entityManager.getOrCreateCronStream(
-    parsed.expression,
-    parsed.timezone
-  )
-  return json({ streamUrl: streamPath })
-}