@electric-ax/agents-server 0.3.0

package/src/entrypoint-lib.ts

@@ -0,0 +1,295 @@
+import { DurableStreamTestServer } from '@durable-streams/server'
+import {
+  createDevAssertedAuthenticateRequest,
+  devAssertedAuthOptionsFromEnv,
+} from './dev-asserted-auth.js'
+import { ElectricAgentsServer } from './server.js'
+import type { ElectricAgentsServerOptions } from './server.js'
+
+const DEFAULT_HOST = `0.0.0.0`
+const DEFAULT_PORT = 4437
+const DEFAULT_STREAMS_HOST = `127.0.0.1`
+
+type EnvSource = Record<string, string | undefined>
+
+export interface ElectricAgentsEntrypointOptions
+  extends ElectricAgentsServerOptions {}
+
+export interface ElectricAgentsEntrypointServer {
+  start: () => Promise<string>
+  stop: () => Promise<void>
+}
+
+export interface RunElectricAgentsEntrypointOptions {
+  env?: EnvSource
+  cwd?: string
+  createServer?: (
+    options: ElectricAgentsEntrypointOptions
+  ) => ElectricAgentsEntrypointServer
+}
+
+function readEnv(env: EnvSource, names: Array<string>): string | undefined {
+  for (const name of names) {
+    const value = env[name]?.trim()
+    if (value) {
+      return value
+    }
+  }
+  return undefined
+}
+
+function readRequiredEnv(
+  env: EnvSource,
+  names: Array<string>,
+  description: string
+): string {
+  const value = readEnv(env, names)
+  if (value) {
+    return value
+  }
+
+  throw new Error(
+    `Missing ${description}. Set one of: ${names.map((name) => `"${name}"`).join(`, `)}`
+  )
+}
+
+function readPort(env: EnvSource): number {
+  const raw = readEnv(env, [`ELECTRIC_AGENTS_PORT`, `PORT`])
+  if (!raw) {
+    return DEFAULT_PORT
+  }
+
+  const port = Number(raw)
+  if (!Number.isInteger(port) || port < 1 || port > 65_535) {
+    throw new Error(
+      `Invalid ELECTRIC_AGENTS port "${raw}". Expected an integer between 1 and 65535.`
+    )
+  }
+
+  return port
+}
+
+function validateUrl(name: string, value: string): string {
+  try {
+    new URL(value)
+    return value
+  } catch {
+    throw new Error(`Invalid ${name}: "${value}"`)
+  }
+}
+
+function readOptionalPositiveInteger(
+  env: EnvSource,
+  names: Array<string>,
+  description: string
+): number | undefined {
+  const raw = readEnv(env, names)
+  if (!raw) return undefined
+
+  const value = Number(raw)
+  if (!Number.isInteger(value) || value <= 0) {
+    throw new Error(
+      `Invalid ${description} "${raw}". Expected a positive integer.`
+    )
+  }
+
+  return value
+}
+
+function readOptionalPort(
+  env: EnvSource,
+  names: Array<string>,
+  description: string
+): number | undefined {
+  const raw = readEnv(env, names)
+  if (!raw) {
+    return undefined
+  }
+
+  const port = Number(raw)
+  if (!Number.isInteger(port) || port < 0 || port > 65_535) {
+    throw new Error(
+      `Invalid ${description} "${raw}". Expected an integer between 0 and 65535.`
+    )
+  }
+
+  return port
+}
+
+export function resolveElectricAgentsEntrypointOptions(
+  env: EnvSource = process.env,
+  cwd = process.cwd()
+): ElectricAgentsEntrypointOptions {
+  const durableStreamsUrl = readEnv(env, [
+    `ELECTRIC_AGENTS_DURABLE_STREAMS_URL`,
+    `DURABLE_STREAMS_URL`,
+    `STREAMS_URL`,
+  ])
+  const durableStreamsBearer = readEnv(env, [
+    `ELECTRIC_AGENTS_DURABLE_STREAMS_BEARER`,
+    `DURABLE_STREAMS_BEARER`,
+  ])
+  const postgresUrl = validateUrl(
+    `Postgres URL`,
+    readRequiredEnv(
+      env,
+      [`ELECTRIC_AGENTS_DATABASE_URL`, `DATABASE_URL`],
+      `Postgres connection URL`
+    )
+  )
+
+  const electricUrl = readEnv(env, [
+    `ELECTRIC_AGENTS_ELECTRIC_URL`,
+    `ELECTRIC_URL`,
+  ])
+  const electricSecret = readEnv(env, [`ELECTRIC_AGENTS_ELECTRIC_SECRET`])
+  const baseUrl = readEnv(env, [`ELECTRIC_AGENTS_BASE_URL`, `BASE_URL`])
+  const authenticateRequest = createDevAssertedAuthenticateRequest(
+    devAssertedAuthOptionsFromEnv(env)
+  )
+
+  return {
+    ...(authenticateRequest ? { authenticateRequest } : {}),
+    service: readEnv(env, [`ELECTRIC_AGENTS_SERVICE`, `SERVICE`]),
+    tenantId: readEnv(env, [`ELECTRIC_AGENTS_TENANT_ID`, `TENANT_ID`]),
+    baseUrl: baseUrl ? validateUrl(`base URL`, baseUrl) : undefined,
+    durableStreamsUrl: durableStreamsUrl
+      ? validateUrl(`durable streams URL`, durableStreamsUrl)
+      : undefined,
+    ...(durableStreamsBearer ? { durableStreamsBearer } : {}),
+    postgresUrl,
+    electricUrl: electricUrl
+      ? validateUrl(`Electric URL`, electricUrl)
+      : undefined,
+    electricSecret,
+    host: readEnv(env, [`ELECTRIC_AGENTS_HOST`, `HOST`]) ?? DEFAULT_HOST,
+    port: readPort(env),
+    workingDirectory:
+      readEnv(env, [
+        `ELECTRIC_AGENTS_WORKING_DIRECTORY`,
+        `WORKING_DIRECTORY`,
+      ]) ?? cwd,
+    dispatchRecoveryIntervalMs: readOptionalPositiveInteger(
+      env,
+      [`ELECTRIC_AGENTS_DISPATCH_RECOVERY_INTERVAL_MS`],
+      `dispatch recovery interval`
+    ),
+    staleOutstandingWakeAfterMs: readOptionalPositiveInteger(
+      env,
+      [`ELECTRIC_AGENTS_STALE_OUTSTANDING_WAKE_AFTER_MS`],
+      `stale outstanding wake age`
+    ),
+  }
+}
+
+function createEmbeddedStreamsServer(
+  env: EnvSource,
+  cwd: string
+): DurableStreamTestServer | undefined {
+  const externalUrl = readEnv(env, [
+    `ELECTRIC_AGENTS_DURABLE_STREAMS_URL`,
+    `DURABLE_STREAMS_URL`,
+    `STREAMS_URL`,
+  ])
+  if (externalUrl) {
+    return undefined
+  }
+
+  const dataDir =
+    readEnv(env, [`ELECTRIC_AGENTS_STREAMS_DATA_DIR`, `STREAMS_DATA_DIR`]) ??
+    `${cwd}/.streams-data`
+
+  return new DurableStreamTestServer({
+    host:
+      readEnv(env, [`ELECTRIC_AGENTS_STREAMS_HOST`, `STREAMS_HOST`]) ??
+      DEFAULT_STREAMS_HOST,
+    port:
+      readOptionalPort(
+        env,
+        [`ELECTRIC_AGENTS_STREAMS_PORT`, `STREAMS_PORT`],
+        `embedded streams port`
+      ) ?? 0,
+    dataDir,
+    webhooks: true,
+  })
+}
+
+export async function runElectricAgentsEntrypoint({
+  env = process.env,
+  cwd = process.cwd(),
+  createServer = (options) => new ElectricAgentsServer(options),
+}: RunElectricAgentsEntrypointOptions = {}): Promise<{
+  options: ElectricAgentsEntrypointOptions
+  server: ElectricAgentsEntrypointServer
+  url: string
+}> {
+  const embeddedStreamsServer = createEmbeddedStreamsServer(env, cwd)
+  const options = {
+    ...resolveElectricAgentsEntrypointOptions(env, cwd),
+    durableStreamsServer: embeddedStreamsServer,
+  }
+  const server = createServer(options)
+  const url = await server.start()
+
+  return { options, server, url }
+}
+
+export async function main(): Promise<void> {
+  try {
+    process.loadEnvFile()
+  } catch {}
+
+  let server: ElectricAgentsEntrypointServer | null = null
+  let stopping: Promise<void> | null = null
+
+  const stop = async (exitCode: number): Promise<never> => {
+    if (!stopping) {
+      stopping =
+        server?.stop().catch((error) => {
+          console.error(`[agent-server] failed to stop cleanly`, error)
+        }) ?? Promise.resolve()
+    }
+
+    await stopping
+    process.exit(exitCode)
+  }
+
+  try {
+    const started = await runElectricAgentsEntrypoint()
+    server = started.server
+
+    console.log(`Electric Agents server running at ${started.url}`)
+    console.log(
+      `Durable Streams: ${started.options.durableStreamsUrl ?? `(embedded DurableStreamTestServer)`}`
+    )
+    console.log(`Postgres: ${started.options.postgresUrl}`)
+    if (started.options.electricUrl) {
+      console.log(`Electric: ${started.options.electricUrl}`)
+    }
+    console.log(`Working directory: ${started.options.workingDirectory}`)
+
+    process.on(`SIGINT`, () => {
+      void stop(0)
+    })
+
+    process.on(`SIGTERM`, () => {
+      void stop(0)
+    })
+
+    process.on(`uncaughtException`, (error) => {
+      console.error(error)
+      void stop(1)
+    })
+
+    process.on(`unhandledRejection`, (error) => {
+      console.error(error)
+      void stop(1)
+    })
+  } catch (error) {
+    console.error(error)
+    if (server) {
+      await server.stop().catch(() => {})
+    }
+    process.exit(1)
+  }
+}

package/src/entrypoint.ts

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+
+import { pathToFileURL } from 'node:url'
+import { main } from './entrypoint-lib.js'
+
+if (
+  process.argv[1] &&
+  import.meta.url === pathToFileURL(process.argv[1]).href
+) {
+  void main()
+}

package/src/host.ts

@@ -0,0 +1,323 @@
+import { randomUUID } from 'node:crypto'
+import { PostgresRegistry } from './entity-registry.js'
+import { EntityProjector } from './entity-projector.js'
+import { ElectricAgentsTenantRuntime } from './runtime.js'
+import { PostgresSchedulerClient, Scheduler } from './scheduler.js'
+import { StreamClient, durableStreamsServiceUrl } from './stream-client.js'
+import { TagStreamOutboxDrainer } from './tag-stream-outbox-drainer.js'
+import { DEFAULT_TENANT_ID, UnregisteredTenantError } from './tenant.js'
+import { WakeRegistry } from './wake-registry.js'
+import type { DrizzleDB, PgClient } from './db/index.js'
+import type { DurableStreamsBearerProvider } from './stream-client.js'
+
+export interface AgentsHostTenantConfig {
+  serviceId: string
+  durableStreamsUrl?: string
+  durableStreamsBearer?: DurableStreamsBearerProvider
+  streamClient?: StreamClient
+}
+
+export type AgentsHostTenantRuntime = ElectricAgentsTenantRuntime
+
+export interface AgentsHostOptions {
+  db: DrizzleDB
+  pgClient: PgClient
+  electricUrl?: string
+  electricSecret?: string
+  instanceId?: string
+  wakeRegistry?: WakeRegistry
+  entityProjector?: EntityProjector
+  startEntityBridgeManager?: boolean
+  rehydrateTenantOnStart?: boolean
+}
+
+export class AgentsHost {
+  readonly db: DrizzleDB
+  readonly pgClient: PgClient
+  readonly wakeRegistry: WakeRegistry
+  readonly entityProjector: EntityProjector
+  readonly scheduler: Scheduler
+  readonly tagStreamOutboxDrainer: TagStreamOutboxDrainer
+
+  private readonly electricUrl?: string
+  private readonly electricSecret?: string
+  private readonly instanceId: string
+  private readonly ownsWakeRegistry: boolean
+  private readonly ownsEntityProjector: boolean
+  private readonly startEntityBridgeManager: boolean
+  private readonly rehydrateTenantOnStart: boolean
+  private readonly tenantRegistrations = new Map<
+    string,
+    Promise<AgentsHostTenantRuntime>
+  >()
+  private readonly tenantRuntimes = new Map<string, AgentsHostTenantRuntime>()
+  private readonly tenantOperations = new Map<string, Promise<void>>()
+  private running = false
+
+  constructor(options: AgentsHostOptions) {
+    this.db = options.db
+    this.pgClient = options.pgClient
+    this.electricUrl = options.electricUrl
+    this.electricSecret = options.electricSecret
+    this.instanceId = options.instanceId ?? randomUUID()
+    this.ownsWakeRegistry = !options.wakeRegistry
+    this.wakeRegistry = options.wakeRegistry ?? new WakeRegistry(this.db, null)
+    this.ownsEntityProjector = !options.entityProjector
+    this.startEntityBridgeManager = options.startEntityBridgeManager ?? true
+    this.rehydrateTenantOnStart = options.rehydrateTenantOnStart ?? true
+    this.entityProjector =
+      options.entityProjector ??
+      new EntityProjector({
+        db: this.db,
+        electricUrl: this.electricUrl,
+        electricSecret: this.electricSecret,
+        streamClientForTenant: (tenantId) =>
+          this.requireTenantForSharedProcess(tenantId, `entity-projector`)
+            .streamClient,
+        tenantIds: () => this.registeredTenantIds(),
+      })
+
+    this.scheduler = new Scheduler({
+      pgClient: this.pgClient,
+      instanceId: `${this.instanceId}:scheduler`,
+      tenantId: null,
+      tenantIds: () => this.registeredTenantIds(),
+      executors: {
+        delayed_send: async (payload, taskId, tenantId) => {
+          const runtime = this.requireTenantForSharedProcess(
+            tenantId,
+            `scheduler delayed_send`
+          )
+          await runtime.executeDelayedSend(payload, taskId)
+        },
+        cron_tick: async (payload, tickNumber, _taskId, tenantId) => {
+          const runtime = this.requireTenantForSharedProcess(
+            tenantId,
+            `scheduler cron_tick`
+          )
+          await runtime.executeCronTick(payload, tickNumber)
+        },
+      },
+    })
+
+    this.tagStreamOutboxDrainer = new TagStreamOutboxDrainer(
+      new PostgresRegistry(this.db, DEFAULT_TENANT_ID),
+      (tenantId) =>
+        this.requireTenantForSharedProcess(tenantId, `tag-outbox`).streamClient,
+      { tenantId: null, tenantIds: () => this.registeredTenantIds() }
+    )
+  }
+
+  async start(): Promise<void> {
+    if (this.running) return
+    this.running = true
+
+    try {
+      if (this.electricUrl) {
+        await this.wakeRegistry.startSync(this.electricUrl, this.electricSecret)
+      } else {
+        await this.wakeRegistry.loadRegistrations()
+      }
+      if (this.startEntityBridgeManager) {
+        await this.entityProjector.start()
+      }
+      await this.startRegisteredTenants()
+      this.tagStreamOutboxDrainer.start()
+      await this.scheduler.start()
+    } catch (error) {
+      await this.stop().catch(() => {})
+      throw error
+    }
+  }
+
+  async stop(): Promise<void> {
+    if (!this.running) return
+    this.running = false
+
+    await this.scheduler.stop()
+    await this.tagStreamOutboxDrainer.stop()
+
+    const runtimes = await Promise.allSettled(this.tenantRegistrations.values())
+    await Promise.allSettled(
+      runtimes
+        .filter(
+          (result): result is PromiseFulfilledResult<AgentsHostTenantRuntime> =>
+            result.status === `fulfilled`
+        )
+        .map((result) => result.value.stop())
+    )
+    this.tenantRegistrations.clear()
+    this.tenantRuntimes.clear()
+
+    if (this.ownsWakeRegistry) {
+      await this.wakeRegistry.stopSync()
+    }
+    if (this.ownsEntityProjector) {
+      await this.entityProjector.stop()
+    }
+  }
+
+  async registerTenant(
+    config: AgentsHostTenantConfig
+  ): Promise<AgentsHostTenantRuntime> {
+    const serviceId = config.serviceId
+    return await this.withTenantOperation(serviceId, async () => {
+      const runtime = this.tenantRuntimes.get(serviceId)
+      if (runtime) return runtime
+
+      const existing = this.tenantRegistrations.get(serviceId)
+      if (existing) return existing
+
+      const runtimePromise = this.createTenantRuntime(config)
+      this.tenantRegistrations.set(serviceId, runtimePromise)
+
+      try {
+        const registeredRuntime = await runtimePromise
+        this.tenantRuntimes.set(serviceId, registeredRuntime)
+        if (this.running) {
+          await this.startTenantRuntime(registeredRuntime)
+          this.scheduler.wake()
+        }
+        return registeredRuntime
+      } catch (error) {
+        if (this.tenantRegistrations.get(serviceId) === runtimePromise) {
+          this.tenantRegistrations.delete(serviceId)
+        }
+        if (this.tenantRuntimes.get(serviceId)) {
+          this.tenantRuntimes.delete(serviceId)
+        }
+        throw error
+      }
+    })
+  }
+
+  getTenant(
+    serviceId = DEFAULT_TENANT_ID
+  ): AgentsHostTenantRuntime | undefined {
+    return this.tenantRuntimes.get(serviceId)
+  }
+
+  requireTenant(serviceId = DEFAULT_TENANT_ID): AgentsHostTenantRuntime {
+    const runtime = this.getTenant(serviceId)
+    if (!runtime) {
+      throw new Error(`AgentsHost tenant "${serviceId}" is not registered`)
+    }
+    return runtime
+  }
+
+  async unregisterTenant(serviceId = DEFAULT_TENANT_ID): Promise<void> {
+    await this.withTenantOperation(serviceId, async () => {
+      const registration = this.tenantRegistrations.get(serviceId)
+      const runtime = this.tenantRuntimes.get(serviceId)
+
+      this.tenantRegistrations.delete(serviceId)
+      this.tenantRuntimes.delete(serviceId)
+
+      const resolvedRuntime =
+        runtime ??
+        (registration ? await registration.catch(() => undefined) : undefined)
+      if (!resolvedRuntime) return
+
+      await resolvedRuntime.stop()
+      if (this.running) {
+        this.scheduler.wake()
+      }
+    })
+  }
+
+  private async withTenantOperation<T>(
+    serviceId: string,
+    operation: () => Promise<T>
+  ): Promise<T> {
+    const previous = this.tenantOperations.get(serviceId) ?? Promise.resolve()
+    const result = previous.catch(() => {}).then(operation)
+    const current = result.then(
+      () => undefined,
+      () => undefined
+    )
+    this.tenantOperations.set(serviceId, current)
+
+    try {
+      return await result
+    } finally {
+      if (this.tenantOperations.get(serviceId) === current) {
+        this.tenantOperations.delete(serviceId)
+      }
+    }
+  }
+
+  private registeredTenantIds(): Array<string> {
+    return [...this.tenantRuntimes.keys()]
+  }
+
+  private requireTenantForSharedProcess(
+    serviceId: string,
+    processName: string
+  ): AgentsHostTenantRuntime {
+    const runtime = this.getTenant(serviceId)
+    if (!runtime) {
+      throw new UnregisteredTenantError(serviceId, processName)
+    }
+    return runtime
+  }
+
+  private async startRegisteredTenants(): Promise<void> {
+    const runtimes = await Promise.all(this.tenantRegistrations.values())
+    for (const runtime of runtimes) {
+      await this.startTenantRuntime(runtime)
+    }
+  }
+
+  private async startTenantRuntime(
+    runtime: AgentsHostTenantRuntime
+  ): Promise<void> {
+    if (this.rehydrateTenantOnStart) {
+      await runtime.rehydrateCronSchedules()
+    }
+    if (this.startEntityBridgeManager) {
+      await this.entityProjector.loadTenantBridges(
+        runtime.serviceId,
+        runtime.registry
+      )
+    }
+  }
+
+  private async createTenantRuntime(
+    config: AgentsHostTenantConfig
+  ): Promise<AgentsHostTenantRuntime> {
+    const serviceId = config.serviceId
+    const streamClient = this.createStreamClient(config)
+    const registry = new PostgresRegistry(this.db, serviceId)
+    const scheduler = new PostgresSchedulerClient(
+      this.pgClient,
+      serviceId,
+      () => this.scheduler.wake()
+    )
+    const runtime = new ElectricAgentsTenantRuntime({
+      service: serviceId,
+      db: this.db,
+      registry,
+      streamClient,
+      wakeRegistry: this.wakeRegistry,
+      scheduler,
+      entityBridgeManager: this.entityProjector.forTenant(serviceId, registry),
+    })
+
+    return runtime
+  }
+
+  private createStreamClient(config: AgentsHostTenantConfig): StreamClient {
+    if (config.streamClient) return config.streamClient
+    if (config.durableStreamsUrl) {
+      return new StreamClient(
+        durableStreamsServiceUrl(config.durableStreamsUrl, config.serviceId),
+        { bearer: config.durableStreamsBearer }
+      )
+    }
+    throw new Error(
+      `AgentsHost tenant "${config.serviceId}" must provide a streamClient or durableStreamsUrl`
+    )
+  }
+}
+
+export { AgentsHost as ElectricAgentsHost }

package/src/index.ts

@@ -0,0 +1,49 @@
+export { createDb, runMigrations } from './db/index.js'
+export type { DrizzleDB, PgClient } from './db/index.js'
+export { AgentsHost } from './host.js'
+export type {
+  AgentsHostOptions,
+  AgentsHostTenantConfig,
+  AgentsHostTenantRuntime,
+} from './host.js'
+export { StreamClient } from './stream-client.js'
+export type {
+  DurableStreamsBearerProvider,
+  StreamClientOptions,
+  SubscriptionClaimResponse,
+  SubscriptionCreateInput,
+  SubscriptionResponse,
+  SubscriptionStreamInfo,
+} from './stream-client.js'
+export type {
+  AuthenticatedRequestUser,
+  AuthenticateRequest,
+  ConsumerClaim,
+  DispatchPolicy,
+  DispatchTarget,
+  ElectricAgentsRunner,
+  ElectricAgentsUser,
+  EntityDispatchState,
+  PublicWakeNotification,
+  RegisterRunnerRequest,
+  RunnerAdminStatus,
+  RunnerHeartbeatRequest,
+  RunnerKind,
+  RunnerLiveness,
+  SourceStreamOffset,
+  WakeNotificationRow,
+} from './electric-agents-types.js'
+export { globalRouter } from './routing/global-router.js'
+export type { GlobalRoutes } from './routing/global-router.js'
+export type { TenantContext } from './routing/context.js'
+export { pathPrefixedSingleTenantDurableStreamsRoutingAdapter } from './routing/durable-streams-routing-adapter.js'
+export type {
+  DurableStreamsRoutingAdapter,
+  DurableStreamsRoutingInput,
+} from './routing/durable-streams-routing-adapter.js'
+export type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
+export {
+  DEFAULT_TENANT_ID,
+  UnregisteredTenantError,
+  isUnregisteredTenantError,
+} from './tenant.js'