@electric-ax/agents-server 0.3.0

package/src/tag-stream-outbox-drainer.ts

@@ -0,0 +1,188 @@
+import { randomUUID } from 'node:crypto'
+import { entityStateSchema } from '@electric-ax/agents-runtime'
+import { serverLog } from './utils/log.js'
+import { DEFAULT_TENANT_ID, isUnregisteredTenantError } from './tenant.js'
+import type { ChangeEvent } from '@durable-streams/state'
+import type { PostgresRegistry, TagStreamOutboxRow } from './entity-registry.js'
+import type { StreamClient } from './stream-client.js'
+
+const DRAIN_INTERVAL_MS = 500
+const MAX_FAILURE_ATTEMPTS = 10
+
+type StreamClientResolver = (
+  tenantId: string
+) => StreamClient | Promise<StreamClient>
+type TenantIdsProvider = () => Iterable<string>
+
+interface TagStreamOutboxDrainerOptions {
+  tenantId?: string | null
+  tenantIds?: TenantIdsProvider
+}
+
+export class TagStreamOutboxDrainer {
+  private timer: NodeJS.Timeout | null = null
+  private draining = false
+  private activeDrain: Promise<void> | null = null
+  private stopping = false
+  private workerId = randomUUID()
+  private readonly streamClientForTenant: StreamClientResolver
+  private readonly tenantId: string | null
+  private readonly tenantIds?: TenantIdsProvider
+
+  constructor(
+    private registry: PostgresRegistry,
+    streamClient: StreamClient | StreamClientResolver,
+    options?: TagStreamOutboxDrainerOptions
+  ) {
+    this.streamClientForTenant =
+      typeof streamClient === `function` ? streamClient : () => streamClient
+    this.tenantId =
+      options?.tenantId !== undefined
+        ? options.tenantId
+        : (registry.tenantId ?? DEFAULT_TENANT_ID)
+    this.tenantIds = options?.tenantIds
+  }
+
+  start(): void {
+    if (this.timer) return
+    this.timer = setInterval(() => {
+      void this.runDrain().catch((error) => {
+        serverLog.warn(`[tag-outbox] drain failed:`, error)
+      })
+    }, DRAIN_INTERVAL_MS)
+  }
+
+  async stop(): Promise<void> {
+    this.stopping = true
+    if (this.timer) {
+      clearInterval(this.timer)
+      this.timer = null
+    }
+    try {
+      await this.activeDrain
+    } finally {
+      await this.registry.releaseTagOutboxClaims(this.workerId, this.tenantId)
+    }
+  }
+
+  async drainOnce(): Promise<void> {
+    await this.runDrain()
+  }
+
+  private async runDrain(): Promise<void> {
+    if (this.stopping) return
+    if (this.draining) return
+    this.draining = true
+    const drainPromise = (async () => {
+      const rows = await this.claimRows(25)
+      for (const row of rows) {
+        await this.publishRow(row).catch((error) => {
+          return this.handlePublishFailure(row, error)
+        })
+      }
+    })()
+    this.activeDrain = drainPromise
+
+    try {
+      await drainPromise
+    } finally {
+      this.activeDrain = null
+      this.draining = false
+    }
+  }
+
+  private async claimRows(limit: number): Promise<Array<TagStreamOutboxRow>> {
+    const tenantIds = this.sharedTenantIds()
+    if (!tenantIds) {
+      return await this.registry.claimTagOutboxRows(
+        this.workerId,
+        limit,
+        this.tenantId
+      )
+    }
+    if (tenantIds.length === 0) return []
+
+    const rows: Array<TagStreamOutboxRow> = []
+    for (const tenantId of tenantIds) {
+      const remaining = limit - rows.length
+      if (remaining <= 0) break
+      rows.push(
+        ...(await this.registry.claimTagOutboxRows(
+          this.workerId,
+          remaining,
+          tenantId
+        ))
+      )
+    }
+    return rows
+  }
+
+  // Per-row producer identity (producerId = tag-outbox-${row.id}, epoch=0,
+  // seq=0) is deliberate: the protocol requires same-epoch seqs to be
+  // contiguous (PROTOCOL.md), so a shared per-entity producerId with
+  // seq = outbox.id would 409 the first time another entity's row
+  // interleaves. Each row is its own single-append producer; retries
+  // replay the same triple and dedupe server-side as 204.
+  private async publishRow(row: TagStreamOutboxRow): Promise<void> {
+    const event = buildTagChangeEvent(row)
+    const streamClient = await this.streamClientForTenant(row.tenantId)
+    await streamClient.appendWithProducerHeaders(
+      `${row.entityUrl}/main`,
+      JSON.stringify(event),
+      {
+        producerId: `tag-outbox-${row.id}`,
+        epoch: 0,
+        seq: 0,
+      }
+    )
+    await this.registry.deleteTagOutboxRow(row.id, row.tenantId)
+  }
+
+  private async handlePublishFailure(
+    row: TagStreamOutboxRow,
+    error: unknown
+  ): Promise<void> {
+    const message = error instanceof Error ? error.message : String(error)
+    if (isUnregisteredTenantError(error)) {
+      await this.registry.releaseTagOutboxClaims(this.workerId, row.tenantId)
+      serverLog.warn(
+        `[tag-outbox] skipped row ${row.id} for unregistered tenant "${row.tenantId}": ${message}`
+      )
+      return
+    }
+
+    const result = await this.registry.failTagOutboxRow(
+      row.id,
+      this.workerId,
+      message,
+      MAX_FAILURE_ATTEMPTS,
+      row.tenantId
+    )
+
+    const logLine = `[tag-outbox] row ${row.id} failed (attempt ${result.attemptCount}/${MAX_FAILURE_ATTEMPTS})`
+    if (result.deadLettered) {
+      serverLog.error(`${logLine}; dead-lettered: ${message}`)
+      return
+    }
+    serverLog.warn(`${logLine}: ${message}`)
+  }
+
+  private sharedTenantIds(): Array<string> | null {
+    if (this.tenantId !== null || !this.tenantIds) return null
+    return [...new Set(this.tenantIds())]
+  }
+}
+
+function buildTagChangeEvent(row: TagStreamOutboxRow): ChangeEvent<unknown> {
+  const headers = { timestamp: new Date().toISOString() }
+
+  if (row.op === `delete`) {
+    return entityStateSchema.tags.delete({ key: row.key, headers })
+  }
+
+  const value = row.rowData ?? { key: row.key, value: `` }
+  if (row.op === `insert`) {
+    return entityStateSchema.tags.insert({ key: row.key, value, headers })
+  }
+  return entityStateSchema.tags.update({ key: row.key, value, headers })
+}

package/src/tenant.ts

@@ -0,0 +1,25 @@
+export const DEFAULT_TENANT_ID = `default`
+
+export class UnregisteredTenantError extends Error {
+  constructor(
+    readonly tenantId: string,
+    readonly processName: string
+  ) {
+    super(
+      `tenant "${tenantId}" is not registered on this host for ${processName}`
+    )
+    this.name = `UnregisteredTenantError`
+  }
+}
+
+export function isUnregisteredTenantError(
+  error: unknown
+): error is UnregisteredTenantError {
+  return (
+    error instanceof UnregisteredTenantError ||
+    (typeof error === `object` &&
+      error !== null &&
+      `name` in error &&
+      (error as { name?: unknown }).name === `UnregisteredTenantError`)
+  )
+}

package/src/tracing.ts

@@ -0,0 +1,57 @@
+import { SpanStatusCode, context, propagation, trace } from '@opentelemetry/api'
+import type { Context, Span, SpanOptions, Tracer } from '@opentelemetry/api'
+
+export const tracer: Tracer = trace.getTracer(`agent-server`)
+
+export const ATTR = {
+  ENTITY_URL: `electric_agents.entity.url`,
+  ENTITY_TYPE: `electric_agents.entity.type`,
+  PARENT_URL: `electric_agents.entity.parent`,
+  WAKE_SOURCE: `electric_agents.wake.source`,
+  WAKE_SUBSCRIBER: `electric_agents.wake.subscriber`,
+  WAKE_KIND: `electric_agents.wake.kind`,
+  STREAM_PATH: `electric_agents.stream.path`,
+  STREAM_OP: `electric_agents.stream.op`,
+  DB_OP: `electric_agents.db.op`,
+  HTTP_METHOD: `http.method`,
+  HTTP_ROUTE: `http.route`,
+  HTTP_STATUS: `http.status_code`,
+} as const
+
+/**
+ * Run `fn` inside an active span. Errors are recorded + status set to ERROR,
+ * then re-thrown. Span ends in a finally block.
+ */
+export async function withSpan<T>(
+  name: string,
+  fn: (span: Span) => Promise<T>,
+  opts?: SpanOptions
+): Promise<T> {
+  return await tracer.startActiveSpan(name, opts ?? {}, async (span) => {
+    try {
+      return await fn(span)
+    } catch (err) {
+      span.recordException(err as Error)
+      span.setStatus({
+        code: SpanStatusCode.ERROR,
+        message: err instanceof Error ? err.message : String(err),
+      })
+      throw err
+    } finally {
+      span.end()
+    }
+  })
+}
+
+export function injectTraceHeaders(
+  headers: Record<string, string>,
+  ctx: Context = context.active()
+): void {
+  propagation.inject(ctx, headers)
+}
+
+export function extractTraceContext(
+  headers: Record<string, string | Array<string> | undefined>
+): Context {
+  return propagation.extract(context.active(), headers)
+}

package/src/utils/electric-url.ts

@@ -0,0 +1,15 @@
+export function applyElectricUrlQueryParams(
+  target: URL,
+  electricUrl: string
+): void {
+  const configured = new URL(electricUrl)
+  configured.searchParams.forEach((value, key) => {
+    target.searchParams.set(key, value)
+  })
+}
+
+export function electricUrlWithPath(electricUrl: string, path: string): URL {
+  const target = new URL(path, electricUrl)
+  applyElectricUrlQueryParams(target, electricUrl)
+  return target
+}

package/src/utils/log.ts

@@ -0,0 +1,95 @@
+import path from 'node:path'
+import fs from 'node:fs'
+import pino from 'pino'
+
+const LOG_LEVEL = process.env.ELECTRIC_AGENTS_LOG_LEVEL ?? `info`
+const IS_ELECTRON_MAIN = Boolean(process.versions.electron)
+const USE_FILE_LOGS = process.env.ELECTRIC_AGENTS_LOG_FILE !== `false`
+const USE_PRETTY_LOGS =
+  LOG_LEVEL !== `silent` && !process.env.VITEST && !IS_ELECTRON_MAIN
+
+const LOG_DIR = USE_FILE_LOGS
+  ? (process.env.ELECTRIC_AGENTS_LOG_DIR ?? path.resolve(process.cwd(), `logs`))
+  : undefined
+const LOG_FILE = LOG_DIR
+  ? path.join(LOG_DIR, `agent-server-${Date.now()}.jsonl`)
+  : undefined
+
+if (LOG_DIR) {
+  fs.mkdirSync(LOG_DIR, { recursive: true })
+}
+
+export const LOG_FILE_PATH = LOG_FILE
+
+const streams: Array<pino.StreamEntry> = []
+if (LOG_FILE) {
+  streams.push({
+    stream: pino.destination({
+      dest: LOG_FILE,
+      sync: IS_ELECTRON_MAIN,
+    }),
+  })
+}
+if (USE_PRETTY_LOGS) {
+  streams.push({
+    stream: pino.transport({
+      target: `pino-pretty`,
+      options: {
+        colorize: true,
+        ignore: `pid,hostname,name`,
+        translateTime: `SYS:HH:MM:ss`,
+      },
+    }),
+  })
+}
+
+const logger =
+  streams.length > 0
+    ? pino(
+        {
+          base: undefined,
+          level: LOG_LEVEL,
+        },
+        pino.multistream(streams)
+      )
+    : pino({
+        base: undefined,
+        enabled: false,
+        level: LOG_LEVEL,
+      })
+
+function formatArgs(args: Array<unknown>): { err?: Error; msg: string } {
+  const errors: Array<Error> = []
+  const parts: Array<string> = []
+  for (const a of args) {
+    if (a instanceof Error) errors.push(a)
+    else parts.push(typeof a === `string` ? a : JSON.stringify(a))
+  }
+  return {
+    err: errors[0],
+    msg: parts.join(` `),
+  }
+}
+
+export const serverLog = {
+  info(...args: Array<unknown>): void {
+    const { msg } = formatArgs(args)
+    logger.info(msg)
+  },
+
+  warn(...args: Array<unknown>): void {
+    const { err, msg } = formatArgs(args)
+    if (err) logger.warn({ err }, msg)
+    else logger.warn(msg)
+  },
+
+  error(...args: Array<unknown>): void {
+    const { err, msg } = formatArgs(args)
+    if (err) logger.error({ err }, msg)
+    else logger.error(msg)
+  },
+
+  event(obj: Record<string, unknown>, msg: string): void {
+    logger.info(obj, msg)
+  },
+}

package/src/utils/server-utils.ts

@@ -0,0 +1,245 @@
+import { access } from 'node:fs/promises'
+import path from 'node:path'
+import {
+  applyElectricUrlQueryParams,
+  electricUrlWithPath,
+} from './electric-url.js'
+import { resolveDurableStreamsRoutingAdapter } from '../routing/durable-streams-routing-adapter.js'
+import { applyDurableStreamsBearer } from '../stream-client.js'
+import type { Agent } from 'undici'
+import type { DurableStreamsRoutingAdapter } from '../routing/durable-streams-routing-adapter.js'
+import type { DurableStreamsBearerProvider } from '../stream-client.js'
+
+export function contentTypeForStaticFile(filePath: string): string {
+  const ext = path.extname(filePath).toLowerCase()
+  switch (ext) {
+    case `.html`:
+      return `text/html; charset=utf-8`
+    case `.js`:
+      return `text/javascript; charset=utf-8`
+    case `.css`:
+      return `text/css; charset=utf-8`
+    case `.json`:
+      return `application/json; charset=utf-8`
+    case `.svg`:
+      return `image/svg+xml`
+    case `.png`:
+      return `image/png`
+    case `.jpg`:
+    case `.jpeg`:
+      return `image/jpeg`
+    case `.ico`:
+      return `image/x-icon`
+    case `.map`:
+      return `application/json; charset=utf-8`
+    default:
+      return `application/octet-stream`
+  }
+}
+
+export function cacheControlForAgentUiFile(filePath: string): string {
+  return filePath.includes(`${path.sep}assets${path.sep}`)
+    ? `public, max-age=31536000, immutable`
+    : `no-cache`
+}
+
+export function resolveAgentUiPath(
+  agentUiDistDir: string,
+  relativePath: string
+): string {
+  const normalized = relativePath.replace(/^\/+/, ``)
+  return path.resolve(agentUiDistDir, normalized)
+}
+
+export async function pickAgentUiFile(
+  agentUiDistDir: string,
+  filePath: string,
+  fallbackToIndex: boolean
+): Promise<string | null> {
+  if (isAgentUiPath(agentUiDistDir, filePath) && (await fileExists(filePath))) {
+    return filePath
+  }
+
+  if (!fallbackToIndex) {
+    return null
+  }
+
+  const indexPath = path.join(agentUiDistDir, `index.html`)
+  if (!(await fileExists(indexPath))) {
+    return null
+  }
+  return indexPath
+}
+
+export function isAgentUiPath(
+  agentUiDistDir: string,
+  filePath: string
+): boolean {
+  return (
+    filePath === agentUiDistDir ||
+    filePath.startsWith(`${agentUiDistDir}${path.sep}`)
+  )
+}
+
+export async function fileExists(filePath: string): Promise<boolean> {
+  try {
+    await access(filePath)
+    return true
+  } catch {
+    return false
+  }
+}
+
+export function buildElectricProxyTarget(options: {
+  incomingUrl: URL
+  electricUrl: string
+  electricSecret?: string
+  tenantId: string
+}): URL {
+  const targetPath = options.incomingUrl.pathname.replace(
+    `/_electric/electric`,
+    ``
+  )
+  const target = electricUrlWithPath(options.electricUrl, targetPath)
+  options.incomingUrl.searchParams.forEach((value, key) => {
+    target.searchParams.append(key, value)
+  })
+  applyElectricUrlQueryParams(target, options.electricUrl)
+
+  if (targetPath !== `/v1/shape`) {
+    return target
+  }
+
+  if (options.electricSecret) {
+    target.searchParams.set(`secret`, options.electricSecret)
+  }
+
+  const table = options.incomingUrl.searchParams.get(`table`)
+  if (table === `entities`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","url","type","status","dispatch_policy","tags","spawn_args","parent","type_revision","inbox_schemas","state_schemas","created_at","updated_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  } else if (table === `entity_types`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","name","description","creation_schema","inbox_schemas","state_schemas","serve_endpoint","default_dispatch_policy","revision","created_at","updated_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  } else if (table === `runners`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","id","owner_user_id","label","kind","admin_status","wake_stream","wake_stream_offset","last_seen_at","liveness_lease_expires_at","created_at","updated_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  } else if (table === `entity_dispatch_state`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","entity_url","pending_source_streams","pending_reason","pending_since","outstanding_wake_id","outstanding_wake_target","outstanding_wake_created_at","active_consumer_id","active_runner_id","active_epoch","active_claimed_at","active_lease_expires_at","last_wake_id","last_claimed_at","last_released_at","last_completed_at","last_error","updated_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  } else if (table === `wake_notifications`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","wake_id","entity_url","target_type","target_runner_id","target_webhook_url","target_worker_pool_id","runner_wake_stream","runner_wake_stream_offset","notification_public","delivery_status","claim_status","created_at","delivered_at","claimed_at","resolved_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  } else if (table === `consumer_claims`) {
+    target.searchParams.set(
+      `columns`,
+      `"tenant_id","consumer_id","epoch","wake_id","entity_url","stream_path","runner_id","status","claimed_at","last_heartbeat_at","lease_expires_at","released_at","acked_streams","updated_at"`
+    )
+    applyTenantShapeWhere(target, options.tenantId)
+  }
+
+  return target
+}
+
+export async function forwardFetchRequest(options: {
+  request: {
+    method: string
+    url: string
+    headers: Headers
+  }
+  durableStreamsUrl: string
+  durableStreamsRouting?: DurableStreamsRoutingAdapter
+  serviceId: string
+  body?: Uint8Array
+  dispatcher?: Agent
+  route?: `stream` | `stream-meta`
+  durableStreamsBearer?: DurableStreamsBearerProvider
+  durableStreamsBearerMode?: `overwrite` | `if-missing` | `none`
+}): Promise<Response> {
+  const routingAdapter = resolveDurableStreamsRoutingAdapter(
+    options.durableStreamsRouting
+  )
+  const routingInput = {
+    durableStreamsUrl: options.durableStreamsUrl,
+    serviceId: options.serviceId,
+    requestUrl: options.request.url,
+  }
+  const upstreamUrl =
+    options.route === `stream-meta`
+      ? routingAdapter.streamMetaUrl(routingInput)
+      : routingAdapter.streamUrl(routingInput)
+
+  const headers = new Headers(options.request.headers)
+  if (options.durableStreamsBearerMode !== `none`) {
+    await applyDurableStreamsBearer(headers, options.durableStreamsBearer, {
+      overwrite: options.durableStreamsBearerMode !== `if-missing`,
+    })
+  }
+
+  const init: RequestInit & { duplex?: `half`; dispatcher?: Agent } = {
+    method: options.request.method,
+    headers,
+  }
+  if (options.body !== undefined) {
+    headers.delete(`content-length`)
+    init.body = bodyFromBytes(options.body)
+    init.duplex = `half`
+  }
+  if (options.dispatcher) {
+    init.dispatcher = options.dispatcher
+  }
+
+  return await fetch(upstreamUrl, init as RequestInit)
+}
+
+function bodyFromBytes(body: Uint8Array): ArrayBuffer {
+  return body.buffer.slice(
+    body.byteOffset,
+    body.byteOffset + body.byteLength
+  ) as ArrayBuffer
+}
+
+export function decodeJsonObject(
+  body: Uint8Array
+): Record<string, unknown> | null {
+  if (body.length === 0) return null
+
+  try {
+    const parsed = JSON.parse(new TextDecoder().decode(body)) as unknown
+    if (parsed && typeof parsed === `object` && !Array.isArray(parsed)) {
+      return parsed as Record<string, unknown>
+    }
+  } catch {
+    // Not JSON; callers can fall back to raw bytes.
+  }
+
+  return null
+}
+
+function applyTenantShapeWhere(target: URL, tenantId: string): void {
+  const tenantWhere = `tenant_id = ${sqlStringLiteral(tenantId)}`
+  const existingWhere = target.searchParams.get(`where`)
+  target.searchParams.set(
+    `where`,
+    existingWhere ? `${tenantWhere} AND (${existingWhere})` : tenantWhere
+  )
+}
+
+function sqlStringLiteral(value: string): string {
+  return `'${value.replace(/'/g, `''`)}'`
+}

package/src/utils/webhook-url.ts

@@ -0,0 +1,33 @@
+export function rewriteLoopbackWebhookUrl(
+  value: string | undefined
+): string | undefined {
+  if (!value) return undefined
+
+  const rewriteTarget =
+    process.env.ELECTRIC_AGENTS_REWRITE_LOOPBACK_WEBHOOKS_TO?.trim()
+  if (!rewriteTarget) return value
+
+  const url = new URL(value)
+  if (!isLoopbackHostname(url.hostname)) {
+    return value
+  }
+
+  if (rewriteTarget.includes(`://`)) {
+    const target = new URL(rewriteTarget)
+    url.protocol = target.protocol
+    url.username = target.username
+    url.password = target.password
+    url.hostname = target.hostname
+    url.port = target.port
+    return url.toString()
+  }
+
+  url.host = rewriteTarget
+  return url.toString()
+}
+
+function isLoopbackHostname(hostname: string): boolean {
+  return (
+    hostname === `localhost` || hostname === `127.0.0.1` || hostname === `::1`
+  )
+}