@electric-ax/agents-server 0.4.6 → 0.4.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@electric-ax/agents-server",
-  "version": "0.4.6",
+  "version": "0.4.9",
   "description": "Electric Agents entity runtime server",
   "author": "Durable Stream contributors",
   "bin": {
@@ -39,7 +39,7 @@
     "@durable-streams/client": "https://pkg.pr.new/durable-streams/durable-streams/@durable-streams/client@5d5c217",
     "@durable-streams/server": "https://pkg.pr.new/durable-streams/durable-streams/@durable-streams/server@eac712f",
     "@durable-streams/state": "https://pkg.pr.new/durable-streams/durable-streams/@durable-streams/state@5d5c217",
-    "@electric-sql/client": "^1.5.18",
+    "@electric-sql/client": "^1.5.19",
     "@mariozechner/pi-agent-core": "^0.70.2",
     "@opentelemetry/api": "^1.9.1",
     "@sinclair/typebox": "^0.34.48",
@@ -54,7 +54,7 @@
     "pino-pretty": "^13.0.0",
     "postgres": "^3.4.0",
     "undici": "^7.24.7",
-    "@electric-ax/agents-runtime": "0.3.1"
+    "@electric-ax/agents-runtime": "0.3.4"
   },
   "devDependencies": {
     "@types/node": "^22.19.15",
@@ -65,9 +65,9 @@
     "tsx": "^4.19.0",
     "typescript": "^5.0.0",
     "vitest": "^4.1.0",
-    "@electric-ax/agents": "0.4.5",
-    "@electric-ax/agents-server-conformance-tests": "0.1.6",
-    "@electric-ax/agents-server-ui": "0.4.6"
+    "@electric-ax/agents": "0.4.8",
+    "@electric-ax/agents-server-conformance-tests": "0.1.7",
+    "@electric-ax/agents-server-ui": "0.4.9"
   },
   "files": [
     "dist",

package/src/db/schema.ts

@@ -66,7 +66,7 @@ export const entities = pgTable(
     index(`entities_tags_index_gin`).using(`gin`, table.tagsIndex),
     check(
       `chk_entities_status`,
-      sql`${table.status} IN ('spawning', 'running', 'idle', 'stopped')`
+      sql`${table.status} IN ('spawning', 'running', 'idle', 'paused', 'stopping', 'stopped', 'killed')`
     ),
   ]
 )

package/src/electric-agents-types.ts

@@ -15,15 +15,39 @@ export type AuthenticateRequest = (
   request: Request
 ) => Promise<Principal | null> | Principal | null
 
-export type EntityStatus = `spawning` | `running` | `idle` | `stopped`
+export type EntityStatus =
+  | `spawning`
+  | `running`
+  | `idle`
+  | `paused`
+  | `stopping`
+  | `stopped`
+  | `killed`
+
+export const ENTITY_SIGNALS = [
+  `SIGINT`,
+  `SIGHUP`,
+  `SIGTERM`,
+  `SIGKILL`,
+  `SIGSTOP`,
+  `SIGCONT`,
+  `SIGUSR`,
+] as const
+
+export type EntitySignal = (typeof ENTITY_SIGNALS)[number]
 
 const VALID_ENTITY_STATUSES = new Set<string>([
   `spawning`,
   `running`,
   `idle`,
+  `paused`,
+  `stopping`,
   `stopped`,
+  `killed`,
 ])
 
+const VALID_ENTITY_SIGNALS = new Set<string>(ENTITY_SIGNALS)
+
 export function assertEntityStatus(s: string): EntityStatus {
   if (!VALID_ENTITY_STATUSES.has(s)) {
     throw new Error(`Invalid entity status: "${s}"`)
@@ -236,6 +260,41 @@ export interface ConsumerClaim {
   updated_at: string
 }
 
+export function assertEntitySignal(s: string): EntitySignal {
+  if (!VALID_ENTITY_SIGNALS.has(s)) {
+    throw new Error(`Invalid entity signal: "${s}"`)
+  }
+  return s as EntitySignal
+}
+
+export function isTerminalEntityStatus(status: EntityStatus): boolean {
+  return status === `stopped` || status === `killed`
+}
+
+export function rejectsNormalWrites(status: EntityStatus): boolean {
+  return status === `stopping` || isTerminalEntityStatus(status)
+}
+
+export function expectedSignalStatus(
+  status: EntityStatus,
+  signal: EntitySignal
+): EntityStatus {
+  switch (signal) {
+    case `SIGKILL`:
+      return `killed`
+    case `SIGTERM`:
+      return status === `idle` ? `stopped` : `stopping`
+    case `SIGSTOP`:
+      return status === `idle` ? `paused` : status
+    case `SIGCONT`:
+      return status === `paused` ? `idle` : status
+    case `SIGINT`:
+    case `SIGHUP`:
+    case `SIGUSR`:
+      return status
+  }
+}
+
 export interface ElectricAgentsEntity {
   url: string
   type: string
@@ -341,6 +400,7 @@ export interface TypedSpawnRequest {
     debounceMs?: number
     timeoutMs?: number
     includeResponse?: boolean
+    manifestKey?: string
   }
 }
 
@@ -353,6 +413,21 @@ export interface SendRequest {
   position?: string
 }
 
+export interface SignalRequest {
+  signal: EntitySignal
+  reason?: string
+  payload?: unknown
+}
+
+export interface SignalResponse {
+  url: string
+  signal: EntitySignal
+  previous_state: EntityStatus
+  new_state: EntityStatus
+  created_at: number
+  txid: number
+}
+
 export interface SetTagRequest {
   value: string
 }
@@ -369,6 +444,7 @@ export const ErrCodeNoSubscription = `NO_SUBSCRIPTION`
 export const ErrCodeNotFound = `NOT_FOUND`
 export const ErrCodeNotRunning = `NOT_RUNNING`
 export const ErrCodeInvalidRequest = `INVALID_REQUEST`
+export const ErrCodeInvalidSignal = `INVALID_SIGNAL`
 export const ErrCodeUnknownEntityType = `UNKNOWN_ENTITY_TYPE`
 export const ErrCodeSchemaValidationFailed = `SCHEMA_VALIDATION_FAILED`
 export const ErrCodeUnknownMessageType = `UNKNOWN_MESSAGE_TYPE`

package/src/entity-manager.ts

@@ -6,6 +6,7 @@ import {
   getCronStreamPath,
   getSharedStateStreamPath,
   getNextCronFireAt,
+  eventSourceSubscriptionManifestKey,
   manifestChildKey,
   manifestSharedStateKey,
   manifestSourceKey,
@@ -16,6 +17,7 @@ import {
   ErrCodeEntityPersistFailed,
   ErrCodeForkInProgress,
   ErrCodeForkWaitTimeout,
+  ErrCodeInvalidSignal,
   ErrCodeInvalidRequest,
   ErrCodeNotFound,
   ErrCodeNotRunning,
@@ -25,6 +27,8 @@ import {
   ErrCodeUnknownEntityType,
   ErrCodeUnknownEventType,
   ErrCodeUnknownMessageType,
+  isTerminalEntityStatus,
+  rejectsNormalWrites,
 } from './electric-agents-types.js'
 import { parseDispatchPolicy } from './dispatch-policy-schema.js'
 import { applyTypeDefaultSubscriptionScope } from './routing/dispatch-policy.js'
@@ -47,6 +51,7 @@ import type { queueAsPromised } from 'fastq'
 import type { SchedulerClient } from './scheduler.js'
 import type { WakeEvalResult, WakeRegistry } from './wake-registry.js'
 import type { WakeMessage } from '@electric-ax/agents-runtime'
+import type { EventSourceSubscription } from '@electric-ax/agents-runtime'
 import type { PostgresRegistry } from './entity-registry.js'
 import type { SchemaValidator } from './electric-agents/schema-validator.js'
 import type { StreamClient } from './stream-client.js'
@@ -54,9 +59,12 @@ import type {
   DispatchPolicy,
   ElectricAgentsEntity,
   ElectricAgentsEntityType,
+  EntitySignal,
   RegisterEntityTypeRequest,
   SendRequest,
   SetTagRequest,
+  SignalRequest,
+  SignalResponse,
   TypedSpawnRequest,
 } from './electric-agents-types.js'
 import type { EntityBridgeCoordinator } from './entity-bridge-manager.js'
@@ -72,6 +80,36 @@ type WriteTokenValidator = (
   entity: ElectricAgentsEntity,
   token: string
 ) => boolean
+type ServerSignalOutcome = `transitioned` | `ignored`
+type ServerSignalHandling = {
+  status: ElectricAgentsEntity[`status`]
+  handled: boolean
+  outcome: ServerSignalOutcome
+  unregisterWakes: boolean
+}
+type ServerSignalValue = {
+  signal: EntitySignal
+  status: `handled` | `unhandled`
+  sender: typeof SERVER_SIGNAL_SENDER
+  timestamp: string
+  reason?: string
+  payload?: unknown
+  handled_at?: string
+  handled_by?: typeof SERVER_SIGNAL_SENDER
+  outcome?: ServerSignalOutcome
+  previous_state?: ElectricAgentsEntity[`status`]
+  new_state?: ElectricAgentsEntity[`status`]
+}
+type ServerSignalEvent = {
+  type: `signal`
+  key: string
+  value: ServerSignalValue
+  headers: {
+    operation: `insert`
+    timestamp: string
+    txid: string
+  }
+}
 
 function createInitialQueuePosition(date: Date): string {
   return `${String(date.getTime()).padStart(16, `0`)}:a0`
@@ -103,6 +141,8 @@ type ForkResult = {
 const DEFAULT_FORK_WAIT_TIMEOUT_MS = 120_000
 const DEFAULT_FORK_WAIT_POLL_MS = 250
 
+const SERVER_SIGNAL_SENDER = `/_electric/server`
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
@@ -516,6 +556,7 @@ export class EntityManager {
         timeoutMs: req.wake.timeoutMs,
         oneShot: false,
         includeResponse: req.wake.includeResponse,
+        manifestKey: req.wake.manifestKey,
       })
     }
 
@@ -928,16 +969,20 @@ export class EntityManager {
       }
 
       const subtree = await this.listEntitySubtree(root)
-      const stopped = subtree.find((entity) => entity.status === `stopped`)
+      const stopped = subtree.find((entity) =>
+        isTerminalEntityStatus(entity.status)
+      )
       if (stopped) {
         throw new ElectricAgentsError(
           ErrCodeNotRunning,
-          `Cannot fork stopped entity "${stopped.url}"`,
+          `Cannot fork terminal entity "${stopped.url}"`,
           409
         )
       }
 
-      let active = subtree.filter((entity) => entity.status !== `idle`)
+      let active = subtree.filter(
+        (entity) => entity.status !== `idle` && entity.status !== `paused`
+      )
       if (active.length === 0) {
         this.addForkLocks(
           this.forkWorkLockedEntities,
@@ -959,7 +1004,7 @@ export class EntityManager {
           workLocks
         )
         const lockedActive = lockedSubtree.filter(
-          (entity) => entity.status !== `idle`
+          (entity) => entity.status !== `idle` && entity.status !== `paused`
         )
         if (lockedActive.length === 0) {
           return lockedSubtree
@@ -1673,6 +1718,12 @@ export class EntityManager {
       value.processed_at = now
     }
 
+    const wakePausedEntity = entity.status === `paused` && req.mode !== `paused`
+    if (wakePausedEntity) {
+      await this.registry.updateStatus(entityUrl, `idle`)
+      await this.entityBridgeManager?.onEntityChanged(entityUrl)
+    }
+
     const envelope = entityStateSchema.inbox.insert({
       key,
       value,
@@ -1725,8 +1776,12 @@ export class EntityManager {
     if (!entity) {
       throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
     }
-    if (entity.status === `stopped`) {
-      throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
     }
 
     const now = new Date().toISOString()
@@ -1763,8 +1818,12 @@ export class EntityManager {
     if (!entity) {
       throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
     }
-    if (entity.status === `stopped`) {
-      throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
     }
 
     const envelope = entityStateSchema.inbox.delete({ key } as any)
@@ -1796,8 +1855,12 @@ export class EntityManager {
         401
       )
     }
-    if (entity.status === `stopped`) {
-      throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
     }
 
     if (typeof req.value !== `string`) {
@@ -1842,8 +1905,12 @@ export class EntityManager {
         401
       )
     }
-    if (entity.status === `stopped`) {
-      throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
     }
 
     const result = await this.registry.removeEntityTag(entityUrl, key)
@@ -2098,6 +2165,67 @@ export class EntityManager {
     return { txid }
   }
 
+  async upsertEventSourceSubscription(
+    entityUrl: string,
+    req: {
+      subscription: EventSourceSubscription
+      manifest: Record<string, unknown>
+    }
+  ): Promise<{ txid: string; subscription: EventSourceSubscription }> {
+    const manifestKey = req.subscription.manifestKey
+    const txid = randomUUID()
+    await this.writeManifestEntry(
+      entityUrl,
+      manifestKey,
+      `upsert`,
+      req.manifest,
+      {
+        txid,
+      }
+    )
+
+    // The manifest is the durable source of truth. Register side effects after
+    // it is appended so failures can be repaired by manifest replay.
+    await this.wakeRegistry.unregisterByManifestKey(
+      entityUrl,
+      manifestKey,
+      this.tenantId
+    )
+    await this.wakeRegistry.register({
+      tenantId: this.tenantId,
+      subscriberUrl: entityUrl,
+      sourceUrl: req.subscription.sourceUrl,
+      condition: {
+        on: `change`,
+        collections: [`webhook_event`],
+        ops: [`insert`],
+      },
+      oneShot: false,
+      manifestKey,
+    })
+
+    return { txid, subscription: req.subscription }
+  }
+
+  async deleteEventSourceSubscription(
+    entityUrl: string,
+    req: { id: string }
+  ): Promise<{ txid: string }> {
+    const manifestKey = eventSourceSubscriptionManifestKey(req.id)
+    const txid = randomUUID()
+    await this.writeManifestEntry(entityUrl, manifestKey, `delete`, undefined, {
+      txid,
+    })
+
+    await this.wakeRegistry.unregisterByManifestKey(
+      entityUrl,
+      manifestKey,
+      this.tenantId
+    )
+
+    return { txid }
+  }
+
   // ==========================================================================
   // Wake Evaluation
   // ==========================================================================
@@ -2354,37 +2482,194 @@ export class EntityManager {
   }
 
   // ==========================================================================
-  // Kill
+  // Signals
   // ==========================================================================
 
-  async kill(entityUrl: string): Promise<{ txid: number }> {
+  async signal(entityUrl: string, req: SignalRequest): Promise<SignalResponse> {
     const entity = await this.registry.getEntity(entityUrl)
     if (!entity) {
       throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
     }
 
-    await this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId)
-    await this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId)
+    if (isTerminalEntityStatus(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeInvalidSignal,
+        `Cannot signal a ${entity.status} entity`,
+        409
+      )
+    }
 
-    const txid = await this.registry.updateStatusWithTxid(entityUrl, `stopped`)
-    if (this.entityBridgeManager) {
-      await this.entityBridgeManager.onEntityChanged(entityUrl)
+    const now = new Date()
+    const previousState = entity.status
+    const handling = this.serverHandlingForSignal(previousState, req.signal)
+    const txid =
+      handling.status === previousState
+        ? await this.registry.touchEntityWithTxid(entityUrl)
+        : await this.registry.updateStatusWithTxid(entityUrl, handling.status)
+    if (txid === null) {
+      throw new ElectricAgentsError(
+        ErrCodeInvalidSignal,
+        `Cannot signal entity because it is already terminal`,
+        409
+      )
     }
 
-    // Append entity_stopped to main/error streams and close them.
-    const stoppedEvent = entityStateSchema.entityStopped.insert({
-      key: `stopped`,
-      value: {
-        timestamp: new Date().toISOString(),
+    const key = `sig-${now.getTime()}-${randomUUID().slice(0, 8)}`
+    const signalValue: ServerSignalValue = {
+      signal: req.signal,
+      status: handling.handled ? `handled` : `unhandled`,
+      sender: SERVER_SIGNAL_SENDER,
+      timestamp: now.toISOString(),
+    }
+    if (req.reason !== undefined) signalValue.reason = req.reason
+    if (req.payload !== undefined) signalValue.payload = req.payload
+    if (handling.handled) {
+      signalValue.handled_at = now.toISOString()
+      signalValue.handled_by = SERVER_SIGNAL_SENDER
+      signalValue.outcome = handling.outcome
+      signalValue.previous_state = previousState
+      signalValue.new_state = handling.status
+    }
+
+    const signalEvent: ServerSignalEvent = {
+      type: `signal`,
+      key,
+      value: signalValue,
+      headers: {
+        operation: `insert`,
+        timestamp: now.toISOString(),
+        txid: String(txid),
       },
-    } as any)
-    const eofData = this.encodeChangeEvent(
-      stoppedEvent as Record<string, unknown>
+    }
+
+    const shouldCloseStreams = isTerminalEntityStatus(handling.status)
+    await this.appendSignalEvent(entity, signalEvent, shouldCloseStreams)
+    if (!shouldCloseStreams) {
+      await this.evaluateWakes(
+        entityUrl,
+        signalEvent as unknown as Record<string, unknown>
+      )
+    }
+
+    if (handling.unregisterWakes) {
+      await this.wakeRegistry.unregisterBySubscriber(entityUrl, this.tenantId)
+      await this.wakeRegistry.unregisterBySource(entityUrl, this.tenantId)
+    }
+
+    if (handling.status !== previousState && this.entityBridgeManager) {
+      await this.entityBridgeManager.onEntityChanged(entityUrl)
+    }
+
+    return {
+      url: entityUrl,
+      signal: req.signal,
+      previous_state: previousState,
+      new_state: handling.status,
+      created_at: now.getTime(),
+      txid,
+    }
+  }
+
+  async kill(entityUrl: string): Promise<{ txid: number }> {
+    const response = await this.signal(entityUrl, {
+      signal: `SIGKILL`,
+      reason: `Legacy kill command`,
+    })
+    return { txid: response.txid }
+  }
+
+  private serverHandlingForSignal(
+    status: ElectricAgentsEntity[`status`],
+    signal: EntitySignal
+  ): ServerSignalHandling {
+    if (signal === `SIGKILL`) {
+      return {
+        status: `killed`,
+        handled: true,
+        outcome: `transitioned`,
+        unregisterWakes: true,
+      }
+    }
+    if (signal === `SIGTERM`) {
+      if (status === `idle` || status === `paused`) {
+        return {
+          status: `stopped`,
+          handled: true,
+          outcome: `transitioned`,
+          unregisterWakes: true,
+        }
+      }
+      if (status === `running`) {
+        return {
+          status: `stopping`,
+          handled: false,
+          outcome: `transitioned`,
+          unregisterWakes: false,
+        }
+      }
+    }
+    if (status === `paused` && signal !== `SIGCONT`) {
+      return {
+        status,
+        handled: true,
+        outcome: `ignored`,
+        unregisterWakes: false,
+      }
+    }
+    if (signal === `SIGSTOP` && (status === `idle` || status === `running`)) {
+      return {
+        status: `paused`,
+        handled: status === `idle`,
+        outcome: `transitioned`,
+        unregisterWakes: false,
+      }
+    }
+    if (signal === `SIGCONT` && status === `paused`) {
+      return {
+        status: `idle`,
+        handled: false,
+        outcome: `transitioned`,
+        unregisterWakes: false,
+      }
+    }
+
+    return {
+      status,
+      handled: false,
+      outcome: `ignored`,
+      unregisterWakes: false,
+    }
+  }
+
+  private async appendSignalEvent(
+    entity: ElectricAgentsEntity,
+    signalEvent: ServerSignalEvent,
+    closeStreams: boolean
+  ): Promise<void> {
+    const signalData = this.encodeChangeEvent(
+      signalEvent as unknown as Record<string, unknown>
     )
+    if (!closeStreams) {
+      await this.streamClient.append(entity.streams.main, signalData)
+      return
+    }
 
-    for (const streamPath of [entity.streams.main, entity.streams.error]) {
+    const errorCloseEvent = {
+      type: `signal`,
+      key: signalEvent.key,
+      value: signalEvent.value,
+      headers: signalEvent.headers,
+    }
+    const errorSignalData = this.encodeChangeEvent(
+      errorCloseEvent as unknown as Record<string, unknown>
+    )
+
+    for (const [streamPath, data] of [
+      [entity.streams.main, signalData],
+      [entity.streams.error, errorSignalData],
+    ] as const) {
       try {
-        await this.streamClient.append(streamPath, eofData, { close: true })
+        await this.streamClient.append(streamPath, data, { close: true })
       } catch (err) {
         const message = err instanceof Error ? err.message : String(err)
         if (
@@ -2398,8 +2683,6 @@ export class EntityManager {
         throw err
       }
     }
-
-    return { txid }
   }
 
   // ==========================================================================
@@ -2627,8 +2910,12 @@ export class EntityManager {
     if (!entity) {
       throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
     }
-    if (entity.status === `stopped`) {
-      throw new ElectricAgentsError(ErrCodeNotRunning, `Entity is stopped`, 409)
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
     }
 
     if (req.type && entity.type) {