@electric-ax/agents-server 0.4.11 → 0.4.13

package/src/entity-manager.ts

@@ -1,4 +1,4 @@
-import { randomUUID } from 'node:crypto'
+import { createHash, randomUUID } from 'node:crypto'
 import fastq from 'fastq'
 import {
   assertTags,
@@ -110,6 +110,48 @@ type ServerSignalEvent = {
     txid: string
   }
 }
+type AttachmentSubjectType = `inbox` | `run` | `text` | `tool_call` | `context`
+type AttachmentRole = `input` | `output`
+
+export interface CreateAttachmentRequest {
+  id?: string
+  bytes: Uint8Array
+  mimeType: string
+  filename?: string
+  subject: {
+    type: AttachmentSubjectType
+    key: string
+  }
+  role?: AttachmentRole
+  createdBy?: string
+  meta?: Record<string, unknown>
+}
+
+export interface ReadAttachmentResult {
+  attachment: ManifestAttachmentEntry
+  bytes: Uint8Array
+}
+
+type ManifestAttachmentEntry = {
+  key: string
+  kind: `attachment`
+  id: string
+  streamPath: string
+  status: `pending` | `complete` | `failed`
+  subject: {
+    type: AttachmentSubjectType
+    key: string
+  }
+  role: AttachmentRole
+  mimeType: string
+  filename?: string
+  byteLength?: number
+  sha256?: string
+  createdAt: string
+  createdBy?: string
+  error?: string
+  meta?: Record<string, unknown>
+}
 
 function createInitialQueuePosition(date: Date): string {
   return `${String(date.getTime()).padStart(16, `0`)}:a0`
@@ -142,11 +184,98 @@ const DEFAULT_FORK_WAIT_TIMEOUT_MS = 120_000
 const DEFAULT_FORK_WAIT_POLL_MS = 250
 
 const SERVER_SIGNAL_SENDER = `/_electric/server`
+const DEFAULT_MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024
 
 function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
 
+function maxAttachmentBytes(): number {
+  const configured = Number(process.env.ELECTRIC_AGENTS_MAX_ATTACHMENT_BYTES)
+  return Number.isFinite(configured) && configured > 0
+    ? Math.floor(configured)
+    : DEFAULT_MAX_ATTACHMENT_BYTES
+}
+
+function manifestAttachmentKey(id: string): string {
+  return `attachment:${id}`
+}
+
+function getEntityAttachmentStreamPath(
+  entityUrl: string,
+  attachmentId: string
+): string {
+  return `${entityUrl.replace(/\/+$/, ``)}/attachments/${attachmentId}`
+}
+
+function isStreamCreateConflict(error: unknown): boolean {
+  return (
+    !!error &&
+    typeof error === `object` &&
+    ((`status` in error && error.status === 409) ||
+      (`code` in error && error.code === `CONFLICT_SEQ`))
+  )
+}
+
+function assertCanonicalAttachmentStreamPath(
+  entityUrl: string,
+  attachment: ManifestAttachmentEntry
+): void {
+  const expected = getEntityAttachmentStreamPath(entityUrl, attachment.id)
+  if (attachment.streamPath === expected) return
+  throw new ElectricAgentsError(
+    ErrCodeInvalidRequest,
+    `Attachment stream path does not match its entity and id`,
+    409
+  )
+}
+
+function validateAttachmentId(id: string): void {
+  if (!id || id.includes(`/`) || id.startsWith(`.`)) {
+    throw new ElectricAgentsError(
+      ErrCodeInvalidRequest,
+      `attachment id must not be empty, start with ".", or contain forward slashes`,
+      400
+    )
+  }
+}
+
+function validateAttachmentSubject(
+  subject: CreateAttachmentRequest[`subject`]
+): void {
+  if (!subject.key) {
+    throw new ElectricAgentsError(
+      ErrCodeInvalidRequest,
+      `attachment subject key is required`,
+      400
+    )
+  }
+  if (
+    subject.type !== `inbox` &&
+    subject.type !== `run` &&
+    subject.type !== `text` &&
+    subject.type !== `tool_call` &&
+    subject.type !== `context`
+  ) {
+    throw new ElectricAgentsError(
+      ErrCodeInvalidRequest,
+      `invalid attachment subject type`,
+      400
+    )
+  }
+}
+
+function concatByteMessages(messages: Array<{ data: Uint8Array }>): Uint8Array {
+  const total = messages.reduce((sum, message) => sum + message.data.length, 0)
+  const bytes = new Uint8Array(total)
+  let offset = 0
+  for (const message of messages) {
+    bytes.set(message.data, offset)
+    offset += message.data.length
+  }
+  return bytes
+}
+
 function omitUndefined<T extends Record<string, unknown>>(value: T): T {
   return Object.fromEntries(
     Object.entries(value).filter(([, entry]) => entry !== undefined)
@@ -811,6 +940,26 @@ export class EntityManager {
           createdStreams.push(forkPath)
         }
 
+        for (const plan of entityPlans) {
+          const manifests =
+            snapshot.manifestsByEntity.get(plan.source.url) ?? new Map()
+          for (const manifest of manifests.values()) {
+            if (
+              manifest.kind !== `attachment` ||
+              typeof manifest.streamPath !== `string` ||
+              typeof manifest.id !== `string`
+            ) {
+              continue
+            }
+            const forkPath = getEntityAttachmentStreamPath(
+              plan.fork.url,
+              manifest.id
+            )
+            await this.streamClient.fork(forkPath, manifest.streamPath)
+            createdStreams.push(forkPath)
+          }
+        }
+
         for (const plan of entityPlans) {
           const reconciliation = this.buildForkReconciliation(
             plan,
@@ -1474,6 +1623,21 @@ export class EntityManager {
       return { key: String(next.key), value: next, changed: true }
     }
 
+    if (
+      next.kind === `attachment` &&
+      typeof next.streamPath === `string` &&
+      typeof next.id === `string`
+    ) {
+      for (const [sourceUrl, forkUrl] of entityUrlMap) {
+        const prefix = `${sourceUrl}/attachments/`
+        if (!next.streamPath.startsWith(prefix)) {
+          continue
+        }
+        next.streamPath = getEntityAttachmentStreamPath(forkUrl, next.id)
+        return { key, value: next, changed: true }
+      }
+    }
+
     if (next.kind === `schedule` && next.scheduleType === `future_send`) {
       let changed = false
       if (typeof next.targetUrl === `string`) {
@@ -1833,6 +1997,192 @@ export class EntityManager {
     )
   }
 
+  // ==========================================================================
+  // Attachments
+  // ==========================================================================
+
+  isAttachmentStreamPath(path: string): boolean {
+    return /^\/[^/]+\/[^/]+\/attachments\/[^/]+$/.test(path)
+  }
+
+  async createAttachment(
+    entityUrl: string,
+    req: CreateAttachmentRequest
+  ): Promise<{ txid: string; attachment: ManifestAttachmentEntry }> {
+    const entity = await this.registry.getEntity(entityUrl)
+    if (!entity) {
+      throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
+    }
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
+    }
+    if (this.isForkWorkLockedEntity(entityUrl)) {
+      this.assertEntityNotForkWorkLocked(entityUrl)
+    }
+
+    const id = req.id ?? randomUUID()
+    validateAttachmentId(id)
+    validateAttachmentSubject(req.subject)
+
+    const limit = maxAttachmentBytes()
+    if (req.bytes.length > limit) {
+      throw new ElectricAgentsError(
+        ErrCodeInvalidRequest,
+        `Attachment exceeds maximum size of ${limit} bytes`,
+        413
+      )
+    }
+
+    const mimeType = req.mimeType.trim() || `application/octet-stream`
+    const streamPath = getEntityAttachmentStreamPath(entityUrl, id)
+    const manifestKey = manifestAttachmentKey(id)
+    const txid = randomUUID()
+    const now = new Date().toISOString()
+    const sha256 = createHash(`sha256`).update(req.bytes).digest(`hex`)
+    const attachment: ManifestAttachmentEntry = {
+      key: manifestKey,
+      kind: `attachment`,
+      id,
+      streamPath,
+      status: `complete`,
+      subject: req.subject,
+      role: req.role ?? `input`,
+      mimeType,
+      ...(req.filename ? { filename: req.filename } : {}),
+      byteLength: req.bytes.length,
+      sha256,
+      createdAt: now,
+      ...(req.createdBy ? { createdBy: req.createdBy } : {}),
+      ...(req.meta
+        ? { meta: req.meta as ManifestAttachmentEntry[`meta`] }
+        : {}),
+    }
+
+    let streamCreated = false
+    try {
+      await this.streamClient.create(streamPath, {
+        contentType: mimeType,
+        body: req.bytes,
+        closed: true,
+      })
+      streamCreated = true
+      await this.writeManifestEntry(
+        entityUrl,
+        manifestKey,
+        `upsert`,
+        attachment as unknown as Record<string, unknown>,
+        { txid }
+      )
+    } catch (error) {
+      if (streamCreated) {
+        await this.streamClient.delete(streamPath).catch(() => undefined)
+      }
+      if (!streamCreated && isStreamCreateConflict(error)) {
+        throw new ElectricAgentsError(
+          ErrCodeInvalidRequest,
+          `Attachment already exists at id "${id}"`,
+          409
+        )
+      }
+      throw error
+    }
+
+    return { txid, attachment }
+  }
+
+  async getAttachment(
+    entityUrl: string,
+    id: string
+  ): Promise<ManifestAttachmentEntry | null> {
+    validateAttachmentId(id)
+    const entity = await this.registry.getEntity(entityUrl)
+    if (!entity) {
+      throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
+    }
+    const events = await this.streamClient.readJson<Record<string, unknown>>(
+      entity.streams.main
+    )
+    const manifest = this.reduceStateRows(events, `manifest`).get(
+      manifestAttachmentKey(id)
+    )
+    if (!manifest || manifest.kind !== `attachment`) {
+      return null
+    }
+    return manifest as unknown as ManifestAttachmentEntry
+  }
+
+  async readAttachment(
+    entityUrl: string,
+    id: string
+  ): Promise<ReadAttachmentResult> {
+    const attachment = await this.getAttachment(entityUrl, id)
+    if (!attachment) {
+      throw new ElectricAgentsError(
+        ErrCodeNotFound,
+        `Attachment not found`,
+        404
+      )
+    }
+    if (attachment.status !== `complete`) {
+      throw new ElectricAgentsError(
+        ErrCodeInvalidRequest,
+        `Attachment is not complete`,
+        409
+      )
+    }
+    assertCanonicalAttachmentStreamPath(entityUrl, attachment)
+
+    const result = await this.streamClient.read(attachment.streamPath)
+    return {
+      attachment,
+      bytes: concatByteMessages(result.messages),
+    }
+  }
+
+  async deleteAttachment(
+    entityUrl: string,
+    id: string
+  ): Promise<{ txid: string }> {
+    const entity = await this.registry.getEntity(entityUrl)
+    if (!entity) {
+      throw new ElectricAgentsError(ErrCodeNotFound, `Entity not found`, 404)
+    }
+    if (rejectsNormalWrites(entity.status)) {
+      throw new ElectricAgentsError(
+        ErrCodeNotRunning,
+        `Entity is not accepting writes`,
+        409
+      )
+    }
+    if (this.isForkWorkLockedEntity(entityUrl)) {
+      this.assertEntityNotForkWorkLocked(entityUrl)
+    }
+
+    const attachment = await this.getAttachment(entityUrl, id)
+    if (!attachment) {
+      throw new ElectricAgentsError(
+        ErrCodeNotFound,
+        `Attachment not found`,
+        404
+      )
+    }
+    assertCanonicalAttachmentStreamPath(entityUrl, attachment)
+    const txid = randomUUID()
+    await this.writeManifestEntry(
+      entityUrl,
+      manifestAttachmentKey(id),
+      `delete`,
+      undefined,
+      { txid }
+    )
+    await this.streamClient.delete(attachment.streamPath).catch(() => undefined)
+    return { txid }
+  }
+
   // ==========================================================================
   // Tag Updates
   // ==========================================================================

package/src/routing/durable-streams-router.ts

@@ -608,8 +608,11 @@ async function proxyPassThrough(
   request: IRequest,
   ctx: TenantContext
 ): Promise<Response> {
-  const upstream = await forwardToDurableStreams(ctx, request)
   const streamPath = new URL(request.url).pathname
+  if (ctx.entityManager?.isAttachmentStreamPath(streamPath)) {
+    return new Response(null, { status: 404 })
+  }
+  const upstream = await forwardToDurableStreams(ctx, request)
   const method = request.method.toUpperCase()
   const endTrackedRead =
     method === `GET`

package/src/routing/entities-router.ts

@@ -25,6 +25,7 @@ import {
   shouldLinkDispatchBeforeInitialMessage,
   unlinkEntityDispatchSubscription,
 } from './dispatch-policy.js'
+import { ElectricAgentsError } from '../entity-manager.js'
 import { routeBody, withSchema } from './schema.js'
 import type { ElectricAgentsEntity } from '../electric-agents-types.js'
 import type { JsonRouteRequest } from './schema.js'
@@ -203,6 +204,25 @@ type ScheduleBody = Static<typeof scheduleBodySchema>
 type EventSourceSubscriptionBody = Static<
   typeof eventSourceSubscriptionBodySchema
 >
+type AttachmentSubjectType = `inbox` | `run` | `text` | `tool_call` | `context`
+type AttachmentRole = `input` | `output`
+type ParsedAttachmentForm = {
+  id?: string
+  bytes: Uint8Array
+  mimeType: string
+  filename?: string
+  subject: { type: AttachmentSubjectType; key: string }
+  role?: AttachmentRole
+  meta?: Record<string, unknown>
+}
+
+const attachmentSubjectTypes = new Set<AttachmentSubjectType>([
+  `inbox`,
+  `run`,
+  `text`,
+  `tool_call`,
+  `context`,
+])
 
 export const entitiesRouter: EntitiesRoutes = Router<
   AgentsRouteRequest,
@@ -234,6 +254,21 @@ entitiesRouter.post(
   withSchema(sendBodySchema),
   sendEntity
 )
+entitiesRouter.post(
+  `/:type/:instanceId/attachments`,
+  withExistingEntity,
+  createAttachment
+)
+entitiesRouter.get(
+  `/:type/:instanceId/attachments/:attachmentId`,
+  withExistingEntity,
+  readAttachment
+)
+entitiesRouter.delete(
+  `/:type/:instanceId/attachments/:attachmentId`,
+  withExistingEntity,
+  deleteAttachment
+)
 entitiesRouter.patch(
   `/:type/:instanceId/inbox/:messageKey`,
   withExistingEntity,
@@ -318,6 +353,131 @@ function requireExistingEntityRoute(
   return request.entityRoute
 }
 
+function invalidAttachmentRequest(message: string): never {
+  throw new ElectricAgentsError(ErrCodeInvalidRequest, message, 400)
+}
+
+function formString(form: FormData, key: string): string | undefined {
+  const value = form.get(key)
+  if (typeof value !== `string`) return undefined
+  const trimmed = value.trim()
+  return trimmed || undefined
+}
+
+function parseJsonFormField<T>(form: FormData, key: string): T | undefined {
+  const raw = formString(form, key)
+  if (!raw) return undefined
+  try {
+    return JSON.parse(raw) as T
+  } catch {
+    invalidAttachmentRequest(`Invalid JSON field: ${key}`)
+  }
+}
+
+function parseAttachmentSubject(
+  form: FormData
+): ParsedAttachmentForm[`subject`] {
+  const explicit = parseJsonFormField<unknown>(form, `subject`)
+  if (explicit !== undefined) {
+    if (!explicit || typeof explicit !== `object` || Array.isArray(explicit)) {
+      invalidAttachmentRequest(`attachment subject must be an object`)
+    }
+    const subject = explicit as Record<string, unknown>
+    const type = subject.type
+    const key = subject.key
+    if (typeof type !== `string` || typeof key !== `string`) {
+      invalidAttachmentRequest(`attachment subject requires type and key`)
+    }
+    if (!attachmentSubjectTypes.has(type as AttachmentSubjectType)) {
+      invalidAttachmentRequest(`invalid attachment subject type`)
+    }
+    return { type: type as AttachmentSubjectType, key }
+  }
+
+  const type = formString(form, `subjectType`)
+  const key = formString(form, `subjectKey`)
+  if (!type || !key) {
+    invalidAttachmentRequest(`attachment subject is required`)
+  }
+  if (!attachmentSubjectTypes.has(type as AttachmentSubjectType)) {
+    invalidAttachmentRequest(`invalid attachment subject type`)
+  }
+  return { type: type as AttachmentSubjectType, key }
+}
+
+type UploadedFormFile = {
+  arrayBuffer: () => Promise<ArrayBuffer>
+  type?: string
+  name?: string
+}
+
+function getUploadedFormFile(
+  value: FormDataEntryValue | null
+): UploadedFormFile | null {
+  if (
+    value !== null &&
+    typeof value === `object` &&
+    `arrayBuffer` in value &&
+    typeof (value as { arrayBuffer?: unknown }).arrayBuffer === `function`
+  ) {
+    return value as unknown as UploadedFormFile
+  }
+  return null
+}
+
+async function parseAttachmentForm(
+  request: AgentsRouteRequest
+): Promise<ParsedAttachmentForm> {
+  const contentType = request.headers.get(`content-type`)?.toLowerCase() ?? ``
+  if (!contentType.includes(`multipart/form-data`)) {
+    invalidAttachmentRequest(`Attachment uploads must use multipart/form-data`)
+  }
+
+  let form: FormData
+  try {
+    form = await (request as unknown as Request).formData()
+  } catch {
+    invalidAttachmentRequest(`Invalid multipart form data`)
+  }
+
+  const file = getUploadedFormFile(form.get(`file`))
+  if (!file) {
+    invalidAttachmentRequest(`Missing file field`)
+  }
+
+  const role = formString(form, `role`)
+  if (role !== undefined && role !== `input` && role !== `output`) {
+    invalidAttachmentRequest(`invalid attachment role`)
+  }
+
+  const fileName =
+    formString(form, `filename`) ??
+    (typeof file.name === `string` ? file.name : undefined)
+  const mimeType =
+    formString(form, `mimeType`) ||
+    (typeof file.type === `string` ? file.type : undefined) ||
+    `application/octet-stream`
+  const meta = parseJsonFormField<Record<string, unknown>>(form, `meta`)
+  if (meta !== undefined && (typeof meta !== `object` || Array.isArray(meta))) {
+    invalidAttachmentRequest(`attachment meta must be an object`)
+  }
+
+  return {
+    id: formString(form, `id`),
+    bytes: new Uint8Array(await file.arrayBuffer()),
+    mimeType,
+    filename: fileName,
+    subject: parseAttachmentSubject(form),
+    role,
+    meta,
+  }
+}
+
+function contentDisposition(filename: string): string {
+  const fallback = filename.replace(/["\\\r\n]/g, `_`)
+  return `attachment; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(filename)}`
+}
+
 function rejectPrincipalEntityMutation(
   request: AgentsRouteRequest,
   action: string
@@ -695,6 +855,72 @@ async function sendEntity(
   return status(204)
 }
 
+async function createAttachment(
+  request: AgentsRouteRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const principalMutationError = rejectPrincipalEntityMutation(
+    request,
+    `given attachments`
+  )
+  if (principalMutationError) return principalMutationError
+
+  const { entityUrl } = requireExistingEntityRoute(request)
+  const form = await parseAttachmentForm(request)
+  const result = await ctx.entityManager.createAttachment(entityUrl, {
+    id: form.id,
+    bytes: form.bytes,
+    mimeType: form.mimeType,
+    filename: form.filename,
+    subject: form.subject,
+    role: form.role,
+    createdBy: ctx.principal.url,
+    meta: form.meta,
+  })
+  return json(result, { status: 201 })
+}
+
+async function readAttachment(
+  request: AgentsRouteRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const { entityUrl } = requireExistingEntityRoute(request)
+  const result = await ctx.entityManager.readAttachment(
+    entityUrl,
+    decodeURIComponent(request.params.attachmentId)
+  )
+  const headers = new Headers({
+    'content-type': result.attachment.mimeType,
+    'content-length': String(result.bytes.length),
+    'cache-control': `private, max-age=31536000, immutable`,
+  })
+  if (result.attachment.filename) {
+    headers.set(
+      `content-disposition`,
+      contentDisposition(result.attachment.filename)
+    )
+  }
+  return new Response(result.bytes, { status: 200, headers })
+}
+
+async function deleteAttachment(
+  request: AgentsRouteRequest,
+  ctx: TenantContext
+): Promise<Response> {
+  const principalMutationError = rejectPrincipalEntityMutation(
+    request,
+    `stripped of attachments`
+  )
+  if (principalMutationError) return principalMutationError
+
+  const { entityUrl } = requireExistingEntityRoute(request)
+  const result = await ctx.entityManager.deleteAttachment(
+    entityUrl,
+    decodeURIComponent(request.params.attachmentId)
+  )
+  return json(result)
+}
+
 async function updateInboxMessage(
   request: AgentsRouteRequest,
   ctx: TenantContext

package/src/routing/stream-append.ts

@@ -91,6 +91,9 @@ async function handleStreamAppend(
   const { manager } = runtime
   const entity = await manager.registry.getEntityByStream(path)
   const isSharedState = path.startsWith(`/_electric/shared-state/`)
+  if (!entity && manager.isAttachmentStreamPath(path)) {
+    return apiError(401, ErrCodeUnauthorized, `Invalid write token`)
+  }
   if (!entity && !isSharedState) {
     return undefined
   }