devpush 0.0.0-stage → 1.0.0-beta.4

package/src/server/db/sqlite-connection.ts

@@ -0,0 +1,94 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+import { mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+
+import Sqlite from 'better-sqlite3'
+import { drizzle } from 'drizzle-orm/sqlite-proxy'
+import type {
+  AsyncRemoteCallback,
+  SqliteRemoteDatabase,
+} from 'drizzle-orm/sqlite-proxy'
+
+import { sqliteParams } from './sqlite-params'
+
+export function openSqlite<TSchema extends Record<string, unknown>>(
+  path: string,
+  schema: TSchema,
+) {
+  if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true })
+  const connection = new Sqlite(path)
+  connection.pragma('foreign_keys = ON')
+  connection.pragma('journal_mode = WAL')
+  connection.pragma('synchronous = FULL')
+  connection.pragma('busy_timeout = 5000')
+
+  // Callbacks may await between statements. Serialize unrelated operations on
+  // this connection so they cannot join another request's uncommitted work.
+  const transactionScope = new AsyncLocalStorage<{ active: boolean }>()
+  let tail = Promise.resolve()
+  async function exclusive<T>(work: () => T | Promise<T>): Promise<T> {
+    if (transactionScope.getStore()?.active) return work()
+    const previous = tail
+    let release = () => {}
+    tail = new Promise<void>((resolve) => {
+      release = resolve
+    })
+    await previous
+    const scope = { active: true }
+    try {
+      return await transactionScope.run(scope, work)
+    } finally {
+      scope.active = false
+      release()
+    }
+  }
+  function query(
+    sql: string,
+    params: unknown[],
+    method: Parameters<AsyncRemoteCallback>[2],
+    batch = false,
+  ) {
+    const statement = connection.prepare(sql)
+    const values = sqliteParams(params)
+    if (method === 'run') {
+      statement.run(...values)
+      return { rows: [] }
+    }
+    const rows = statement.raw(true).all(...values) as unknown[][]
+    // Drizzle's callback type requires an array even though get() uses an
+    // undefined first row to represent no match. Keep that library contract.
+    return { rows: method === 'get' && !batch ? (rows[0] as unknown[]) : rows }
+  }
+  const database = drizzle<TSchema>(
+    (sql, params: unknown[], method) =>
+      exclusive(() => query(sql, params, method)),
+    (queries) =>
+      exclusive(() =>
+        connection
+          .transaction(() =>
+            queries.map((entry) =>
+              query(entry.sql, entry.params, entry.method, true),
+            ),
+          )
+          .immediate(),
+      ),
+    { schema },
+  )
+  const db = new Proxy(database, {
+    get(target, property) {
+      if (property === 'transaction') {
+        const transaction: SqliteRemoteDatabase<TSchema>['transaction'] = (
+          callback,
+          options,
+        ) =>
+          exclusive(() =>
+            target.transaction(callback, { ...options, behavior: 'immediate' }),
+          )
+        return transaction
+      }
+      const value: unknown = Reflect.get(target, property, target)
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+  return { db, connection, close: () => exclusive(() => connection.close()) }
+}

package/src/server/db/sqlite-files.ts

@@ -0,0 +1,27 @@
+import { existsSync, realpathSync, statSync } from 'node:fs'
+import { basename, dirname, resolve } from 'node:path'
+
+export function assertSeparateSqliteFiles(mainPath: string, logPath: string) {
+  // Each :memory: connection owns an independent database.
+  if (mainPath === ':memory:' || logPath === ':memory:') return
+  let same = canonicalPath(mainPath) === canonicalPath(logPath)
+  if (!same && existsSync(mainPath) && existsSync(logPath)) {
+    const main = statSync(mainPath)
+    const logs = statSync(logPath)
+    same = main.dev === logs.dev && main.ino === logs.ino
+  }
+  if (same) {
+    throw new Error(
+      'DEVPUSH_DATABASE_PATH and DEVPUSH_LOG_DATABASE_PATH must identify separate SQLite files.',
+    )
+  }
+}
+
+// Resolve existing parent symlinks even before the database files are created.
+function canonicalPath(path: string): string {
+  const absolute = resolve(path)
+  if (existsSync(absolute)) return realpathSync(absolute)
+  const parent = dirname(absolute)
+  if (parent === absolute) return absolute
+  return resolve(canonicalPath(parent), basename(absolute))
+}

package/src/server/db/sqlite-params.ts

@@ -0,0 +1,19 @@
+export function sqliteParams(
+  params: unknown[],
+): Array<string | number | bigint | Uint8Array | null> {
+  return params.map((value) => {
+    if (value instanceof Date) return value.getTime()
+    if (typeof value === 'boolean') return Number(value)
+    if (
+      value === null ||
+      typeof value === 'string' ||
+      typeof value === 'number' ||
+      typeof value === 'bigint' ||
+      value instanceof Uint8Array
+    )
+      return value
+    throw new Error(
+      'Unsupported SQLite bound parameter; encode values at the schema boundary.',
+    )
+  })
+}

package/src/server/db/worker-connection.ts

@@ -0,0 +1,47 @@
+import { createClient } from '@libsql/client/web'
+import type { D1Database } from '@cloudflare/workers-types'
+import { openD1 } from './d1-connection'
+import { openLibsql } from './libsql-connection'
+import { remoteDatabaseConfig } from './remote-config'
+import * as schema from './schema'
+import * as logSchema from './log-schema'
+
+type DatabaseEnvironment = Parameters<typeof remoteDatabaseConfig>[0] & {
+  DEVPUSH_DATABASE?: D1Database
+  DEVPUSH_LOG_DATABASE?: D1Database
+}
+
+export function openWorkerDatabases(env: DatabaseEnvironment) {
+  const remote = remoteDatabaseConfig(env)
+  if (remote)
+    return {
+      db: openLibsql(() => createClient(remote.main), schema).db,
+      logsDb: openLibsql(() => createClient(remote.logs), logSchema).db,
+    }
+  if (!env.DEVPUSH_DATABASE || !env.DEVPUSH_LOG_DATABASE)
+    throw new Error('Configure both D1 bindings or both remote database URLs.')
+  return {
+    db: openD1(env.DEVPUSH_DATABASE, schema),
+    logsDb: openD1(env.DEVPUSH_LOG_DATABASE, logSchema),
+  }
+}
+
+// Cloudflare's upload validation can evaluate modules before resource bindings
+// are available. Resolve the stores when application code first uses them.
+export function createWorkerDatabases(env: DatabaseEnvironment) {
+  let stores: ReturnType<typeof openWorkerDatabases> | undefined
+  const getStores = () => (stores ??= openWorkerDatabases(env))
+  function lazy<TStore extends object>(getStore: () => TStore): TStore {
+    return new Proxy({} as TStore, {
+      get(_target, property) {
+        const store = getStore()
+        const value: unknown = Reflect.get(store, property, store)
+        return typeof value === 'function' ? value.bind(store) : value
+      },
+    })
+  }
+  return {
+    db: lazy(() => getStores().db),
+    logsDb: lazy(() => getStores().logsDb),
+  }
+}

package/src/server/deleted-resources.ts

@@ -0,0 +1,133 @@
+import { sql } from 'drizzle-orm'
+import { removeOwnedAvatar } from '#devpush/server/uploads/avatar'
+import { db } from '#devpush/server/db'
+import type { AtomicStore } from '#devpush/server/db/atomic'
+
+// Keep terminal job records long enough for an agent to retry a lost acknowledgement.
+// Unresolved external resources are never treated as successfully cleaned up.
+export const deletedResourceRetentionMs = 30 * 24 * 60 * 60 * 1000
+
+const resolvedJob = sql`(status = 'succeeded' or (status = 'canceled' and type not in ('deprovision', 'purge') and (attempt = 0
+  or json_extract(case when json_valid(payload) then payload else '{}' end, '$.operatorResolution.kind') = 'confirmed-cleanup')))`
+
+export async function pruneDeletedResources({
+  store = db,
+  now = new Date(),
+  limit = 100,
+}: {
+  store?: AtomicStore
+  now?: Date
+  limit?: number
+} = {}) {
+  const cutoff = now.getTime() - deletedResourceRetentionMs
+  const batchLimit = Math.max(1, Math.min(100, Math.trunc(limit)))
+  // Keep the owning row if object storage is unavailable, so a later sweep
+  // can retry rather than losing the only reference to an uploaded avatar.
+  for (const table of ['projects', 'teams'] as const) {
+    const rows =
+      await store.all(sql`select id, avatar_path from ${sql.raw(table)}
+      where status = 'deleted' and updated_at <= ${cutoff} and avatar_path is not null
+      order by id limit ${batchLimit}`)
+    for (const row of rows) {
+      const values = Array.isArray(row)
+        ? row
+        : Object.values(row as Record<string, unknown>)
+      const [id, path] = values
+      if (typeof id !== 'string' || typeof path !== 'string')
+        throw new Error('Invalid deleted avatar row.')
+      try {
+        await removeOwnedAvatar(
+          path,
+          {
+            filePrefix: table === 'projects' ? 'project' : 'team',
+            resourceId: id,
+          },
+          { strict: true },
+        )
+        await store.run(sql`update ${sql.raw(table)} set avatar_path = null where id = ${id}
+          and status = 'deleted' and avatar_path = ${path}`)
+      } catch {
+        console.error('Unable to remove deleted resource avatar.', {
+          resourceId: id,
+          table,
+        })
+      }
+    }
+  }
+  const unprovisionedProjects = sql`select id from projects where status = 'deleted'
+    and (exists(select 1 from aliases a where a.project_id = projects.id)
+      or exists(select 1 from domains d where d.project_id = projects.id))
+    and not exists(select 1 from environments e where e.project_id = projects.id and e.status <> 'deleted')
+    and not exists(select 1 from agent_jobs j where j.type = 'deprovision' and j.status <> 'succeeded'
+      and (j.project_id = projects.id or (j.team_id = projects.team_id and j.project_id is null)))
+    order by id limit ${batchLimit}`
+  await store.batch([
+    store.run(
+      sql`delete from aliases where project_id in (${unprovisionedProjects})`,
+    ),
+    store.run(
+      sql`delete from domains where project_id in (${unprovisionedProjects})`,
+    ),
+  ])
+  const expiredJobs = sql`${resolvedJob} and coalesce(finished_at, updated_at) <= ${cutoff}`
+  const projectEligible = sql`avatar_path is null and status = 'deleted' and updated_at <= ${cutoff}
+    and not exists(select 1 from environments e where e.project_id = projects.id and e.status <> 'deleted')
+    and not exists(select 1 from backup_runs r where r.project_id = projects.id and (
+      r.status in ('queued', 'running') or r.cleanup_queued_at is not null
+      or exists(select 1 from backup_artifacts a where a.backup_run_id = r.id)))
+    and not exists(select 1 from agent_jobs j where
+      (j.project_id = projects.id or (j.team_id = projects.team_id and j.project_id is null))
+      and not (${expiredJobs}))`
+  const serverEligible = sql`status = 'deleted' and updated_at <= ${cutoff}
+    and not exists(select 1 from projects p where p.default_server_id = servers.id)
+    and not exists(select 1 from environments e where e.server_id = servers.id)
+    and not exists(select 1 from deployments d where d.server_id = servers.id)
+    and not exists(select 1 from backup_runs r where r.server_id = servers.id)
+    and not exists(select 1 from agent_jobs j where j.server_id = servers.id and not (${expiredJobs}))
+    and not exists(select 1 from agent_runtime_observations o where o.server_id = servers.id
+      and not exists(select 1 from agent_jobs j,
+        json_each(case when json_valid(j.payload) then j.payload else '{}' end, '$.runtimeIds') target
+        where j.server_id = servers.id and j.type = 'purge' and j.status = 'succeeded'
+          and j.finished_at >= o.last_seen_at and target.value = o.runtime_id))`
+  // No asynchronous reads authorize a delete. Each mutation repeats its guard in
+  // the atomic batch; foreign-key cascades then remove owned settings/credentials.
+  const projectIds = sql`select id from projects where ${projectEligible} order by id limit ${batchLimit}`
+  const serverIds = sql`select id from servers where ${serverEligible} order by id limit ${batchLimit}`
+  const teamEligible = sql`avatar_path is null and status = 'deleted' and updated_at <= ${cutoff}
+    and not exists(select 1 from projects p where p.team_id = teams.id)
+    and not exists(select 1 from servers s where s.team_id = teams.id)
+    and not exists(select 1 from agent_jobs j where j.team_id = teams.id and not (${expiredJobs}))`
+  const teamIds = sql`select id from teams where ${teamEligible} order by id limit ${batchLimit}`
+  const userEligible = sql`status = 'deleted' and updated_at <= ${cutoff}
+    and not exists(select 1 from team_members m where m.user_id = user.id)
+    and exists(select 1 from user_cleanup_jobs j where j.user_id = user.id and j.status = 'succeeded')
+    and not exists(select 1 from user_cleanup_jobs j where j.user_id = user.id and j.status <> 'succeeded')`
+  const results = await store.batch([
+    store.run(
+      sql`delete from agent_jobs where project_id in (${projectIds}) and ${expiredJobs}
+        and exists(select 1 from servers s where s.id = agent_jobs.server_id and s.status <> 'deleted')`,
+    ),
+    store.all(
+      sql`delete from projects where id in (${projectIds}) returning id`,
+    ),
+    store.run(
+      sql`delete from agent_runtime_observations where server_id in (${serverIds})`,
+    ),
+    store.run(
+      sql`delete from agent_jobs where server_id in (${serverIds}) and ${expiredJobs}`,
+    ),
+    store.all(sql`delete from servers where id in (${serverIds}) returning id`),
+    store.run(
+      sql`delete from agent_jobs where team_id in (${teamIds}) and ${expiredJobs}`,
+    ),
+    store.all(sql`delete from teams where id in (${teamIds}) returning id`),
+    store.all(sql`delete from user where id in (select id from user where ${userEligible}
+      order by id limit ${batchLimit}) returning id`),
+  ])
+  return {
+    projects: results[1].length,
+    servers: results[4].length,
+    teams: results[6].length,
+    users: results[7].length,
+  }
+}

package/src/server/deletion-recovery.ts

@@ -0,0 +1,272 @@
+import { sql } from 'drizzle-orm'
+import type { SQL } from 'drizzle-orm'
+import type { AtomicStore } from './db/atomic'
+import { runtimeCleanupInsert } from './agents/cleanup'
+
+export type RecoveryAction =
+  | 'retry-job'
+  | 'confirm-job'
+  | 'cleanup-project'
+  | 'confirm-artifact'
+export type RecoveryTarget = 'project' | 'server' | 'team' | 'job' | 'artifact'
+type Row = Record<string, unknown>
+
+export class RecoveryInputError extends Error {}
+
+export function recoveryRows(rows: unknown[]): Row[] {
+  return rows.map((row) => {
+    const encoded = Array.isArray(row)
+      ? row[0]
+      : row && typeof row === 'object' && 'record' in row
+        ? row.record
+        : undefined
+    if (typeof encoded === 'string') return JSON.parse(encoded) as Row
+    if (!row || typeof row !== 'object' || Array.isArray(row))
+      throw new RecoveryInputError('Recovery requires named database results.')
+    return row as Row
+  })
+}
+
+export function listDeletedResourcesQuery() {
+  return jsonRows(
+    sql`select 'project' as target, id, name, updated_at from projects where status = 'deleted'
+    union all select 'team', id, name, updated_at from teams where status = 'deleted'
+    union all select 'server', id, name, updated_at from servers where status = 'deleted'
+    order by updated_at, target, id limit 101`,
+    ['target', 'id', 'name', 'updated_at'],
+  )
+}
+
+export function recoveryTargetQuery(target: RecoveryTarget, id: string) {
+  const table = {
+    project: 'projects',
+    server: 'servers',
+    team: 'teams',
+    job: 'agent_jobs',
+    artifact: 'backup_artifacts',
+  }[target]
+  const fields =
+    target === 'job'
+      ? [
+          'id',
+          'type',
+          'status',
+          'server_id',
+          'team_id',
+          'project_id',
+          'payload',
+          'attempt',
+          'max_attempts',
+          'updated_at',
+          'claimed_at',
+          'available_at',
+        ]
+      : target === 'artifact'
+        ? ['id', 'backup_run_id', 'storage', 'object_key', 'checksum_sha256']
+        : target === 'team'
+          ? ['id', 'status', 'updated_at']
+          : ['id', 'status', 'updated_at', 'team_id']
+  return jsonRows(sql`select * from ${sql.raw(table)} where id = ${id}`, fields)
+}
+
+function jsonRows(query: SQL, fields: string[]) {
+  return sql`select json_object(${sql.join(
+    fields.flatMap((field) => [sql`${field}`, sql.raw(field)]),
+    sql`, `,
+  )}) as record from (${query})`
+}
+
+// Whitelist output: job payloads and resource rows can contain credentials.
+export async function inspectDeletion(
+  store: AtomicStore,
+  target: RecoveryTarget,
+  row: Row,
+) {
+  const id = String(row.id)
+  const projectIds =
+    target === 'project'
+      ? sql`select id from projects where id = ${id}`
+      : target === 'team'
+        ? sql`select id from projects where team_id = ${id}`
+        : target === 'server'
+          ? sql`select id from projects where default_server_id = ${id}
+      union select project_id from environments where server_id = ${id}
+      union select project_id from deployments where server_id = ${id}`
+          : sql`select null where 0`
+  const jobs = await store.all(
+    jsonRows(
+      sql`select id, type, status, attempt, max_attempts, server_id, project_id,
+    available_at, claimed_at, updated_at, json_extract(payload, '$.runtimeIds') as runtime_ids,
+    json_extract(payload, '$.backupRunIds') as backup_run_ids, json_extract(payload, '$.localObjectKeys') as local_object_keys,
+    json_extract(payload, '$.remoteObjectKeys') as remote_object_keys, json_extract(payload, '$.operatorResolution') as resolution from agent_jobs where
+    ${target === 'job' ? sql`id = ${id}` : target === 'server' ? sql`server_id = ${id}` : target === 'team' ? sql`team_id = ${id} or project_id in (${projectIds})` : sql`project_id in (${projectIds}) or (project_id is null and team_id = ${row.team_id ?? null})`}
+    order by id limit 101`,
+      [
+        'id',
+        'type',
+        'status',
+        'attempt',
+        'max_attempts',
+        'server_id',
+        'project_id',
+        'available_at',
+        'claimed_at',
+        'updated_at',
+        'runtime_ids',
+        'backup_run_ids',
+        'local_object_keys',
+        'remote_object_keys',
+        'resolution',
+      ],
+    ),
+  )
+  const artifacts = await store.all(
+    jsonRows(
+      sql`select a.id, a.backup_run_id, a.storage, a.object_key, a.checksum_sha256,
+    a.size_bytes, r.project_id, r.server_id, json_extract(r.destination, '$.bucket') as bucket,
+    json_extract(r.destination, '$.storageId') as storage_id from backup_artifacts a join backup_runs r on r.id = a.backup_run_id
+    where ${target === 'artifact' ? sql`a.id = ${id}` : target === 'server' ? sql`r.server_id = ${id}` : sql`r.project_id in (${projectIds})`}
+    order by a.id limit 101`,
+      [
+        'id',
+        'backup_run_id',
+        'storage',
+        'object_key',
+        'checksum_sha256',
+        'size_bytes',
+        'project_id',
+        'server_id',
+        'bucket',
+        'storage_id',
+      ],
+    ),
+  )
+  const environments = await store.all(
+    jsonRows(
+      sql`select id, project_id, runtime_id, status from environments
+    where project_id in (${projectIds}) order by id limit 101`,
+      ['id', 'project_id', 'runtime_id', 'status'],
+    ),
+  )
+  return {
+    target,
+    id,
+    status: row.status,
+    jobs: recoveryRows(jobs),
+    artifacts: recoveryRows(artifacts),
+    environments: recoveryRows(environments),
+    note: 'Each list is limited to 101 rows. Inspect individual IDs; never treat this as a complete archive inventory.',
+  }
+}
+
+export function deletionRecoveryPlan(input: {
+  action: RecoveryAction
+  row: Row
+  operator: string
+  reason: string
+  now: Date
+}): [SQL, ...SQL[]] {
+  const { action, row, now } = input
+  if (!input.operator.trim() || !input.reason.trim())
+    throw new RecoveryInputError('Operator and evidence/reason are required.')
+  const id = String(row.id),
+    time = now.getTime()
+  const resolution = JSON.stringify({
+    kind: 'confirmed-cleanup',
+    operator: input.operator,
+    reason: input.reason,
+    at: time,
+  })
+  const statements: SQL[] = []
+  let guard: SQL
+  let team: SQL
+  if (action === 'cleanup-project') {
+    if (row.status !== 'deleted')
+      throw new RecoveryInputError('Only deleted projects can be cleaned up.')
+    guard = sql`exists(select 1 from projects where id = ${id} and status = 'deleted' and updated_at = ${row.updated_at}
+      and exists(select 1 from environments where project_id = ${id} and status <> 'deleted'))`
+    team = sql`select team_id from projects where id = ${id}`
+    statements.push(
+      runtimeCleanupInsert({
+        projectIds: sql`select id from projects where id = ${id}`,
+        projectId: id,
+        guard,
+        reason: `Operator ${input.operator}: ${input.reason}`,
+        now,
+      }),
+    )
+    statements.push(sql`update environments set status = 'deleted', is_production = 0, updated_at = ${time}
+      where project_id = ${id} and ${guard}`)
+  } else if (action === 'confirm-artifact') {
+    guard = sql`exists(select 1 from backup_artifacts a join backup_runs r on r.id = a.backup_run_id
+      join projects p on p.id = r.project_id where a.id = ${id} and p.status = 'deleted'
+      and a.object_key = ${row.object_key} and a.checksum_sha256 = ${row.checksum_sha256}
+      and a.backup_run_id = ${row.backup_run_id} and a.storage = ${row.storage}
+      and r.status not in ('queued','running') and r.cleanup_queued_at is null)`
+    team = sql`select p.team_id from projects p join backup_runs r on r.project_id = p.id where r.id = ${row.backup_run_id}`
+    statements.push(sql`update backup_runs set manifest = json_set(coalesce(manifest, '{}'), '$.operatorRecovery', json(${resolution})),
+      updated_at = ${time} where id = ${row.backup_run_id} and ${guard}`)
+    statements.push(
+      sql`delete from backup_artifacts where id = ${id} and ${guard}`,
+    )
+  } else {
+    const cleanup = row.type === 'deprovision' || row.type === 'purge'
+    if (row.status === 'succeeded')
+      throw new RecoveryInputError('This job has already completed.')
+    const staleCleanup =
+      cleanup &&
+      row.status === 'running' &&
+      (row.claimed_at === null ||
+        Number(row.claimed_at) < time - 60 * 60 * 1000)
+    if (row.status === 'running' && !staleCleanup)
+      throw new RecoveryInputError(
+        'Wait for the running job to finish or become stale before recovery.',
+      )
+    if (action === 'retry-job' && !cleanup)
+      throw new RecoveryInputError(
+        'Only container cleanup jobs can be retried; do not replay deployments, backups or restores.',
+      )
+    if (
+      action === 'retry-job' &&
+      row.status !== 'failed' &&
+      row.status !== 'abandoned' &&
+      row.status !== 'canceled' &&
+      !staleCleanup
+    )
+      throw new RecoveryInputError(
+        'Only terminal unsuccessful cleanup jobs can be retried.',
+      )
+    guard = sql`exists(select 1 from agent_jobs j where j.id = ${id} and j.status = ${row.status}
+      and j.type = ${row.type} and j.server_id = ${row.server_id}
+      and j.project_id is ${row.project_id ?? null} and j.team_id is ${row.team_id ?? null}
+      and j.available_at = ${row.available_at} and j.attempt = ${row.attempt} and j.claimed_at is ${row.claimed_at ?? null} and j.updated_at = ${row.updated_at} and j.payload = ${row.payload}
+      and (exists(select 1 from projects p where p.id = j.project_id and p.status = 'deleted')
+        or exists(select 1 from servers s where s.id = j.server_id and s.status = 'deleted')
+        or exists(select 1 from teams t where t.id = j.team_id and t.status = 'deleted')))`
+    team = sql`select coalesce(j.team_id, p.team_id, s.team_id) from agent_jobs j
+      left join projects p on p.id = j.project_id join servers s on s.id = j.server_id where j.id = ${id}`
+    if (action === 'retry-job') {
+      statements.push(sql`update agent_jobs set status = 'queued', claimed_at = null, finished_at = null,
+        available_at = max(available_at, ${time}), updated_at = ${time}, max_attempts = max(max_attempts, attempt + 10),
+        error_message = null, payload = json_set(payload, '$.operatorRetry', json(${resolution})) where id = ${id} and ${guard}`)
+    } else {
+      if (row.type === 'backup_cleanup')
+        statements.push(sql`update backup_runs set cleanup_queued_at = null, cleanup_error_message = null, updated_at = ${time}
+          where project_id = ${row.project_id} and server_id = ${row.server_id}
+          and id in (select value from json_each(${String(row.payload)}, '$.backupRunIds')) and ${guard}`)
+      if (row.type === 'backup')
+        statements.push(sql`update backup_runs set status = 'canceled', finished_at = ${time}, updated_at = ${time}
+          where id = json_extract(${String(row.payload)}, '$.backupRunId') and project_id = ${row.project_id}
+          and server_id = ${row.server_id} and status in ('queued','running') and ${guard}`)
+      // A confirmed cleanup is a real success; failed executions remain canceled,
+      // with an explicit operator resolution instead of a fabricated backup success.
+      statements.push(sql`update agent_jobs set status = ${cleanup ? 'succeeded' : 'canceled'},
+        finished_at = ${time}, updated_at = ${time}, error_message = ${`Operator resolution: ${input.reason}`},
+        payload = json_set(payload, '$.operatorResolution', json(${resolution})) where id = ${id} and ${guard}`)
+    }
+  }
+  const audit = sql`insert into api_audit (id, team_id, user_id, credential_id, request_id, operation, resource_id, outcome, created_at)
+    select ${crypto.randomUUID()}, coalesce((${team}), 'installation'), ${input.operator}, 'operator-cli',
+      ${crypto.randomUUID()}, ${`deletion.${action}`}, ${id}, 'succeeded', ${time} where ${guard} returning id`
+  return [audit, ...statements]
+}

package/src/server/email/layout.tsx

@@ -0,0 +1,85 @@
+import type { ReactNode } from 'react'
+import {
+  Body,
+  Container,
+  Head,
+  Heading,
+  Html,
+  Img,
+  Link,
+  Preview,
+  Section,
+  Tailwind,
+  Text,
+} from '@react-email/components'
+import { config } from '#devpush/server/config'
+import { emailTheme } from './theme'
+
+export function EmailLayout({
+  title,
+  preview,
+  to,
+  children,
+  allowIgnore = true,
+}: {
+  title: string
+  preview: string
+  to: string
+  children: ReactNode
+  allowIgnore?: boolean
+}) {
+  return (
+    <Html>
+      <Head />
+      <Preview>{preview}</Preview>
+      <Tailwind>
+        <Body
+          className="my-auto mx-auto px-2 font-sans antialiased"
+          style={{
+            backgroundColor: emailTheme.background,
+            color: emailTheme.foreground,
+          }}
+        >
+          <Container className="my-[40px] mx-auto max-w-[465px] p-[20px]">
+            <Section className="mt-[24px]">
+              <Img
+                alt="devpush"
+                className="my-0 mx-auto"
+                width="32"
+                height="32"
+                src={new URL(
+                  '/logo-email.png',
+                  config.BETTER_AUTH_URL,
+                ).toString()}
+              />
+            </Section>
+            <Heading
+              className="my-[30px] mx-0 p-0 text-center text-[24px] font-semibold tracking-tight"
+              style={{ color: emailTheme.foreground }}
+            >
+              {title}
+            </Heading>
+            {children}
+            <Text
+              className="mt-[36px] text-[14px] leading-[24px]"
+              style={{ color: emailTheme.mutedForeground }}
+            >
+              This email was intended for{' '}
+              <Link
+                className="underline"
+                href={`mailto:${to}`}
+                style={{ color: emailTheme.mutedLink }}
+              >
+                {to}
+              </Link>
+              .
+              {allowIgnore
+                ? ' If you think this is a mistake, you can safely ignore this email.'
+                : ''}
+            </Text>
+          </Container>
+        </Body>
+      </Tailwind>
+    </Html>
+  )
+}