devpush 0.0.0-stage → 1.0.0-beta.3

package/src/server/agents/cleanup.ts

@@ -0,0 +1,52 @@
+import { sql } from 'drizzle-orm'
+import type { SQL } from 'drizzle-orm'
+
+export type DeprovisionJobPayload = {
+  schemaVersion: 1
+  runtimeIds: string[]
+  removeVolumes: boolean
+  reason: string
+}
+
+export const cleanupVolumeRetentionMs = 7 * 24 * 60 * 60 * 1000
+
+// Derive targets inside the caller's deletion batch. One noisy project cannot
+// produce an unbounded agent payload; every target is kept in a chunk of 200.
+export function runtimeCleanupInsert(input: {
+  projectIds: SQL
+  guard: SQL
+  projectId?: string
+  reason: string
+  now: Date
+}) {
+  const now = input.now.getTime()
+  return sql`with targets as materialized (
+    select p.team_id, trim(coalesce(e.server_id, p.default_server_id)) as server_id,
+      trim(e.runtime_id) as runtime_id
+    from environments e join projects p on p.id = e.project_id
+    where p.id in (${input.projectIds}) and ${input.guard}
+    union
+    select p.team_id, trim(d.server_id), trim(d.runtime_id)
+    from deployments d join environments e on e.id = d.environment_id
+      join projects p on p.id = e.project_id
+    where p.id in (${input.projectIds}) and ${input.guard}
+  ), numbered as (
+    select *, (row_number() over (partition by team_id, server_id order by runtime_id) - 1) / 200 as chunk
+    from targets where server_id is not null and server_id <> '' and runtime_id <> ''
+  ), groups as (
+    select team_id, server_id, chunk, json_group_array(runtime_id) as runtime_ids
+    from (select * from numbered order by team_id, server_id, chunk, runtime_id)
+    group by team_id, server_id, chunk
+  ), kinds as (select 'deprovision' as type union all select 'purge')
+  insert into agent_jobs
+    (id, type, status, server_id, team_id, project_id, payload, max_attempts, available_at, created_at, updated_at)
+  select lower(hex(randomblob(16))), kinds.type, 'queued', groups.server_id, groups.team_id,
+    ${input.projectId ?? null},
+    json_object('schemaVersion', 1, 'runtimeIds', json(groups.runtime_ids),
+      'removeVolumes', json(case when kinds.type = 'purge' then 'true' else 'false' end),
+      'reason', case when kinds.type = 'purge'
+        then ${`${input.reason} Purge retained volumes after the retention window.`} else ${input.reason} end),
+    10, case when kinds.type = 'purge' then ${now + cleanupVolumeRetentionMs} else ${now} end,
+    ${now}, ${now}
+  from groups cross join kinds`
+}

package/src/server/agents/compatibility.ts

@@ -0,0 +1,116 @@
+export const currentAgentProtocolVersion = 1
+
+export const currentAgentCapabilities = [
+  'jobs.compose.deploy.v1',
+  'jobs.compose.cleanup.v2',
+  'jobs.backup.v1',
+  'jobs.backup.archive.v1',
+  'jobs.backup.cleanup.v1',
+  'jobs.restore.v1',
+  'runtime.logs.v1',
+  'metrics.history.v1',
+  'ingress.certificates.v1',
+] as const
+
+export type AgentCapability = (typeof currentAgentCapabilities)[number]
+
+export type AgentUpdate = {
+  status: 'current' | 'available' | 'recommended' | 'required'
+  latestVersion: string | null
+  reason: string | null
+}
+
+export function getAgentUpdate({
+  agentVersion,
+  capabilities,
+  latestVersion,
+  protocolVersion,
+  recommendedVersion,
+}: {
+  agentVersion: string | null
+  capabilities: Array<string> | null
+  latestVersion: string | null
+  protocolVersion: number | null
+  recommendedVersion: string | null
+}): AgentUpdate {
+  if (!canAgentRunDeployments({ capabilities, protocolVersion })) {
+    return {
+      status: 'required',
+      latestVersion,
+      reason:
+        'This agent can connect, but it cannot run the current deployment protocol.',
+    }
+  }
+
+  if (!capabilities?.includes('jobs.compose.cleanup.v2')) {
+    return {
+      status: 'required',
+      latestVersion,
+      reason: 'Update this agent to run cleanup jobs with attempt protection.',
+    }
+  }
+
+  if (isOlderVersion(agentVersion, recommendedVersion) && recommendedVersion) {
+    return {
+      status: 'recommended',
+      latestVersion,
+      reason: `Agent ${recommendedVersion} or newer is recommended.`,
+    }
+  }
+
+  if (isOlderVersion(agentVersion, latestVersion)) {
+    return {
+      status: 'available',
+      latestVersion,
+      reason: latestVersion ? `Agent ${latestVersion} is available.` : null,
+    }
+  }
+
+  return { status: 'current', latestVersion, reason: null }
+}
+
+export function canAgentRunDeployments({
+  capabilities,
+  protocolVersion,
+}: {
+  capabilities: Array<string> | null
+  protocolVersion: number | null
+}) {
+  return (
+    protocolVersion === currentAgentProtocolVersion &&
+    (capabilities?.includes('jobs.compose.deploy.v1') ?? false)
+  )
+}
+
+export function hasAgentCapability(
+  capabilities: Array<string> | undefined,
+  capability: AgentCapability,
+) {
+  return capabilities?.includes(capability) ?? false
+}
+
+function isOlderVersion(current: string | null, target: string | null) {
+  const currentParts = parseVersion(current)
+  const targetParts = parseVersion(target)
+
+  if (!currentParts || !targetParts) {
+    return false
+  }
+
+  for (let index = 0; index < targetParts.length; index += 1) {
+    const currentPart = currentParts[index]
+    const targetPart = targetParts[index]
+    if (currentPart === undefined || targetPart === undefined) return false
+    const difference = currentPart - targetPart
+    if (difference !== 0) {
+      return difference < 0
+    }
+  }
+
+  return false
+}
+
+function parseVersion(value: string | null) {
+  const match = value?.match(/^v?(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/)
+  return match ? [Number(match[1]), Number(match[2]), Number(match[3])] : null
+}

package/src/server/agents/completion.ts

@@ -0,0 +1,83 @@
+import { notifyAfterAgentCompletion } from '#devpush/server/notifications/observe'
+import {
+  completeBackupCleanupJob,
+  completeBackupJob,
+  completeCleanupJob,
+  completeDeploymentJob,
+  completeRestoreJob,
+} from '#devpush/server/agents/jobs'
+import type { CompleteAgentJobPayload } from './protocol'
+
+export async function completeAgentJob(payload: CompleteAgentJobPayload) {
+  const result = await completeJob(payload)
+  await notifyAfterAgentCompletion(payload.serverId)
+  return result
+}
+
+async function completeJob(payload: CompleteAgentJobPayload) {
+  if (payload.type === 'deploy') {
+    return completeDeploymentJob({
+      serverId: payload.serverId,
+      token: payload.token,
+      completion: {
+        deploymentId: payload.deploymentId,
+        status: payload.status,
+        gitSha: payload.gitSha,
+        logStartSeq: payload.logStartSeq,
+        logs: payload.logs,
+        events: payload.events,
+      },
+    })
+  }
+
+  if (payload.type === 'backup') {
+    return completeBackupJob({
+      serverId: payload.serverId,
+      token: payload.token,
+      completion: {
+        jobId: payload.jobId,
+        backupRunId: payload.backupRunId,
+        status: payload.status,
+        message: payload.message,
+        manifest: payload.manifest,
+        artifacts: payload.artifacts,
+      },
+    })
+  }
+
+  if (payload.type === 'backup_cleanup') {
+    return completeBackupCleanupJob({
+      serverId: payload.serverId,
+      token: payload.token,
+      completion: {
+        jobId: payload.jobId,
+        backupRunIds: payload.backupRunIds,
+        status: payload.status,
+        message: payload.message,
+      },
+    })
+  }
+
+  if (payload.type === 'restore') {
+    return completeRestoreJob({
+      serverId: payload.serverId,
+      token: payload.token,
+      completion: {
+        jobId: payload.jobId,
+        status: payload.status,
+        message: payload.message,
+      },
+    })
+  }
+
+  return completeCleanupJob({
+    serverId: payload.serverId,
+    token: payload.token,
+    completion: {
+      jobId: payload.jobId,
+      attempt: payload.attempt,
+      status: payload.status,
+      message: payload.message,
+    },
+  })
+}

package/src/server/agents/container-cleanup-state.ts

@@ -0,0 +1,51 @@
+import { sql } from 'drizzle-orm'
+import type { AtomicStore } from '#devpush/server/db/atomic'
+import type { agentJobs } from '#devpush/server/db/schema'
+import type { AgentCleanupCompletion } from './protocol'
+
+export async function recoverStaleCleanupJobs(
+  store: AtomicStore,
+  input: {
+    serverId: string
+    cleanupJobId?: string
+    now: Date
+  },
+) {
+  const now = input.now.getTime()
+  const stale = sql`server_id = ${input.serverId} and type in ('deprovision','purge') and status = 'running'
+    and (claimed_at is null or claimed_at < ${now - 60 * 60 * 1000})
+    ${input.cleanupJobId ? sql`and id <> ${input.cleanupJobId}` : sql``}`
+  await store.batch([
+    store.run(sql`update agent_jobs set status = 'abandoned', error_message = 'Cleanup stopped reporting after its final attempt.',
+      finished_at = ${now},updated_at = ${now} where ${stale} and attempt >= max_attempts`),
+    store.run(sql`update agent_jobs set status = 'queued', error_message = 'Cleanup stopped reporting and will be retried.',
+      claimed_at = null,available_at = ${now},finished_at = null,updated_at = ${now} where ${stale} and attempt < max_attempts`),
+  ])
+}
+
+export async function finishContainerCleanup(
+  store: AtomicStore,
+  input: {
+    job: typeof agentJobs.$inferSelect
+    completion: AgentCleanupCompletion
+    now: Date
+  },
+) {
+  const job = input.job,
+    now = input.now.getTime()
+  const retry =
+    input.completion.status === 'failed' && job.attempt < job.maxAttempts
+  const status = retry
+    ? 'queued'
+    : input.completion.status === 'failed'
+      ? 'abandoned'
+      : input.completion.status
+  const rows = await store.all(sql`update agent_jobs set status = ${status},
+    error_message = ${input.completion.status === 'failed' ? (input.completion.message ?? 'Cleanup failed.') : null},
+    claimed_at = ${retry ? null : (job.claimedAt?.getTime() ?? null)},
+    available_at = ${retry ? now + 60_000 : job.availableAt.getTime()},finished_at = ${retry ? null : now},updated_at = ${now}
+    where id = ${job.id} and server_id = ${job.serverId} and type = ${job.type} and type in ('deprovision','purge')
+    and status = 'running' and attempt = ${input.completion.attempt} and attempt = ${job.attempt} and claimed_at is ${job.claimedAt?.getTime() ?? null}
+    and updated_at = ${job.updatedAt.getTime()} returning id`)
+  return rows.length === 1
+}

package/src/server/agents/deployment-state.ts

@@ -0,0 +1,111 @@
+import { and, eq, or, sql } from 'drizzle-orm'
+import type { BaseSQLiteDatabase } from 'drizzle-orm/sqlite-core'
+import type { AtomicStore } from '#devpush/server/db/atomic'
+import { deployments } from '#devpush/server/db/schema'
+import * as schema from '#devpush/server/db/schema'
+
+export async function finishDeployment(
+  store: Pick<BaseSQLiteDatabase<'async', unknown, typeof schema>, 'update'> &
+    Pick<AtomicStore, 'batch'>,
+  input: {
+    deployment: typeof deployments.$inferSelect
+    projectId: string
+    status: 'succeeded' | 'failed' | 'canceled'
+    phase: string
+    gitSha: string
+    errorMessage: string | null
+    now: Date
+  },
+) {
+  const { aliases } = schema
+  const scope = and(
+    eq(deployments.id, input.deployment.id),
+    eq(deployments.serverId, input.deployment.serverId),
+    eq(deployments.status, 'running'),
+  )
+  const finish = store
+    .update(deployments)
+    .set({
+      status: input.status,
+      phase: input.phase,
+      gitSha: input.gitSha,
+      errorMessage: input.errorMessage,
+      finishedAt: input.now,
+    })
+    .where(scope)
+    .returning({ id: deployments.id })
+  if (input.status !== 'succeeded') return (await finish).length === 1
+  // Move aliases while the guarded deployment is still running, then finish
+  // it in the same batch. A terminal retry cannot move an alias a second time.
+  const [, completed] = await store.batch([
+    store
+      .update(aliases)
+      .set({
+        previousDeploymentId: sql`${aliases.deploymentId}`,
+        deploymentId: input.deployment.id,
+        updatedAt: input.now,
+      })
+      .where(
+        and(
+          eq(aliases.projectId, input.projectId),
+          eq(aliases.environmentId, input.deployment.environmentId),
+          or(
+            eq(aliases.type, 'environment'),
+            and(
+              eq(aliases.type, 'branch'),
+              eq(aliases.branch, input.deployment.gitBranch),
+            ),
+          ),
+          sql`exists(select 1 from ${deployments} where ${scope})`,
+        ),
+      ),
+    finish,
+  ])
+  return completed.length === 1
+}
+
+export async function failRunningDeployments(
+  store: AtomicStore,
+  input: {
+    ids: string[]
+    message: string
+    phase: string
+    now: Date
+    startedBefore?: Date
+    heartbeatBefore?: Date
+  },
+) {
+  const failedIds: string[] = []
+  for (let offset = 0; offset < input.ids.length; offset += 80) {
+    const ids = input.ids.slice(offset, offset + 80)
+    const guard = sql`${deployments.id} in (${sql.join(
+      ids.map((id) => sql`${id}`),
+      sql`, `,
+    )})
+      and ${deployments.status} = 'running'
+      ${input.startedBefore ? sql`and ${deployments.startedAt} < ${input.startedBefore.getTime()}` : sql``}
+      ${
+        input.heartbeatBefore
+          ? sql`and exists(select 1 from servers where servers.id = ${deployments.serverId}
+        and (servers.last_seen_at is null or servers.last_seen_at < ${input.heartbeatBefore.getTime()}))`
+          : sql``
+      }`
+    const [changed] = await store.batch([
+      store.all(sql`update ${deployments} set status = 'failed', phase = ${input.phase}, error_message = ${input.message},
+        finished_at = ${input.now.getTime()} where ${guard} returning id`),
+    ])
+    // Raw SQLite proxy results are arrays; D1 returns named objects. Normalize
+    // only this raw SQL boundary, without relying on either driver's shape.
+    for (const row of changed) {
+      const id = Array.isArray(row)
+        ? row[0]
+        : typeof row === 'object' && row !== null && 'id' in row
+          ? row.id
+          : undefined
+      if (typeof id !== 'string')
+        throw new Error('Invalid deployment mutation result.')
+      failedIds.push(id)
+    }
+  }
+  return failedIds
+}

package/src/server/agents/job-payloads.ts

@@ -0,0 +1,159 @@
+import type {
+  BackupCleanupJobPayload,
+  BackupJobPayload,
+  BackupRestoreJobPayload,
+} from '#devpush/server/backups/service'
+import type { DeprovisionJobPayload } from './cleanup'
+
+export function parseDeprovisionPayload(
+  value: unknown,
+): DeprovisionJobPayload | null {
+  if (
+    typeof value !== 'object' ||
+    value === null ||
+    !('schemaVersion' in value) ||
+    !('runtimeIds' in value) ||
+    !('removeVolumes' in value) ||
+    !('reason' in value)
+  ) {
+    return null
+  }
+
+  const payload = value as Record<string, unknown>
+
+  if (
+    payload.schemaVersion !== 1 ||
+    !Array.isArray(payload.runtimeIds) ||
+    typeof payload.removeVolumes !== 'boolean' ||
+    typeof payload.reason !== 'string'
+  ) {
+    return null
+  }
+
+  const runtimeIds = payload.runtimeIds.filter(
+    (runtimeId): runtimeId is string =>
+      typeof runtimeId === 'string' && runtimeId.trim() !== '',
+  )
+
+  if (runtimeIds.length === 0) {
+    return null
+  }
+
+  return {
+    schemaVersion: 1,
+    runtimeIds: Array.from(new Set(runtimeIds)).sort(),
+    removeVolumes: payload.removeVolumes,
+    reason: payload.reason,
+  }
+}
+
+export function parseBackupPayload(value: unknown): BackupJobPayload | null {
+  if (typeof value !== 'object' || value === null) {
+    return null
+  }
+
+  const payload = value as Record<string, unknown>
+
+  if (
+    payload.schemaVersion !== 1 ||
+    typeof payload.backupRunId !== 'string' ||
+    typeof payload.artifactPrefix !== 'string' ||
+    typeof payload.localBasePath !== 'string' ||
+    typeof payload.projectId !== 'string' ||
+    typeof payload.environmentId !== 'string' ||
+    typeof payload.runtimeId !== 'string' ||
+    !isRecord(payload.destination) ||
+    !isRecord(payload.retention) ||
+    !isRecord(payload.volumes) ||
+    !Array.isArray(payload.databaseDumps)
+  ) {
+    return null
+  }
+
+  return payload as BackupJobPayload
+}
+
+export function parseBackupCleanupPayload(
+  value: unknown,
+): BackupCleanupJobPayload | null {
+  if (typeof value !== 'object' || value === null) {
+    return null
+  }
+
+  const payload = value as Record<string, unknown>
+
+  if (
+    payload.schemaVersion !== 1 ||
+    !Array.isArray(payload.backupRunIds) ||
+    !Array.isArray(payload.localObjectKeys) ||
+    !Array.isArray(payload.remoteObjectKeys) ||
+    typeof payload.reason !== 'string'
+  ) {
+    return null
+  }
+
+  if (payload.remoteObjectKeys.length > 0 && !payload.destination) {
+    return null
+  }
+
+  return payload as BackupCleanupJobPayload
+}
+
+export function parseRestorePayload(
+  value: unknown,
+): BackupRestoreJobPayload | null {
+  if (typeof value !== 'object' || value === null) {
+    return null
+  }
+
+  const payload = value as Record<string, unknown>
+
+  if (
+    payload.schemaVersion !== 1 ||
+    typeof payload.backupRunId !== 'string' ||
+    typeof payload.projectId !== 'string' ||
+    typeof payload.environmentId !== 'string' ||
+    typeof payload.runtimeId !== 'string' ||
+    !Array.isArray(payload.artifacts) ||
+    payload.artifacts.length === 0
+  ) {
+    return null
+  }
+
+  for (const artifact of payload.artifacts) {
+    if (
+      !isRecord(artifact) ||
+      typeof artifact.id !== 'string' ||
+      typeof artifact.objectKey !== 'string' ||
+      typeof artifact.checksumSha256 !== 'string' ||
+      (artifact.storage !== 'local' && artifact.storage !== 'remote')
+    ) {
+      return null
+    }
+
+    if (artifact.type === 'volume') {
+      if (typeof artifact.targetVolume !== 'string') {
+        return null
+      }
+    } else if (artifact.type === 'database') {
+      if (
+        typeof artifact.targetService !== 'string' ||
+        typeof artifact.restoreCommand !== 'string'
+      ) {
+        return null
+      }
+    } else {
+      return null
+    }
+
+    if (artifact.storage === 'remote' && !payload.destination) {
+      return null
+    }
+  }
+
+  return payload as BackupRestoreJobPayload
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null
+}