devpush 0.0.0-stage → 1.0.0-beta.4

package/src/server/resources/color.ts

@@ -0,0 +1,13 @@
+const resourceColors = ['lime', 'sky', 'blue', 'green'] as const
+
+export type ResourceColor = (typeof resourceColors)[number]
+
+export function getResourceColor(id: string): ResourceColor {
+  let hash = 0
+
+  for (const char of id) {
+    hash = (hash * 31 + char.charCodeAt(0)) >>> 0
+  }
+
+  return resourceColors[hash % resourceColors.length] ?? resourceColors[0]
+}

package/src/server/runtime/background.ts

@@ -0,0 +1,21 @@
+import { safeErrorDiagnostics } from '#devpush/server/http/error-diagnostics'
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+type Defer = (promise: Promise<unknown>) => void
+const scope = new AsyncLocalStorage<Defer>()
+
+export function withBackgroundTasks<T>(defer: Defer, work: () => T): T {
+  return scope.run(defer, work)
+}
+/** Workers keep the task alive with waitUntil; Node runs it independently of the response. */
+export function deferBackgroundTask(work: () => Promise<unknown>) {
+  const pending = Promise.resolve()
+    .then(work)
+    .catch((error: unknown) => {
+      console.warn(
+        'Background task failed; persisted work will be retried.',
+        safeErrorDiagnostics(error),
+      )
+    })
+  scope.getStore()?.(pending)
+}

package/src/server/runtime/node-asset-cache-plugin.ts

@@ -0,0 +1,12 @@
+import { definePlugin } from 'nitro'
+
+export function preventAssetErrorCaching(response: Response, url: URL) {
+  if (response.status >= 400 && url.pathname.startsWith('/assets/'))
+    response.headers.set('Cache-Control', 'no-store')
+}
+
+export default definePlugin((app) => {
+  app.hooks.hook('response', (response, event) => {
+    preventAssetErrorCaching(response, new URL(event.req.url))
+  })
+})

package/src/server/runtime/node-scheduler-plugin.ts

@@ -0,0 +1,17 @@
+import { withPluginScope } from '#devpush/server/plugins/scope'
+import { definePlugin } from 'nitro'
+import { assertRuntimeSecurity, config } from '#devpush/server/config'
+import { withDatabaseScope } from '#devpush/server/db'
+import { runScheduledTasks } from '#devpush/server/scheduled-tasks'
+import { startNodeScheduler } from './node-scheduler'
+
+export default definePlugin((app) => {
+  if (process.env.NODE_ENV !== 'production') return
+  assertRuntimeSecurity(config)
+  const stop = startNodeScheduler((maintenance) =>
+    withPluginScope(() =>
+      withDatabaseScope(() => runScheduledTasks(maintenance)),
+    ),
+  )
+  app.hooks.hook('close', stop)
+})

package/src/server/runtime/node-scheduler.ts

@@ -0,0 +1,51 @@
+import { safeErrorDiagnostics } from '#devpush/server/http/error-diagnostics'
+import { ScheduledTaskError } from './scheduled-task-error'
+/** One minute-aligned timer, with no overlapping runs and retry after failure. */
+export function startNodeScheduler(
+  run: (maintenance: boolean) => Promise<void>,
+) {
+  let stopped = false
+  let timer: ReturnType<typeof setTimeout> | undefined
+  let lastMaintenanceHour: number | undefined
+  let active: Promise<void> = Promise.resolve()
+  function schedule() {
+    if (stopped) return
+    timer = setTimeout(
+      () => {
+        const hour = Math.floor(Date.now() / 3_600_000)
+        const maintenance = hour !== lastMaintenanceHour
+        active = Promise.resolve()
+          .then(() => run(maintenance))
+          .then(() => {
+            if (maintenance) lastMaintenanceHour = hour
+          })
+          .catch((error: unknown) => {
+            if (
+              maintenance &&
+              error instanceof ScheduledTaskError &&
+              error.maintenanceCompleted
+            )
+              lastMaintenanceHour = hour
+            console.error(
+              'Scheduled tasks failed; retrying on the next minute tick.',
+              {
+                errors: (error instanceof AggregateError
+                  ? error.errors.slice(0, 4)
+                  : [error]
+                ).map(safeErrorDiagnostics),
+              },
+            )
+          })
+          .finally(schedule)
+      },
+      60_000 - (Date.now() % 60_000),
+    )
+    timer.unref()
+  }
+  schedule()
+  return async () => {
+    stopped = true
+    clearTimeout(timer)
+    await active
+  }
+}

package/src/server/runtime/platform-env.cloudflare.ts

@@ -0,0 +1,17 @@
+import { env } from 'cloudflare:workers'
+
+export function isCloudflareRuntime() {
+  return true
+}
+
+export function getR2ObjectStore() {
+  return env.DEVPUSH_OBJECTS
+}
+
+export function getWorkerVersionMetadata() {
+  return env.CF_VERSION_METADATA
+}
+
+export function getWorkspaceUpdates() {
+  return env.DEVPUSH_WORKSPACE_UPDATES
+}

package/src/server/runtime/platform-env.ts

@@ -0,0 +1,19 @@
+import { nodeBroadcaster } from '#devpush/server/realtime/node-broadcaster'
+
+export function isCloudflareRuntime() {
+  return false
+}
+
+export function getR2ObjectStore(): R2Bucket | undefined {
+  return undefined
+}
+
+export function getWorkerVersionMetadata():
+  | { id: string; tag: string }
+  | undefined {
+  return undefined
+}
+
+export function getWorkspaceUpdates(): WorkspaceUpdatesNamespace | undefined {
+  return nodeBroadcaster
+}

package/src/server/runtime/scheduled-task-error.ts

@@ -0,0 +1,9 @@
+/** Preserve hourly progress when an unrelated minute task fails. */
+export class ScheduledTaskError extends AggregateError {
+  constructor(
+    errors: unknown[],
+    readonly maintenanceCompleted: boolean,
+  ) {
+    super(errors, 'Scheduled tasks failed.')
+  }
+}

package/src/server/scheduled-tasks.ts

@@ -0,0 +1,28 @@
+import { ScheduledTaskError } from '#devpush/server/runtime/scheduled-task-error'
+import { maintainOperationalNotifications } from '#devpush/server/notifications/observe'
+import { config } from '#devpush/server/config'
+import { queueDueBackupSchedules } from '#devpush/server/backups/service'
+import { runMaintenance } from '#devpush/server/maintenance'
+
+/** Shared by the Node timer and Cloudflare scheduled handler. */
+export async function runScheduledTasks(maintenance: boolean) {
+  const tasks: Array<{ name: string; run: () => Promise<unknown> }> = [
+    { name: 'notifications', run: maintainOperationalNotifications },
+    ...(config.DEVPUSH_BACKUPS_ENABLED
+      ? [{ name: 'backups', run: queueDueBackupSchedules }]
+      : []),
+    ...(maintenance ? [{ name: 'maintenance', run: runMaintenance }] : []),
+  ]
+  const results = await Promise.allSettled(
+    tasks.map((task) => Promise.resolve().then(() => task.run())),
+  )
+  const failures = results.filter((result) => result.status === 'rejected')
+  if (failures.length)
+    throw new ScheduledTaskError(
+      failures.map((result) => result.reason),
+      tasks.some(
+        (task, index) =>
+          task.name === 'maintenance' && results[index]?.status === 'fulfilled',
+      ),
+    )
+}

package/src/server/secrets/crypto.ts

@@ -0,0 +1,56 @@
+import {
+  createCipheriv,
+  createDecipheriv,
+  createHash,
+  randomBytes,
+} from 'node:crypto'
+
+import { config } from '#devpush/server/config'
+
+export function encryptSecret(value: string) {
+  const iv = randomBytes(12)
+  const cipher = createCipheriv('aes-256-gcm', encryptionKey(), iv)
+  const encrypted = Buffer.concat([
+    cipher.update(value, 'utf8'),
+    cipher.final(),
+  ])
+  const tag = cipher.getAuthTag()
+
+  return [
+    'v1',
+    iv.toString('base64url'),
+    tag.toString('base64url'),
+    encrypted.toString('base64url'),
+  ].join(':')
+}
+
+export function decryptSecret(payload: string) {
+  const [version, ivValue, tagValue, encryptedValue] = payload.split(':')
+
+  if (version !== 'v1' || !ivValue || !tagValue || !encryptedValue) {
+    throw new Error('Invalid encrypted secret payload.')
+  }
+
+  const decipher = createDecipheriv(
+    'aes-256-gcm',
+    encryptionKey(),
+    Buffer.from(ivValue, 'base64url'),
+  )
+  decipher.setAuthTag(Buffer.from(tagValue, 'base64url'))
+  const decrypted = Buffer.concat([
+    decipher.update(Buffer.from(encryptedValue, 'base64url')),
+    decipher.final(),
+  ])
+
+  return decrypted.toString('utf8')
+}
+
+function encryptionKey() {
+  if (!config.DEVPUSH_SECRET_ENCRYPTION_KEY) {
+    throw new Error('DEVPUSH_SECRET_ENCRYPTION_KEY is required.')
+  }
+
+  return createHash('sha256')
+    .update(config.DEVPUSH_SECRET_ENCRYPTION_KEY)
+    .digest()
+}

package/src/server/servers/functions.ts

@@ -0,0 +1,233 @@
+import {
+  getAuthSession,
+  requireAuthSession,
+} from '#devpush/server/auth/session.server'
+import { requireTeam } from '#devpush/server/teams/service'
+import {
+  requireServerForTeam,
+  requireServerMetricHistoryForTeam,
+  createPendingServer,
+  createServerInstallCommand,
+  deleteTeamServer,
+  disableTeamServer,
+  enableTeamServer,
+  getServerDeleteDiagnostic,
+  updateTeamServer,
+} from '#devpush/server/servers/service'
+import { settingsVersionSchema } from '#devpush/lib/settings-version'
+import { publicErrorsMiddleware } from '#devpush/server/http/public-errors'
+import { serverNameSchema } from '#devpush/lib/server-name-schema'
+import { createServerFn } from '@tanstack/react-start'
+import { z } from 'zod'
+
+const createServerSchema = z.object({
+  teamSlug: z.string().min(1),
+  name: serverNameSchema,
+})
+
+const serverInstallCommandSchema = z.object({
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+})
+
+const updateServerSchema = z.object({
+  settingsVersion: settingsVersionSchema,
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+  name: serverNameSchema,
+})
+
+const disableServerSchema = z.object({
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+  confirmationName: z.string().min(1),
+})
+
+const enableServerSchema = z.object({
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+})
+
+const deleteServerDiagnosticSchema = z.object({
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+})
+
+const deleteServerSchema = z.object({
+  teamSlug: z.string().min(1),
+  serverId: z.string().min(1),
+  confirmationName: z.string().min(1),
+  cancelOngoingDeployments: z.boolean(),
+})
+
+export const createServer = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: { name: string; teamSlug: string }) =>
+    createServerSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession('Sign in before adding a server.')
+
+    return createPendingServer({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      name: data.name,
+    })
+  })
+
+export const createInstallCommand = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: { serverId: string; teamSlug: string }) =>
+    serverInstallCommandSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before enrolling a server.',
+    )
+
+    return createServerInstallCommand({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+    })
+  })
+
+export const updateServer = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator(
+    (data: {
+      settingsVersion: number
+      name: string
+      serverId: string
+      teamSlug: string
+    }) => updateServerSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before updating a server.',
+    )
+
+    return updateTeamServer({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+      settingsVersion: data.settingsVersion,
+      name: data.name,
+    })
+  })
+
+export const disableServer = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator(
+    (data: { confirmationName: string; serverId: string; teamSlug: string }) =>
+      disableServerSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before disabling a server.',
+    )
+
+    await disableTeamServer({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+      confirmationName: data.confirmationName,
+    })
+  })
+
+export const enableServer = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: { serverId: string; teamSlug: string }) =>
+    enableServerSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before enabling a server.',
+    )
+
+    await enableTeamServer({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+    })
+  })
+
+export const getDeleteServerDiagnostic = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: { serverId: string; teamSlug: string }) =>
+    deleteServerDiagnosticSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before deleting a server.',
+    )
+
+    return getServerDeleteDiagnostic({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+    })
+  })
+
+export const deleteServer = createServerFn({ method: 'POST' })
+  .middleware([publicErrorsMiddleware])
+  .validator(
+    (data: {
+      cancelOngoingDeployments: boolean
+      confirmationName: string
+      serverId: string
+      teamSlug: string
+    }) => deleteServerSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await requireAuthSession(
+      'Sign in before deleting a server.',
+    )
+
+    await deleteTeamServer({
+      userId: session.user.id,
+      teamSlug: data.teamSlug,
+      serverId: data.serverId,
+      confirmationName: data.confirmationName,
+      cancelOngoingDeployments: data.cancelOngoingDeployments,
+    })
+  })
+
+export const getTeamServerContext = createServerFn({ method: 'GET' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: z.input<typeof serverInstallCommandSchema>) =>
+    serverInstallCommandSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await getAuthSession()
+
+    if (!session) {
+      return null
+    }
+
+    const team = await requireTeam(session.user.id, data.teamSlug)
+    const server = await requireServerForTeam({
+      teamId: team.id,
+      serverId: data.serverId,
+    })
+
+    return {
+      team,
+      server,
+    }
+  })
+
+export const getTeamServerMetricHistory = createServerFn({ method: 'GET' })
+  .middleware([publicErrorsMiddleware])
+  .validator((data: z.input<typeof serverInstallCommandSchema>) =>
+    serverInstallCommandSchema.parse(data),
+  )
+  .handler(async ({ data }) => {
+    const session = await getAuthSession()
+    if (!session) return null
+    const team = await requireTeam(session.user.id, data.teamSlug)
+    return requireServerMetricHistoryForTeam({
+      teamId: team.id,
+      serverId: data.serverId,
+    })
+  })

package/src/server/servers/lifecycle.ts

@@ -0,0 +1,122 @@
+import { releaseBackupCleanupRuns } from '#devpush/server/backups/queue-state'
+import type { ApiMutationAdmission } from '#devpush/server/api/admission'
+import { sql } from 'drizzle-orm'
+import type { AtomicStore } from '#devpush/server/db/atomic'
+import {
+  serverAvailabilityWindows,
+  servers,
+  teamMembers,
+  teams,
+  user,
+} from '#devpush/server/db/schema'
+
+export async function changeServerEnabled(
+  store: AtomicStore,
+  input: {
+    userId: string
+    teamId: string
+    serverId: string
+    enabled: boolean
+    now: Date
+    admission?: ApiMutationAdmission
+  },
+) {
+  const now = input.now.getTime()
+  const mayManage = sql`exists(select 1 from ${teamMembers} as member
+    join ${user} as manager on manager.id = member.user_id
+    join ${teams} as managed_team on managed_team.id = member.team_id
+    where member.team_id = ${input.teamId} and member.user_id = ${input.userId}
+      and member.role in ('owner', 'admin') and manager.status = 'active' and managed_team.status = 'active') and ${input.admission?.guard ?? sql`1`}`
+  if (input.enabled) {
+    // Insert before changing status, so a racing request cannot open a second
+    // interval after the first request has already enabled the server.
+    const [, changed] = await store.batch([
+      store.run(sql`insert into ${serverAvailabilityWindows} (id, server_id, started_at, created_at)
+        select ${crypto.randomUUID()}, id, ${now}, ${now} from ${servers}
+        where id = ${input.serverId} and team_id = ${input.teamId} and status = 'disabled' and ${mayManage}
+        and not exists(select 1 from ${serverAvailabilityWindows} where server_id = ${input.serverId} and ended_at is null)`),
+      store.all(sql`update ${servers} set status = 'pending', updated_at = ${now}
+        where id = ${input.serverId} and team_id = ${input.teamId} and status = 'disabled' and ${mayManage} returning id`),
+      ...(input.admission?.statements ?? []),
+    ])
+    return changed.length === 1
+  }
+  const [changed] = await store.batch([
+    store.all(sql`update ${servers} set status = 'disabled', enrollment_token_hash = null,
+      enrollment_token_expires_at = null, updated_at = ${now}
+      where id = ${input.serverId} and team_id = ${input.teamId} and status <> 'deleted' and ${mayManage} returning id`),
+    ...(input.admission?.statements ?? []),
+    store.run(sql`update ${serverAvailabilityWindows} set ended_at = ${now}
+      where server_id = ${input.serverId} and ended_at is null and ${mayManage}
+      and exists(select 1 from ${servers} where id = ${input.serverId} and team_id = ${input.teamId} and status = 'disabled')`),
+  ])
+  return changed.length === 1
+}
+
+export async function deleteServerState(
+  store: AtomicStore,
+  input: {
+    userId: string
+    teamId: string
+    serverId: string
+    cancelOngoingDeployments: boolean
+    now: Date
+    admission?: ApiMutationAdmission
+  },
+) {
+  const now = input.now.getTime()
+  const guard = sql`exists(select 1 from servers s
+    join teams t on t.id = s.team_id
+    join team_members m on m.team_id = t.id
+    join user u on u.id = m.user_id
+    where s.id = ${input.serverId} and s.team_id = ${input.teamId} and s.status <> 'deleted'
+      and t.status = 'active' and u.status = 'active'
+      and m.user_id = ${input.userId} and m.role in ('owner', 'admin')) and ${input.admission?.guard ?? sql`1`}`
+  const message = 'Canceled because the server was deleted.'
+  const changes: [
+    ReturnType<AtomicStore['run']>,
+    ...ReturnType<AtomicStore['run']>[],
+  ] = [
+    store.run(
+      releaseBackupCleanupRuns({
+        jobIds: sql`select id from agent_jobs where server_id = ${input.serverId}`,
+        guard,
+        message,
+        now: input.now,
+      }),
+    ),
+    store.run(sql`update agent_jobs set status = 'canceled', error_message = ${message},
+      finished_at = ${now}, updated_at = ${now} where server_id = ${input.serverId}
+      and type in ('backup', 'restore', 'backup_cleanup') and status in ('queued', 'running') and ${guard}`),
+    store.run(sql`update backup_runs set status = 'canceled', error_message = ${message},
+      finished_at = ${now}, updated_at = ${now} where server_id = ${input.serverId}
+      and status in ('queued', 'running') and ${guard}`),
+  ]
+  if (input.cancelOngoingDeployments)
+    changes.push(
+      store.run(sql`update deployments set status = 'canceled', phase = 'canceled',
+      error_message = ${message}, finished_at = ${now} where server_id = ${input.serverId}
+      and status in ('queued', 'running') and ${guard}`),
+      store.run(sql`update agent_jobs set status = 'canceled', error_message = ${message},
+      finished_at = ${now}, updated_at = ${now} where server_id = ${input.serverId}
+      and type = 'deploy' and status in ('queued', 'running') and ${guard}`),
+    )
+  const results = await store.batch([
+    ...changes,
+    store.run(
+      sql`delete from server_metrics where server_id = ${input.serverId} and ${guard}`,
+    ),
+    store.run(
+      sql`delete from server_heartbeats where server_id = ${input.serverId} and ${guard}`,
+    ),
+    store.run(sql`update server_availability_windows set ended_at = ${now}
+      where server_id = ${input.serverId} and ended_at is null and ${guard}`),
+    store.all(sql`update servers set status = 'deleted', updated_at = ${now}
+      ${input.cancelOngoingDeployments ? sql`, enrollment_token_hash = null, enrollment_token_expires_at = null` : sql``}
+      where id = ${input.serverId} and ${guard} returning id`),
+    ...(input.admission?.statements ?? []),
+  ])
+  // The optional deployment cancellations change the number of statements.
+  const changed = results.at(-1 - (input.admission?.statements.length ?? 0))
+  return Array.isArray(changed) && changed.length === 1
+}

package/src/server/servers/metric-history-schema.ts

@@ -0,0 +1,24 @@
+import { z } from 'zod'
+
+const metricValuesSchema = z.object({
+  cpuPercent: z.number().min(0).max(100).optional(),
+  memoryUsedBytes: z.number().int().min(0).optional(),
+  memoryTotalBytes: z.number().int().min(0).optional(),
+  diskUsedBytes: z.number().int().min(0).optional(),
+  diskTotalBytes: z.number().int().min(0).optional(),
+  networkReceiveBytesPerSecond: z.number().min(0).optional(),
+  networkTransmitBytesPerSecond: z.number().min(0).optional(),
+  diskReadBytesPerSecond: z.number().min(0).optional(),
+  diskWriteBytesPerSecond: z.number().min(0).optional(),
+})
+
+export const metricSampleSchema = z.object({
+  id: z.string().regex(/^[a-f0-9]{32}$/),
+  observedAt: z.string().datetime(),
+  metrics: metricValuesSchema.refine(
+    (values) => Object.values(values).some(Number.isFinite),
+    'Metric sample must contain a measurement.',
+  ),
+})
+
+export type AgentMetricSample = z.infer<typeof metricSampleSchema>