devpush 0.0.0-stage → 1.0.0-beta.3

package/src/server/db/d1-connection.ts

@@ -0,0 +1,83 @@
+import type { D1Database } from '@cloudflare/workers-types'
+import { drizzle } from 'drizzle-orm/d1'
+import type { BatchItem, BatchResponse } from 'drizzle-orm/batch'
+import { sqliteParams } from './sqlite-params'
+
+type PreparedBatchQuery = {
+  getQuery: () => { sql: string; params: unknown[] }
+  mapResult: (result: unknown, isFromBatch: boolean) => unknown
+}
+
+export function openD1<TSchema extends Record<string, unknown>>(
+  binding: D1Database,
+  schema: TSchema,
+) {
+  // Raw SQL interpolations bypass Drizzle's column encoders. Keep their
+  // Date/boolean representation consistent with the other SQLite drivers.
+  const client = new Proxy(binding, {
+    get(target, property) {
+      if (property === 'prepare') {
+        return (query: string) =>
+          new Proxy(target.prepare(query), {
+            get(statement, method) {
+              if (method === 'bind') {
+                return (...params: unknown[]) =>
+                  statement.bind(...sqliteParams(params))
+              }
+              const value: unknown = Reflect.get(statement, method, statement)
+              return typeof value === 'function' ? value.bind(statement) : value
+            },
+          })
+      }
+      const value: unknown = Reflect.get(target, property, target)
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+  const database = drizzle(client, { schema })
+  return new Proxy(database, {
+    get(target, property) {
+      if (property === 'batch') {
+        // Drizzle's D1 batch implementation accesses preparedQuery.stmt, which
+        // raw SQL queries do not have. Prepare through the binding for both
+        // raw SQL and builders, then keep Drizzle's result mapping.
+        return async <
+          TItem extends BatchItem<'sqlite'>,
+          T extends Readonly<[TItem, ...TItem[]]>,
+        >(
+          queries: T,
+        ): Promise<BatchResponse<T>> => {
+          // RunnableQuery omits the internal preparation method that both
+          // Drizzle batch implementations require. Keep that boundary here.
+          const prepared = queries.map((query) =>
+            (
+              query as TItem & { _prepare: () => PreparedBatchQuery }
+            )._prepare(),
+          )
+          const statements = prepared.map((query) => {
+            const built = query.getQuery()
+            return client.prepare(built.sql).bind(...built.params)
+          })
+          const results = await binding.batch(statements)
+          // The library's mapper returns unknown; the query tuple defines the
+          // corresponding result types, just as its built-in batch does.
+          return results.map((result, index) => {
+            const query = prepared[index]
+            if (!query) throw new Error('Unexpected D1 batch result count.')
+            return query.mapResult(result, true)
+          }) as BatchResponse<T>
+        }
+      }
+      // D1 only supports transactions as a prebuilt atomic batch. Never let
+      // Drizzle's callback API issue BEGIN or simulate non-atomic statements.
+      if (property === 'transaction') {
+        return () => {
+          throw new Error(
+            'D1 requires atomic SQL batches; interactive transaction callbacks are unsupported.',
+          )
+        }
+      }
+      const value: unknown = Reflect.get(target, property, target)
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+}

package/src/server/db/errors.ts

@@ -0,0 +1,17 @@
+export function isUniqueConstraintError(error: unknown): boolean {
+  if (typeof error !== 'object' || error === null) return false
+  if (
+    'code' in error &&
+    (error.code === 'SQLITE_CONSTRAINT_UNIQUE' ||
+      error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY')
+  )
+    return true
+  // D1 includes SQLite's failure in its message rather than a native code.
+  if (
+    'message' in error &&
+    typeof error.message === 'string' &&
+    /UNIQUE constraint failed:/i.test(error.message)
+  )
+    return true
+  return 'cause' in error && isUniqueConstraintError(error.cause)
+}

package/src/server/db/index.cloudflare.ts

@@ -0,0 +1,15 @@
+import { env } from 'cloudflare:workers'
+import { createWorkerDatabases } from './worker-connection'
+
+export const { db, logsDb } = createWorkerDatabases(env)
+
+export async function withDatabaseScope<TResult>(
+  callback: () => TResult | Promise<TResult>,
+  _defer?: (promise: Promise<unknown>) => void,
+) {
+  return callback()
+}
+
+export async function closeDb() {
+  // D1 has no pool; remote clients close after each operation.
+}

package/src/server/db/index.ts

@@ -0,0 +1,35 @@
+import { createClient } from '@libsql/client/web'
+import { openLibsql } from './libsql-connection'
+import { remoteDatabaseConfig } from './remote-config'
+import { config } from '#devpush/server/config'
+import { openSqlite } from './sqlite-connection'
+import * as schema from './schema'
+import * as logSchema from './log-schema'
+import { assertSeparateSqliteFiles } from './sqlite-files'
+
+const remote = remoteDatabaseConfig(config)
+if (!remote)
+  assertSeparateSqliteFiles(
+    config.DEVPUSH_DATABASE_PATH,
+    config.DEVPUSH_LOG_DATABASE_PATH,
+  )
+
+const main = remote
+  ? openLibsql(createClient(remote.main), schema)
+  : openSqlite(config.DEVPUSH_DATABASE_PATH, schema)
+export const db = main.db
+const logStore = remote
+  ? openLibsql(createClient(remote.logs), logSchema)
+  : openSqlite(config.DEVPUSH_LOG_DATABASE_PATH, logSchema)
+export const logsDb = logStore.db
+
+export async function withDatabaseScope<TResult>(
+  callback: () => TResult | Promise<TResult>,
+  _defer?: (promise: Promise<unknown>) => void,
+) {
+  return callback()
+}
+
+export async function closeDb() {
+  await Promise.all([main.close(), logStore.close()])
+}

package/src/server/db/libsql-connection.ts

@@ -0,0 +1,72 @@
+import type { Client, ResultSet } from '@libsql/client'
+import { drizzle } from 'drizzle-orm/sqlite-proxy'
+import { sqliteParams } from './sqlite-params'
+
+// Preserve positional columns, including duplicate names in joined rows.
+function rows(result: ResultSet) {
+  return result.rows.map((row) => Array.from(row))
+}
+
+export function openLibsql<TSchema extends Record<string, unknown>>(
+  connection: Client | (() => Client),
+  schema: TSchema,
+) {
+  async function useClient<TResult>(
+    operation: (client: Client) => Promise<TResult>,
+  ) {
+    const client = typeof connection === 'function' ? connection() : connection
+    try {
+      return await operation(client)
+    } finally {
+      // Workers must not retain I/O resources across requests. A factory also
+      // keeps streaming responses and scheduled work independent of fetch scope.
+      if (typeof connection === 'function') client.close()
+    }
+  }
+  const database = drizzle<TSchema>(
+    async (sql, params, method) => {
+      const result = rows(
+        await useClient((client) =>
+          client.execute({ sql, args: sqliteParams(params) }),
+        ),
+      )
+      // Drizzle consumes undefined for a missing get() row, despite requiring
+      // an array in AsyncRemoteCallback's return type.
+      return {
+        rows:
+          method === 'get' ? (result[0] as (typeof result)[number]) : result,
+      }
+    },
+    async (queries) => {
+      const results = await useClient((client) =>
+        client.batch(
+          queries.map((query) => ({
+            sql: query.sql,
+            args: sqliteParams(query.params),
+          })),
+          'write',
+        ),
+      )
+      return results.map((result) => ({ rows: rows(result) }))
+    },
+    { schema },
+  )
+  const db = new Proxy(database, {
+    get(target, property) {
+      if (property === 'transaction')
+        return () => {
+          throw new Error(
+            'Use atomic SQL batches; callback transactions are unsupported by the libSQL adapter.',
+          )
+        }
+      const value: unknown = Reflect.get(target, property, target)
+      return typeof value === 'function' ? value.bind(target) : value
+    },
+  })
+  return {
+    db,
+    close: () => {
+      if (typeof connection !== 'function') connection.close()
+    },
+  }
+}

package/src/server/db/log-schema.ts

@@ -0,0 +1,102 @@
+import { sql } from 'drizzle-orm'
+import {
+  index,
+  integer,
+  sqliteTable,
+  text,
+  uniqueIndex,
+} from 'drizzle-orm/sqlite-core'
+
+const createdAt = () =>
+  integer('created_at', { mode: 'timestamp_ms' })
+    .notNull()
+    .default(sql`(cast(unixepoch('subsec') * 1000 as integer))`)
+
+export const logCleanupPositions = sqliteTable('log_cleanup_positions', {
+  id: text('id').primaryKey(),
+  cursor: text('cursor'),
+})
+
+// Identity is verified against the main database at ingestion and authorization
+// is checked there on every read. Foreign keys cannot cross database files.
+export const deploymentLogs = sqliteTable(
+  'deployment_logs',
+  {
+    projectId: text('project_id').notNull(),
+    environmentId: text('environment_id').notNull(),
+    deploymentId: text('deployment_id').notNull(),
+    serverId: text('server_id').notNull(),
+    branch: text('branch').notNull(),
+    stream: text('stream').notNull(),
+    seq: integer('seq').notNull(),
+    content: text('content').notNull(),
+    searchContent: text('search_content').notNull(),
+    createdAt: createdAt(),
+  },
+  (table) => [
+    uniqueIndex('deployment_logs_deployment_stream_seq_idx').on(
+      table.deploymentId,
+      table.stream,
+      table.seq,
+    ),
+    index('deployment_logs_project_time_idx').on(
+      table.projectId,
+      table.createdAt,
+    ),
+    index('deployment_logs_project_environment_time_idx').on(
+      table.projectId,
+      table.environmentId,
+      table.createdAt,
+    ),
+  ],
+)
+
+export const runtimeLogs = sqliteTable(
+  'runtime_logs',
+  {
+    id: text('id').primaryKey(),
+    projectId: text('project_id').notNull(),
+    environmentId: text('environment_id').notNull(),
+    serverId: text('server_id').notNull(),
+    runtimeId: text('runtime_id').notNull(),
+    service: text('service').notNull(),
+    observedAt: integer('observed_at', { mode: 'timestamp_ms' }).notNull(),
+    content: text('content').notNull(),
+    searchContent: text('search_content').notNull(),
+    contentHash: text('content_hash').notNull(),
+    createdAt: createdAt(),
+  },
+  (table) => [
+    uniqueIndex('runtime_logs_hash_idx').on(
+      table.serverId,
+      table.runtimeId,
+      table.contentHash,
+    ),
+    index('runtime_logs_project_time_idx').on(
+      table.projectId,
+      table.observedAt,
+    ),
+    index('runtime_logs_server_runtime_order_idx').on(
+      table.serverId,
+      table.runtimeId,
+      table.observedAt,
+      table.createdAt,
+      table.id,
+    ),
+    index('runtime_logs_observed_at_idx').on(table.observedAt),
+  ],
+)
+
+export const operationLogs = sqliteTable(
+  'operation_logs',
+  {
+    jobId: text('job_id').notNull(),
+    offset: integer('offset').notNull(),
+    content: text('content').notNull(),
+    createdAt: createdAt(),
+  },
+  (table) => [
+    uniqueIndex('operation_logs_job_offset_idx').on(table.jobId, table.offset),
+    index('operation_logs_time_idx').on(table.createdAt),
+  ],
+)

package/src/server/db/migrate-libsql.ts

@@ -0,0 +1,29 @@
+import type { Client } from '@libsql/client'
+import { drizzle } from 'drizzle-orm/libsql/web'
+import { migrate } from 'drizzle-orm/libsql/migrator'
+import { readMigrationFiles } from 'drizzle-orm/migrator'
+import { assertKnownMigrationHistory } from './migration-history'
+
+export async function migrateLibsql(client: Client, migrationsFolder: string) {
+  const tracked = await client.execute(
+    "select name from sqlite_master where type='table' and name='__drizzle_migrations'",
+  )
+  const applied = tracked.rows.length
+    ? (await client.execute('select hash from __drizzle_migrations')).rows.map(
+        (row) => String(row.hash),
+      )
+    : []
+  const untracked =
+    applied.length === 0 &&
+    (
+      await client.execute(
+        "select name from sqlite_master where type='table' and name not like 'sqlite_%' and name <> '__drizzle_migrations' limit 1",
+      )
+    ).rows.length > 0
+  assertKnownMigrationHistory(
+    applied,
+    readMigrationFiles({ migrationsFolder }).map((migration) => migration.hash),
+    untracked,
+  )
+  await migrate(drizzle(client), { migrationsFolder })
+}

package/src/server/db/migrate-sqlite.ts

@@ -0,0 +1,47 @@
+import { mkdirSync } from 'node:fs'
+import { dirname } from 'node:path'
+
+import Sqlite from 'better-sqlite3'
+import { drizzle } from 'drizzle-orm/better-sqlite3'
+import { migrate } from 'drizzle-orm/better-sqlite3/migrator'
+import { readMigrationFiles } from 'drizzle-orm/migrator'
+import { assertKnownMigrationHistory } from './migration-history'
+
+export function migrateSqliteFile(path: string, migrationsFolder: string) {
+  if (path !== ':memory:') mkdirSync(dirname(path), { recursive: true })
+  const connection = new Sqlite(path)
+  try {
+    const tracked = connection
+      .prepare(
+        "select name from sqlite_master where type='table' and name='__drizzle_migrations'",
+      )
+      .get()
+    const applied = tracked
+      ? (connection.prepare('select hash from __drizzle_migrations').all() as {
+          hash: string
+        }[])
+      : []
+    const untracked =
+      applied.length === 0 &&
+      Boolean(
+        connection
+          .prepare(
+            "select name from sqlite_master where type='table' and name not like 'sqlite_%' and name <> '__drizzle_migrations' limit 1",
+          )
+          .get(),
+      )
+    assertKnownMigrationHistory(
+      applied.map((row) => row.hash),
+      readMigrationFiles({ migrationsFolder }).map(
+        (migration) => migration.hash,
+      ),
+      untracked,
+    )
+    connection.pragma('foreign_keys = ON')
+    connection.pragma('journal_mode = WAL')
+    connection.pragma('synchronous = FULL')
+    migrate(drizzle(connection), { migrationsFolder })
+  } finally {
+    connection.close()
+  }
+}

package/src/server/db/migration-history.ts

@@ -0,0 +1,12 @@
+export function assertKnownMigrationHistory(
+  appliedHashes: string[],
+  availableHashes: string[],
+  hasUntrackedTables: boolean,
+) {
+  const known = new Set(availableHashes)
+  if (hasUntrackedTables || appliedHashes.some((hash) => !known.has(hash))) {
+    throw new Error(
+      'Database migration history does not match this release. The flattened baseline requires fresh main and log databases; back up/export existing data before recreating them. No migration was applied.',
+    )
+  }
+}

package/src/server/db/migrations-logs/0000_initial.sql

@@ -0,0 +1,48 @@
+CREATE TABLE `deployment_logs` (
+	`project_id` text NOT NULL,
+	`environment_id` text NOT NULL,
+	`deployment_id` text NOT NULL,
+	`server_id` text NOT NULL,
+	`branch` text NOT NULL,
+	`stream` text NOT NULL,
+	`seq` integer NOT NULL,
+	`content` text NOT NULL,
+	`search_content` text NOT NULL,
+	`created_at` integer DEFAULT (cast(unixepoch('subsec') * 1000 as integer)) NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `deployment_logs_deployment_stream_seq_idx` ON `deployment_logs` (`deployment_id`,`stream`,`seq`);--> statement-breakpoint
+CREATE INDEX `deployment_logs_project_time_idx` ON `deployment_logs` (`project_id`,`created_at`);--> statement-breakpoint
+CREATE INDEX `deployment_logs_project_environment_time_idx` ON `deployment_logs` (`project_id`,`environment_id`,`created_at`);--> statement-breakpoint
+CREATE TABLE `log_cleanup_positions` (
+	`id` text PRIMARY KEY NOT NULL,
+	`cursor` text
+);
+--> statement-breakpoint
+CREATE TABLE `operation_logs` (
+	`job_id` text NOT NULL,
+	`offset` integer NOT NULL,
+	`content` text NOT NULL,
+	`created_at` integer DEFAULT (cast(unixepoch('subsec') * 1000 as integer)) NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `operation_logs_job_offset_idx` ON `operation_logs` (`job_id`,`offset`);--> statement-breakpoint
+CREATE INDEX `operation_logs_time_idx` ON `operation_logs` (`created_at`);--> statement-breakpoint
+CREATE TABLE `runtime_logs` (
+	`id` text PRIMARY KEY NOT NULL,
+	`project_id` text NOT NULL,
+	`environment_id` text NOT NULL,
+	`server_id` text NOT NULL,
+	`runtime_id` text NOT NULL,
+	`service` text NOT NULL,
+	`observed_at` integer NOT NULL,
+	`content` text NOT NULL,
+	`search_content` text NOT NULL,
+	`content_hash` text NOT NULL,
+	`created_at` integer DEFAULT (cast(unixepoch('subsec') * 1000 as integer)) NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `runtime_logs_hash_idx` ON `runtime_logs` (`server_id`,`runtime_id`,`content_hash`);--> statement-breakpoint
+CREATE INDEX `runtime_logs_project_time_idx` ON `runtime_logs` (`project_id`,`observed_at`);--> statement-breakpoint
+CREATE INDEX `runtime_logs_server_runtime_time_idx` ON `runtime_logs` (`server_id`,`runtime_id`,`observed_at`);--> statement-breakpoint
+CREATE INDEX `runtime_logs_observed_at_idx` ON `runtime_logs` (`observed_at`);

package/src/server/db/migrations-logs/0001_runtime_log_order.sql

@@ -0,0 +1,2 @@
+CREATE INDEX `runtime_logs_server_runtime_order_idx` ON `runtime_logs` (`server_id`,`runtime_id`,`observed_at`,`created_at`,`id`);--> statement-breakpoint
+DROP INDEX `runtime_logs_server_runtime_time_idx`;