@hot-updater/server 1.0.0-rc.1 → 1.0.0-rc.16

package/src/database/sql/sqlSchema.ts

@@ -0,0 +1,205 @@
+import {
+  DatabaseSchemaError,
+  findPhysicalColumn,
+  indexOrderColumns,
+  type PhysicalColumn,
+  type PhysicalIndex,
+  type PhysicalTable,
+} from "@hot-updater/plugin-core/internal";
+
+export type SqlDialect = "postgresql" | "mysql" | "sqlite";
+
+/** Quotes an identifier: backticks on MySQL, double quotes elsewhere. */
+export const quoteSql = (dialect: SqlDialect, name: string) =>
+  dialect === "mysql"
+    ? `\`${name.replaceAll("`", "``")}\``
+    : `"${name.replaceAll('"', '""')}"`;
+
+export const isMultiIndex = (table: PhysicalTable, index: PhysicalIndex) =>
+  index.eq.some((column) => findPhysicalColumn(table, column).multi);
+
+/** Keeps an identifier within PostgreSQL's 63 and MySQL's 64 characters. */
+export const shortSqlName = (name: string) => {
+  if (name.length <= 60) return name;
+  let hash = 0x811c9dc5;
+  for (const char of name)
+    hash = Math.imul(hash ^ char.charCodeAt(0), 0x01000193) >>> 0;
+  return `${name.slice(0, 51)}_${hash.toString(36)}`;
+};
+
+/** The column type; `entry` types one value of a multi-valued column in its index table. */
+const columnType = (
+  dialect: SqlDialect,
+  column: PhysicalColumn,
+  indexed: boolean,
+  entry = false,
+): string => {
+  if ((column.multi && !entry) || column.type === "json") {
+    return { postgresql: "jsonb", mysql: "json", sqlite: "TEXT" }[dialect];
+  }
+  if (column.type === "string") {
+    const length = column.maxLength ?? (indexed ? 255 : undefined);
+    if (dialect === "sqlite") return "TEXT";
+    if (dialect === "postgresql") {
+      return `${column.maxLength ? `varchar(${column.maxLength})` : "text"} COLLATE "C"`;
+    }
+    const charset = column.ascii
+      ? "ascii COLLATE ascii_bin"
+      : "utf8mb4 COLLATE utf8mb4_0900_bin";
+    return `${length ? `varchar(${length})` : "longtext"} CHARACTER SET ${charset}`;
+  }
+  if (column.type === "integer")
+    return dialect === "sqlite" ? "INTEGER" : "bigint";
+  if (column.type === "number") {
+    return { postgresql: "double precision", mysql: "double", sqlite: "REAL" }[
+      dialect
+    ];
+  }
+  return dialect === "sqlite" ? "INTEGER" : "boolean";
+};
+
+/** InnoDB's limit on the bytes of one key or index. */
+const MYSQL_KEY_BYTES = 3072;
+
+/** Refuses a MySQL key or index over InnoDB's limit before any DDL runs. */
+const checkMysqlKey = (
+  table: PhysicalTable,
+  name: string,
+  columns: readonly string[],
+) => {
+  const bytes = columns.reduce((sum, column) => {
+    const { type, maxLength, ascii } = findPhysicalColumn(table, column);
+    if (type === "string") return sum + (maxLength ?? 255) * (ascii ? 1 : 4);
+    return sum + (type === "boolean" ? 1 : 8);
+  }, 0);
+  if (bytes > MYSQL_KEY_BYTES) {
+    throw new DatabaseSchemaError(
+      `${table.name} ${name} needs ${bytes} bytes on MySQL, over its ${MYSQL_KEY_BYTES}; shorten its strings or mark ASCII ones ascii.`,
+    );
+  }
+};
+
+export interface SqlColumnShape {
+  readonly name: string;
+  /** The column's DDL type, collation included. */
+  readonly type: string;
+  readonly notNull: boolean;
+  readonly default?: number;
+}
+
+export interface SqlTableShape {
+  readonly name: string;
+  readonly columns: readonly SqlColumnShape[];
+  readonly key: readonly string[];
+  /** Indexes, and index tables for multi-valued fields, in declaration order. */
+  readonly indexes: readonly (
+    | {
+        readonly kind: "index";
+        readonly name: string;
+        readonly unique: boolean;
+        readonly columns: readonly string[];
+      }
+    | { readonly kind: "table"; readonly table: SqlTableShape }
+  )[];
+}
+
+/**
+ * Each table as the DDL creates it, for the DDL below and for ORM schema
+ * generators: binary collation, one index per declared index, and an index
+ * table `<table>__<index>` for each index over a multi-valued field, keyed by
+ * every column it holds.
+ */
+export const sqlTableShapes = (
+  dialect: SqlDialect,
+  tables: readonly PhysicalTable[],
+  tablePrefix = "",
+): SqlTableShape[] =>
+  tables.map((table) => {
+    const name = tablePrefix + table.name;
+    const indexed = new Set([
+      ...table.key,
+      ...table.indexes.flatMap(({ eq, sort }) => [...eq, ...sort]),
+    ]);
+    if (dialect === "mysql") checkMysqlKey(table, "key", table.key);
+    const indexes = table.indexes.flatMap((index): SqlTableShape["indexes"] => {
+      const columns = [...index.eq, ...indexOrderColumns(table, index)];
+      if (dialect === "mysql") {
+        checkMysqlKey(table, `index ${index.name}`, columns);
+      }
+      if (isMultiIndex(table, index)) {
+        const entries = columns.map((column) => ({
+          name: column,
+          type: columnType(
+            dialect,
+            findPhysicalColumn(table, column),
+            true,
+            true,
+          ),
+          notNull: true,
+        }));
+        const indexTable = {
+          name: `${name}__${index.name}`,
+          columns: entries,
+          key: columns,
+          indexes: [],
+        };
+        return [{ kind: "table", table: indexTable }];
+      }
+      if (!index.unique && columns.join() === table.key.join()) return [];
+      return [
+        {
+          kind: "index",
+          name: shortSqlName(`${name}_${index.name}`),
+          unique: index.unique === true,
+          columns: index.unique ? index.eq : columns,
+        },
+      ];
+    });
+    return {
+      name,
+      columns: table.columns.map((column) => ({
+        name: column.name,
+        type: columnType(dialect, column, indexed.has(column.name)),
+        notNull: !column.nullable,
+        ...(column.default === undefined ? {} : { default: column.default }),
+      })),
+      key: table.key,
+      indexes,
+    };
+  });
+
+/** DDL for the tables' shapes: each table, then its indexes and index tables. */
+export const createTableStatements = (
+  dialect: SqlDialect,
+  tables: readonly PhysicalTable[],
+  tablePrefix = "",
+): string[] => {
+  const quote = (name: string) => quoteSql(dialect, name);
+  const list = (columns: readonly string[]) => columns.map(quote).join(", ");
+  const render = (shape: SqlTableShape): string[] => {
+    const definitions = shape.columns.map(
+      (column) =>
+        `${quote(column.name)} ${column.type}${column.notNull ? " NOT NULL" : ""}${column.default === undefined ? "" : ` DEFAULT ${column.default}`}`,
+    );
+    definitions.push(`PRIMARY KEY (${list(shape.key)})`);
+    const after: string[] = [];
+    for (const index of shape.indexes) {
+      if (index.kind === "table") {
+        after.push(...render(index.table));
+      } else if (dialect === "mysql") {
+        definitions.push(
+          `${index.unique ? "UNIQUE " : ""}INDEX ${quote(index.name)} (${list(index.columns)})`,
+        );
+      } else {
+        after.push(
+          `CREATE ${index.unique ? "UNIQUE " : ""}INDEX IF NOT EXISTS ${quote(index.name)} ON ${quote(shape.name)} (${list(index.columns)})`,
+        );
+      }
+    }
+    return [
+      `CREATE TABLE IF NOT EXISTS ${quote(shape.name)} (${definitions.join(", ")})`,
+      ...after,
+    ];
+  };
+  return sqlTableShapes(dialect, tables, tablePrefix).flatMap(render);
+};

package/src/database/sql/sqlTestExecutors.ts

@@ -0,0 +1,183 @@
+import type { DatabaseSync, SqliteValue } from "node:sqlite";
+
+import type { PGlite } from "@electric-sql/pglite";
+import type { Pool as MysqlPool, PoolConnection } from "mysql2/promise";
+import type { Pool as PgPool, PoolClient } from "pg";
+
+import type {
+  SqlConnection,
+  SqlExecutor,
+  SqlResult,
+  SqlStatement,
+} from "./sqlAdapter";
+
+/** Test executors for the SQL core; provider executors arrive with D1–D3. */
+
+export const pgliteExecutor = (db: PGlite): SqlExecutor => {
+  const over = (runner: Pick<PGlite, "query">): SqlConnection => ({
+    execute: async ({ sql, params }) => {
+      const result = await runner.query<Record<string, unknown>>(sql, [
+        ...params,
+      ]);
+      return { rows: result.rows, changes: result.affectedRows ?? 0 };
+    },
+  });
+  return {
+    dialect: "postgresql",
+    ...over(db),
+    transaction: (fn) => db.transaction((tx) => fn(over(tx))),
+  };
+};
+
+/** One connection, so statements and transactions run one at a time. */
+export const sqliteExecutor = (db: DatabaseSync): SqlExecutor => {
+  let queue: Promise<unknown> = Promise.resolve();
+  const serial = <T>(task: () => Promise<T>): Promise<T> => {
+    const next = queue.then(task);
+    queue = next.catch(() => undefined);
+    return next;
+  };
+  const execute = async ({ sql, params }: SqlStatement): Promise<SqlResult> => {
+    const statement = db.prepare(sql);
+    const values = params as SqliteValue[];
+    if (statement.columns().length > 0) {
+      return { rows: statement.all(...values), changes: 0 };
+    }
+    return { rows: [], changes: Number(statement.run(...values).changes) };
+  };
+  return {
+    dialect: "sqlite",
+    execute: (statement) => serial(() => execute(statement)),
+    transaction: (fn) =>
+      serial(async () => {
+        db.exec("BEGIN IMMEDIATE");
+        try {
+          const result = await fn({ execute });
+          db.exec("COMMIT");
+          return result;
+        } catch (error) {
+          db.exec("ROLLBACK");
+          throw error;
+        }
+      }),
+  };
+};
+
+/**
+ * An executor without interactive transactions, as D1's: writes run as one
+ * atomic batch, here in a SQLite transaction that rolls back on any error.
+ */
+export const sqliteBatchExecutor = (db: DatabaseSync): SqlExecutor => {
+  const executor = sqliteExecutor(db);
+  return {
+    dialect: "sqlite",
+    execute: executor.execute,
+    transaction: () => {
+      throw new Error("A batch executor has no interactive transactions.");
+    },
+    batch: (statements) =>
+      executor.transaction(async (connection) => {
+        const results: SqlResult[] = [];
+        for (const statement of statements) {
+          results.push(await connection.execute(statement));
+        }
+        return results;
+      }),
+  };
+};
+
+/** PGlite without interactive transactions, as an RPC applying a batch would be. */
+export const pgliteBatchExecutor = (db: PGlite): SqlExecutor => {
+  const executor = pgliteExecutor(db);
+  return {
+    dialect: "postgresql",
+    execute: executor.execute,
+    transaction: () => {
+      throw new Error("A batch executor has no interactive transactions.");
+    },
+    batch: (statements) =>
+      executor.transaction(async (connection) => {
+        const results: SqlResult[] = [];
+        for (const statement of statements) {
+          results.push(await connection.execute(statement));
+        }
+        return results;
+      }),
+  };
+};
+
+export const pgExecutor = (pool: PgPool): SqlExecutor => {
+  const over = (runner: PgPool | PoolClient): SqlConnection => ({
+    execute: async ({ sql, params }) => {
+      const result = await runner.query(sql, [...params]);
+      return { rows: result.rows, changes: result.rowCount ?? 0 };
+    },
+  });
+  return {
+    dialect: "postgresql",
+    ...over(pool),
+    async transaction(fn) {
+      const client = await pool.connect();
+      try {
+        await client.query("BEGIN");
+        const result = await fn(over(client));
+        await client.query("COMMIT");
+        return result;
+      } catch (error) {
+        await client.query("ROLLBACK").catch(() => undefined);
+        throw error;
+      } finally {
+        client.release();
+      }
+    },
+  };
+};
+
+/** A pooled PostgreSQL batch: one READ COMMITTED transaction per write, as Supabase's apply RPC runs. */
+export const pgBatchExecutor = (pool: PgPool): SqlExecutor => {
+  const executor = pgExecutor(pool);
+  return {
+    dialect: "postgresql",
+    execute: executor.execute,
+    transaction: () => {
+      throw new Error("A batch executor has no interactive transactions.");
+    },
+    batch: (statements) =>
+      executor.transaction(async (connection) => {
+        const results: SqlResult[] = [];
+        for (const statement of statements) {
+          results.push(await connection.execute(statement));
+        }
+        return results;
+      }),
+  };
+};
+
+export const mysqlExecutor = (pool: MysqlPool): SqlExecutor => {
+  const over = (runner: MysqlPool | PoolConnection): SqlConnection => ({
+    execute: async ({ sql, params }) => {
+      const [result] = await runner.query(sql, [...params]);
+      return Array.isArray(result)
+        ? { rows: result as Record<string, unknown>[], changes: 0 }
+        : { rows: [], changes: result.affectedRows };
+    },
+  });
+  return {
+    dialect: "mysql",
+    ...over(pool),
+    async transaction(fn) {
+      const connection = await pool.getConnection();
+      try {
+        await connection.beginTransaction();
+        const result = await fn(over(connection));
+        await connection.commit();
+        return result;
+      } catch (error) {
+        await connection.rollback().catch(() => undefined);
+        throw error;
+      } finally {
+        connection.release();
+      }
+    },
+  };
+};

package/src/databaseTestSuite.spec.ts

@@ -0,0 +1,39 @@
+import { createMemoryAdapter } from "@hot-updater/plugin-core/internal";
+import {
+  setupDatabaseTestSuite,
+  startHttpTestServer,
+} from "@hot-updater/test-utils";
+
+import { createDatabasePluginApis } from "./assembly/databasePlugins";
+import { createHotUpdater } from "./index";
+import { createInsightsModel, insights } from "./plugins/insights";
+
+let adapter = createMemoryAdapter();
+
+setupDatabaseTestSuite({
+  name: "in-memory engine database",
+  migrate: () => undefined,
+  createDatabase: () => ({
+    name: "memory",
+    // Each test's reset swaps in an empty adapter.
+    adapter: new Proxy({} as typeof adapter, {
+      get: (_target, key) => Reflect.get(adapter, key),
+    }),
+  }),
+  reset: () => {
+    adapter = createMemoryAdapter();
+  },
+  dispose: () => undefined,
+  createHttpClient: (options) =>
+    startHttpTestServer(
+      createHotUpdater({
+        ...options,
+        plugins: [insights()],
+        clientAccess: "public",
+      }).handlers,
+    ),
+  createInsightsModel: (database) =>
+    createInsightsModel(
+      createDatabasePluginApis(database, [insights()]).insights,
+    ),
+});

package/src/db/builtInDatabase.ts

@@ -0,0 +1,149 @@
+import type { DatabaseAdapter } from "@hot-updater/plugin-core/internal";
+
+import { coreModule, HOT_UPDATER_SCHEMA_VERSION } from "../core/schema";
+import type { ModelShape } from "../database/definitions";
+import {
+  ENGINE_SCHEMA_KEY,
+  ENGINE_SCHEMA_VERSION,
+  isMissingSchemaError,
+  readSettings,
+  withSchemaFence,
+  type SchemaSettings,
+} from "../database/fence";
+import { resolveSchema, type SchemaModule } from "../database/resolveSchema";
+import { apiKeys, apiKeysSchema } from "../plugins/api-keys";
+import { builtInPlugin } from "../plugins/builtIn";
+import { insights, insightsSchema } from "../plugins/insights";
+import { createEngineMigrator } from "./engineMigrator";
+import { migrateSchema } from "./schemaSettings";
+import type { ToolingDatabase, ToolingTarget } from "./types";
+
+/** Core and the built-in plugins: their tables keep their names. */
+export const builtInModules: readonly SchemaModule[] = [
+  coreModule,
+  { id: "insights", schema: insightsSchema },
+  { id: "apiKeys", schema: apiKeysSchema },
+];
+
+/**
+ * Core's tables and the built-in plugins' (Insights and API keys): the tables
+ * every provider's tooling creates, whichever plugins a server runs.
+ */
+export const builtInSchema = resolveSchema(builtInModules);
+
+/** The settings rows a provider's database is fenced by. */
+export const builtInSettings: SchemaSettings = {
+  [ENGINE_SCHEMA_KEY]: ENGINE_SCHEMA_VERSION,
+  "schema.core": HOT_UPDATER_SCHEMA_VERSION,
+  "schema.insights": insights().schemaVersion,
+  "schema.apiKeys": apiKeys().schemaVersion,
+};
+
+/** Creates the built-in tables, then writes their settings rows. */
+export const migrateBuiltInSchema = (adapter: DatabaseAdapter, name: string) =>
+  migrateSchema(adapter, name, builtInSchema.tables, builtInSettings);
+
+/** What `hot-updater db` creates for a server without third-party plugins. */
+export const builtInTarget: ToolingTarget = {
+  schema: builtInSchema,
+  settings: builtInSettings,
+};
+
+/** A plugin as the tooling reads it: its id, its tables, and their version. */
+export interface PluginTables {
+  readonly id: string;
+  readonly schemaVersion: string;
+  readonly schema: Readonly<Record<string, ModelShape>>;
+}
+
+const addedPlugins = (plugins: readonly PluginTables[]) =>
+  plugins.filter((plugin) => !(builtInPlugin in plugin));
+
+/** Each third-party plugin's `schema.<id>` settings row. */
+export const addedSettings = (
+  plugins: readonly PluginTables[],
+): SchemaSettings =>
+  Object.fromEntries(
+    addedPlugins(plugins).map(({ id, schemaVersion }) => [
+      `schema.${id}`,
+      schemaVersion,
+    ]),
+  );
+
+/**
+ * What `hot-updater db` creates for a server's plugins: the built-in tables,
+ * then each third-party plugin's tables with its `schema.<id>` settings row.
+ */
+export const toolingTargetOf = (
+  plugins: readonly PluginTables[],
+): ToolingTarget => {
+  const settings = addedSettings(plugins);
+  if (Object.keys(settings).length === 0) return builtInTarget;
+  return {
+    schema: resolveSchema([
+      ...builtInModules,
+      ...addedPlugins(plugins).map(({ id, schema }) => ({
+        id,
+        schema,
+        namespace: id,
+      })),
+    ]),
+    settings: { ...builtInSettings, ...settings },
+  };
+};
+
+export interface EngineDatabaseOptions {
+  readonly name: string;
+  readonly adapter: DatabaseAdapter;
+  /**
+   * Purges a CDN's copies of the cacheable client routes. Core calls it after
+   * a committed write that changes what those routes answer, since only core
+   * knows which of its tables they read.
+   */
+  readonly onCachedRoutesChange?: () => Promise<void>;
+}
+
+/**
+ * A provider's database on the storage engine: its adapter behind the schema
+ * fence, which checks the built-in settings rows before the first read or
+ * write and names `hot-updater db migrate` when they are missing or stale.
+ * When the adapter creates its own tables, `hot-updater db migrate` creates
+ * the built-in tables and the server's plugin tables, then writes their
+ * settings rows.
+ */
+export const createEngineDatabase = ({
+  name,
+  adapter,
+  onCachedRoutesChange,
+}: EngineDatabaseOptions): ToolingDatabase => {
+  const fenced = withSchemaFence(adapter, name, builtInSettings);
+  const createMigrator = ({ schema, settings } = builtInTarget) =>
+    createEngineMigrator({
+      adapterName: name,
+      adapter,
+      schema,
+      settings,
+      readSettings: async () => {
+        const rows = await readSettings(adapter, Object.keys(settings)).catch(
+          (error: unknown) => {
+            if (isMissingSchemaError(error)) return [];
+            throw error;
+          },
+        );
+        return new Map(
+          rows.flatMap((row) =>
+            row === null ? [] : [[String(row.key), row.value]],
+          ),
+        );
+      },
+    });
+  return {
+    name,
+    adapter: fenced,
+    ...(onCachedRoutesChange === undefined ? {} : { onCachedRoutesChange }),
+    ...(adapter.dispose === undefined
+      ? {}
+      : { dispose: () => adapter.dispose!() }),
+    ...(adapter.migrations === undefined ? {} : { createMigrator }),
+  };
+};

package/src/db/bundleRows.spec.ts

@@ -13,10 +13,10 @@ import {
 const createBundle = (id: string): Bundle => ({
   id,
   platform: "ios",
-  fileHash: `hash-${id}`,
   gitCommitHash: null,
-  storageUri: `s3://bucket/${id}.zip`,
-  archiveByteSize: 3_000_000_001,
+  manifestStorageUri: `s3://bucket/${id}/manifest.json`,
+  manifestFileHash: `manifest-hash-${id}`,
+  assetBaseStorageUri: "s3://bucket/assets",
 });
 
 const toRow = (bundle: Bundle): BundleRow => bundleToRow(bundle);