@deterministic-code/migraters 0.0.0-stage → 0.0.1

package/src/cli-spec.ts

@@ -0,0 +1,64 @@
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parse } from 'yaml';
+
+export type MigrateVerb = 'init' | 'setup' | 'up' | 'down' | 'create';
+
+export type CliArg = {
+  flag: string;
+  placeholder?: string;
+  required: boolean;
+};
+
+export type CliCommand = {
+  verb: MigrateVerb;
+  args: CliArg[];
+};
+
+export type CliSpec = {
+  command: string;
+  providers: string[];
+  provider_aliases: Record<string, string>;
+  aliases: Record<string, string>;
+  env_var_list_separator: string;
+  connection_environment: Record<string, string[]>;
+  commands: CliCommand[];
+};
+
+export const cliYamlPath = (): string => {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const candidates = [
+    join(here, '..', 'cli.yaml'),
+    join(here, '..', '..', 'cli.yaml'),
+    join(process.cwd(), 'cli.yaml'),
+  ];
+  const found = candidates.find((candidate) => existsSync(candidate));
+  if (!found) {
+    throw new Error('cli.yaml not found');
+  }
+  return found;
+};
+
+export const cliSpec = parse(await readFile(cliYamlPath(), 'utf8')) as CliSpec;
+
+export const migrateCommand = (verb: MigrateVerb): string =>
+  cliSpec.command.replace('{verb}', verb);
+
+const providers = cliSpec.providers.join('|');
+
+const argUsage = (arg: CliArg): string => {
+  const ph = arg.placeholder === 'providers' ? providers : arg.placeholder;
+  const body = ph ? `${arg.flag} <${ph}>` : arg.flag;
+  return arg.required ? body : `[${body}]`;
+};
+
+export const usageLine = (verb: MigrateVerb): string => {
+  const command = cliSpec.commands.find((c) => c.verb === verb);
+  if (!command) throw new Error(`cli.yaml: missing command ${verb}`);
+  return `Usage: ${migrateCommand(verb)} ${command.args.map(argUsage).join(' ')}`;
+};
+
+export const programUsageText = (): string =>
+  `${cliSpec.commands.map((c) => usageLine(c.verb)).join('\n')}\n`;

package/src/cli.ts

@@ -0,0 +1,52 @@
+import { connectionResolver, defaultDialectFactory } from './infrastructure/default-factory.ts';
+import type { ISqlDialect } from './abstractions/sql-dialect.ts';
+import { cliSpec } from './cli-spec.ts';
+import { fillTemplate, loadMessage } from './infrastructure/message-templates.ts';
+
+const missingProviderMsg = await loadMessage('errors/missing-provider');
+const unsupportedProviderTpl = await loadMessage('errors/unsupported-provider');
+const missingConnectionTpl = await loadMessage('errors/missing-connection');
+
+export { missingProviderMsg };
+
+export const requireDialect = (provider: string | null): ISqlDialect => {
+  if (!provider) {
+    process.stderr.write(missingProviderMsg);
+    process.exit(2);
+  }
+  const dialect = defaultDialectFactory.tryGet(provider);
+  if (!dialect) {
+    process.stderr.write(fillTemplate(unsupportedProviderTpl, { provider }));
+    process.exit(2);
+  }
+  return dialect;
+};
+
+export const requireConnection = (dialect: ISqlDialect, fromFlag: string | null): string => {
+  const connection = connectionResolver.resolve(dialect, fromFlag);
+  if (!connection) {
+    process.stderr.write(
+      fillTemplate(missingConnectionTpl, {
+        envVars: dialect.connectionEnvironmentVariables.join(cliSpec.env_var_list_separator),
+      }),
+    );
+    process.exit(2);
+  }
+  return connection;
+};
+
+export const takeFlag = (
+  rest: string[],
+  i: number,
+  arg: string,
+  flag: string,
+): { value: string; next: number } | undefined => {
+  const eq = `${flag}=`;
+  if (arg === flag) {
+    const value = rest[i + 1];
+    if (value === undefined) return undefined;
+    return { value, next: i + 1 };
+  }
+  if (arg.startsWith(eq)) return { value: arg.slice(eq.length), next: i };
+  return undefined;
+};

package/src/create-sequence.ts

@@ -0,0 +1,17 @@
+export const MIGRATION_NAME_RE = /^[a-z][a-z0-9_]*$/;
+const SEQ_RE = /^(\d{1,4})_.*_up\.sql$/;
+
+export const isMigrationName = (name: string): boolean => MIGRATION_NAME_RE.test(name);
+
+export const nextSequence = (filenames: Iterable<string>): number => {
+  let max = 0;
+  for (const name of filenames) {
+    const m = SEQ_RE.exec(name);
+    if (!m) continue;
+    const n = Number.parseInt(m[1] as string, 10);
+    if (n > max) max = n;
+  }
+  return max + 1;
+};
+
+export const pad4 = (n: number): string => String(n).padStart(4, '0');

package/src/datasource-migrate.ts

@@ -0,0 +1,314 @@
+import { createHash } from 'node:crypto';
+import { readFile, readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { defaultDialectFactory } from './infrastructure/default-factory.ts';
+import { splitOnGo } from './infrastructure/split-on-go.ts';
+import { fillTemplate, loadMessage } from './infrastructure/message-templates.ts';
+import type { ISqlDialect } from './abstractions/sql-dialect.ts';
+import type { MigrationClient, MigrationRowValue } from './migration-client.ts';
+import type { SqlDialect } from './sql.ts';
+
+export { normalizeDialect } from './sql.ts';
+export type {
+  MigrationClient,
+  MigrationRow,
+  MigrationSqlParam,
+  SqliteRawDatabase,
+} from './migration-client.ts';
+export { applyMysqlDdlViaTextProtocol } from './dialects/mysql-dialect.ts';
+
+/** One discovered migration: its up script and the rollback sibling when present. */
+export interface MigrationDescriptor {
+  name: string;
+  upPath: string;
+  downPath: string | null;
+}
+
+const UP_SUFFIX = '_up.sql';
+const DOWN_SUFFIX = '_down.sql';
+
+const [
+  checksumDriftTpl,
+  checksumDriftLineTpl,
+  rollbackNotInPathMsg,
+  rollbackNoSiblingTpl,
+  migratePathNotDirTpl,
+] = await Promise.all([
+  loadMessage('checksum-drift'),
+  loadMessage('checksum-drift-line'),
+  loadMessage('errors/rollback-not-in-path'),
+  loadMessage('errors/rollback-no-rollback-sibling'),
+  loadMessage('errors/migrate-path-not-dir'),
+]);
+
+const missingMigrationsDir = (path: string): Error =>
+  new Error(fillTemplate(migratePathNotDirTpl, { path }).trimEnd());
+
+export async function discoverMigrations({
+  migratePath,
+  dialect,
+}: {
+  migratePath: string;
+  dialect: string;
+}): Promise<MigrationDescriptor[]> {
+  requireDialect(dialect);
+  let st;
+  try {
+    st = await stat(migratePath);
+  } catch {
+    throw missingMigrationsDir(migratePath);
+  }
+  if (!st.isDirectory()) throw missingMigrationsDir(migratePath);
+
+  const entries = await readdir(migratePath, { withFileTypes: true });
+  const files = new Set(entries.filter((e) => e.isFile()).map((e) => e.name));
+  const ups: MigrationDescriptor[] = [];
+  for (const name of files) {
+    if (!name.endsWith(UP_SUFFIX)) continue;
+    const stem = name.slice(0, -UP_SUFFIX.length);
+    const downName = `${stem}${DOWN_SUFFIX}`;
+    ups.push({
+      name: stem,
+      upPath: join(migratePath, name),
+      downPath: files.has(downName) ? join(migratePath, downName) : null,
+    });
+  }
+  ups.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+  return ups;
+}
+
+/** Strip pre-refactor `-- deterministic-content-hash: <sha>` header if present so upgrading users carrying the legacy header keep the same checksum migrate:up recorded. */
+const LEGACY_HEADER_RE = /^(?:\/\/|--|#)\s+deterministic-content-hash:\s+[0-9a-f]{64}\s*\r?\n?/;
+
+export function checksum(text: string): string {
+  const body = text.replace(LEGACY_HEADER_RE, '');
+  return createHash('sha256').update(body, 'utf8').digest('hex');
+}
+
+export async function applyOneUp({
+  client,
+  migrations,
+  env = process.env,
+}: {
+  client: MigrationClient;
+  migrations: MigrationDescriptor[];
+  env?: NodeJS.ProcessEnv;
+}): Promise<{ applied: boolean; name: string | null }> {
+  const applied = await appliedRecords(client);
+  await assertNoChecksumDrift({ migrations, applied, env });
+  const next = migrations.find((m) => !applied.has(m.name));
+  if (!next) return { applied: false, name: null };
+
+  const sqlText = await readFile(next.upPath, 'utf8');
+  const sum = checksum(sqlText);
+
+  const logId = await insertLogStarted(client, next.name, 'up');
+  const startedAt = Date.now();
+
+  try {
+    await client.transaction(async () => {
+      const stmts = splitOnGo(await readFile(next.upPath, 'utf8'));
+      for (const stmt of stmts) {
+        await client.exec(stmt);
+      }
+      await insertMigrate(client, next.name, sum);
+    });
+  } catch (e) {
+    const err = e as Error;
+    await updateLogTerminal({
+      client,
+      id: logId,
+      status: 'error',
+      durationMs: Date.now() - startedAt,
+      errorMessage: String(err?.message ?? e),
+    });
+    throw e;
+  }
+
+  await updateLogTerminal({
+    client,
+    id: logId,
+    status: 'success',
+    durationMs: Date.now() - startedAt,
+    errorMessage: null,
+  });
+  return { applied: true, name: next.name };
+}
+
+export async function applyOneDown({
+  client,
+  migrations,
+  env = process.env,
+}: {
+  client: MigrationClient;
+  migrations: MigrationDescriptor[];
+  env?: NodeJS.ProcessEnv;
+}): Promise<{ rolledBack: boolean; name: string | null }> {
+  const applied = await appliedRecords(client);
+  await assertNoChecksumDrift({ migrations, applied, env });
+  if (applied.size === 0) return { rolledBack: false, name: null };
+
+  const lastApplied = [...migrations].reverse().find((m) => applied.has(m.name));
+  if (!lastApplied) {
+    throw new Error(rollbackNotInPathMsg.trimEnd());
+  }
+  if (!lastApplied.downPath) {
+    throw new Error(
+      fillTemplate(rollbackNoSiblingTpl, { name: lastApplied.name }).trimEnd(),
+    );
+  }
+  const downPath = lastApplied.downPath;
+
+  const logId = await insertLogStarted(client, lastApplied.name, 'down');
+  const startedAt = Date.now();
+
+  try {
+    await client.transaction(async () => {
+      const stmts = splitOnGo(await readFile(downPath, 'utf8'));
+      for (const stmt of stmts) {
+        await client.exec(stmt);
+      }
+      await deleteMigrate(client, lastApplied.name);
+    });
+  } catch (e) {
+    const err = e as Error;
+    await updateLogTerminal({
+      client,
+      id: logId,
+      status: 'error',
+      durationMs: Date.now() - startedAt,
+      errorMessage: String(err?.message ?? e),
+    });
+    throw e;
+  }
+
+  await updateLogTerminal({
+    client,
+    id: logId,
+    status: 'success',
+    durationMs: Date.now() - startedAt,
+    errorMessage: null,
+  });
+  return { rolledBack: true, name: lastApplied.name };
+}
+
+function dialectOf(client: MigrationClient): ISqlDialect {
+  return defaultDialectFactory.get(client.dialect);
+}
+
+async function appliedRecords(client: MigrationClient): Promise<Map<string, string | null>> {
+  const d = dialectOf(client);
+  const q = (ident: string) => d.quoteIdent(ident);
+  const rows = await client.query(
+    `SELECT ${q('name')} AS name, ${q('checksum')} AS checksum FROM ${q('migrates')}`,
+  );
+  return new Map(
+    rows.map(
+      (r) => [r.name ?? r.NAME, r.checksum ?? r.CHECKSUM ?? null] as [string, string | null],
+    ),
+  );
+}
+
+async function assertNoChecksumDrift({
+  migrations,
+  applied,
+  env = process.env,
+}: {
+  migrations: MigrationDescriptor[];
+  applied: Map<string, string | null>;
+  env?: NodeJS.ProcessEnv;
+}): Promise<void> {
+  if (env.MIGRATE_ALLOW_CHECKSUM_DRIFT === '1') return;
+  const drifted: { name: string; stored: string; current: string }[] = [];
+  for (const m of migrations) {
+    if (!applied.has(m.name)) continue;
+    const stored = applied.get(m.name);
+    if (stored == null) continue;
+    const current = checksum(await readFile(m.upPath, 'utf8'));
+    if (current !== stored) drifted.push({ name: m.name, stored, current });
+  }
+  if (drifted.length === 0) return;
+  const lines = drifted
+    .map((d) =>
+      fillTemplate(checksumDriftLineTpl, {
+        name: d.name,
+        stored: d.stored.slice(0, 12),
+        current: d.current.slice(0, 12),
+      }).trimEnd(),
+    )
+    .join('\n');
+  throw new Error(fillTemplate(checksumDriftTpl, { lines }).trimEnd());
+}
+
+async function insertMigrate(client: MigrationClient, name: string, sum: string): Promise<void> {
+  const d = dialectOf(client);
+  const t = d.quoteIdent('migrates');
+  await client.run(
+    `INSERT INTO ${t} (${d.quoteIdent('name')}, ${d.quoteIdent('checksum')}) VALUES (?, ?)`,
+    [name, sum],
+  );
+}
+
+async function deleteMigrate(client: MigrationClient, name: string): Promise<void> {
+  const d = dialectOf(client);
+  const t = d.quoteIdent('migrates');
+  await client.run(`DELETE FROM ${t} WHERE ${d.quoteIdent('name')} = ?`, [name]);
+}
+
+async function insertLogStarted(
+  client: MigrationClient,
+  name: string,
+  direction: string,
+): Promise<MigrationRowValue> {
+  const d = dialectOf(client);
+  const t = d.quoteIdent('migrate_logs');
+  if (d.usesLastInsertRowid) {
+    const stmt = `INSERT INTO ${t} (${d.quoteIdent('migrate_name')}, ${d.quoteIdent('direction')}, ${d.quoteIdent('status')}) VALUES (?, ?, 'started')`;
+    const db = client._raw;
+    if (db) {
+      const info = db.prepare(stmt).run(name, direction);
+      return info.lastInsertRowid;
+    }
+    await client.run(stmt, [name, direction]);
+    const rows = await client.query(
+      `SELECT ${d.quoteIdent('id')} AS id FROM ${t} WHERE ${d.quoteIdent('migrate_name')} = ? AND ${d.quoteIdent('direction')} = ? ORDER BY id DESC ${d.limitClause(1)}`,
+      [name, direction],
+    );
+    return rows[0]?.id;
+  }
+  await client.run(
+    `INSERT INTO ${t} (${d.quoteIdent('migrate_name')}, ${d.quoteIdent('direction')}, ${d.quoteIdent('status')}) VALUES (?, ?, 'started')`,
+    [name, direction],
+  );
+  const rows = await client.query(
+    `SELECT ${d.quoteIdent('id')} AS id FROM ${t} WHERE ${d.quoteIdent('migrate_name')} = ? AND ${d.quoteIdent('direction')} = ? ORDER BY ${d.quoteIdent('id')} DESC ${d.limitClause(1)}`,
+    [name, direction],
+  );
+  return rows[0]?.id ?? rows[0]?.ID;
+}
+
+async function updateLogTerminal({
+  client,
+  id,
+  status,
+  durationMs,
+  errorMessage,
+}: {
+  client: MigrationClient;
+  id: MigrationRowValue;
+  status: string;
+  durationMs: number;
+  errorMessage: string | null;
+}): Promise<void> {
+  const d = dialectOf(client);
+  const t = d.quoteIdent('migrate_logs');
+  const setNow = d.nowExpr();
+  await client.run(
+    `UPDATE ${t} SET ${d.quoteIdent('status')} = ?, ${d.quoteIdent('finished_at')} = ${setNow}, ${d.quoteIdent('duration_ms')} = ?, ${d.quoteIdent('error_message')} = ?, ${d.quoteIdent('updated')} = ${setNow} WHERE ${d.quoteIdent('id')} = ?`,
+    [status, durationMs, errorMessage, id],
+  );
+}
+
+function requireDialect(dialect: string): SqlDialect {
+  return defaultDialectFactory.get(dialect).name;
+}

package/src/dialects/mysql-dialect.test.ts

@@ -0,0 +1,93 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+
+const mysql = vi.hoisted(() => {
+  const state = {
+    failRollback: false,
+    ended: false,
+    queries: [] as string[],
+    executes: [] as { sql: string; params?: unknown[] }[],
+  };
+  const conn = {
+    async query(sql: string) {
+      state.queries.push(sql);
+    },
+    async execute(sql: string, params?: unknown[]) {
+      state.executes.push({ sql, params });
+      return [[{ n: 1 }]];
+    },
+    async beginTransaction() {},
+    async commit() {},
+    async rollback() {
+      if (state.failRollback) throw new Error('rollback failed');
+    },
+    async end() {
+      state.ended = true;
+    },
+  };
+  return {
+    state,
+    createConnection: async () => conn,
+  };
+});
+
+vi.mock('mysql2/promise', () => ({
+  createConnection: mysql.createConnection,
+}));
+
+import { applyMysqlDdlViaTextProtocol, MysqlDialect } from './mysql-dialect.ts';
+
+const dialect = new MysqlDialect();
+
+describe('MysqlDialect', () => {
+  afterEach(() => {
+    mysql.state.failRollback = false;
+    mysql.state.ended = false;
+    mysql.state.queries = [];
+    mysql.state.executes = [];
+  });
+
+  it('registers as mysql with backtick identifiers', async () => {
+    expect(dialect.name).toBe('mysql');
+    expect([...dialect.connectionEnvironmentVariables]).toEqual(['MYSQL_URL', 'DATABASE_URL']);
+    expect(dialect.quoteIdent('t')).toBe('`t`');
+    expect(dialect.nowExpr()).toBe('CURRENT_TIMESTAMP');
+    expect(dialect.limitClause(2)).toBe('LIMIT 2');
+    expect(dialect.translatePlaceholders('?')).toBe('?');
+    expect(dialect.migratesDdl).toMatch(/migrates/i);
+    expect(dialect.migrateLogsDdl).toMatch(/migrate_logs/i);
+    expect(await dialect.prerequisiteError('mysql://x')).toBeNull();
+    await dialect.prepareSetup('mysql://x');
+  });
+
+  it('applyMysqlDdlViaTextProtocol uses query, not execute', async () => {
+    const seen: string[] = [];
+    await applyMysqlDdlViaTextProtocol({ query: async (sql: string) => { seen.push(sql); } }, 'CREATE TABLE t (id INT)');
+    expect(seen).toEqual(['CREATE TABLE t (id INT)']);
+  });
+
+  it('createClient uses text-protocol exec and execute for query/run', async () => {
+    const client = await dialect.createClient('mysql://localhost/db');
+    expect(client.dialect).toBe('mysql');
+    await client.exec('DROP PROCEDURE p');
+    expect(mysql.state.queries).toContain('DROP PROCEDURE p');
+    expect(await client.query('SELECT ?', [1])).toEqual([{ n: 1 }]);
+    await client.run('UPDATE t SET x = ?', [2]);
+    expect(await client.transaction(async () => 9)).toBe(9);
+    await expect(
+      client.transaction(async () => {
+        throw new Error('tx');
+      }),
+    ).rejects.toThrow('tx');
+    mysql.state.failRollback = true;
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+    await expect(
+      client.transaction(async () => {
+        throw new Error('tx2');
+      }),
+    ).rejects.toThrow('tx2');
+    expect(warn).toHaveBeenCalled();
+    warn.mockRestore();
+    await client.close();
+    expect(mysql.state.ended).toBe(true);
+  });
+});

package/src/dialects/mysql-dialect.ts

@@ -0,0 +1,67 @@
+import type { Connection as MysqlConnection } from 'mysql2/promise';
+import type { MigrationClient, MigrationRow } from '../migration-client.ts';
+import { readSqlTemplate } from '../infrastructure/sql-templates.ts';
+import { SqlDialectBase } from '../abstractions/sql-dialect-base.ts';
+
+const [migratesDdl, migrateLogsDdl] = await Promise.all([
+  readSqlTemplate('mysql', 'migrates'),
+  readSqlTemplate('mysql', 'migrate_logs'),
+]);
+
+export async function applyMysqlDdlViaTextProtocol(
+  conn: Pick<MysqlConnection, 'query'>,
+  sql: string,
+): Promise<void> {
+  // why: DDL statements (DROP/CREATE PROCEDURE) reject prepared-statement protocol (MySQL 1295)
+  await conn.query(sql);
+}
+
+export class MysqlDialect extends SqlDialectBase {
+  readonly name = 'mysql' as const;
+  readonly connectionEnvironmentVariables = ['MYSQL_URL', 'DATABASE_URL'] as const;
+  readonly migratesDdl = migratesDdl;
+  readonly migrateLogsDdl = migrateLogsDdl;
+
+  quoteIdent(ident: string): string {
+    return `\`${ident}\``;
+  }
+
+  async createClient(connection: string): Promise<MigrationClient> {
+    const mysql = await import('mysql2/promise');
+    const conn = await mysql.createConnection(connection);
+    return {
+      dialect: 'mysql',
+      async exec(sql) {
+        await applyMysqlDdlViaTextProtocol(conn, sql);
+      },
+      async query(sql, params = []) {
+        const [rows] = await conn.execute(sql, params);
+        return rows as MigrationRow[];
+      },
+      async run(sql, params = []) {
+        await conn.execute(sql, params);
+      },
+      async transaction(fn) {
+        await conn.beginTransaction();
+        try {
+          const r = await fn();
+          await conn.commit();
+          return r;
+        } catch (e) {
+          try {
+            await conn.rollback();
+          } catch (rollbackErr) {
+            console.warn(
+              'mysql rollback failed after transaction error; original error rethrown',
+              rollbackErr,
+            );
+          }
+          throw e;
+        }
+      },
+      async close() {
+        await conn.end();
+      },
+    };
+  }
+}