@deterministic-code/migraters 0.0.4 → 0.0.5

package/orchestrator/dist/config.js

@@ -3,29 +3,51 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { stringify, parse } from 'yaml';
 import { binDirFor, normalizeLanguage } from "./languages.js";
+import { providerEntries } from "./provider-location.js";
 export const CONFIG_NAME = '.migraters.yaml';
 export const configPath = (dir) => join(dir, CONFIG_NAME);
-export const renderConfig = (language, dialects) => stringify({
+export const renderConfig = (language, dialects, configDir = '') => stringify({
     language,
-    dialects,
+    providers: providerEntries(language, dialects, configDir),
     binDir: binDirFor(language),
 });
 export const writeConfig = async (dir, language, dialects) => {
     const path = configPath(dir);
-    await writeFile(path, renderConfig(language, dialects));
+    await writeFile(path, renderConfig(language, dialects, dir));
+    if (language === 'typescript') {
+        const tsDir = join(dir, 'typescript');
+        if (existsSync(tsDir)) {
+            await writeFile(configPath(tsDir), renderConfig(language, dialects, tsDir));
+        }
+    }
     return path;
 };
 export const parseConfig = (raw) => {
     const doc = parse(raw);
-    const language = typeof doc?.language === 'string' ? normalizeLanguage(doc.language) : undefined;
-    if (!language) {
+    if (!doc || typeof doc.language !== 'string') {
         throw new Error(`${CONFIG_NAME} is missing a valid language`);
     }
-    if (!Array.isArray(doc?.dialects) || doc.dialects.some((d) => typeof d !== 'string')) {
-        throw new Error(`${CONFIG_NAME} is missing a dialects list`);
+    const language = normalizeLanguage(doc.language);
+    if (!language) {
+        throw new Error(`${CONFIG_NAME} is missing a valid language`);
     }
+    const providers = parseProviders(doc);
     const binDir = typeof doc.binDir === 'string' && doc.binDir.length > 0 ? doc.binDir : binDirFor(language);
-    return { language, dialects: doc.dialects, binDir };
+    return { language, providers, binDir };
+};
+const parseProviders = (doc) => {
+    if (Array.isArray(doc.providers)) {
+        return doc.providers.map((item) => {
+            if (item &&
+                typeof item === 'object' &&
+                typeof item.name === 'string' &&
+                typeof item.location === 'string') {
+                return { name: item.name, location: item.location };
+            }
+            throw new Error(`${CONFIG_NAME} is missing a providers list`);
+        });
+    }
+    throw new Error(`${CONFIG_NAME} is missing a providers list`);
 };
 export const findConfigPath = (start, fallbackRoot) => {
     let dir = start;

package/orchestrator/dist/manifests.js

@@ -1,9 +1,9 @@
 const DIALECT_PROJECTS = {
-    sqlite: 'Dialects/Sqlite/MigrateSqlite.csproj',
-    postgres: 'Dialects/Postgres/MigratePostgres.csproj',
-    mysql: 'Dialects/Mysql/MigrateMysql.csproj',
-    sqlserver: 'Dialects/SqlServer/MigrateSqlServer.csproj',
-    oracle: 'Dialects/Oracle/MigrateOracle.csproj',
+    sqlite: 'Providers/Sqlite/MigrateSqlite.csproj',
+    postgres: 'Providers/Postgres/MigratePostgres.csproj',
+    mysql: 'Providers/Mysql/MigrateMysql.csproj',
+    sqlserver: 'Providers/SqlServer/MigrateSqlServer.csproj',
+    oracle: 'Providers/Oracle/MigrateOracle.csproj',
 };
 export const cargoDefaultFeaturesLine = (dialects) => `default = [${dialects.map((name) => `"${name}"`).join(', ')}]`;
 export const setCargoDefaultFeatures = (cargoToml, dialects) => {
@@ -25,7 +25,7 @@ export const setCargoDefaultFeatures = (cargoToml, dialects) => {
 };
 export const rustDialectCrate = (dialect) => `migraters-${dialect}`;
 export const rustDialectIdent = (dialect) => rustDialectCrate(dialect).replaceAll('-', '_');
-export const rustDialectPath = (dialect) => `dialects/${dialect}`;
+export const rustDialectPath = (dialect) => `providers/${dialect}`;
 export const renderAttachedDialects = (dialects) => [
     '// Generated by migrate-init. Do not edit.',
     '#![allow(unused_imports)]',

package/orchestrator/dist/provider-location.d.ts

@@ -0,0 +1,7 @@
+import type { Language } from './languages.ts';
+export type ProviderEntry = {
+    name: string;
+    location: string;
+};
+export declare const providerLocation: (language: Language, name: string, configDir?: string) => string;
+export declare const providerEntries: (language: Language, names: string[], configDir?: string) => ProviderEntry[];

package/orchestrator/dist/provider-location.js

@@ -0,0 +1,23 @@
+import { basename } from 'node:path';
+const csharpType = {
+    sqlite: { dll: 'MigrateSqlite.dll', type: 'Deterministic.MigrateRunner.SqliteProvider' },
+    postgres: { dll: 'MigratePostgres.dll', type: 'Deterministic.MigrateRunner.PostgresProvider' },
+    mysql: { dll: 'MigrateMysql.dll', type: 'Deterministic.MigrateRunner.MysqlProvider' },
+    sqlserver: { dll: 'MigrateSqlServer.dll', type: 'Deterministic.MigrateRunner.SqlServerProvider' },
+    oracle: { dll: 'MigrateOracle.dll', type: 'Deterministic.MigrateRunner.OracleProvider' },
+};
+export const providerLocation = (language, name, configDir = '') => {
+    if (language === 'csharp') {
+        const entry = csharpType[name];
+        if (!entry)
+            throw new Error(name);
+        return `${entry.dll}!${entry.type}`;
+    }
+    if (language === 'typescript') {
+        return basename(configDir) === 'typescript'
+            ? `providers/${name}/src/index.ts`
+            : `typescript/providers/${name}/src/index.ts`;
+    }
+    return `providers/${name}`;
+};
+export const providerEntries = (language, names, configDir = '') => names.map((name) => ({ name, location: providerLocation(language, name, configDir) }));

package/orchestrator/dist/run-init.js

@@ -10,7 +10,7 @@ const dialectPackage = (name) => `@deterministic-code/migraters-${name}`;
 const dialectRuntimeDeps = async (tsRoot, dialects) => {
     const names = new Set();
     for (const dialect of dialects) {
-        const raw = await readFile(join(tsRoot, 'dialects', dialect, 'package.json'), 'utf8');
+        const raw = await readFile(join(tsRoot, 'providers', dialect, 'package.json'), 'utf8');
         const pkg = JSON.parse(raw);
         for (const name of Object.keys(pkg.dependencies ?? {})) {
             if (!name.startsWith('@types/'))
@@ -22,7 +22,7 @@ const dialectRuntimeDeps = async (tsRoot, dialects) => {
 const initTypescript = async (root, dialects) => {
     const tsRoot = join(root, 'typescript');
     requireStatus(runCommand(npmCmd(), ['install', ...npmFlags], { cwd: tsRoot }));
-    const specs = dialects.map((name) => join(tsRoot, 'dialects', name));
+    const specs = dialects.map((name) => join(tsRoot, 'providers', name));
     if (specs.length > 0) {
         requireStatus(runCommand(npmCmd(), ['install', ...npmFlags, '--no-save', ...specs], { cwd: tsRoot }));
     }

package/orchestrator/prepare.js

@@ -5,6 +5,7 @@ import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 const orch = dirname(fileURLToPath(import.meta.url));
+const root = join(orch, '..');
 const src = join(orch, 'src', 'migrate-init.ts');
 const built = join(orch, 'dist', 'migrate-init.js');
 if (!existsSync(src)) {
@@ -14,8 +15,20 @@ if (!existsSync(src)) {
   }
   process.exit(0);
 }
-const result = spawnSync('npx', ['tsc', '-p', join(orch, 'tsconfig.json')], {
+const tsc = join(root, 'node_modules', 'typescript', 'bin', 'tsc');
+if (!existsSync(tsc)) {
+  process.stderr.write('typescript is not installed; run npm ci at the repo root\n');
+  process.exit(1);
+}
+const result = spawnSync(process.execPath, [tsc, '-p', join(orch, 'tsconfig.json')], {
   stdio: 'inherit',
-  shell: process.platform === 'win32',
 });
-process.exit(result.status ?? 1);
+if ((result.status ?? 1) !== 0) {
+  process.exit(result.status ?? 1);
+}
+const gitDir = join(root, '.git');
+const hooks = join(root, '.githooks');
+if (existsSync(src) && existsSync(gitDir) && existsSync(hooks)) {
+  spawnSync('git', ['config', 'core.hooksPath', '.githooks'], { cwd: root, stdio: 'ignore' });
+}
+process.exit(0);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@deterministic-code/migraters",
-  "version": "0.0.4",
+  "version": "0.0.5",
   "description": "Migrate executables for deterministic-code (typescript, rust, csharp).",
   "type": "module",
   "publishConfig": {
@@ -20,18 +20,20 @@
     "typescript/package.json",
     "typescript/tsconfig.build.json",
     "typescript/src",
-    "typescript/dialects",
+    "typescript/providers",
     "rust/Cargo.toml",
     "rust/Cargo.lock",
     "rust/rustfmt.toml",
     "rust/src",
     "rust/api",
-    "rust/dialects",
+    "rust/providers",
     "csharp/Abstractions",
-    "csharp/Dialects",
+    "csharp/Providers",
     "csharp/Infrastructure",
     "csharp/*.cs",
     "csharp/*.csproj",
+    "csharp/Directory.Build.props",
+    "csharp/Directory.Build.targets",
     "cli.yaml",
     "bundle.yaml",
     "shared"
@@ -40,10 +42,13 @@
     "prepare": "node orchestrator/prepare.js",
     "prepublishOnly": "node orchestrator/prepare.js",
     "typecheck": "tsc -p orchestrator/tsconfig.json --noEmit && npm run typecheck --prefix typescript",
-    "test": "npm run prepare && npm run test:orchestrator && npm run test --prefix typescript && npm run test:rust && npm run test:csharp",
+    "test": "npm run test:orchestrator && npm run test --prefix typescript && npm run test:rust && npm run test:csharp",
     "test:orchestrator": "vitest run --config orchestrator/vitest.config.ts",
-    "test:rust": "node orchestrator/dist/migrate-init.js --language rust --all && CARGO_TARGET_DIR=rust/target cargo llvm-cov --manifest-path rust/Cargo.toml --lib --fail-under-lines 100 --fail-under-functions 100 --fail-under-regions 100 --ignore-filename-regex 'src/(abstractions|argv|checksum|cli_spec|help|infrastructure|messages|migrater|migrations|provider)|lib\\.rs'",
-    "test:csharp": "node orchestrator/dist/migrate-init.js --language csharp --all && dotnet test csharp/MigrateRunner.Tests/MigrateRunner.Tests.csproj",
+    "test:rust": "node orchestrator/attach-sqlite.js rust-core && CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/Cargo.toml --lib",
+    "test:csharp": "node orchestrator/attach-sqlite.js csharp && dotnet test csharp/MigrateRunner.Tests/MigrateRunner.Tests.csproj",
+    "integration": "npm run prepare && npm run integration --prefix typescript && npm run integration:rust && npm run integration:csharp",
+    "integration:rust": "CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/providers/postgres/Cargo.toml --lib && CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/providers/mysql/Cargo.toml --lib && CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/providers/sqlserver/Cargo.toml --lib && CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/providers/oracle/Cargo.toml --lib && CARGO_TARGET_DIR=rust/target cargo test --manifest-path rust/providers/sqlite/Cargo.toml --lib",
+    "integration:csharp": "node orchestrator/dist/migrate-init.js --language csharp --all && dotnet test csharp/MigrateRunner.ProviderTests/MigrateRunner.ProviderTests.csproj",
     "e2e:sqlite": "bash proof/e2e-sqlite.sh",
     "e2e:postgres": "bash proof/e2e-postgres.sh",
     "e2e:mysql": "bash proof/e2e-mysql.sh",
@@ -53,7 +58,8 @@
     "e2e:typescript": "bash proof/e2e-typescript.sh",
     "e2e:rust": "bash proof/e2e-rust.sh",
     "e2e:csharp": "bash proof/e2e-csharp.sh",
-    "test:e2e": "docker compose -f proof/docker-compose.yml up -d --wait postgres mysql sqlserver oracle && rm -rf sql && cp -R proof/sql sql && npm run e2e:typescript && npm run e2e:rust && npm run e2e:csharp",
+    "e2e": "node proof/e2e.mjs",
+    "test:e2e": "node proof/e2e.mjs",
     "generate:help": "npm run generate:help --prefix typescript",
     "format": "npm run format --prefix typescript"
   },

package/rust/Cargo.lock

@@ -72,6 +72,7 @@ dependencies = [
 name = "deterministic-migraters"
 version = "0.0.1"
 dependencies = [
+ "async-trait",
  "dotenvy",
  "migraters-api",
  "sha2",

package/rust/Cargo.toml

@@ -27,3 +27,6 @@ time = "=0.3.47"
 sha2 = "0.10"
 # migrate-init-dialects-start
 # migrate-init-dialects-end
+
+[dev-dependencies]
+async-trait = "0.1"

package/rust/api/src/lib.rs

@@ -8,18 +8,19 @@ pub fn catalog_name(stored: &str) -> &str {
     stored.strip_suffix("_up.sql").unwrap_or(stored)
 }
 
-pub trait SqlDialect: Send + Sync {
+pub trait IDatasourceProvider: Send + Sync {
     fn name(&self) -> &'static str;
     fn connection_env_vars(&self) -> &'static [&'static str];
+    fn normalize_connection(&self, connection: &str) -> String;
+    fn prerequisite_error(&self, connection: &str) -> Option<String>;
+    fn prepare_setup(&self, connection: &str) -> std::io::Result<()>;
+
+    // SQL catalog text used by the locked rust/src/migrater.rs until that file can be thinned.
     fn migrates_ddl(&self) -> &'static str;
     fn migrate_logs_ddl(&self) -> &'static str;
     fn use_transaction(&self) -> bool;
     fn insert_applied_sql(&self) -> &'static str;
     fn delete_applied_sql(&self) -> &'static str;
-    fn normalize_connection(&self, connection: &str) -> String;
-    fn prerequisite_error(&self, connection: &str) -> Option<String>;
-    fn prepare_setup(&self, connection: &str) -> std::io::Result<()>;
-
     fn quote_ident(&self, ident: &str) -> String {
         format!("\"{ident}\"")
     }
@@ -39,7 +40,83 @@ pub trait SqlDialect: Send + Sync {
     fn connect<'a>(
         &'a self,
         connection: &'a str,
-    ) -> Pin<Box<dyn Future<Output = Result<Box<dyn MigrationSession>, BoxError>> + Send + 'a>>;
+    ) -> Pin<Box<dyn Future<Output = Result<Box<dyn MigrationDriver>, BoxError>> + Send + 'a>>;
+
+    fn setup<'a>(
+        &'a self,
+        connection: &'a str,
+    ) -> Pin<Box<dyn Future<Output = Result<(), BoxError>> + Send + 'a>> {
+        Box::pin(async move {
+            let driver = self.connect(connection).await?;
+            driver.setup().await?;
+            driver.close().await
+        })
+    }
+
+    fn applied<'a>(
+        &'a self,
+        connection: &'a str,
+    ) -> Pin<Box<dyn Future<Output = Result<HashMap<String, Option<String>>, BoxError>> + Send + 'a>>
+    {
+        Box::pin(async move {
+            let driver = self.connect(connection).await?;
+            let map = driver.applied().await?;
+            driver.close().await?;
+            Ok(map)
+        })
+    }
+
+    fn migrate_up<'a>(
+        &'a self,
+        connection: &'a str,
+        name: &'a str,
+        body: &'a str,
+        checksum: &'a str,
+    ) -> Pin<Box<dyn Future<Output = Result<(), BoxError>> + Send + 'a>> {
+        Box::pin(async move {
+            let driver = self.connect(connection).await?;
+            driver.migrate_up(name, body, checksum).await?;
+            driver.close().await
+        })
+    }
+
+    fn migrate_down<'a>(
+        &'a self,
+        connection: &'a str,
+        name: &'a str,
+        body: &'a str,
+    ) -> Pin<Box<dyn Future<Output = Result<(), BoxError>> + Send + 'a>> {
+        Box::pin(async move {
+            let driver = self.connect(connection).await?;
+            driver.migrate_down(name, body).await?;
+            driver.close().await
+        })
+    }
+
+    fn log<'a>(
+        &'a self,
+        connection: &'a str,
+        id: Option<i64>,
+        name: &'a str,
+        direction: &'a str,
+        status: Option<&'a str>,
+        duration_ms: i64,
+        error: Option<&'a str>,
+    ) -> Pin<Box<dyn Future<Output = Result<i64, BoxError>> + Send + 'a>> {
+        Box::pin(async move {
+            let driver = self.connect(connection).await?;
+            let result = if let Some(id) = id {
+                driver
+                    .log_finished(id, status.unwrap_or("started"), duration_ms, error)
+                    .await?;
+                id
+            } else {
+                driver.log_started(name, direction).await?
+            };
+            driver.close().await?;
+            Ok(result)
+        })
+    }
 
     fn create_database<'a>(
         &'a self,
@@ -49,169 +126,40 @@ pub trait SqlDialect: Send + Sync {
     }
 }
 
+pub use IDatasourceProvider as Dialect;
+pub use IDatasourceProvider as SqlDialect;
+
 #[async_trait::async_trait]
-pub trait MigrationSession: Send + Sync {
-    async fn execute(&self, sql: &str) -> Result<(), BoxError>;
-    async fn fetch_applied(&self, sql: &str) -> Result<HashMap<String, Option<String>>, BoxError>;
-    async fn apply_in_transaction(
-        &self,
-        stmts: &[String],
-        catalog_sql: &str,
-        p1: &str,
-        p2: Option<&str>,
-    ) -> Result<(), BoxError>;
-    async fn catalog_write(
+pub trait MigrationDriver: Send + Sync {
+    async fn setup(&self) -> Result<(), BoxError>;
+    async fn applied(&self) -> Result<HashMap<String, Option<String>>, BoxError>;
+    async fn migrate_up(&self, name: &str, body: &str, checksum: &str) -> Result<(), BoxError>;
+    async fn migrate_down(&self, name: &str, body: &str) -> Result<(), BoxError>;
+    async fn log_started(&self, name: &str, direction: &str) -> Result<i64, BoxError>;
+    async fn log_finished(
         &self,
-        catalog_sql: &str,
-        p1: &str,
-        p2: Option<&str>,
-    ) -> Result<(), BoxError>;
-    async fn fetch_id(&self, sql: &str, p1: &str, p2: &str) -> Result<i64, BoxError>;
-    async fn update_log_exec(
-        &self,
-        sql: &str,
+        id: i64,
         status: &str,
         duration_ms: i64,
-        error_message: Option<&str>,
-        id: i64,
+        error: Option<&str>,
     ) -> Result<(), BoxError>;
+    async fn close(&self) -> Result<(), BoxError>;
 }
 
 pub struct DialectPlugin {
-    pub dialect: &'static dyn SqlDialect,
+    pub dialect: &'static dyn IDatasourceProvider,
 }
 
 inventory::collect!(DialectPlugin);
 
-pub fn registered_dialects() -> Vec<&'static dyn SqlDialect> {
+pub fn registered_dialects() -> Vec<&'static dyn IDatasourceProvider> {
     inventory::iter::<DialectPlugin>
         .into_iter()
         .map(|p| p.dialect)
         .collect()
 }
 
-pub fn register_try_get(
-    name: &str,
-    normalize: impl Fn(&str) -> Option<&str>,
-) -> Option<&'static dyn SqlDialect> {
-    let key = normalize(name)?;
-    registered_dialects()
-        .into_iter()
-        .find(|d| d.name() == key)
-}
-
-pub fn split_on_go(sql: &str) -> Vec<String> {
-    let mut out = Vec::new();
-    let mut buf = String::new();
-    for line in sql.replace("\r\n", "\n").split('\n') {
-        if line.trim().eq_ignore_ascii_case("GO") {
-            let s = buf.trim();
-            if !s.is_empty() {
-                out.push(s.to_string());
-            }
-            buf.clear();
-            continue;
-        }
-        if !buf.is_empty() {
-            buf.push('\n');
-        }
-        buf.push_str(line);
-    }
-    let tail = buf.trim();
-    if !tail.is_empty() {
-        out.push(tail.to_string());
-    }
-    out
-}
-
-pub fn placeholders(dialect: &str, n: usize) -> Vec<String> {
-    match dialect {
-        "postgres" => (1..=n).map(|i| format!("${i}")).collect(),
-        "sqlserver" => (1..=n).map(|i| format!("@P{i}")).collect(),
-        "oracle" => (1..=n).map(|i| format!(":{i}")).collect(),
-        _ => (0..n).map(|_| "?".into()).collect(),
-    }
-}
-
-pub async fn apply_statements(
-    session: &dyn MigrationSession,
-    dialect: &dyn SqlDialect,
-    sql: &str,
-    catalog_sql: &str,
-    p1: &str,
-    p2: Option<&str>,
-) -> Result<(), BoxError> {
-    let stmts = split_on_go(sql);
-    if dialect.use_transaction() {
-        session
-            .apply_in_transaction(&stmts, catalog_sql, p1, p2)
-            .await
-    } else {
-        for stmt in &stmts {
-            session.execute(stmt).await?;
-        }
-        session.catalog_write(catalog_sql, p1, p2).await
-    }
-}
-
-pub async fn insert_log_started(
-    session: &dyn MigrationSession,
-    dialect: &dyn SqlDialect,
-    name: &str,
-    direction: &str,
-) -> Result<i64, BoxError> {
-    let t = dialect.quote_ident("migrate_logs");
-    let migrate_name = dialect.quote_ident("migrate_name");
-    let dir_col = dialect.quote_ident("direction");
-    let status = dialect.quote_ident("status");
-    let id_col = dialect.quote_ident("id");
-    let ph = placeholders(dialect.name(), 2);
-    let insert = format!(
-        "INSERT INTO {t} ({migrate_name}, {dir_col}, {status}) VALUES ({}, {}, 'started')",
-        ph[0], ph[1]
-    );
-    session
-        .catalog_write(&insert, name, Some(direction))
-        .await?;
-    let select = format!(
-        "SELECT {id_col} AS id FROM {t} WHERE {migrate_name} = {} AND {dir_col} = {} ORDER BY {id_col} DESC {}",
-        ph[0],
-        ph[1],
-        dialect.limit_clause(1)
-    );
-    session.fetch_id(&select, name, direction).await
-}
-
-pub async fn update_log(
-    session: &dyn MigrationSession,
-    dialect: &dyn SqlDialect,
-    id: i64,
-    status: &str,
-    duration_ms: i64,
-    error_message: Option<&str>,
-) -> Result<(), BoxError> {
-    let t = dialect.quote_ident("migrate_logs");
-    let now = dialect.now_expr();
-    let ph = placeholders(dialect.name(), 4);
-    let sql = format!(
-        "UPDATE {t} SET {status_col} = {p1}, {finished} = {now}, {duration} = {p2}, {err} = {p3}, {updated} = {now} WHERE {id_col} = {p4}",
-        status_col = dialect.quote_ident("status"),
-        finished = dialect.quote_ident("finished_at"),
-        duration = dialect.quote_ident("duration_ms"),
-        err = dialect.quote_ident("error_message"),
-        updated = dialect.quote_ident("updated"),
-        id_col = dialect.quote_ident("id"),
-        p1 = ph[0],
-        p2 = ph[1],
-        p3 = ph[2],
-        p4 = ph[3],
-    );
-    session
-        .update_log_exec(&sql, status, duration_ms, error_message, id)
-        .await
-}
-
-pub fn assert_dialect_surface(d: &dyn SqlDialect) {
+pub fn assert_dialect_surface(d: &dyn IDatasourceProvider) {
     assert!(!d.name().is_empty());
     assert!(!d.connection_env_vars().is_empty());
     assert!(d.migrates_ddl().to_ascii_lowercase().contains("migrates"));