@pgpmjs/core 3.0.0

package/esm/migrate/sql/procedures.sql

@@ -0,0 +1,258 @@
+-- Register a package (auto-called by deploy if needed)
+CREATE PROCEDURE pgpm_migrate.register_package(p_package TEXT)
+LANGUAGE plpgsql AS $$
+BEGIN
+    INSERT INTO pgpm_migrate.packages (package) 
+    VALUES (p_package)
+    ON CONFLICT (package) DO NOTHING;
+END;
+$$;
+
+-- Check if a change is deployed (handles both local and cross-package dependencies)
+CREATE FUNCTION pgpm_migrate.is_deployed(
+    p_package TEXT,
+    p_change_name TEXT
+)
+RETURNS BOOLEAN
+LANGUAGE plpgsql STABLE AS $$
+DECLARE
+    v_actual_package TEXT;
+    v_actual_change TEXT;
+    v_colon_pos INT;
+BEGIN
+    -- Check if change_name contains a package prefix (cross-package dependency)
+    v_colon_pos := position(':' in p_change_name);
+    
+    IF v_colon_pos > 0 THEN
+        -- Split into package and change name
+        v_actual_package := substring(p_change_name from 1 for v_colon_pos - 1);
+        v_actual_change := substring(p_change_name from v_colon_pos + 1);
+    ELSE
+        -- Use provided package as default
+        v_actual_package := p_package;
+        v_actual_change := p_change_name;
+    END IF;
+    
+    RETURN EXISTS (
+        SELECT 1 FROM pgpm_migrate.changes 
+        WHERE package = v_actual_package 
+        AND change_name = v_actual_change
+    );
+END;
+$$;
+
+-- Deploy a change
+CREATE PROCEDURE pgpm_migrate.deploy(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_script_hash TEXT,
+    p_requires TEXT[],
+    p_deploy_sql TEXT,
+    p_log_only BOOLEAN DEFAULT FALSE
+)
+LANGUAGE plpgsql AS $$
+DECLARE
+    v_change_id TEXT;
+BEGIN
+    -- Ensure package exists
+    CALL pgpm_migrate.register_package(p_package);
+    
+    -- Generate simple ID
+    v_change_id := encode(sha256((p_package || p_change_name || p_script_hash)::bytea), 'hex');
+    
+    -- Check if already deployed
+    IF pgpm_migrate.is_deployed(p_package, p_change_name) THEN
+        -- Check if it's the same script (by hash)
+        IF EXISTS (
+            SELECT 1 FROM pgpm_migrate.changes 
+            WHERE package = p_package 
+            AND change_name = p_change_name 
+            AND script_hash = p_script_hash
+        ) THEN
+            -- Same change with same content, skip silently
+            RETURN;
+        ELSE
+            -- Different content, this is an error
+            RAISE EXCEPTION 'Change % already deployed in package % with different content', p_change_name, p_package;
+        END IF;
+    END IF;
+    
+    -- Check dependencies
+    IF p_requires IS NOT NULL THEN
+        DECLARE
+            missing_changes TEXT[];
+        BEGIN
+            SELECT array_agg(req) INTO missing_changes
+            FROM unnest(p_requires) AS req
+            WHERE NOT pgpm_migrate.is_deployed(p_package, req);
+            
+            IF array_length(missing_changes, 1) > 0 THEN
+                RAISE EXCEPTION 'Missing required changes for %: %', p_change_name, array_to_string(missing_changes, ', ');
+            END IF;
+        END;
+    END IF;
+    
+    -- Execute deploy (skip if log-only mode)
+    IF NOT p_log_only THEN
+        BEGIN
+            EXECUTE p_deploy_sql;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE;
+        END;
+    END IF;
+    
+    -- Record deployment
+    INSERT INTO pgpm_migrate.changes (change_id, change_name, package, script_hash)
+    VALUES (v_change_id, p_change_name, p_package, p_script_hash);
+    
+    -- Record dependencies (INSERTED AFTER SUCCESSFUL DEPLOYMENT)
+    IF p_requires IS NOT NULL THEN
+        INSERT INTO pgpm_migrate.dependencies (change_id, requires)
+        SELECT v_change_id, req FROM unnest(p_requires) AS req;
+    END IF;
+    
+    -- Log success
+    INSERT INTO pgpm_migrate.events (event_type, change_name, package)
+    VALUES ('deploy', p_change_name, p_package);
+END;
+$$;
+
+-- Revert a change
+CREATE PROCEDURE pgpm_migrate.revert(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_revert_sql TEXT
+)
+LANGUAGE plpgsql AS $$
+BEGIN
+    -- Check if deployed
+    IF NOT pgpm_migrate.is_deployed(p_package, p_change_name) THEN
+        RAISE EXCEPTION 'Change % not deployed in package %', p_change_name, p_package;
+    END IF;
+    
+    -- Check if other changes depend on this (including cross-package dependencies)
+    IF EXISTS (
+        SELECT 1 FROM pgpm_migrate.dependencies d
+        JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+        WHERE (
+            -- Local dependency within same package
+            (d.requires = p_change_name AND c.package = p_package)
+            OR
+            -- Cross-package dependency
+            (d.requires = p_package || ':' || p_change_name)
+        )
+    ) THEN
+        -- Get list of dependent changes for better error message
+        DECLARE
+            dependent_changes TEXT;
+        BEGIN
+            SELECT string_agg(
+                CASE 
+                    WHEN d.requires = p_change_name THEN c.change_name
+                    ELSE c.package || ':' || c.change_name
+                END, 
+                ', '
+            ) INTO dependent_changes
+            FROM pgpm_migrate.dependencies d
+            JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+            WHERE (
+                (d.requires = p_change_name AND c.package = p_package)
+                OR
+                (d.requires = p_package || ':' || p_change_name)
+            );
+            
+            RAISE EXCEPTION 'Cannot revert %: required by %', p_change_name, dependent_changes;
+        END;
+    END IF;
+    
+    -- Execute revert
+    EXECUTE p_revert_sql;
+    
+    -- Remove from deployed
+    DELETE FROM pgpm_migrate.changes 
+    WHERE package = p_package AND change_name = p_change_name;
+    
+    -- Log revert
+    INSERT INTO pgpm_migrate.events (event_type, change_name, package)
+    VALUES ('revert', p_change_name, p_package);
+END;
+$$;
+
+-- Verify a change
+CREATE FUNCTION pgpm_migrate.verify(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_verify_sql TEXT
+)
+RETURNS BOOLEAN
+LANGUAGE plpgsql AS $$
+BEGIN
+    EXECUTE p_verify_sql;
+    RETURN TRUE;
+EXCEPTION WHEN OTHERS THEN
+    RETURN FALSE;
+END;
+$$;
+
+-- List deployed changes
+CREATE FUNCTION pgpm_migrate.deployed_changes(
+    p_package TEXT DEFAULT NULL
+)
+RETURNS TABLE(package TEXT, change_name TEXT, deployed_at TIMESTAMPTZ)
+LANGUAGE sql STABLE AS $$
+    SELECT package, change_name, deployed_at 
+    FROM pgpm_migrate.changes 
+    WHERE p_package IS NULL OR package = p_package
+    ORDER BY deployed_at;
+$$;
+
+-- Get changes that depend on a given change
+CREATE FUNCTION pgpm_migrate.get_dependents(
+    p_package TEXT,
+    p_change_name TEXT
+)
+RETURNS TABLE(package TEXT, change_name TEXT, dependency TEXT)
+LANGUAGE sql STABLE AS $$
+    SELECT c.package, c.change_name, d.requires as dependency
+    FROM pgpm_migrate.dependencies d
+    JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+    WHERE (
+        -- Local dependency within same package
+        (d.requires = p_change_name AND c.package = p_package)
+        OR
+        -- Cross-package dependency
+        (d.requires = p_package || ':' || p_change_name)
+    )
+    ORDER BY c.package, c.change_name;
+$$;
+
+-- Get deployment status
+CREATE FUNCTION pgpm_migrate.status(
+    p_package TEXT DEFAULT NULL
+)
+RETURNS TABLE(
+    package TEXT,
+    total_deployed INTEGER,
+    last_change TEXT,
+    last_deployed TIMESTAMPTZ
+)
+LANGUAGE sql STABLE AS $$
+    WITH latest AS (
+        SELECT DISTINCT ON (package) 
+            package,
+            change_name,
+            deployed_at
+        FROM pgpm_migrate.changes
+        WHERE p_package IS NULL OR package = p_package
+        ORDER BY package, deployed_at DESC
+    )
+    SELECT 
+        c.package,
+        COUNT(*)::INTEGER AS total_deployed,
+        l.change_name AS last_change,
+        l.deployed_at AS last_deployed
+    FROM pgpm_migrate.changes c
+    JOIN latest l ON l.package = c.package
+    WHERE p_package IS NULL OR c.package = p_package
+    GROUP BY c.package, l.change_name, l.deployed_at;
+$$;

package/esm/migrate/sql/schema.sql

@@ -0,0 +1,37 @@
+-- Create schema
+CREATE SCHEMA pgpm_migrate;
+
+-- 1. Packages (minimal - just name and timestamp)
+CREATE TABLE pgpm_migrate.packages (
+    package         TEXT        PRIMARY KEY,
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+);
+
+-- 2. Deployed changes (what's currently deployed)
+CREATE TABLE pgpm_migrate.changes (
+    change_id       TEXT        PRIMARY KEY,
+    change_name     TEXT        NOT NULL,
+    package         TEXT        NOT NULL REFERENCES pgpm_migrate.packages(package),
+    script_hash     TEXT        NOT NULL,
+    deployed_at     TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+    UNIQUE(package, change_name),
+    UNIQUE(package, script_hash)
+);
+
+-- 3. Dependencies (what depends on what)
+CREATE TABLE pgpm_migrate.dependencies (
+    change_id       TEXT        NOT NULL REFERENCES pgpm_migrate.changes(change_id) ON DELETE CASCADE,
+    requires        TEXT        NOT NULL,
+    PRIMARY KEY (change_id, requires)
+);
+
+-- 4. Event log (minimal history for rollback)
+CREATE TABLE pgpm_migrate.events (
+    event_id        SERIAL      PRIMARY KEY,
+    event_type      TEXT        NOT NULL CHECK (event_type IN ('deploy', 'revert', 'verify')),
+    change_name     TEXT        NOT NULL,
+    package         TEXT        NOT NULL,
+    occurred_at     TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+    error_message   TEXT,
+    error_code      TEXT
+);

package/esm/migrate/types.js

@@ -0,0 +1 @@
+export {};

package/esm/migrate/utils/event-logger.js

@@ -0,0 +1,28 @@
+import { Logger } from '@pgpmjs/logger';
+import { getPgPool } from 'pg-cache';
+const log = new Logger('migrate:event-logger');
+export class EventLogger {
+    pool;
+    constructor(config) {
+        this.pool = getPgPool(config);
+    }
+    async logEvent(entry) {
+        try {
+            await this.pool.query(`
+        INSERT INTO pgpm_migrate.events 
+        (event_type, change_name, package, error_message, error_code)
+        VALUES ($1::TEXT, $2::TEXT, $3::TEXT, $4::TEXT, $5::TEXT)
+      `, [
+                entry.eventType,
+                entry.changeName,
+                entry.package,
+                entry.errorMessage || null,
+                entry.errorCode || null
+            ]);
+            log.debug(`Logged ${entry.eventType} event for ${entry.package}:${entry.changeName}`);
+        }
+        catch (error) {
+            log.error(`Failed to log event: ${error.message}`);
+        }
+    }
+}

package/esm/migrate/utils/hash.js

@@ -0,0 +1,27 @@
+import { createHash } from 'crypto';
+import { readFile } from 'fs/promises';
+import { parse } from 'pgsql-parser';
+import { cleanTree } from '../../packaging/package';
+/**
+ * Generate SHA256 hash of a file's contents
+ */
+export async function hashFile(filePath) {
+    const content = await readFile(filePath, 'utf-8');
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Generate SHA256 hash of a string
+ */
+export function hashString(content) {
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Generate SHA256 hash of a SQL file's parsed and cleaned AST
+ */
+export async function hashSqlFile(filePath) {
+    const content = await readFile(filePath, 'utf-8');
+    const parsed = await parse(content);
+    const cleaned = cleanTree(parsed);
+    const astString = JSON.stringify(cleaned);
+    return hashString(astString);
+}

package/esm/migrate/utils/transaction.js

@@ -0,0 +1,125 @@
+import { Logger } from '@pgpmjs/logger';
+const log = new Logger('migrate:transaction');
+/**
+ * Execute a function within a transaction context
+ * If useTransaction is true, wraps the execution in a transaction
+ * If false, uses the pool directly without transaction
+ */
+export async function withTransaction(pool, options, fn) {
+    const queryHistory = [];
+    const addQuery = (query, params, startTime) => {
+        queryHistory.push({
+            query,
+            params,
+            timestamp: Date.now(),
+            duration: startTime ? Date.now() - startTime : undefined
+        });
+    };
+    if (!options.useTransaction) {
+        // No transaction - use pool directly
+        log.debug('Executing without transaction');
+        return fn({ client: pool, isTransaction: false, queryHistory, addQuery });
+    }
+    // Use transaction
+    const client = await pool.connect();
+    const transactionStartTime = Date.now();
+    log.debug('Starting transaction');
+    try {
+        const beginTime = Date.now();
+        await client.query('BEGIN');
+        addQuery('BEGIN', [], beginTime);
+        const result = await fn({ client, isTransaction: true, queryHistory, addQuery });
+        const commitTime = Date.now();
+        await client.query('COMMIT');
+        addQuery('COMMIT', [], commitTime);
+        const transactionDuration = Date.now() - transactionStartTime;
+        log.debug(`Transaction committed successfully in ${transactionDuration}ms`);
+        return result;
+    }
+    catch (error) {
+        const rollbackTime = Date.now();
+        try {
+            await client.query('ROLLBACK');
+            addQuery('ROLLBACK', [], rollbackTime);
+        }
+        catch (rollbackError) {
+            log.error('Failed to rollback transaction:', rollbackError);
+        }
+        const transactionDuration = Date.now() - transactionStartTime;
+        // Enhanced error logging with context
+        const errorLines = [];
+        errorLines.push(`Transaction rolled back due to error after ${transactionDuration}ms:`);
+        errorLines.push(`Error Code: ${error.code || 'N/A'}`);
+        errorLines.push(`Error Message: ${error.message || 'N/A'}`);
+        // Log query history for debugging
+        if (queryHistory.length > 0) {
+            errorLines.push('Query history for this transaction:');
+            queryHistory.forEach((entry, index) => {
+                const duration = entry.duration ? ` (${entry.duration}ms)` : '';
+                const params = entry.params && entry.params.length > 0
+                    ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
+                    : '';
+                errorLines.push(`  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`);
+            });
+        }
+        // For transaction aborted errors, provide additional context
+        if (error.code === '25P02') {
+            errorLines.push('🔍 Debug Info: Transaction was aborted due to a previous error.');
+            errorLines.push('   This usually means a previous command in the transaction failed.');
+            errorLines.push('   Check the query history above to identify the failing command.');
+        }
+        // Log the consolidated error message
+        log.error(errorLines.join('\n'));
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Helper to execute a query within a transaction context with enhanced logging
+ */
+export async function executeQuery(context, query, params) {
+    const startTime = Date.now();
+    try {
+        const result = await context.client.query(query, params);
+        const duration = Date.now() - startTime;
+        // Add to query history
+        context.addQuery(query, params, startTime);
+        // Log slow queries for debugging
+        if (duration > 1000) {
+            log.warn(`Slow query detected (${duration}ms): ${query.split('\n')[0].trim()}`);
+        }
+        return result;
+    }
+    catch (error) {
+        const duration = Date.now() - startTime;
+        // Add failed query to history
+        context.addQuery(query, params, startTime);
+        // Enhanced error logging
+        const errorLines = [];
+        errorLines.push(`Query failed after ${duration}ms:`);
+        errorLines.push(`  Query: ${query.split('\n')[0].trim()}`);
+        if (params && params.length > 0) {
+            errorLines.push(`  Params: ${JSON.stringify(params.slice(0, 3))}${params.length > 3 ? '...' : ''}`);
+        }
+        errorLines.push(`  Error Code: ${error.code || 'N/A'}`);
+        errorLines.push(`  Error Message: ${error.message || 'N/A'}`);
+        // Provide debugging hints for common errors
+        if (error.code === '42P01') {
+            errorLines.push('💡 Hint: Relation (table/view) does not exist. Check if migrations are applied in correct order.');
+        }
+        else if (error.code === '42883') {
+            errorLines.push('💡 Hint: Function does not exist. Check if required extensions or functions are installed.');
+        }
+        else if (error.code === '23505') {
+            errorLines.push('💡 Hint: Unique constraint violation. Check for duplicate data.');
+        }
+        else if (error.code === '23503') {
+            errorLines.push('💡 Hint: Foreign key constraint violation. Check referential integrity.');
+        }
+        // Log the consolidated error message
+        log.error(errorLines.join('\n'));
+        throw error;
+    }
+}

package/esm/modules/modules.js

@@ -0,0 +1,49 @@
+import { getLatestChange } from '../files';
+import { errors } from '@pgpmjs/types';
+/**
+ * Get the latest change from the pgpm.plan file for a specific module.
+ */
+export const latestChange = (sqlmodule, modules, basePath) => {
+    const module = modules[sqlmodule];
+    if (!module) {
+        throw errors.MODULE_NOT_FOUND({ name: sqlmodule });
+    }
+    const planPath = `${basePath}/${module.path}/pgpm.plan`;
+    return getLatestChange(planPath);
+};
+/**
+ * Get the latest change and version for a specific module.
+ */
+export const latestChangeAndVersion = (sqlmodule, modules, basePath) => {
+    const module = modules[sqlmodule];
+    if (!module) {
+        throw errors.MODULE_NOT_FOUND({ name: sqlmodule });
+    }
+    const planPath = `${basePath}/${module.path}/pgpm.plan`;
+    const change = getLatestChange(planPath);
+    const pkg = require(`${basePath}/${module.path}/package.json`);
+    return { change, version: pkg.version };
+};
+/**
+ * Get extensions and modules required by a specific module.
+ */
+export const getExtensionsAndModules = (sqlmodule, modules) => {
+    const module = modules[sqlmodule];
+    if (!module) {
+        throw errors.MODULE_NOT_FOUND({ name: sqlmodule });
+    }
+    const native = module.requires.filter((req) => !Object.keys(modules).includes(req));
+    const sqitch = module.requires.filter((req) => Object.keys(modules).includes(req));
+    return { native, sqitch };
+};
+/**
+ * Get extensions and modules with their latest changes and versions.
+ */
+export const getExtensionsAndModulesChanges = (sqlmodule, modules, basePath) => {
+    const { native, sqitch } = getExtensionsAndModules(sqlmodule, modules);
+    const sqitchWithDetails = sqitch.map((mod) => {
+        const { change, version } = latestChangeAndVersion(mod, modules, basePath);
+        return { name: mod, latest: change, version };
+    });
+    return { native, sqitch: sqitchWithDetails };
+};

package/esm/packaging/package.js

@@ -0,0 +1,96 @@
+import { Logger } from '@pgpmjs/logger';
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
+import { relative } from 'path';
+import { deparse } from 'pgsql-deparser';
+import { parse } from 'pgsql-parser';
+import { getExtensionName } from '../files';
+import { resolve, resolveWithPlan } from '../resolution/resolve';
+import { transformProps } from './transform';
+const log = new Logger('package');
+const noop = () => undefined;
+export const cleanTree = (tree) => {
+    return transformProps(tree, {
+        stmt_len: noop,
+        stmt_location: noop,
+        location: noop,
+    });
+};
+const filterStatements = (stmts, extension) => {
+    if (!extension)
+        return stmts;
+    return stmts.filter(node => {
+        const stmt = node.stmt;
+        return !stmt.hasOwnProperty('TransactionStmt') &&
+            !stmt.hasOwnProperty('CreateExtensionStmt');
+    });
+};
+export const packageModule = async (packageDir, { usePlan = true, extension = true, pretty = true, functionDelimiter = '$EOFCODE$' } = {}) => {
+    const resolveFn = usePlan ? resolveWithPlan : resolve;
+    const sql = resolveFn(packageDir);
+    if (!sql?.trim()) {
+        log.warn(`⚠️ No SQL generated for module at ${packageDir}. Skipping.`);
+        return { sql: '' };
+    }
+    const extname = getExtensionName(packageDir);
+    try {
+        const parsed = await parse(sql);
+        parsed.stmts = filterStatements(parsed.stmts, extension);
+        const topLine = extension
+            ? `\\echo Use "CREATE EXTENSION ${extname}" to load this file. \\quit\n`
+            : '';
+        const finalSql = await deparse(parsed, {
+            pretty,
+            functionDelimiter
+        });
+        const tree1 = parsed.stmts;
+        const tree2 = await parse(finalSql);
+        const results = {
+            sql: `${topLine}${finalSql}`,
+        };
+        const diff = JSON.stringify(cleanTree(tree1)) !== JSON.stringify(cleanTree(tree2));
+        if (diff) {
+            results.diff = true;
+            results.tree1 = JSON.stringify(cleanTree(tree1), null, 2);
+            results.tree2 = JSON.stringify(cleanTree(tree2), null, 2);
+        }
+        return results;
+    }
+    catch (e) {
+        log.error(`❌ Failed to parse SQL for ${packageDir}`);
+        console.error(e);
+        throw e;
+    }
+};
+export const writePackage = async ({ version, extension = true, usePlan = true, packageDir, }) => {
+    const pkgPath = `${packageDir}/package.json`;
+    const pkg = require(pkgPath);
+    const extname = await getExtensionName(packageDir);
+    const makePath = `${packageDir}/Makefile`;
+    const controlPath = `${packageDir}/${extname}.control`;
+    const sqlFileName = `${extname}--${version}.sql`;
+    const Makefile = readFileSync(makePath, 'utf-8');
+    const control = readFileSync(controlPath, 'utf-8');
+    const { sql, diff, tree1, tree2 } = await packageModule(packageDir, {
+        extension,
+        usePlan,
+    });
+    const outPath = extension ? `${packageDir}/sql` : `${packageDir}/out`;
+    rmSync(outPath, { recursive: true, force: true });
+    mkdirSync(outPath, { recursive: true });
+    if (extension) {
+        writeFileSync(controlPath, control.replace(/default_version = '[0-9\.]+'/, `default_version = '${version}'`));
+        pkg.version = version;
+        writeFileSync(pkgPath, JSON.stringify(pkg, null, 2));
+        const regex = new RegExp(`${extname}--[0-9.]+.sql`);
+        writeFileSync(makePath, Makefile.replace(regex, sqlFileName));
+    }
+    if (diff) {
+        log.warn(`⚠️ SQL diff exists! Review the ${relative(packageDir, outPath)}/ folder.`);
+        // Uncomment if needed:
+        // writeFileSync(`${outPath}/orig.${sqlFileName}.tree.json`, tree1);
+        // writeFileSync(`${outPath}/parsed.${sqlFileName}.tree.json`, tree2);
+    }
+    const writePath = `${outPath}/${sqlFileName}`;
+    writeFileSync(writePath, sql);
+    log.success(`${relative(packageDir, writePath)} written`);
+};