@pgpmjs/core 3.0.0

package/migrate/sql/procedures.sql

@@ -0,0 +1,258 @@
+-- Register a package (auto-called by deploy if needed)
+CREATE PROCEDURE pgpm_migrate.register_package(p_package TEXT)
+LANGUAGE plpgsql AS $$
+BEGIN
+    INSERT INTO pgpm_migrate.packages (package) 
+    VALUES (p_package)
+    ON CONFLICT (package) DO NOTHING;
+END;
+$$;
+
+-- Check if a change is deployed (handles both local and cross-package dependencies)
+CREATE FUNCTION pgpm_migrate.is_deployed(
+    p_package TEXT,
+    p_change_name TEXT
+)
+RETURNS BOOLEAN
+LANGUAGE plpgsql STABLE AS $$
+DECLARE
+    v_actual_package TEXT;
+    v_actual_change TEXT;
+    v_colon_pos INT;
+BEGIN
+    -- Check if change_name contains a package prefix (cross-package dependency)
+    v_colon_pos := position(':' in p_change_name);
+    
+    IF v_colon_pos > 0 THEN
+        -- Split into package and change name
+        v_actual_package := substring(p_change_name from 1 for v_colon_pos - 1);
+        v_actual_change := substring(p_change_name from v_colon_pos + 1);
+    ELSE
+        -- Use provided package as default
+        v_actual_package := p_package;
+        v_actual_change := p_change_name;
+    END IF;
+    
+    RETURN EXISTS (
+        SELECT 1 FROM pgpm_migrate.changes 
+        WHERE package = v_actual_package 
+        AND change_name = v_actual_change
+    );
+END;
+$$;
+
+-- Deploy a change
+CREATE PROCEDURE pgpm_migrate.deploy(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_script_hash TEXT,
+    p_requires TEXT[],
+    p_deploy_sql TEXT,
+    p_log_only BOOLEAN DEFAULT FALSE
+)
+LANGUAGE plpgsql AS $$
+DECLARE
+    v_change_id TEXT;
+BEGIN
+    -- Ensure package exists
+    CALL pgpm_migrate.register_package(p_package);
+    
+    -- Generate simple ID
+    v_change_id := encode(sha256((p_package || p_change_name || p_script_hash)::bytea), 'hex');
+    
+    -- Check if already deployed
+    IF pgpm_migrate.is_deployed(p_package, p_change_name) THEN
+        -- Check if it's the same script (by hash)
+        IF EXISTS (
+            SELECT 1 FROM pgpm_migrate.changes 
+            WHERE package = p_package 
+            AND change_name = p_change_name 
+            AND script_hash = p_script_hash
+        ) THEN
+            -- Same change with same content, skip silently
+            RETURN;
+        ELSE
+            -- Different content, this is an error
+            RAISE EXCEPTION 'Change % already deployed in package % with different content', p_change_name, p_package;
+        END IF;
+    END IF;
+    
+    -- Check dependencies
+    IF p_requires IS NOT NULL THEN
+        DECLARE
+            missing_changes TEXT[];
+        BEGIN
+            SELECT array_agg(req) INTO missing_changes
+            FROM unnest(p_requires) AS req
+            WHERE NOT pgpm_migrate.is_deployed(p_package, req);
+            
+            IF array_length(missing_changes, 1) > 0 THEN
+                RAISE EXCEPTION 'Missing required changes for %: %', p_change_name, array_to_string(missing_changes, ', ');
+            END IF;
+        END;
+    END IF;
+    
+    -- Execute deploy (skip if log-only mode)
+    IF NOT p_log_only THEN
+        BEGIN
+            EXECUTE p_deploy_sql;
+        EXCEPTION WHEN OTHERS THEN
+            RAISE;
+        END;
+    END IF;
+    
+    -- Record deployment
+    INSERT INTO pgpm_migrate.changes (change_id, change_name, package, script_hash)
+    VALUES (v_change_id, p_change_name, p_package, p_script_hash);
+    
+    -- Record dependencies (INSERTED AFTER SUCCESSFUL DEPLOYMENT)
+    IF p_requires IS NOT NULL THEN
+        INSERT INTO pgpm_migrate.dependencies (change_id, requires)
+        SELECT v_change_id, req FROM unnest(p_requires) AS req;
+    END IF;
+    
+    -- Log success
+    INSERT INTO pgpm_migrate.events (event_type, change_name, package)
+    VALUES ('deploy', p_change_name, p_package);
+END;
+$$;
+
+-- Revert a change
+CREATE PROCEDURE pgpm_migrate.revert(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_revert_sql TEXT
+)
+LANGUAGE plpgsql AS $$
+BEGIN
+    -- Check if deployed
+    IF NOT pgpm_migrate.is_deployed(p_package, p_change_name) THEN
+        RAISE EXCEPTION 'Change % not deployed in package %', p_change_name, p_package;
+    END IF;
+    
+    -- Check if other changes depend on this (including cross-package dependencies)
+    IF EXISTS (
+        SELECT 1 FROM pgpm_migrate.dependencies d
+        JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+        WHERE (
+            -- Local dependency within same package
+            (d.requires = p_change_name AND c.package = p_package)
+            OR
+            -- Cross-package dependency
+            (d.requires = p_package || ':' || p_change_name)
+        )
+    ) THEN
+        -- Get list of dependent changes for better error message
+        DECLARE
+            dependent_changes TEXT;
+        BEGIN
+            SELECT string_agg(
+                CASE 
+                    WHEN d.requires = p_change_name THEN c.change_name
+                    ELSE c.package || ':' || c.change_name
+                END, 
+                ', '
+            ) INTO dependent_changes
+            FROM pgpm_migrate.dependencies d
+            JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+            WHERE (
+                (d.requires = p_change_name AND c.package = p_package)
+                OR
+                (d.requires = p_package || ':' || p_change_name)
+            );
+            
+            RAISE EXCEPTION 'Cannot revert %: required by %', p_change_name, dependent_changes;
+        END;
+    END IF;
+    
+    -- Execute revert
+    EXECUTE p_revert_sql;
+    
+    -- Remove from deployed
+    DELETE FROM pgpm_migrate.changes 
+    WHERE package = p_package AND change_name = p_change_name;
+    
+    -- Log revert
+    INSERT INTO pgpm_migrate.events (event_type, change_name, package)
+    VALUES ('revert', p_change_name, p_package);
+END;
+$$;
+
+-- Verify a change
+CREATE FUNCTION pgpm_migrate.verify(
+    p_package TEXT,
+    p_change_name TEXT,
+    p_verify_sql TEXT
+)
+RETURNS BOOLEAN
+LANGUAGE plpgsql AS $$
+BEGIN
+    EXECUTE p_verify_sql;
+    RETURN TRUE;
+EXCEPTION WHEN OTHERS THEN
+    RETURN FALSE;
+END;
+$$;
+
+-- List deployed changes
+CREATE FUNCTION pgpm_migrate.deployed_changes(
+    p_package TEXT DEFAULT NULL
+)
+RETURNS TABLE(package TEXT, change_name TEXT, deployed_at TIMESTAMPTZ)
+LANGUAGE sql STABLE AS $$
+    SELECT package, change_name, deployed_at 
+    FROM pgpm_migrate.changes 
+    WHERE p_package IS NULL OR package = p_package
+    ORDER BY deployed_at;
+$$;
+
+-- Get changes that depend on a given change
+CREATE FUNCTION pgpm_migrate.get_dependents(
+    p_package TEXT,
+    p_change_name TEXT
+)
+RETURNS TABLE(package TEXT, change_name TEXT, dependency TEXT)
+LANGUAGE sql STABLE AS $$
+    SELECT c.package, c.change_name, d.requires as dependency
+    FROM pgpm_migrate.dependencies d
+    JOIN pgpm_migrate.changes c ON c.change_id = d.change_id
+    WHERE (
+        -- Local dependency within same package
+        (d.requires = p_change_name AND c.package = p_package)
+        OR
+        -- Cross-package dependency
+        (d.requires = p_package || ':' || p_change_name)
+    )
+    ORDER BY c.package, c.change_name;
+$$;
+
+-- Get deployment status
+CREATE FUNCTION pgpm_migrate.status(
+    p_package TEXT DEFAULT NULL
+)
+RETURNS TABLE(
+    package TEXT,
+    total_deployed INTEGER,
+    last_change TEXT,
+    last_deployed TIMESTAMPTZ
+)
+LANGUAGE sql STABLE AS $$
+    WITH latest AS (
+        SELECT DISTINCT ON (package) 
+            package,
+            change_name,
+            deployed_at
+        FROM pgpm_migrate.changes
+        WHERE p_package IS NULL OR package = p_package
+        ORDER BY package, deployed_at DESC
+    )
+    SELECT 
+        c.package,
+        COUNT(*)::INTEGER AS total_deployed,
+        l.change_name AS last_change,
+        l.deployed_at AS last_deployed
+    FROM pgpm_migrate.changes c
+    JOIN latest l ON l.package = c.package
+    WHERE p_package IS NULL OR c.package = p_package
+    GROUP BY c.package, l.change_name, l.deployed_at;
+$$;

package/migrate/sql/schema.sql

@@ -0,0 +1,37 @@
+-- Create schema
+CREATE SCHEMA pgpm_migrate;
+
+-- 1. Packages (minimal - just name and timestamp)
+CREATE TABLE pgpm_migrate.packages (
+    package         TEXT        PRIMARY KEY,
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
+);
+
+-- 2. Deployed changes (what's currently deployed)
+CREATE TABLE pgpm_migrate.changes (
+    change_id       TEXT        PRIMARY KEY,
+    change_name     TEXT        NOT NULL,
+    package         TEXT        NOT NULL REFERENCES pgpm_migrate.packages(package),
+    script_hash     TEXT        NOT NULL,
+    deployed_at     TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+    UNIQUE(package, change_name),
+    UNIQUE(package, script_hash)
+);
+
+-- 3. Dependencies (what depends on what)
+CREATE TABLE pgpm_migrate.dependencies (
+    change_id       TEXT        NOT NULL REFERENCES pgpm_migrate.changes(change_id) ON DELETE CASCADE,
+    requires        TEXT        NOT NULL,
+    PRIMARY KEY (change_id, requires)
+);
+
+-- 4. Event log (minimal history for rollback)
+CREATE TABLE pgpm_migrate.events (
+    event_id        SERIAL      PRIMARY KEY,
+    event_type      TEXT        NOT NULL CHECK (event_type IN ('deploy', 'revert', 'verify')),
+    change_name     TEXT        NOT NULL,
+    package         TEXT        NOT NULL,
+    occurred_at     TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
+    error_message   TEXT,
+    error_code      TEXT
+);

package/migrate/types.d.ts

@@ -0,0 +1,67 @@
+export interface MigrateChange {
+    name: string;
+    dependencies: string[];
+    timestamp?: string;
+    planner?: string;
+    email?: string;
+    comment?: string;
+}
+export interface MigratePlanFile {
+    package: string;
+    uri?: string;
+    changes: MigrateChange[];
+}
+export interface DeployOptions {
+    modulePath: string;
+    /**
+     * Target change name or tag (e.g., "changeName" or "@tagName").
+     * Note: Project name is already resolved upstream by LaunchQLPackage.
+     */
+    toChange?: string;
+    useTransaction?: boolean;
+    debug?: boolean;
+    logOnly?: boolean;
+    /**
+     * When true, construct dependency resolution purely from plan files
+     * rather than scanning deploy/*.sql files.
+     */
+    usePlan?: boolean;
+}
+export interface RevertOptions {
+    modulePath: string;
+    /**
+     * Target change name or tag (e.g., "changeName" or "@tagName").
+     * Note: Project name is already resolved upstream by LaunchQLPackage.
+     */
+    toChange?: string;
+    useTransaction?: boolean;
+    debug?: boolean;
+}
+export interface VerifyOptions {
+    modulePath: string;
+    /**
+     * Target change name or tag (e.g., "changeName" or "@tagName").
+     * Note: Project name is already resolved upstream by LaunchQLPackage.
+     */
+    toChange?: string;
+}
+export interface DeployResult {
+    deployed: string[];
+    skipped: string[];
+    failed?: string;
+}
+export interface RevertResult {
+    reverted: string[];
+    skipped: string[];
+    failed?: string;
+}
+export interface VerifyResult {
+    verified: string[];
+    failed: string[];
+}
+export interface StatusResult {
+    package: string;
+    totalDeployed: number;
+    lastChange: string;
+    lastDeployed: Date;
+}

package/migrate/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/migrate/utils/event-logger.d.ts

@@ -0,0 +1,13 @@
+import { PgConfig } from 'pg-env';
+export interface EventLogEntry {
+    eventType: 'deploy' | 'revert' | 'verify';
+    changeName: string;
+    package: string;
+    errorMessage?: string;
+    errorCode?: string;
+}
+export declare class EventLogger {
+    private pool;
+    constructor(config: PgConfig);
+    logEvent(entry: EventLogEntry): Promise<void>;
+}

package/migrate/utils/event-logger.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EventLogger = void 0;
+const logger_1 = require("@pgpmjs/logger");
+const pg_cache_1 = require("pg-cache");
+const log = new logger_1.Logger('migrate:event-logger');
+class EventLogger {
+    pool;
+    constructor(config) {
+        this.pool = (0, pg_cache_1.getPgPool)(config);
+    }
+    async logEvent(entry) {
+        try {
+            await this.pool.query(`
+        INSERT INTO pgpm_migrate.events 
+        (event_type, change_name, package, error_message, error_code)
+        VALUES ($1::TEXT, $2::TEXT, $3::TEXT, $4::TEXT, $5::TEXT)
+      `, [
+                entry.eventType,
+                entry.changeName,
+                entry.package,
+                entry.errorMessage || null,
+                entry.errorCode || null
+            ]);
+            log.debug(`Logged ${entry.eventType} event for ${entry.package}:${entry.changeName}`);
+        }
+        catch (error) {
+            log.error(`Failed to log event: ${error.message}`);
+        }
+    }
+}
+exports.EventLogger = EventLogger;

package/migrate/utils/hash.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Generate SHA256 hash of a file's contents
+ */
+export declare function hashFile(filePath: string): Promise<string>;
+/**
+ * Generate SHA256 hash of a string
+ */
+export declare function hashString(content: string): string;
+/**
+ * Generate SHA256 hash of a SQL file's parsed and cleaned AST
+ */
+export declare function hashSqlFile(filePath: string): Promise<string>;

package/migrate/utils/hash.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hashFile = hashFile;
+exports.hashString = hashString;
+exports.hashSqlFile = hashSqlFile;
+const crypto_1 = require("crypto");
+const promises_1 = require("fs/promises");
+const pgsql_parser_1 = require("pgsql-parser");
+const package_1 = require("../../packaging/package");
+/**
+ * Generate SHA256 hash of a file's contents
+ */
+async function hashFile(filePath) {
+    const content = await (0, promises_1.readFile)(filePath, 'utf-8');
+    return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
+}
+/**
+ * Generate SHA256 hash of a string
+ */
+function hashString(content) {
+    return (0, crypto_1.createHash)('sha256').update(content).digest('hex');
+}
+/**
+ * Generate SHA256 hash of a SQL file's parsed and cleaned AST
+ */
+async function hashSqlFile(filePath) {
+    const content = await (0, promises_1.readFile)(filePath, 'utf-8');
+    const parsed = await (0, pgsql_parser_1.parse)(content);
+    const cleaned = (0, package_1.cleanTree)(parsed);
+    const astString = JSON.stringify(cleaned);
+    return hashString(astString);
+}

package/migrate/utils/transaction.d.ts

@@ -0,0 +1,27 @@
+import { Pool, PoolClient } from 'pg';
+export interface TransactionOptions {
+    useTransaction: boolean;
+}
+export interface TransactionContext {
+    client: PoolClient | Pool;
+    isTransaction: boolean;
+    queryHistory: QueryHistoryEntry[];
+    addQuery: (query: string, params?: any[], startTime?: number) => void;
+}
+export interface QueryHistoryEntry {
+    query: string;
+    params?: any[];
+    timestamp: number;
+    duration?: number;
+    error?: any;
+}
+/**
+ * Execute a function within a transaction context
+ * If useTransaction is true, wraps the execution in a transaction
+ * If false, uses the pool directly without transaction
+ */
+export declare function withTransaction<T>(pool: Pool, options: TransactionOptions, fn: (context: TransactionContext) => Promise<T>): Promise<T>;
+/**
+ * Helper to execute a query within a transaction context with enhanced logging
+ */
+export declare function executeQuery(context: TransactionContext, query: string, params?: any[]): Promise<any>;

package/migrate/utils/transaction.js

@@ -0,0 +1,129 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withTransaction = withTransaction;
+exports.executeQuery = executeQuery;
+const logger_1 = require("@pgpmjs/logger");
+const log = new logger_1.Logger('migrate:transaction');
+/**
+ * Execute a function within a transaction context
+ * If useTransaction is true, wraps the execution in a transaction
+ * If false, uses the pool directly without transaction
+ */
+async function withTransaction(pool, options, fn) {
+    const queryHistory = [];
+    const addQuery = (query, params, startTime) => {
+        queryHistory.push({
+            query,
+            params,
+            timestamp: Date.now(),
+            duration: startTime ? Date.now() - startTime : undefined
+        });
+    };
+    if (!options.useTransaction) {
+        // No transaction - use pool directly
+        log.debug('Executing without transaction');
+        return fn({ client: pool, isTransaction: false, queryHistory, addQuery });
+    }
+    // Use transaction
+    const client = await pool.connect();
+    const transactionStartTime = Date.now();
+    log.debug('Starting transaction');
+    try {
+        const beginTime = Date.now();
+        await client.query('BEGIN');
+        addQuery('BEGIN', [], beginTime);
+        const result = await fn({ client, isTransaction: true, queryHistory, addQuery });
+        const commitTime = Date.now();
+        await client.query('COMMIT');
+        addQuery('COMMIT', [], commitTime);
+        const transactionDuration = Date.now() - transactionStartTime;
+        log.debug(`Transaction committed successfully in ${transactionDuration}ms`);
+        return result;
+    }
+    catch (error) {
+        const rollbackTime = Date.now();
+        try {
+            await client.query('ROLLBACK');
+            addQuery('ROLLBACK', [], rollbackTime);
+        }
+        catch (rollbackError) {
+            log.error('Failed to rollback transaction:', rollbackError);
+        }
+        const transactionDuration = Date.now() - transactionStartTime;
+        // Enhanced error logging with context
+        const errorLines = [];
+        errorLines.push(`Transaction rolled back due to error after ${transactionDuration}ms:`);
+        errorLines.push(`Error Code: ${error.code || 'N/A'}`);
+        errorLines.push(`Error Message: ${error.message || 'N/A'}`);
+        // Log query history for debugging
+        if (queryHistory.length > 0) {
+            errorLines.push('Query history for this transaction:');
+            queryHistory.forEach((entry, index) => {
+                const duration = entry.duration ? ` (${entry.duration}ms)` : '';
+                const params = entry.params && entry.params.length > 0
+                    ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
+                    : '';
+                errorLines.push(`  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`);
+            });
+        }
+        // For transaction aborted errors, provide additional context
+        if (error.code === '25P02') {
+            errorLines.push('🔍 Debug Info: Transaction was aborted due to a previous error.');
+            errorLines.push('   This usually means a previous command in the transaction failed.');
+            errorLines.push('   Check the query history above to identify the failing command.');
+        }
+        // Log the consolidated error message
+        log.error(errorLines.join('\n'));
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+}
+/**
+ * Helper to execute a query within a transaction context with enhanced logging
+ */
+async function executeQuery(context, query, params) {
+    const startTime = Date.now();
+    try {
+        const result = await context.client.query(query, params);
+        const duration = Date.now() - startTime;
+        // Add to query history
+        context.addQuery(query, params, startTime);
+        // Log slow queries for debugging
+        if (duration > 1000) {
+            log.warn(`Slow query detected (${duration}ms): ${query.split('\n')[0].trim()}`);
+        }
+        return result;
+    }
+    catch (error) {
+        const duration = Date.now() - startTime;
+        // Add failed query to history
+        context.addQuery(query, params, startTime);
+        // Enhanced error logging
+        const errorLines = [];
+        errorLines.push(`Query failed after ${duration}ms:`);
+        errorLines.push(`  Query: ${query.split('\n')[0].trim()}`);
+        if (params && params.length > 0) {
+            errorLines.push(`  Params: ${JSON.stringify(params.slice(0, 3))}${params.length > 3 ? '...' : ''}`);
+        }
+        errorLines.push(`  Error Code: ${error.code || 'N/A'}`);
+        errorLines.push(`  Error Message: ${error.message || 'N/A'}`);
+        // Provide debugging hints for common errors
+        if (error.code === '42P01') {
+            errorLines.push('💡 Hint: Relation (table/view) does not exist. Check if migrations are applied in correct order.');
+        }
+        else if (error.code === '42883') {
+            errorLines.push('💡 Hint: Function does not exist. Check if required extensions or functions are installed.');
+        }
+        else if (error.code === '23505') {
+            errorLines.push('💡 Hint: Unique constraint violation. Check for duplicate data.');
+        }
+        else if (error.code === '23503') {
+            errorLines.push('💡 Hint: Foreign key constraint violation. Check referential integrity.');
+        }
+        // Log the consolidated error message
+        log.error(errorLines.join('\n'));
+        throw error;
+    }
+}

package/modules/modules.d.ts

@@ -0,0 +1,31 @@
+import { Module } from '../files';
+export type ModuleMap = Record<string, Module>;
+/**
+ * Get the latest change from the pgpm.plan file for a specific module.
+ */
+export declare const latestChange: (sqlmodule: string, modules: ModuleMap, basePath: string) => string;
+/**
+ * Get the latest change and version for a specific module.
+ */
+export declare const latestChangeAndVersion: (sqlmodule: string, modules: ModuleMap, basePath: string) => {
+    change: string;
+    version: string;
+};
+/**
+ * Get extensions and modules required by a specific module.
+ */
+export declare const getExtensionsAndModules: (sqlmodule: string, modules: ModuleMap) => {
+    native: string[];
+    sqitch: string[];
+};
+/**
+ * Get extensions and modules with their latest changes and versions.
+ */
+export declare const getExtensionsAndModulesChanges: (sqlmodule: string, modules: ModuleMap, basePath: string) => {
+    native: string[];
+    sqitch: {
+        name: string;
+        latest: string;
+        version: string;
+    }[];
+};