@pgpmjs/core 4.13.3 → 4.14.0

package/README.md

@@ -22,6 +22,37 @@ pgpm Core is the main package for pgpm, providing tools for database migrations,
 - Reading and writing SQL scripts
 - Resolving dependencies between migrations
 
+## Configuration
+
+### Error Output Configuration
+
+When migrations fail, pgpm provides detailed error output including query history. For large deployments with many migrations, this output can be verbose. The following environment variables control error output formatting:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `PGPM_ERROR_QUERY_HISTORY_LIMIT` | `30` | Maximum number of queries to show in error output. Earlier queries are omitted with a count. |
+| `PGPM_ERROR_MAX_LENGTH` | `10000` | Maximum characters for error output. Output exceeding this is truncated. |
+| `PGPM_ERROR_VERBOSE` | `false` | Set to `true` to disable all limiting and show full error output. |
+
+#### Smart Query Collapsing
+
+Error output automatically collapses consecutive identical queries (like repeated `pgpm_migrate.deploy` calls) into a summary showing:
+- The range of query numbers (e.g., "2-57")
+- The total count (e.g., "56 calls")
+- For deploy calls: the first and last change names for context
+
+Example collapsed output:
+```
+Query history for this transaction:
+  1. BEGIN
+  2-57. CALL pgpm_migrate.deploy($1::TEXT, ...) (56 calls)
+       First: schemas/metaschema_public/tables/extension/table
+       Last:  schemas/metaschema_modules_public/tables/permissions_module/table
+  58. ROLLBACK
+```
+
+To see full uncompressed output, set `PGPM_ERROR_VERBOSE=true`.
+
 ---
 
 ## Education and Tutorials

package/esm/migrate/utils/errors.js

@@ -0,0 +1,169 @@
+import { getEnvVars } from '@pgpmjs/env';
+import { pgpmDefaults } from '@pgpmjs/types';
+/**
+ * Get error output configuration from environment variables with defaults.
+ * Uses centralized env var parsing from @pgpmjs/env and defaults from @pgpmjs/types.
+ */
+export const getErrorOutputConfig = () => {
+    const envVars = getEnvVars();
+    const defaults = pgpmDefaults.errorOutput;
+    return {
+        queryHistoryLimit: envVars.errorOutput?.queryHistoryLimit ?? defaults.queryHistoryLimit,
+        maxLength: envVars.errorOutput?.maxLength ?? defaults.maxLength,
+        verbose: envVars.errorOutput?.verbose ?? defaults.verbose,
+    };
+};
+const errorConfig = getErrorOutputConfig();
+/**
+ * Extract a friendly name from pgpm_migrate.deploy params for better error context
+ */
+function extractDeployChangeName(params) {
+    if (!params || params.length < 2)
+        return null;
+    // params[1] is the change name (e.g., "schemas/metaschema_public/tables/extension/table")
+    return typeof params[1] === 'string' ? params[1] : null;
+}
+/**
+ * Group consecutive queries by their query template (ignoring params)
+ */
+function groupConsecutiveQueries(history) {
+    if (history.length === 0)
+        return [];
+    const groups = [];
+    let currentGroup = {
+        query: history[0].query,
+        startIndex: 0,
+        endIndex: 0,
+        count: 1,
+        entries: [history[0]]
+    };
+    for (let i = 1; i < history.length; i++) {
+        const entry = history[i];
+        if (entry.query === currentGroup.query) {
+            // Same query template, extend the group
+            currentGroup.endIndex = i;
+            currentGroup.count++;
+            currentGroup.entries.push(entry);
+        }
+        else {
+            // Different query, start a new group
+            groups.push(currentGroup);
+            currentGroup = {
+                query: entry.query,
+                startIndex: i,
+                endIndex: i,
+                count: 1,
+                entries: [entry]
+            };
+        }
+    }
+    groups.push(currentGroup);
+    return groups;
+}
+/**
+ * Format a single query entry for display
+ */
+function formatQueryEntry(entry, index) {
+    const duration = entry.duration ? ` (${entry.duration}ms)` : '';
+    const params = entry.params && entry.params.length > 0
+        ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
+        : '';
+    return `  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`;
+}
+/**
+ * Format a group of queries for display, collapsing repetitive queries
+ */
+function formatQueryGroup(group) {
+    const lines = [];
+    const queryPreview = group.query.split('\n')[0].trim();
+    if (group.count === 1) {
+        // Single query, format normally
+        lines.push(formatQueryEntry(group.entries[0], group.startIndex));
+    }
+    else {
+        // Multiple consecutive identical queries, collapse them
+        const isPgpmDeploy = queryPreview.includes('pgpm_migrate.deploy');
+        // Show range and count
+        lines.push(`  ${group.startIndex + 1}-${group.endIndex + 1}. ${queryPreview} (${group.count} calls)`);
+        // For pgpm_migrate.deploy, show first and last change names for context
+        if (isPgpmDeploy) {
+            const firstChange = extractDeployChangeName(group.entries[0].params);
+            const lastChange = extractDeployChangeName(group.entries[group.entries.length - 1].params);
+            if (firstChange) {
+                lines.push(`       First: ${firstChange}`);
+            }
+            if (lastChange && lastChange !== firstChange) {
+                lines.push(`       Last:  ${lastChange}`);
+            }
+        }
+        else {
+            // For other queries, show first and last params
+            const firstParams = group.entries[0].params;
+            const lastParams = group.entries[group.entries.length - 1].params;
+            if (firstParams && firstParams.length > 0) {
+                lines.push(`       First params: ${JSON.stringify(firstParams.slice(0, 2))}${firstParams.length > 2 ? '...' : ''}`);
+            }
+            if (lastParams && lastParams.length > 0 && JSON.stringify(lastParams) !== JSON.stringify(firstParams)) {
+                lines.push(`       Last params:  ${JSON.stringify(lastParams.slice(0, 2))}${lastParams.length > 2 ? '...' : ''}`);
+            }
+        }
+    }
+    return lines;
+}
+/**
+ * Format query history with smart collapsing and limiting
+ */
+export function formatQueryHistory(history) {
+    if (history.length === 0)
+        return [];
+    // In verbose mode, show everything without collapsing
+    if (errorConfig.verbose) {
+        return history.map((entry, index) => formatQueryEntry(entry, index));
+    }
+    // Group consecutive identical queries
+    const groups = groupConsecutiveQueries(history);
+    // Apply limit - keep last N queries worth of groups
+    let totalQueries = 0;
+    let startGroupIndex = groups.length;
+    for (let i = groups.length - 1; i >= 0; i--) {
+        totalQueries += groups[i].count;
+        if (totalQueries > errorConfig.queryHistoryLimit) {
+            startGroupIndex = i + 1;
+            break;
+        }
+        startGroupIndex = i;
+    }
+    const lines = [];
+    // If we're truncating, show how many were omitted
+    if (startGroupIndex > 0) {
+        let omittedCount = 0;
+        for (let i = 0; i < startGroupIndex; i++) {
+            omittedCount += groups[i].count;
+        }
+        lines.push(`  ... (${omittedCount} earlier queries omitted, set PGPM_ERROR_VERBOSE=true to see all)`);
+    }
+    // Format the remaining groups
+    for (let i = startGroupIndex; i < groups.length; i++) {
+        lines.push(...formatQueryGroup(groups[i]));
+    }
+    return lines;
+}
+/**
+ * Truncate error output if it exceeds the max length
+ */
+export function truncateErrorOutput(lines) {
+    if (errorConfig.verbose)
+        return lines;
+    const joined = lines.join('\n');
+    if (joined.length <= errorConfig.maxLength)
+        return lines;
+    // Truncate and add notice
+    const truncated = joined.slice(0, errorConfig.maxLength);
+    const lastNewline = truncated.lastIndexOf('\n');
+    const cleanTruncated = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
+    return [
+        ...cleanTruncated.split('\n'),
+        `... (output truncated at ${errorConfig.maxLength} chars, total was ${joined.length} chars)`,
+        `    Set PGPM_ERROR_VERBOSE=true to see full output`
+    ];
+}

package/esm/migrate/utils/transaction.js

@@ -1,5 +1,8 @@
 import { Logger } from '@pgpmjs/logger';
 import { extractPgErrorFields, formatPgErrorFields } from '@pgpmjs/types';
+import { formatQueryHistory, truncateErrorOutput } from './errors';
+// Re-export error formatting functions for backward compatibility
+export { formatQueryHistory, truncateErrorOutput } from './errors';
 const log = new Logger('migrate:transaction');
 /**
  * Execute a function within a transaction context
@@ -60,16 +63,11 @@ export async function withTransaction(pool, options, fn) {
                 errorLines.push(...fieldLines);
             }
         }
-        // Log query history for debugging
+        // Log query history for debugging (with smart collapsing and limiting)
         if (queryHistory.length > 0) {
             errorLines.push('Query history for this transaction:');
-            queryHistory.forEach((entry, index) => {
-                const duration = entry.duration ? ` (${entry.duration}ms)` : '';
-                const params = entry.params && entry.params.length > 0
-                    ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
-                    : '';
-                errorLines.push(`  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`);
-            });
+            const historyLines = formatQueryHistory(queryHistory);
+            errorLines.push(...historyLines);
         }
         // For transaction aborted errors, provide additional context
         if (error.code === '25P02') {
@@ -77,8 +75,9 @@ export async function withTransaction(pool, options, fn) {
             errorLines.push('   This usually means a previous command in the transaction failed.');
             errorLines.push('   Check the query history above to identify the failing command.');
         }
-        // Log the consolidated error message
-        log.error(errorLines.join('\n'));
+        // Apply total output length limit and log the consolidated error message
+        const finalLines = truncateErrorOutput(errorLines);
+        log.error(finalLines.join('\n'));
         throw error;
     }
     finally {

package/migrate/utils/errors.d.ts

@@ -0,0 +1,18 @@
+import { QueryHistoryEntry } from './transaction';
+/**
+ * Get error output configuration from environment variables with defaults.
+ * Uses centralized env var parsing from @pgpmjs/env and defaults from @pgpmjs/types.
+ */
+export declare const getErrorOutputConfig: () => {
+    queryHistoryLimit: number;
+    maxLength: number;
+    verbose: boolean;
+};
+/**
+ * Format query history with smart collapsing and limiting
+ */
+export declare function formatQueryHistory(history: QueryHistoryEntry[]): string[];
+/**
+ * Truncate error output if it exceeds the max length
+ */
+export declare function truncateErrorOutput(lines: string[]): string[];

package/migrate/utils/errors.js

@@ -0,0 +1,175 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getErrorOutputConfig = void 0;
+exports.formatQueryHistory = formatQueryHistory;
+exports.truncateErrorOutput = truncateErrorOutput;
+const env_1 = require("@pgpmjs/env");
+const types_1 = require("@pgpmjs/types");
+/**
+ * Get error output configuration from environment variables with defaults.
+ * Uses centralized env var parsing from @pgpmjs/env and defaults from @pgpmjs/types.
+ */
+const getErrorOutputConfig = () => {
+    const envVars = (0, env_1.getEnvVars)();
+    const defaults = types_1.pgpmDefaults.errorOutput;
+    return {
+        queryHistoryLimit: envVars.errorOutput?.queryHistoryLimit ?? defaults.queryHistoryLimit,
+        maxLength: envVars.errorOutput?.maxLength ?? defaults.maxLength,
+        verbose: envVars.errorOutput?.verbose ?? defaults.verbose,
+    };
+};
+exports.getErrorOutputConfig = getErrorOutputConfig;
+const errorConfig = (0, exports.getErrorOutputConfig)();
+/**
+ * Extract a friendly name from pgpm_migrate.deploy params for better error context
+ */
+function extractDeployChangeName(params) {
+    if (!params || params.length < 2)
+        return null;
+    // params[1] is the change name (e.g., "schemas/metaschema_public/tables/extension/table")
+    return typeof params[1] === 'string' ? params[1] : null;
+}
+/**
+ * Group consecutive queries by their query template (ignoring params)
+ */
+function groupConsecutiveQueries(history) {
+    if (history.length === 0)
+        return [];
+    const groups = [];
+    let currentGroup = {
+        query: history[0].query,
+        startIndex: 0,
+        endIndex: 0,
+        count: 1,
+        entries: [history[0]]
+    };
+    for (let i = 1; i < history.length; i++) {
+        const entry = history[i];
+        if (entry.query === currentGroup.query) {
+            // Same query template, extend the group
+            currentGroup.endIndex = i;
+            currentGroup.count++;
+            currentGroup.entries.push(entry);
+        }
+        else {
+            // Different query, start a new group
+            groups.push(currentGroup);
+            currentGroup = {
+                query: entry.query,
+                startIndex: i,
+                endIndex: i,
+                count: 1,
+                entries: [entry]
+            };
+        }
+    }
+    groups.push(currentGroup);
+    return groups;
+}
+/**
+ * Format a single query entry for display
+ */
+function formatQueryEntry(entry, index) {
+    const duration = entry.duration ? ` (${entry.duration}ms)` : '';
+    const params = entry.params && entry.params.length > 0
+        ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
+        : '';
+    return `  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`;
+}
+/**
+ * Format a group of queries for display, collapsing repetitive queries
+ */
+function formatQueryGroup(group) {
+    const lines = [];
+    const queryPreview = group.query.split('\n')[0].trim();
+    if (group.count === 1) {
+        // Single query, format normally
+        lines.push(formatQueryEntry(group.entries[0], group.startIndex));
+    }
+    else {
+        // Multiple consecutive identical queries, collapse them
+        const isPgpmDeploy = queryPreview.includes('pgpm_migrate.deploy');
+        // Show range and count
+        lines.push(`  ${group.startIndex + 1}-${group.endIndex + 1}. ${queryPreview} (${group.count} calls)`);
+        // For pgpm_migrate.deploy, show first and last change names for context
+        if (isPgpmDeploy) {
+            const firstChange = extractDeployChangeName(group.entries[0].params);
+            const lastChange = extractDeployChangeName(group.entries[group.entries.length - 1].params);
+            if (firstChange) {
+                lines.push(`       First: ${firstChange}`);
+            }
+            if (lastChange && lastChange !== firstChange) {
+                lines.push(`       Last:  ${lastChange}`);
+            }
+        }
+        else {
+            // For other queries, show first and last params
+            const firstParams = group.entries[0].params;
+            const lastParams = group.entries[group.entries.length - 1].params;
+            if (firstParams && firstParams.length > 0) {
+                lines.push(`       First params: ${JSON.stringify(firstParams.slice(0, 2))}${firstParams.length > 2 ? '...' : ''}`);
+            }
+            if (lastParams && lastParams.length > 0 && JSON.stringify(lastParams) !== JSON.stringify(firstParams)) {
+                lines.push(`       Last params:  ${JSON.stringify(lastParams.slice(0, 2))}${lastParams.length > 2 ? '...' : ''}`);
+            }
+        }
+    }
+    return lines;
+}
+/**
+ * Format query history with smart collapsing and limiting
+ */
+function formatQueryHistory(history) {
+    if (history.length === 0)
+        return [];
+    // In verbose mode, show everything without collapsing
+    if (errorConfig.verbose) {
+        return history.map((entry, index) => formatQueryEntry(entry, index));
+    }
+    // Group consecutive identical queries
+    const groups = groupConsecutiveQueries(history);
+    // Apply limit - keep last N queries worth of groups
+    let totalQueries = 0;
+    let startGroupIndex = groups.length;
+    for (let i = groups.length - 1; i >= 0; i--) {
+        totalQueries += groups[i].count;
+        if (totalQueries > errorConfig.queryHistoryLimit) {
+            startGroupIndex = i + 1;
+            break;
+        }
+        startGroupIndex = i;
+    }
+    const lines = [];
+    // If we're truncating, show how many were omitted
+    if (startGroupIndex > 0) {
+        let omittedCount = 0;
+        for (let i = 0; i < startGroupIndex; i++) {
+            omittedCount += groups[i].count;
+        }
+        lines.push(`  ... (${omittedCount} earlier queries omitted, set PGPM_ERROR_VERBOSE=true to see all)`);
+    }
+    // Format the remaining groups
+    for (let i = startGroupIndex; i < groups.length; i++) {
+        lines.push(...formatQueryGroup(groups[i]));
+    }
+    return lines;
+}
+/**
+ * Truncate error output if it exceeds the max length
+ */
+function truncateErrorOutput(lines) {
+    if (errorConfig.verbose)
+        return lines;
+    const joined = lines.join('\n');
+    if (joined.length <= errorConfig.maxLength)
+        return lines;
+    // Truncate and add notice
+    const truncated = joined.slice(0, errorConfig.maxLength);
+    const lastNewline = truncated.lastIndexOf('\n');
+    const cleanTruncated = lastNewline > 0 ? truncated.slice(0, lastNewline) : truncated;
+    return [
+        ...cleanTruncated.split('\n'),
+        `... (output truncated at ${errorConfig.maxLength} chars, total was ${joined.length} chars)`,
+        `    Set PGPM_ERROR_VERBOSE=true to see full output`
+    ];
+}

package/migrate/utils/transaction.d.ts

@@ -1,4 +1,5 @@
 import { Pool, PoolClient } from 'pg';
+export { formatQueryHistory, truncateErrorOutput } from './errors';
 export interface TransactionOptions {
     useTransaction: boolean;
 }

package/migrate/utils/transaction.js

@@ -1,9 +1,15 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.truncateErrorOutput = exports.formatQueryHistory = void 0;
 exports.withTransaction = withTransaction;
 exports.executeQuery = executeQuery;
 const logger_1 = require("@pgpmjs/logger");
 const types_1 = require("@pgpmjs/types");
+const errors_1 = require("./errors");
+// Re-export error formatting functions for backward compatibility
+var errors_2 = require("./errors");
+Object.defineProperty(exports, "formatQueryHistory", { enumerable: true, get: function () { return errors_2.formatQueryHistory; } });
+Object.defineProperty(exports, "truncateErrorOutput", { enumerable: true, get: function () { return errors_2.truncateErrorOutput; } });
 const log = new logger_1.Logger('migrate:transaction');
 /**
  * Execute a function within a transaction context
@@ -64,16 +70,11 @@ async function withTransaction(pool, options, fn) {
                 errorLines.push(...fieldLines);
             }
         }
-        // Log query history for debugging
+        // Log query history for debugging (with smart collapsing and limiting)
         if (queryHistory.length > 0) {
             errorLines.push('Query history for this transaction:');
-            queryHistory.forEach((entry, index) => {
-                const duration = entry.duration ? ` (${entry.duration}ms)` : '';
-                const params = entry.params && entry.params.length > 0
-                    ? ` with params: ${JSON.stringify(entry.params.slice(0, 2))}${entry.params.length > 2 ? '...' : ''}`
-                    : '';
-                errorLines.push(`  ${index + 1}. ${entry.query.split('\n')[0].trim()}${params}${duration}`);
-            });
+            const historyLines = (0, errors_1.formatQueryHistory)(queryHistory);
+            errorLines.push(...historyLines);
         }
         // For transaction aborted errors, provide additional context
         if (error.code === '25P02') {
@@ -81,8 +82,9 @@ async function withTransaction(pool, options, fn) {
             errorLines.push('   This usually means a previous command in the transaction failed.');
             errorLines.push('   Check the query history above to identify the failing command.');
         }
-        // Log the consolidated error message
-        log.error(errorLines.join('\n'));
+        // Apply total output length limit and log the consolidated error message
+        const finalLines = (0, errors_1.truncateErrorOutput)(errorLines);
+        log.error(finalLines.join('\n'));
         throw error;
     }
     finally {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pgpmjs/core",
-  "version": "4.13.3",
+  "version": "4.14.0",
   "author": "Constructive <developers@constructive.io>",
   "description": "PGPM Package and Migration Tools",
   "main": "index.js",
@@ -48,21 +48,21 @@
     "makage": "^0.1.10"
   },
   "dependencies": {
-    "@pgpmjs/env": "^2.9.3",
+    "@pgpmjs/env": "^2.9.4",
     "@pgpmjs/logger": "^1.3.7",
-    "@pgpmjs/server-utils": "^2.8.14",
-    "@pgpmjs/types": "^2.14.0",
+    "@pgpmjs/server-utils": "^2.8.15",
+    "@pgpmjs/types": "^2.14.1",
     "csv-to-pg": "^3.4.1",
     "genomic": "^5.2.3",
     "glob": "^13.0.0",
     "komoji": "^0.7.14",
     "parse-package-name": "^1.0.0",
     "pg": "^8.16.3",
-    "pg-cache": "^1.6.14",
+    "pg-cache": "^1.6.15",
     "pg-env": "^1.2.5",
     "pgsql-deparser": "^17.17.2",
     "pgsql-parser": "^17.9.11",
     "yanse": "^0.1.11"
   },
-  "gitHead": "c1708066d429d86bbc1ca3aed778ff51779c8efc"
+  "gitHead": "cb4af2cf6c23dad24cd951c232d3e2006b81aa3d"
 }