@constructive-io/graphql-server 4.10.0 → 4.11.1

package/esm/diagnostics/debug-db-snapshot.js

@@ -0,0 +1,191 @@
+import { buildConnectionString, getPgPool } from 'pg-cache';
+import { getPgEnvOptions } from 'pg-env';
+import { Pool } from 'pg';
+const ACTIVE_ACTIVITY_SQL = `
+  select
+    pid,
+    usename,
+    application_name,
+    state,
+    wait_event_type,
+    wait_event,
+    age(now(), xact_start) as xact_age,
+    age(now(), query_start) as query_age,
+    left(query, 500) as query
+  from pg_stat_activity
+  where datname = current_database()
+    and pid <> pg_backend_pid()
+    and (
+      xact_start is not null
+      or wait_event_type is not null
+      or state <> 'idle'
+    )
+  order by xact_start asc nulls last, query_start asc nulls last
+  limit 50
+`;
+const BLOCKED_ACTIVITY_SQL = `
+  with blocked as (
+    select
+      a.pid as blocked_pid,
+      a.usename as blocked_user,
+      a.application_name as blocked_application,
+      a.state as blocked_state,
+      a.wait_event_type,
+      a.wait_event,
+      age(now(), a.query_start) as blocked_for,
+      left(a.query, 500) as blocked_query,
+      pg_blocking_pids(a.pid) as blocker_pids
+    from pg_stat_activity a
+    where a.datname = current_database()
+      and cardinality(pg_blocking_pids(a.pid)) > 0
+  )
+  select
+    b.blocked_pid,
+    b.blocked_user,
+    b.blocked_application,
+    b.blocked_state,
+    b.wait_event_type,
+    b.wait_event,
+    b.blocked_for,
+    b.blocked_query,
+    blocker.pid as blocker_pid,
+    blocker.usename as blocker_user,
+    blocker.application_name as blocker_application,
+    blocker.state as blocker_state,
+    left(blocker.query, 500) as blocker_query
+  from blocked b
+  left join lateral unnest(b.blocker_pids) blocker_pid on true
+  left join pg_stat_activity blocker on blocker.pid = blocker_pid
+  order by b.blocked_for desc
+`;
+const LOCK_SUMMARY_SQL = `
+  select
+    locktype,
+    mode,
+    granted,
+    count(*)::int as count
+  from pg_locks
+  group by locktype, mode, granted
+  order by granted asc, count desc, locktype asc, mode asc
+`;
+const DATABASE_STATS_SQL = `
+  select
+    numbackends,
+    xact_commit,
+    xact_rollback,
+    blks_read,
+    blks_hit,
+    tup_returned,
+    tup_fetched,
+    tup_inserted,
+    tup_updated,
+    tup_deleted,
+    temp_files,
+    temp_bytes,
+    deadlocks,
+    checksum_failures,
+    stats_reset
+  from pg_stat_database
+  where datname = current_database()
+`;
+const SETTINGS_SQL = `
+  select
+    name,
+    setting,
+    unit
+  from pg_settings
+  where name = any(array[
+    'max_connections',
+    'shared_buffers',
+    'work_mem',
+    'maintenance_work_mem',
+    'effective_cache_size',
+    'statement_timeout',
+    'lock_timeout',
+    'idle_in_transaction_session_timeout'
+  ])
+  order by name asc
+`;
+const NOTIFY_QUEUE_SQL = `
+  select pg_notification_queue_usage() as queue_usage
+`;
+const DIAGNOSTICS_STATEMENT_TIMEOUT_MS = 3_000;
+const DIAGNOSTICS_LOCK_TIMEOUT_MS = 500;
+const diagnosticsPools = new Map();
+const buildDiagnosticsConnectionString = (opts) => {
+    const pgConfig = getPgEnvOptions(opts.pg);
+    return buildConnectionString(pgConfig.user, pgConfig.password, pgConfig.host, pgConfig.port, pgConfig.database);
+};
+const getDiagnosticsPool = (opts) => {
+    const connectionString = buildDiagnosticsConnectionString(opts);
+    const existing = diagnosticsPools.get(connectionString);
+    if (existing) {
+        return existing;
+    }
+    const pool = new Pool({
+        connectionString,
+        max: 1,
+        idleTimeoutMillis: 10_000,
+        connectionTimeoutMillis: 1_500,
+        allowExitOnIdle: true,
+        application_name: 'constructive-debug-snapshot',
+    });
+    diagnosticsPools.set(connectionString, pool);
+    return pool;
+};
+const withDiagnosticsClient = async (opts, fn) => {
+    const diagnosticsPool = getDiagnosticsPool(opts);
+    const client = await diagnosticsPool.connect();
+    try {
+        await client.query('BEGIN');
+        await client.query(`SET LOCAL statement_timeout = '${DIAGNOSTICS_STATEMENT_TIMEOUT_MS}ms'`);
+        await client.query(`SET LOCAL lock_timeout = '${DIAGNOSTICS_LOCK_TIMEOUT_MS}ms'`);
+        const result = await fn(client);
+        await client.query('COMMIT');
+        return result;
+    }
+    catch (error) {
+        try {
+            await client.query('ROLLBACK');
+        }
+        catch {
+            // Best-effort rollback for diagnostics-only transactions.
+        }
+        throw error;
+    }
+    finally {
+        client.release();
+    }
+};
+export const closeDebugDatabasePools = async () => {
+    const pools = [...diagnosticsPools.values()];
+    diagnosticsPools.clear();
+    await Promise.allSettled(pools.map((pool) => pool.end()));
+};
+export const getDebugDatabaseSnapshot = async (opts) => {
+    const appPool = getPgPool(opts.pg);
+    const { activity, blocked, lockSummary, databaseStats, settings, notifyQueue, } = await withDiagnosticsClient(opts, async (client) => ({
+        activity: await client.query(ACTIVE_ACTIVITY_SQL),
+        blocked: await client.query(BLOCKED_ACTIVITY_SQL),
+        lockSummary: await client.query(LOCK_SUMMARY_SQL),
+        databaseStats: await client.query(DATABASE_STATS_SQL),
+        settings: await client.query(SETTINGS_SQL),
+        notifyQueue: await client.query(NOTIFY_QUEUE_SQL),
+    }));
+    return {
+        database: opts.pg?.database ?? null,
+        pool: {
+            max: appPool.options?.max ?? null,
+            totalCount: appPool.totalCount,
+            idleCount: appPool.idleCount,
+            waitingCount: appPool.waitingCount,
+        },
+        activeActivity: activity.rows,
+        blockedActivity: blocked.rows,
+        lockSummary: lockSummary.rows,
+        databaseStats: databaseStats.rows[0] ?? null,
+        settings: settings.rows,
+        notificationQueueUsage: notifyQueue.rows[0]?.queue_usage ?? null,
+        timestamp: new Date().toISOString(),
+    };
+};

package/esm/diagnostics/debug-memory-snapshot.js

@@ -0,0 +1,70 @@
+import os from 'node:os';
+import v8 from 'node:v8';
+import { svcCache, SVC_CACHE_TTL_MS } from '@pgpmjs/server-utils';
+import { getCacheStats } from 'graphile-cache';
+import { getInFlightCount, getInFlightKeys } from '../middleware/graphile';
+import { getGraphileBuildStats } from '../middleware/observability/graphile-build-stats';
+const toMB = (bytes) => `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+export const getDebugMemorySnapshot = () => {
+    const mem = process.memoryUsage();
+    const heapSpaces = v8.getHeapSpaceStatistics().map((space) => ({
+        spaceName: space.space_name,
+        spaceSizeBytes: space.space_size,
+        spaceUsedBytes: space.space_used_size,
+        spaceAvailableBytes: space.space_available_size,
+        physicalSpaceSizeBytes: space.physical_space_size,
+    }));
+    return {
+        pid: process.pid,
+        nodeEnv: process.env.NODE_ENV,
+        memory: {
+            heapUsedBytes: mem.heapUsed,
+            heapTotalBytes: mem.heapTotal,
+            rssBytes: mem.rss,
+            externalBytes: mem.external,
+            arrayBuffersBytes: mem.arrayBuffers,
+            heapUsed: toMB(mem.heapUsed),
+            heapTotal: toMB(mem.heapTotal),
+            rss: toMB(mem.rss),
+            external: toMB(mem.external),
+            arrayBuffers: toMB(mem.arrayBuffers),
+        },
+        cpuUsageMicros: process.cpuUsage(),
+        resourceUsage: process.resourceUsage(),
+        system: {
+            loadAverage: os.loadavg(),
+            freeMemoryBytes: os.freemem(),
+            totalMemoryBytes: os.totalmem(),
+            uptimeSeconds: os.uptime(),
+        },
+        v8: {
+            heapStatistics: v8.getHeapStatistics(),
+            heapSpaces,
+        },
+        graphileCache: getCacheStats(),
+        svcCache: {
+            size: svcCache.size,
+            max: svcCache.max,
+            ttlMs: SVC_CACHE_TTL_MS,
+            // Note: with updateAgeOnGet: true, this is "time since last access" not "time since creation"
+            oldestKeyAgeMs: (() => {
+                let minRemaining = Infinity;
+                for (const key of svcCache.keys()) {
+                    const remaining = svcCache.getRemainingTTL(key);
+                    if (remaining < minRemaining) {
+                        minRemaining = remaining;
+                    }
+                }
+                return Number.isFinite(minRemaining) ? SVC_CACHE_TTL_MS - minRemaining : null;
+            })(),
+            keys: [...svcCache.keys()].slice(0, 200),
+        },
+        inFlight: {
+            count: getInFlightCount(),
+            keys: getInFlightKeys(),
+        },
+        graphileBuilds: getGraphileBuildStats(),
+        uptimeMinutes: process.uptime() / 60,
+        timestamp: new Date().toISOString(),
+    };
+};

package/esm/diagnostics/debug-sampler.js

@@ -0,0 +1,188 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { Logger } from '@pgpmjs/logger';
+import { getDebugDatabaseSnapshot } from './debug-db-snapshot';
+import { getDebugMemorySnapshot } from './debug-memory-snapshot';
+import { isGraphqlDebugSamplerEnabled } from './observability';
+const log = new Logger('debug-sampler');
+const MAX_TOTAL_BYTES = 1024 * 1024 * 1024; // 1 GB
+const getSamplerIntervalMs = () => {
+    const raw = process.env.GRAPHQL_DEBUG_SAMPLER_INTERVAL_MS;
+    const parsed = raw ? Number.parseInt(raw, 10) : 10_000;
+    return Number.isFinite(parsed) && parsed >= 1_000 ? parsed : 10_000;
+};
+const getSamplerRootDir = () => {
+    if (process.env.GRAPHQL_DEBUG_SAMPLER_DIR) {
+        return path.resolve(process.env.GRAPHQL_DEBUG_SAMPLER_DIR);
+    }
+    return path.resolve(__dirname, '../..', 'logs');
+};
+const createSessionLogDir = () => {
+    const rootDir = getSamplerRootDir();
+    const sessionName = `run-${new Date().toISOString().replace(/[:.]/g, '-')}-pid${process.pid}`;
+    return process.env.GRAPHQL_DEBUG_SAMPLER_DIR
+        ? rootDir
+        : path.join(rootDir, sessionName);
+};
+const appendJsonLine = async (filePath, payload) => {
+    await fs.mkdir(path.dirname(filePath), { recursive: true });
+    await fs.appendFile(filePath, `${JSON.stringify(payload)}\n`, 'utf8');
+};
+const getDirSize = async (dirPath) => {
+    let total = 0;
+    const entries = await fs.readdir(dirPath, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dirPath, entry.name);
+        if (entry.isDirectory()) {
+            total += await getDirSize(fullPath);
+        }
+        else {
+            const stat = await fs.stat(fullPath);
+            total += stat.size;
+        }
+    }
+    return total;
+};
+const enforceMaxSize = async (rootDir, currentSessionDir) => {
+    try {
+        const totalSize = await getDirSize(rootDir);
+        if (totalSize <= MAX_TOTAL_BYTES) {
+            return;
+        }
+        const entries = await fs.readdir(rootDir, { withFileTypes: true });
+        const sessionDirs = await Promise.all(entries
+            .filter((e) => e.isDirectory())
+            .map(async (e) => {
+            const fullPath = path.join(rootDir, e.name);
+            const stat = await fs.stat(fullPath);
+            return { path: fullPath, mtimeMs: stat.mtimeMs };
+        }));
+        sessionDirs.sort((a, b) => a.mtimeMs - b.mtimeMs);
+        for (const dir of sessionDirs) {
+            if (dir.path === currentSessionDir) {
+                continue;
+            }
+            log.info(`Rolling cleanup: removing old session dir ${dir.path}`);
+            await fs.rm(dir.path, { recursive: true, force: true });
+            const newSize = await getDirSize(rootDir);
+            if (newSize <= MAX_TOTAL_BYTES) {
+                break;
+            }
+        }
+    }
+    catch (error) {
+        log.error('Failed to enforce max log size', error);
+    }
+};
+export const startDebugSampler = (opts) => {
+    if (!isGraphqlDebugSamplerEnabled(opts.server?.host)) {
+        return null;
+    }
+    const intervalMs = getSamplerIntervalMs();
+    const logDir = createSessionLogDir();
+    const memoryLogPath = path.join(logDir, 'debug-memory.ndjson');
+    const dbLogPath = path.join(logDir, 'debug-db.ndjson');
+    const errorLogPath = path.join(logDir, 'debug-sampler-errors.ndjson');
+    let timer = null;
+    let stopped = false;
+    let inFlight = null;
+    let writeFailureLogged = false;
+    const runBackgroundWrite = (promise, scope) => {
+        promise.catch((error) => {
+            // Avoid recursive attempts to write additional error files when the
+            // underlying storage path is broken or unavailable.
+            if (!writeFailureLogged) {
+                writeFailureLogged = true;
+                log.error(`Debug sampler background write failed (${scope})`, error);
+            }
+        });
+    };
+    const recordError = async (scope, error) => {
+        const payload = {
+            scope,
+            timestamp: new Date().toISOString(),
+            pid: process.pid,
+            error: error instanceof Error ? {
+                name: error.name,
+                message: error.message,
+                stack: error.stack,
+            } : {
+                message: String(error),
+            },
+        };
+        await appendJsonLine(errorLogPath, payload);
+    };
+    const sampleOnce = async () => {
+        if (stopped) {
+            return;
+        }
+        try {
+            await appendJsonLine(memoryLogPath, getDebugMemorySnapshot());
+        }
+        catch (error) {
+            log.error('Failed to capture debug memory snapshot', error);
+            await recordError('memory', error);
+        }
+        try {
+            await appendJsonLine(dbLogPath, await getDebugDatabaseSnapshot(opts));
+        }
+        catch (error) {
+            log.error('Failed to capture debug DB snapshot', error);
+            await recordError('db', error);
+        }
+        await enforceMaxSize(getSamplerRootDir(), logDir);
+    };
+    const tick = () => {
+        if (stopped) {
+            return;
+        }
+        if (inFlight) {
+            runBackgroundWrite(recordError('sampler', new Error('Skipped debug sample because previous sample is still running')), 'record-skip');
+            return;
+        }
+        inFlight = sampleOnce()
+            .catch(async (error) => {
+            log.error('Debug sampler tick failed', error);
+            await recordError('sampler', error);
+        })
+            .finally(() => {
+            inFlight = null;
+        });
+    };
+    const lifecyclePayload = {
+        event: 'sampler_started',
+        intervalMs,
+        logDir,
+        pid: process.pid,
+        timestamp: new Date().toISOString(),
+    };
+    runBackgroundWrite(appendJsonLine(memoryLogPath, lifecyclePayload), 'lifecycle-memory-start');
+    runBackgroundWrite(appendJsonLine(dbLogPath, lifecyclePayload), 'lifecycle-db-start');
+    log.info(`Debug sampler writing snapshots every ${intervalMs}ms to ${logDir}`);
+    tick();
+    timer = setInterval(tick, intervalMs);
+    timer.unref();
+    return {
+        async stop() {
+            stopped = true;
+            if (timer) {
+                clearInterval(timer);
+                timer = null;
+            }
+            if (inFlight) {
+                await inFlight;
+            }
+            const payload = {
+                event: 'sampler_stopped',
+                intervalMs,
+                logDir,
+                pid: process.pid,
+                timestamp: new Date().toISOString(),
+            };
+            await Promise.allSettled([
+                appendJsonLine(memoryLogPath, payload),
+                appendJsonLine(dbLogPath, payload),
+            ]);
+        },
+    };
+};

package/esm/diagnostics/observability.js

@@ -0,0 +1,53 @@
+const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1', '[::1]']);
+const LOOPBACK_ADDRESSES = new Set(['127.0.0.1', '::1']);
+const parseBooleanEnv = (value, fallback) => {
+    if (value == null) {
+        return fallback;
+    }
+    const normalized = value.trim().toLowerCase();
+    if (['1', 'true', 'yes', 'on'].includes(normalized))
+        return true;
+    if (['0', 'false', 'no', 'off'].includes(normalized))
+        return false;
+    return fallback;
+};
+const normalizeHost = (value) => {
+    if (!value) {
+        return null;
+    }
+    const trimmed = value.trim().toLowerCase();
+    if (!trimmed) {
+        return null;
+    }
+    if (trimmed.startsWith('[')) {
+        const closingIndex = trimmed.indexOf(']');
+        return closingIndex === -1 ? trimmed : trimmed.slice(0, closingIndex + 1);
+    }
+    const colonCount = trimmed.split(':').length - 1;
+    if (colonCount === 1) {
+        return trimmed.split(':')[0] || null;
+    }
+    return trimmed;
+};
+const normalizeAddress = (value) => {
+    const normalized = normalizeHost(value);
+    if (!normalized) {
+        return null;
+    }
+    return normalized.startsWith('::ffff:') ? normalized.slice(7) : normalized;
+};
+export const isDevelopmentObservabilityMode = () => process.env.NODE_ENV === 'development';
+export const isLoopbackHost = (value) => {
+    const normalized = normalizeHost(value);
+    return normalized != null && LOOPBACK_HOSTS.has(normalized);
+};
+export const isLoopbackAddress = (value) => {
+    const normalized = normalizeAddress(value);
+    return normalized != null && LOOPBACK_ADDRESSES.has(normalized);
+};
+export const isGraphqlObservabilityRequested = () => parseBooleanEnv(process.env.GRAPHQL_OBSERVABILITY_ENABLED, false);
+export const isGraphqlObservabilityEnabled = (serverHost) => isDevelopmentObservabilityMode() &&
+    isGraphqlObservabilityRequested() &&
+    isLoopbackHost(serverHost);
+export const isGraphqlDebugSamplerEnabled = (serverHost) => isGraphqlObservabilityEnabled(serverHost) &&
+    parseBooleanEnv(process.env.GRAPHQL_DEBUG_SAMPLER_ENABLED, true);

package/esm/middleware/graphile.js

@@ -6,7 +6,9 @@ import { ConstructivePreset, makePgService } from 'graphile-settings';
 import { buildConnectionString } from 'pg-cache';
 import { getPgEnvOptions } from 'pg-env';
 import './types'; // for Request type
+import { isGraphqlObservabilityEnabled } from '../diagnostics/observability';
 import { HandlerCreationError } from '../errors/api-errors';
+import { observeGraphileBuild } from './observability/graphile-build-stats';
 const maskErrorLog = new Logger('graphile:maskError');
 const SAFE_ERROR_CODES = new Set([
     // GraphQL standard
@@ -170,6 +172,7 @@ const buildPreset = (connectionString, schemas, anonRole, roleName) => ({
     },
 });
 export const graphile = (opts) => {
+    const observabilityEnabled = isGraphqlObservabilityEnabled(opts.server?.host);
     return async (req, res, next) => {
         const label = reqLabel(req);
         try {
@@ -233,7 +236,11 @@ export const graphile = (opts) => {
             const connectionString = buildConnectionString(pgConfig.user, pgConfig.password, pgConfig.host, pgConfig.port, pgConfig.database);
             // Create promise and store in in-flight map BEFORE try block
             const preset = buildPreset(connectionString, schema || [], anonRole, roleName);
-            const creationPromise = createGraphileInstance({ preset, cacheKey: key });
+            const creationPromise = observeGraphileBuild({
+                cacheKey: key,
+                serviceKey: key,
+                databaseId: api.databaseId ?? null,
+            }, () => createGraphileInstance({ preset, cacheKey: key }), { enabled: observabilityEnabled });
             creating.set(key, creationPromise);
             try {
                 const instance = await creationPromise;

package/esm/middleware/observability/debug-db.js

@@ -0,0 +1,23 @@
+import { Logger } from '@pgpmjs/logger';
+import { getDebugDatabaseSnapshot } from '../../diagnostics/debug-db-snapshot';
+const log = new Logger('debug-db');
+export const createDebugDatabaseMiddleware = (opts) => {
+    return async (_req, res) => {
+        try {
+            const response = await getDebugDatabaseSnapshot(opts);
+            log.debug('Database debug snapshot', {
+                activeActivity: response.activeActivity.length,
+                blockedActivity: response.blockedActivity.length,
+                lockSummary: response.lockSummary.length,
+            });
+            res.json(response);
+        }
+        catch (error) {
+            log.error('Failed to fetch debug DB snapshot', error);
+            res.status(500).json({
+                error: 'Failed to fetch database debug snapshot',
+                message: error instanceof Error ? error.message : String(error),
+            });
+        }
+    };
+};

package/esm/middleware/observability/debug-memory.js

@@ -0,0 +1,8 @@
+import { Logger } from '@pgpmjs/logger';
+import { getDebugMemorySnapshot } from '../../diagnostics/debug-memory-snapshot';
+const log = new Logger('debug-memory');
+export const debugMemory = (_req, res) => {
+    const response = getDebugMemorySnapshot();
+    log.debug('Memory snapshot:', response);
+    res.json(response);
+};