@arcadialdev/arcality 4.0.2 → 4.1.1

package/src/services/databaseValidationService.mjs

@@ -0,0 +1,598 @@
+import fs from 'fs';
+import path from 'path';
+import { createRequire } from 'module';
+import { load as loadYaml } from 'js-yaml';
+import { callMcpToolWithConfig, extractRowsFromMcpToolResult } from './mcpStdioClient.mjs';
+
+const DEFAULT_TIMEOUT_MS = 8000;
+const DEFAULT_SAMPLE_LIMIT = 3;
+const DEFAULT_CATALOG_PATH = path.join('.arcality', 'db-validations.yaml');
+const DEFAULT_MCP_CONFIG_PATH = path.join('.arcality', 'mcp.postgres.json');
+const SENSITIVE_FIELD_PATTERN = /(password|passwd|pwd|secret|token|api[_-]?key|authorization|auth|credential|session|cookie)/i;
+const SQL_WRITE_PATTERN = /\b(insert|update|delete|drop|alter|create|truncate|grant|revoke|vacuum|copy|call|execute|merge)\b/i;
+
+function toEnvKey(value) {
+    return String(value || '')
+        .trim()
+        .toUpperCase()
+        .replace(/[^A-Z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '');
+}
+
+function isEnabled(value) {
+    return ['1', 'true', 'yes', 'on'].includes(String(value || '').trim().toLowerCase());
+}
+
+function isPlainObject(value) {
+    return !!value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeDbValue(value) {
+    if (value instanceof Date) return value.toISOString();
+    if (typeof value === 'bigint') return value.toString();
+    if (Buffer.isBuffer(value)) return '[binary]';
+    if (Array.isArray(value)) return value.map(normalizeDbValue);
+    if (isPlainObject(value)) {
+        return Object.fromEntries(
+            Object.entries(value).map(([key, entryValue]) => [key, normalizeDbValue(entryValue)])
+        );
+    }
+    return value;
+}
+
+function normalizeRows(rows) {
+    if (!Array.isArray(rows)) return [];
+    return rows.map(row => normalizeDbValue(row));
+}
+
+function normalizeRedactSet(...sources) {
+    const redact = new Set();
+    for (const source of sources) {
+        if (Array.isArray(source)) {
+            for (const field of source) {
+                const name = String(field || '').trim();
+                if (name) redact.add(name);
+            }
+        }
+    }
+    return redact;
+}
+
+function shouldRedactField(field, redactSet) {
+    return redactSet.has(field) || SENSITIVE_FIELD_PATTERN.test(field);
+}
+
+function redactValue(value) {
+    if (value === undefined || value === null) return value;
+    return '[REDACTED]';
+}
+
+function redactObject(value, redactSet) {
+    if (!isPlainObject(value)) return value;
+    return Object.fromEntries(
+        Object.entries(value).map(([key, entryValue]) => [
+            key,
+            shouldRedactField(key, redactSet) ? redactValue(entryValue) : entryValue
+        ])
+    );
+}
+
+function redactRows(rows, redactSet, limit = DEFAULT_SAMPLE_LIMIT) {
+    return normalizeRows(rows)
+        .slice(0, limit)
+        .map(row => redactObject(row, redactSet));
+}
+
+function validateReadOnlySql(sql) {
+    const text = String(sql || '').trim();
+    if (!text) {
+        throw new Error('Database validation query is empty.');
+    }
+
+    const withoutTrailingSemicolon = text.replace(/;\s*$/, '');
+    if (withoutTrailingSemicolon.includes(';')) {
+        throw new Error('Database validation queries must contain a single statement.');
+    }
+
+    if (!/^(select|with)\b/i.test(withoutTrailingSemicolon)) {
+        throw new Error('Database validation queries must start with SELECT or WITH.');
+    }
+
+    if (SQL_WRITE_PATTERN.test(withoutTrailingSemicolon)) {
+        throw new Error('Database validation queries must be read-only.');
+    }
+}
+
+function resolveConfigPath(configPath, fallbackPath, options = {}) {
+    const projectRoot = options.projectRoot || process.env.ARCALITY_PROJECT_ROOT || process.cwd();
+    return path.isAbsolute(configPath)
+        ? configPath
+        : path.join(projectRoot, configPath || fallbackPath);
+}
+
+function readJsonOrYamlFile(filePath) {
+    const raw = fs.readFileSync(filePath, 'utf8');
+    if (/\.json$/i.test(filePath)) {
+        return JSON.parse(raw);
+    }
+    return loadYaml(raw);
+}
+
+export function resolveCatalogPath(options = {}) {
+    const configuredPath = options.catalogPath ||
+        process.env.ARCALITY_DB_CATALOG ||
+        process.env.ARCALITY_DB_VALIDATIONS_FILE ||
+        DEFAULT_CATALOG_PATH;
+    return resolveConfigPath(configuredPath, DEFAULT_CATALOG_PATH, options);
+}
+
+export function resolveMcpConfigPath(options = {}) {
+    const configuredPath = options.mcpConfigPath ||
+        process.env.ARCALITY_DB_MCP_CONFIG ||
+        DEFAULT_MCP_CONFIG_PATH;
+    return resolveConfigPath(configuredPath, DEFAULT_MCP_CONFIG_PATH, options);
+}
+
+export function loadDatabaseValidationCatalog(options = {}) {
+    const catalogPath = resolveCatalogPath(options);
+    if (!fs.existsSync(catalogPath)) {
+        throw new Error(`Database validation catalog not found: ${catalogPath}`);
+    }
+
+    const raw = readJsonOrYamlFile(catalogPath);
+    const catalog = raw && typeof raw === 'object' ? raw : {};
+    const queries = isPlainObject(catalog.queries) ? catalog.queries : {};
+    return { catalogPath, queries };
+}
+
+export function loadDatabaseMcpConfig(options = {}) {
+    const configPath = resolveMcpConfigPath(options);
+    if (!fs.existsSync(configPath)) {
+        throw new Error(`MCP database config not found: ${configPath}`);
+    }
+
+    const raw = readJsonOrYamlFile(configPath);
+    if (!isPlainObject(raw)) {
+        throw new Error(`Invalid MCP database config: ${configPath}`);
+    }
+
+    return { configPath, config: raw };
+}
+
+export function resolveDatabaseProvider(options = {}) {
+    const rawProvider = String(options.provider || process.env.ARCALITY_DB_PROVIDER || 'postgres').trim().toLowerCase();
+    if (['mcp', 'mcp_postgres', 'mcp-postgres'].includes(rawProvider)) {
+        return 'mcp';
+    }
+    return 'postgres';
+}
+
+export function resolveDatabaseConnectionEnv(options = {}) {
+    const profile = String(options.profile || process.env.ARCALITY_DB_PROFILE || 'qa_local').trim();
+    const profileKey = toEnvKey(profile);
+    const profileEnvName = profileKey ? `ARCALITY_DB_${profileKey}_URL` : '';
+    const connectionEnvName = options.connectionEnv ||
+        process.env.ARCALITY_DB_CONNECTION_ENV ||
+        profileEnvName ||
+        'ARCALITY_DB_CONNECTION';
+    const connectionString = process.env[connectionEnvName] ||
+        process.env.ARCALITY_DB_CONNECTION ||
+        process.env.ARCALITY_QA_POSTGRES_URL ||
+        '';
+
+    return {
+        profile,
+        connectionEnvName,
+        hasConnectionString: !!connectionString,
+        connectionString
+    };
+}
+
+function normalizeTimeoutMs(value) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TIMEOUT_MS;
+}
+
+function resolveProfileRegistryPath(options = {}) {
+    const configuredPath = options.registryPath || options.registry_path || path.join('.arcality', 'db-profiles.json');
+    return resolveConfigPath(configuredPath, path.join('.arcality', 'db-profiles.json'), options);
+}
+
+export function registerDatabaseProfile(profileInput = {}, options = {}) {
+    const profile = String(profileInput.profile || options.profile || process.env.ARCALITY_DB_PROFILE || 'qa_local').trim() || 'qa_local';
+    const provider = resolveDatabaseProvider({ provider: profileInput.provider || options.provider || process.env.ARCALITY_DB_PROVIDER });
+    const connection = resolveDatabaseConnectionEnv({
+        profile,
+        connectionEnv: profileInput.connection_env || profileInput.connectionEnv || options.connectionEnv
+    });
+    const catalogPath = resolveCatalogPath({
+        ...options,
+        catalogPath: profileInput.catalog_path || profileInput.catalogPath
+    });
+    const timeoutMs = normalizeTimeoutMs(profileInput.timeout_ms || profileInput.timeoutMs || options.timeoutMs || process.env.ARCALITY_DB_TIMEOUT_MS);
+    const warnings = [];
+    const result = {
+        type: 'database_profile_registration',
+        profile,
+        provider,
+        connection_env: connection.connectionEnvName,
+        has_connection_string: connection.hasConnectionString,
+        catalog_path: catalogPath,
+        catalog_exists: fs.existsSync(catalogPath),
+        timeout_ms: timeoutMs,
+        warnings
+    };
+
+    if (!result.has_connection_string) {
+        warnings.push('Missing PostgreSQL connection string in ' + connection.connectionEnvName + '.');
+    }
+
+    if (!result.catalog_exists) {
+        warnings.push('Database validation catalog not found: ' + catalogPath);
+    }
+
+    if (profileInput.validate_catalog !== false && result.catalog_exists) {
+        try {
+            const catalog = loadDatabaseValidationCatalog({ ...options, catalogPath });
+            result.query_count = Object.keys(catalog.queries || {}).length;
+        } catch (error) {
+            result.query_count = 0;
+            warnings.push(error instanceof Error ? error.message : String(error));
+        }
+    }
+
+    if (provider === 'mcp') {
+        const mcpConfigPath = resolveMcpConfigPath({
+            ...options,
+            mcpConfigPath: profileInput.mcp_config_path || profileInput.mcpConfigPath
+        });
+        result.mcp_config_path = mcpConfigPath;
+        result.mcp_config_exists = fs.existsSync(mcpConfigPath);
+        if (!result.mcp_config_exists) {
+            warnings.push('MCP database config not found: ' + mcpConfigPath);
+        }
+    }
+
+    if (profileInput.persist === true) {
+        const registryPath = resolveProfileRegistryPath({
+            ...options,
+            registryPath: profileInput.registry_path || profileInput.registryPath
+        });
+        let registry = { profiles: {} };
+        if (fs.existsSync(registryPath)) {
+            try {
+                const existing = JSON.parse(fs.readFileSync(registryPath, 'utf8'));
+                if (existing && typeof existing === 'object' && !Array.isArray(existing)) {
+                    registry = existing;
+                    registry.profiles = registry.profiles && typeof registry.profiles === 'object' ? registry.profiles : {};
+                }
+            } catch {}
+        }
+        registry.profiles[profile] = {
+            provider,
+            connection_env: connection.connectionEnvName,
+            catalog_path: catalogPath,
+            timeout_ms: timeoutMs,
+            ...(result.mcp_config_path ? { mcp_config_path: result.mcp_config_path } : {})
+        };
+        fs.mkdirSync(path.dirname(registryPath), { recursive: true });
+        fs.writeFileSync(registryPath, JSON.stringify(registry, null, 2), 'utf8');
+        result.persisted = true;
+        result.registry_path = registryPath;
+    } else {
+        result.persisted = false;
+    }
+
+    result.status = warnings.length === 0 ? 'ready' : 'warning';
+    return result;
+}
+
+function loadPgPool() {
+    const roots = [process.env.ARCALITY_ROOT, process.cwd()].filter(Boolean);
+    for (const root of roots) {
+        try {
+            const requireFromRoot = createRequire(path.join(root, 'package.json'));
+            const pg = requireFromRoot('pg');
+            return pg.Pool;
+        } catch { }
+    }
+
+    throw new Error('PostgreSQL driver "pg" is not installed. Run `npm install pg --save` in this project to enable database validation.');
+}
+
+function getQueryEntry(catalog, queryId) {
+    const entry = catalog.queries[queryId];
+    if (!isPlainObject(entry)) {
+        throw new Error(`Database validation query_id not found in catalog: ${queryId}`);
+    }
+    validateReadOnlySql(entry.sql);
+    return entry;
+}
+
+function buildQueryValues(entry, params) {
+    const paramNames = Array.isArray(entry.params) ? entry.params : [];
+    return paramNames.map(name => {
+        const key = String(name);
+        if (!Object.prototype.hasOwnProperty.call(params || {}, key)) {
+            throw new Error(`Missing database validation param: ${key}`);
+        }
+        return params[key];
+    });
+}
+
+export function evaluateDatabaseValidationRows(validation, rows, queryEntry = {}) {
+    if (!isPlainObject(validation)) {
+        throw new Error('Database validation must be an object.');
+    }
+
+    const normalizedRows = normalizeRows(rows);
+    const expectBlock = isPlainObject(validation.expect) ? validation.expect : {};
+    const expectsFields = isPlainObject(expectBlock.fields);
+    const expectedExists = typeof expectBlock.exists === 'boolean'
+        ? expectBlock.exists
+        : (expectsFields ? true : undefined);
+    const redactSet = normalizeRedactSet(queryEntry.redact, validation.redact);
+    const checks = [];
+
+    if (typeof expectedExists === 'boolean') {
+        const actualExists = normalizedRows.length > 0;
+        checks.push({
+            type: 'exists',
+            expected: expectedExists,
+            actual: actualExists,
+            passed: actualExists === expectedExists
+        });
+    }
+
+    if (expectsFields) {
+        const firstRow = normalizedRows[0] || {};
+        for (const [field, expected] of Object.entries(expectBlock.fields)) {
+            const hasField = Object.prototype.hasOwnProperty.call(firstRow, field);
+            const actual = hasField ? firstRow[field] : undefined;
+            const passed = hasField && actual === normalizeDbValue(expected);
+            const redacted = shouldRedactField(field, redactSet);
+            checks.push({
+                type: 'field_exact',
+                field,
+                expected: redacted ? redactValue(expected) : normalizeDbValue(expected),
+                actual: redacted ? redactValue(actual) : actual,
+                passed
+            });
+        }
+    }
+
+    const passed = checks.length > 0 && checks.every(check => check.passed);
+    return {
+        name: String(validation.name || validation.query_id || validation.queryId || 'database_validation'),
+        query_id: String(validation.query_id || validation.queryId || ''),
+        status: passed ? 'passed' : 'failed',
+        passed,
+        row_count: normalizedRows.length,
+        checks,
+        rows_sample: redactRows(normalizedRows, redactSet)
+    };
+}
+
+async function queryWithTimeout(pool, query, timeoutMs) {
+    let timeoutHandle;
+    const timeout = new Promise((_, reject) => {
+        timeoutHandle = setTimeout(() => {
+            reject(new Error(`Database validation query timed out after ${timeoutMs}ms.`));
+        }, timeoutMs);
+    });
+
+    try {
+        return await Promise.race([pool.query(query), timeout]);
+    } finally {
+        clearTimeout(timeoutHandle);
+    }
+}
+
+async function queryDatabaseRowsViaPostgres({ validation, queryEntry, timeoutMs, startedAt, options = {} }) {
+    const connection = resolveDatabaseConnectionEnv({ profile: validation.profile || options.profile });
+    if (!connection.hasConnectionString) {
+        throw new Error(`Missing PostgreSQL connection string in ${connection.connectionEnvName}.`);
+    }
+
+    const Pool = loadPgPool();
+    const values = buildQueryValues(queryEntry, validation.params || {});
+    const pool = new Pool({
+        connectionString: connection.connectionString,
+        max: 1,
+        idleTimeoutMillis: 1000,
+        connectionTimeoutMillis: Math.min(timeoutMs, 10000),
+        statement_timeout: timeoutMs
+    });
+
+    try {
+        const result = await queryWithTimeout(pool, { text: queryEntry.sql, values }, timeoutMs);
+        return {
+            rows: result.rows || [],
+            metadata: {
+                provider: 'postgres',
+                profile: connection.profile,
+                catalog_path: options.catalogPath || resolveCatalogPath(options),
+                duration_ms: Date.now() - startedAt,
+                executed_at: new Date().toISOString()
+            }
+        };
+    } finally {
+        await pool.end().catch(() => {});
+    }
+}
+
+async function executeDatabaseValidationViaPostgres({ validation, queryEntry, timeoutMs, startedAt, options = {} }) {
+    const { rows, metadata } = await queryDatabaseRowsViaPostgres({ validation, queryEntry, timeoutMs, startedAt, options });
+    const evaluated = evaluateDatabaseValidationRows(validation, rows, queryEntry);
+    return {
+        ...evaluated,
+        ...metadata
+    };
+}
+
+async function queryDatabaseRowsViaMcp({ validation, queryEntry, timeoutMs, startedAt, options = {} }) {
+    const { configPath, config } = loadDatabaseMcpConfig(options);
+    const values = buildQueryValues(queryEntry, validation.params || {});
+    const toolContext = {
+        sql: queryEntry.sql,
+        params: values,
+        paramsNamed: validation.params || {},
+        paramsJson: JSON.stringify(values),
+        queryId: validation.query_id || validation.queryId || '',
+        validationName: validation.name || validation.query_id || validation.queryId || 'database_validation',
+        timeoutMs
+    };
+    const result = await callMcpToolWithConfig(config, toolContext);
+    const rows = extractRowsFromMcpToolResult(result, String(config.resultRowsPath || '').trim());
+    return {
+        rows,
+        metadata: {
+            provider: 'mcp',
+            mcp_config_path: configPath,
+            mcp_tool_name: String(config.toolName || 'query'),
+            catalog_path: options.catalogPath || resolveCatalogPath(options),
+            duration_ms: Date.now() - startedAt,
+            executed_at: new Date().toISOString()
+        }
+    };
+}
+
+async function executeDatabaseValidationViaMcp({ validation, queryEntry, timeoutMs, startedAt, options = {} }) {
+    const { rows, metadata } = await queryDatabaseRowsViaMcp({ validation, queryEntry, timeoutMs, startedAt, options });
+    const evaluated = evaluateDatabaseValidationRows(validation, rows, queryEntry);
+    return {
+        ...evaluated,
+        ...metadata
+    };
+}
+
+export async function executeDatabaseValidation(validation, options = {}) {
+    if (!isEnabled(process.env.ARCALITY_DB_ENABLED) && !options.allowDisabled) {
+        throw new Error('Database validation is disabled. Set ARCALITY_DB_ENABLED=true to use it.');
+    }
+
+    const queryId = String(validation?.query_id || validation?.queryId || '').trim();
+    if (!queryId) {
+        throw new Error('Database validation requires query_id.');
+    }
+
+    const startedAt = Date.now();
+    const catalog = loadDatabaseValidationCatalog(options);
+    const queryEntry = getQueryEntry(catalog, queryId);
+    const timeoutMs = Number(validation.timeout_ms || validation.timeoutMs || process.env.ARCALITY_DB_TIMEOUT_MS || DEFAULT_TIMEOUT_MS);
+    const provider = resolveDatabaseProvider(options);
+
+    if (provider === 'mcp') {
+        return executeDatabaseValidationViaMcp({ validation, queryEntry, timeoutMs, startedAt, options: { ...options, catalogPath: catalog.catalogPath } });
+    }
+
+    return executeDatabaseValidationViaPostgres({ validation, queryEntry, timeoutMs, startedAt, options: { ...options, catalogPath: catalog.catalogPath } });
+}
+
+export function buildDatabaseValidationPreview(validation, rows, queryEntry = {}, metadata = {}) {
+    const redactSet = normalizeRedactSet(queryEntry.redact, validation?.redact);
+    const normalizedRows = normalizeRows(rows);
+    return {
+        type: 'database_validation_preview',
+        name: String(validation?.name || validation?.query_id || validation?.queryId || 'database_validation_preview'),
+        query_id: String(validation?.query_id || validation?.queryId || ''),
+        params: redactObject(normalizeDbValue(validation?.params || {}), redactSet),
+        row_count: normalizedRows.length,
+        rows_sample: redactRows(normalizedRows, redactSet),
+        query: {
+            description: String(queryEntry.description || ''),
+            params: Array.isArray(queryEntry.params) ? queryEntry.params.map(name => String(name)) : []
+        },
+        supports_expect: {
+            exists: true,
+            fields: true
+        },
+        ...metadata
+    };
+}
+
+export async function previewDatabaseValidation(validation, options = {}) {
+    if (!isEnabled(process.env.ARCALITY_DB_ENABLED) && !options.allowDisabled) {
+        throw new Error('Database validation is disabled. Set ARCALITY_DB_ENABLED=true to use it.');
+    }
+
+    const queryId = String(validation?.query_id || validation?.queryId || '').trim();
+    if (!queryId) {
+        throw new Error('Database validation preview requires query_id.');
+    }
+
+    const startedAt = Date.now();
+    const catalog = loadDatabaseValidationCatalog(options);
+    const queryEntry = getQueryEntry(catalog, queryId);
+    const timeoutMs = Number(validation?.timeout_ms || validation?.timeoutMs || process.env.ARCALITY_DB_TIMEOUT_MS || DEFAULT_TIMEOUT_MS);
+    const provider = resolveDatabaseProvider(options);
+    const normalizedValidation = {
+        ...validation,
+        query_id: queryId,
+        name: String(validation?.name || queryId || 'database_validation_preview'),
+        params: isPlainObject(validation?.params) ? validation.params : {}
+    };
+
+    const execution = provider === 'mcp'
+        ? await queryDatabaseRowsViaMcp({ validation: normalizedValidation, queryEntry, timeoutMs, startedAt, options: { ...options, catalogPath: catalog.catalogPath } })
+        : await queryDatabaseRowsViaPostgres({ validation: normalizedValidation, queryEntry, timeoutMs, startedAt, options: { ...options, catalogPath: catalog.catalogPath } });
+
+    return buildDatabaseValidationPreview(normalizedValidation, execution.rows, queryEntry, execution.metadata);
+}
+
+export async function runDatabaseValidations(validations, options = {}) {
+    const list = Array.isArray(validations) ? validations : [];
+    const startedAt = Date.now();
+    const results = [];
+
+    for (const validation of list) {
+        try {
+            results.push(await executeDatabaseValidation(validation, options));
+        } catch (error) {
+            results.push({
+                name: String(validation?.name || validation?.query_id || validation?.queryId || 'database_validation'),
+                query_id: String(validation?.query_id || validation?.queryId || ''),
+                status: 'failed',
+                passed: false,
+                row_count: 0,
+                checks: [],
+                rows_sample: [],
+                error: error instanceof Error ? error.message : String(error),
+                executed_at: new Date().toISOString()
+            });
+        }
+    }
+
+    const passedCount = results.filter(result => result.passed).length;
+    return {
+        type: 'database_validation_report',
+        status: passedCount === results.length ? 'passed' : 'failed',
+        passed: results.length > 0 && passedCount === results.length,
+        summary: {
+            total: results.length,
+            passed: passedCount,
+            failed: results.length - passedCount
+        },
+        duration_ms: Date.now() - startedAt,
+        executed_at: new Date().toISOString(),
+        validations: results
+    };
+}
+
+export function parseDatabaseValidationsFromEnv(env = process.env) {
+    const raw = env.ARCALITY_DB_VALIDATIONS_JSON || env.ARCALITY_DB_VALIDATIONS || '';
+    if (!raw.trim()) return [];
+
+    try {
+        const parsed = JSON.parse(raw);
+        return Array.isArray(parsed) ? parsed : [];
+    } catch {
+        return [];
+    }
+}
+
+export async function runDatabaseValidationsFromEnv(options = {}) {
+    const validations = parseDatabaseValidationsFromEnv();
+    if (validations.length === 0) return null;
+    return runDatabaseValidations(validations, options);
+}