@aopslabs/aops 0.2.0

package/dist/lib/commercial-final-readiness.js

@@ -0,0 +1,504 @@
+import { createHash } from 'node:crypto';
+import { execFile, spawn } from 'node:child_process';
+import { lstatSync, readFileSync, readdirSync, realpathSync, } from 'node:fs';
+import path from 'node:path';
+import { compareCanonicalText, CommercialArtifactError, commercialRoleLaunchSpecBinding, setupReadinessReceiptBinding, verifySetupReadinessFinalReadback, } from '@aopslabs/artifact-trust-contracts';
+function execListenerTool(file, args) {
+    return new Promise((resolve, reject) => {
+        execFile(file, [...args], {
+            windowsHide: true,
+            maxBuffer: 8 * 1024 * 1024,
+            encoding: 'utf8',
+            env: Object.fromEntries(['SystemRoot', 'WINDIR', 'PATH', 'LANG', 'LC_ALL']
+                .flatMap((key) => process.env[key] ? [[key, process.env[key]]] : [])),
+        }, (error, stdout) => error ? reject(error) : resolve(stdout));
+    });
+}
+export const inspectSystemCommercialListeners = async (processes) => {
+    const byPid = new Map([
+        [processes.server, []],
+        [processes.cockpit, []],
+    ]);
+    try {
+        if (process.platform === 'win32') {
+            const systemRoot = process.env.SystemRoot ?? process.env.WINDIR ?? 'C:\\Windows';
+            const output = await execListenerTool(path.join(systemRoot, 'System32', 'netstat.exe'), ['-ano', '-p', 'tcp']);
+            for (const line of output.split(/\r?\n/)) {
+                const match = /^\s*TCP\s+(\S+)\s+\S+\s+LISTENING\s+(\d+)\s*$/i.exec(line);
+                if (!match)
+                    continue;
+                const addresses = byPid.get(Number(match[2]));
+                if (addresses)
+                    addresses.push(match[1]);
+            }
+        }
+        else if (process.platform === 'linux') {
+            const output = await execListenerTool('ss', ['-ltnpH']);
+            for (const line of output.split(/\r?\n/)) {
+                const pidMatch = /pid=(\d+)/.exec(line);
+                if (!pidMatch)
+                    continue;
+                const addresses = byPid.get(Number(pidMatch[1]));
+                if (!addresses)
+                    continue;
+                const columns = line.trim().split(/\s+/);
+                if (columns[3])
+                    addresses.push(columns[3]);
+            }
+        }
+        else if (process.platform === 'darwin') {
+            const output = await execListenerTool('lsof', ['-nP', '-iTCP', '-sTCP:LISTEN']);
+            for (const line of output.split(/\r?\n/).slice(1)) {
+                const columns = line.trim().split(/\s+/);
+                const addresses = byPid.get(Number(columns[1]));
+                const endpoint = columns.at(-2);
+                if (addresses && endpoint)
+                    addresses.push(endpoint.replace(/^TCP\s*/, ''));
+            }
+        }
+        else {
+            fail('final_listener_inspection_unsupported');
+        }
+    }
+    catch {
+        throw new Error('commercial_listener_inspection_failed');
+    }
+    return Object.freeze({
+        server: Object.freeze({ pid: processes.server, addresses: Object.freeze(byPid.get(processes.server) ?? []) }),
+        cockpit: Object.freeze({ pid: processes.cockpit, addresses: Object.freeze(byPid.get(processes.cockpit) ?? []) }),
+    });
+};
+const SECRET_ENV_KEY = 'AOPS_PG_URL';
+const SECRET_PATTERN = /postgres(?:ql)?:\/\/|password|passwd|token|secret/i;
+function fail(reason, decision = 'deny') {
+    throw new CommercialArtifactError('commercial-final-readiness', reason, decision);
+}
+function containsSecretValue(value) {
+    if (typeof value === 'string')
+        return SECRET_PATTERN.test(value);
+    if (Array.isArray(value))
+        return value.some(containsSecretValue);
+    if (value && typeof value === 'object')
+        return Object.values(value).some(containsSecretValue);
+    return false;
+}
+function within(root, candidate) {
+    const relative = path.relative(root, candidate);
+    return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
+}
+function hashStoreEntry(root, candidate, hash) {
+    const stat = lstatSync(candidate);
+    if (stat.isSymbolicLink())
+        fail('final_store_symlink_refused', 'quarantine');
+    const real = realpathSync.native(candidate);
+    if (!within(root, real))
+        fail('final_store_path_escape', 'quarantine');
+    const relative = path.relative(root, candidate).split(path.sep).join('/') || '.';
+    if (stat.isDirectory()) {
+        hash.update(`d\0${relative}\0`);
+        for (const name of readdirSync(candidate).sort(compareCanonicalText)) {
+            hashStoreEntry(root, path.join(candidate, name), hash);
+        }
+        return;
+    }
+    if (!stat.isFile())
+        fail('final_store_entry_unsafe', 'quarantine');
+    const bytes = readFileSync(candidate);
+    hash.update(`f\0${relative}\0${bytes.length}\0`);
+    hash.update(bytes);
+}
+export function digestCommercialStoreTree(storeRoot) {
+    const resolved = path.resolve(storeRoot);
+    const stat = lstatSync(resolved);
+    if (!stat.isDirectory() || stat.isSymbolicLink())
+        fail('final_store_root_unsafe', 'quarantine');
+    const canonical = realpathSync.native(resolved);
+    if (canonical !== resolved)
+        fail('final_store_root_alias_refused', 'quarantine');
+    const hash = createHash('sha256');
+    hashStoreEntry(canonical, canonical, hash);
+    return `sha256:${hash.digest('hex')}`;
+}
+function defaultSpawnAuthority(spec, privateEnvironment) {
+    const child = spawn(spec.command, [...spec.args], {
+        cwd: spec.cwd,
+        shell: spec.shell,
+        windowsHide: spec.windowsHide,
+        env: { ...spec.env, ...privateEnvironment },
+        stdio: ['ignore', 'inherit', 'inherit'],
+    });
+    const exited = new Promise((resolve) => {
+        child.once('exit', (code, signal) => resolve(Object.freeze({ code, signal })));
+        child.once('error', () => resolve(Object.freeze({ code: null, signal: null })));
+    });
+    if (typeof child.pid !== 'number')
+        fail('final_service_spawn_failed');
+    return Object.freeze({
+        pid: child.pid,
+        exited,
+        terminate: () => { child.kill('SIGTERM'); },
+        kill: () => { child.kill('SIGKILL'); },
+    });
+}
+function assertSpecSafe(spec, expectedRole) {
+    const verified = commercialRoleLaunchSpecBinding(spec);
+    if (verified.role !== expectedRole)
+        fail('final_launch_role_mismatch');
+    if (verified.shell !== false || !path.isAbsolute(verified.command) || !path.isAbsolute(verified.args[0] ?? '') ||
+        Object.values(verified.env).some((value) => SECRET_PATTERN.test(value)) ||
+        verified.args.some((value) => SECRET_PATTERN.test(value)) ||
+        verified.args.some((value) => /(?:^|[^0-9])0\.0\.0\.0(?:[^0-9]|$)/.test(value)))
+        fail('final_launch_spec_unsafe', 'quarantine');
+    return verified;
+}
+async function waitForExit(child, timeoutMs) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(false), timeoutMs);
+    });
+    const exited = child.exited.then(() => true);
+    const result = await Promise.race([exited, timeout]);
+    if (timer)
+        clearTimeout(timer);
+    return result;
+}
+export function createCommercialServiceSupervisor(options) {
+    const specs = {
+        server: assertSpecSafe(options.serverSpec, 'server'),
+        cockpit: assertSpecSafe(options.cockpitSpec, 'cockpit'),
+    };
+    const spawnAuthority = options.spawnAuthority ?? defaultSpawnAuthority;
+    const maxRestarts = Math.min(Math.max(options.maxRestartAttempts ?? 2, 0), 10);
+    const restartBackoffMs = Math.min(Math.max(options.restartBackoffMs ?? 250, 0), 30_000);
+    const gracefulStopMs = Math.min(Math.max(options.gracefulStopMs ?? 5_000, 100), 30_000);
+    const states = {
+        server: { child: null, state: 'stopped', generation: 0, restartCount: 0, intentionalStop: false, failureCode: null, restartTimer: null },
+        cockpit: { child: null, state: 'stopped', generation: 0, restartCount: 0, intentionalStop: false, failureCode: null, restartTimer: null },
+    };
+    let shuttingDown = false;
+    let exitHookRegistered = false;
+    const killChildrenOnParentExit = () => {
+        for (const role of ['cockpit', 'server'])
+            states[role].child?.kill();
+    };
+    const spawnRole = async (role) => {
+        const state = states[role];
+        if (shuttingDown)
+            throw new Error('commercial_services_shutting_down');
+        let privateEnvironment = Object.freeze({});
+        if (role === 'server') {
+            let value = '';
+            try {
+                value = await options.readServerPostgresUrl();
+            }
+            catch {
+                throw new Error('commercial_service_postgres_credential_unavailable');
+            }
+            if (!value || /[\u0000\r\n]/.test(value)) {
+                throw new Error('commercial_service_postgres_credential_unavailable');
+            }
+            privateEnvironment = Object.freeze({ [SECRET_ENV_KEY]: value });
+        }
+        const child = spawnAuthority(specs[role], privateEnvironment);
+        state.child = child;
+        state.state = 'running';
+        state.intentionalStop = false;
+        state.failureCode = null;
+        state.generation += 1;
+        void child.exited.then(() => {
+            if (state.child !== child)
+                return;
+            state.child = null;
+            if (state.intentionalStop || shuttingDown) {
+                state.state = 'stopped';
+                return;
+            }
+            if (state.restartCount >= maxRestarts) {
+                state.state = 'failed';
+                state.failureCode = `commercial_${role}_restart_limit_reached`;
+                return;
+            }
+            state.restartCount += 1;
+            state.state = 'restart-wait';
+            state.restartTimer = setTimeout(() => {
+                state.restartTimer = null;
+                void spawnRole(role).catch(() => {
+                    state.state = 'failed';
+                    state.failureCode = `commercial_${role}_restart_failed`;
+                });
+            }, restartBackoffMs * state.restartCount);
+        });
+    };
+    const stopRole = async (role) => {
+        const state = states[role];
+        state.intentionalStop = true;
+        if (state.restartTimer) {
+            clearTimeout(state.restartTimer);
+            state.restartTimer = null;
+        }
+        const child = state.child;
+        if (!child) {
+            state.state = 'stopped';
+            return;
+        }
+        child.terminate();
+        if (!await waitForExit(child, gracefulStopMs)) {
+            child.kill();
+            if (!await waitForExit(child, gracefulStopMs)) {
+                throw new Error(`commercial_${role}_child_cleanup_timeout`);
+            }
+        }
+        if (state.child === child)
+            state.child = null;
+        state.state = 'stopped';
+    };
+    return Object.freeze({
+        start: async () => {
+            if (!exitHookRegistered) {
+                process.once('exit', killChildrenOnParentExit);
+                exitHookRegistered = true;
+            }
+            await spawnRole('server');
+            try {
+                await spawnRole('cockpit');
+            }
+            catch (error) {
+                await stopRole('server').catch(() => undefined);
+                process.removeListener('exit', killChildrenOnParentExit);
+                exitHookRegistered = false;
+                throw error;
+            }
+        },
+        stopRole,
+        restartRole: async (role, nextSpec) => {
+            const replacement = nextSpec === undefined ? null : assertSpecSafe(nextSpec, role);
+            await stopRole(role);
+            if (replacement)
+                specs[role] = replacement;
+            states[role].restartCount = 0;
+            await spawnRole(role);
+        },
+        shutdown: async () => {
+            shuttingDown = true;
+            const results = await Promise.allSettled([stopRole('cockpit'), stopRole('server')]);
+            if (exitHookRegistered) {
+                process.removeListener('exit', killChildrenOnParentExit);
+                exitHookRegistered = false;
+            }
+            if (results.some((result) => result.status === 'rejected')) {
+                throw new Error('commercial_service_shutdown_incomplete');
+            }
+        },
+        status: () => Object.freeze({
+            server: Object.freeze({
+                state: states.server.state,
+                pid: states.server.child?.pid ?? null,
+                generation: states.server.generation,
+                restartCount: states.server.restartCount,
+                failureCode: states.server.failureCode,
+            }),
+            cockpit: Object.freeze({
+                state: states.cockpit.state,
+                pid: states.cockpit.child?.pid ?? null,
+                generation: states.cockpit.generation,
+                restartCount: states.cockpit.restartCount,
+                failureCode: states.cockpit.failureCode,
+            }),
+        }),
+    });
+}
+function loopbackEndpoint(value) {
+    let url;
+    try {
+        url = new URL(value);
+    }
+    catch {
+        return fail('final_health_endpoint_invalid');
+    }
+    if (url.protocol !== 'http:' || !['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname) ||
+        url.username || url.password || url.search || url.hash)
+        fail('final_health_endpoint_not_loopback');
+    return url;
+}
+async function probe(url, fetchFn) {
+    try {
+        const response = await fetchFn(url, { redirect: 'error', cache: 'no-store' });
+        const bytes = Buffer.from(await response.arrayBuffer());
+        return Object.freeze({
+            ok: response.ok,
+            body: bytes.toString('utf8'),
+            bytes,
+            contentType: response.headers.get('content-type') ?? '',
+        });
+    }
+    catch {
+        return Object.freeze({ ok: false, body: '', bytes: Buffer.alloc(0), contentType: '' });
+    }
+}
+async function waitForProbe(url, expected, fetchFn, timeoutMs, axis, marker) {
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const result = await probe(url, fetchFn);
+        const matched = expected === 'up'
+            ? result.ok && (marker === undefined || result.body.includes(marker))
+            : !result.ok;
+        if (matched)
+            return result;
+        if (Date.now() >= deadline)
+            fail(`final_health_${axis}_${expected}_timeout`);
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+}
+async function waitForStaticProbe(url, fetchFn, timeoutMs, expectedSha256, expectedContentType) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(expectedSha256))
+        fail('final_health_cockpit_static_digest_invalid');
+    const deadline = Date.now() + timeoutMs;
+    for (;;) {
+        const result = await probe(url, fetchFn);
+        const mediaType = result.contentType.split(';', 1)[0]?.trim().toLowerCase();
+        const digest = `sha256:${createHash('sha256').update(result.bytes).digest('hex')}`;
+        if (result.ok && result.bytes.length > 0 && mediaType === expectedContentType && digest === expectedSha256)
+            return;
+        if (Date.now() >= deadline)
+            fail('final_health_cockpit-static_up_timeout');
+        await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+}
+async function assertLoopbackListeners(supervisor, inspectListeners) {
+    const status = supervisor.status();
+    if (status.server.pid === null || status.cockpit.pid === null)
+        fail('final_service_process_missing');
+    const expected = Object.freeze({ server: status.server.pid, cockpit: status.cockpit.pid });
+    const observed = await inspectListeners(expected);
+    for (const role of ['server', 'cockpit']) {
+        const entry = observed[role];
+        if (!entry || entry.pid !== expected[role] || !Array.isArray(entry.addresses) || entry.addresses.length === 0) {
+            fail(`final_${role}_listener_readback_invalid`, 'quarantine');
+        }
+        if (entry.addresses.some((address) => !/^(?:127(?:\.[0-9]{1,3}){3}|localhost|\[::1\]):[1-9][0-9]{0,4}$/.test(address))) {
+            fail(`final_${role}_listener_not_loopback`, 'quarantine');
+        }
+    }
+}
+export async function verifyCommercialServiceHealthAndRecovery(options) {
+    const fetchFn = options.fetchFn ?? fetch;
+    const timeoutMs = Math.min(Math.max(options.timeoutMs ?? 30_000, 1_000), 120_000);
+    const urls = Object.freeze({
+        server: loopbackEndpoint(options.endpoints.serverDirect),
+        root: loopbackEndpoint(options.endpoints.cockpitRoot),
+        static: loopbackEndpoint(options.endpoints.cockpitStatic),
+        missingStatic: loopbackEndpoint(options.endpoints.cockpitMissingStatic),
+        proxy: loopbackEndpoint(options.endpoints.cockpitProxy),
+    });
+    if (urls.root.origin !== urls.static.origin || urls.root.origin !== urls.missingStatic.origin || urls.root.origin !== urls.proxy.origin ||
+        urls.server.origin === urls.root.origin || urls.root.pathname === urls.static.pathname ||
+        urls.root.pathname === urls.missingStatic.pathname || urls.root.pathname === urls.proxy.pathname ||
+        urls.static.pathname === urls.missingStatic.pathname || urls.static.pathname === urls.proxy.pathname ||
+        urls.missingStatic.pathname === urls.proxy.pathname)
+        fail('final_health_axes_not_distinct');
+    await waitForProbe(urls.server, 'up', fetchFn, timeoutMs, 'server', options.endpoints.serverResponseMarker);
+    await waitForProbe(urls.root, 'up', fetchFn, timeoutMs, 'cockpit-root');
+    await waitForStaticProbe(urls.static, fetchFn, timeoutMs, options.endpoints.cockpitStaticSha256, options.endpoints.cockpitStaticContentType);
+    await waitForProbe(urls.missingStatic, 'down', fetchFn, timeoutMs, 'cockpit-static-missing');
+    await waitForProbe(urls.proxy, 'up', fetchFn, timeoutMs, 'cockpit-proxy', options.endpoints.serverResponseMarker);
+    await assertLoopbackListeners(options.supervisor, options.inspectListeners);
+    const cockpitBefore = options.supervisor.status().cockpit;
+    await options.supervisor.stopRole('server');
+    await waitForProbe(urls.proxy, 'down', fetchFn, timeoutMs, 'cockpit-proxy');
+    if (options.supervisor.status().cockpit.pid !== cockpitBefore.pid) {
+        fail('final_cockpit_not_independent', 'quarantine');
+    }
+    await options.supervisor.restartRole('server');
+    await waitForProbe(urls.server, 'up', fetchFn, timeoutMs, 'server', options.endpoints.serverResponseMarker);
+    await waitForProbe(urls.proxy, 'up', fetchFn, timeoutMs, 'cockpit-proxy', options.endpoints.serverResponseMarker);
+    await assertLoopbackListeners(options.supervisor, options.inspectListeners);
+    if (options.supervisor.status().cockpit.pid !== cockpitBefore.pid) {
+        fail('final_cockpit_restarted_during_server_recovery', 'quarantine');
+    }
+    const serverBeforeCockpitRecovery = options.supervisor.status().server;
+    await options.supervisor.stopRole('cockpit');
+    await waitForProbe(urls.root, 'down', fetchFn, timeoutMs, 'cockpit-root');
+    await waitForProbe(urls.static, 'down', fetchFn, timeoutMs, 'cockpit-static');
+    await waitForProbe(urls.proxy, 'down', fetchFn, timeoutMs, 'cockpit-proxy');
+    await waitForProbe(urls.server, 'up', fetchFn, timeoutMs, 'server', options.endpoints.serverResponseMarker);
+    if (options.supervisor.status().server.pid !== serverBeforeCockpitRecovery.pid) {
+        fail('final_server_not_independent', 'quarantine');
+    }
+    await options.supervisor.restartRole('cockpit');
+    await waitForProbe(urls.root, 'up', fetchFn, timeoutMs, 'cockpit-root');
+    await waitForStaticProbe(urls.static, fetchFn, timeoutMs, options.endpoints.cockpitStaticSha256, options.endpoints.cockpitStaticContentType);
+    await waitForProbe(urls.missingStatic, 'down', fetchFn, timeoutMs, 'cockpit-static-missing');
+    await waitForProbe(urls.proxy, 'up', fetchFn, timeoutMs, 'cockpit-proxy', options.endpoints.serverResponseMarker);
+    await assertLoopbackListeners(options.supervisor, options.inspectListeners);
+    if (options.supervisor.status().server.pid !== serverBeforeCockpitRecovery.pid) {
+        fail('final_server_restarted_during_cockpit_recovery', 'quarantine');
+    }
+}
+export async function runCommercialFinalReadiness(options) {
+    const serverSpec = assertSpecSafe(options.serverSpec, 'server');
+    const cockpitSpec = assertSpecSafe(options.cockpitSpec, 'cockpit');
+    const readinessReceipt = setupReadinessReceiptBinding(options.readinessReceipt);
+    if (serverSpec.readback.receiptId !== cockpitSpec.readback.receiptId) {
+        fail('final_launch_release_mismatch', 'quarantine');
+    }
+    if (serverSpec.readback.receiptId !== readinessReceipt.releaseReceiptId ||
+        JSON.stringify(serverSpec.readback) !== JSON.stringify(cockpitSpec.readback))
+        fail('final_readiness_release_mismatch', 'quarantine');
+    const before = digestCommercialStoreTree(options.storeRoot);
+    const supervisor = createCommercialServiceSupervisor({
+        serverSpec,
+        cockpitSpec,
+        readServerPostgresUrl: options.readServerPostgresUrl,
+        spawnAuthority: options.spawnAuthority,
+    });
+    try {
+        await supervisor.start();
+        await verifyCommercialServiceHealthAndRecovery({
+            supervisor,
+            endpoints: options.health,
+            inspectListeners: options.inspectListeners ?? inspectSystemCommercialListeners,
+            fetchFn: options.fetchFn,
+            timeoutMs: options.timeoutMs,
+        });
+        await verifySetupReadinessFinalReadback({
+            plan: options.readinessPlan,
+            receipt: readinessReceipt,
+            postgresReceipt: options.postgresReceipt,
+            adapter: options.readinessAdapter,
+        });
+        const after = digestCommercialStoreTree(options.storeRoot);
+        if (after !== before)
+            fail('final_store_digest_changed', 'quarantine');
+        const receipt = Object.freeze({
+            schemaVersion: 1,
+            objectType: 'aops-commercial-final-readiness-receipt-v1',
+            readinessPlanId: readinessReceipt.planId,
+            releaseReceiptId: readinessReceipt.releaseReceiptId,
+            serverLaunchReceiptId: serverSpec.readback.receiptId,
+            cockpitLaunchReceiptId: cockpitSpec.readback.receiptId,
+            storeDigest: after,
+            readinessReadbackDigest: readinessReceipt.readbackDigest,
+            verification: Object.freeze({
+                launchSpecs: 'verified-verbatim',
+                independentSupervision: 'verified',
+                directServerHealth: 'verified',
+                cockpitRoot: 'verified',
+                cockpitStatic: 'verified',
+                cockpitProxyRecovery: 'verified',
+                cockpitIndependentRecovery: 'verified',
+                loopbackOnly: 'verified',
+                finalLiveReadback: 'verified',
+                immutableStore: 'verified',
+                secretsSerialized: false,
+            }),
+        });
+        if (containsSecretValue(receipt)) {
+            fail('final_receipt_secret_detected', 'quarantine');
+        }
+        return Object.freeze({ receipt, supervisor });
+    }
+    catch (error) {
+        await supervisor.shutdown().catch(() => undefined);
+        throw error;
+    }
+}