@steve31415/baselib 2.2.2 → 2.4.0

package/dist/bin/pw-deploy.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin/pw-deploy.js

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+// `pw-deploy` — the fleet-wide routine deploy (design:
+// ~/migration/research/deploy-procedure-design.md). All logic lives in
+// ../deploy/deploy.ts; configuration is the app's deploy.config.json.
+//
+// process.exit (not exitCode) so an in-flight Cloud Build child cannot keep
+// a failed deploy's process alive; the server-side build finishes on its own
+// and is reused by the next attempt.
+import { realRunner } from '../deploy/exec.js';
+import { runDeploy } from '../deploy/deploy.js';
+process.exit(await runDeploy({
+    repoRoot: process.cwd(),
+    runner: realRunner,
+    argv: process.argv.slice(2),
+}));

package/dist/bin/pw-rollback.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/bin/pw-rollback.js

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+// `pw-rollback` — one-command traffic move to a previously released build.
+// All logic lives in ../deploy/rollback.ts.
+import { realRunner } from '../deploy/exec.js';
+import { runRollback } from '../deploy/rollback.js';
+process.exit(await runRollback({
+    repoRoot: process.cwd(),
+    runner: realRunner,
+    argv: process.argv.slice(2),
+}));

package/dist/db.d.ts

@@ -1,10 +1,21 @@
 import pg from 'pg';
-import type { Logger } from './log-core.js';
+import { type Logger } from './log-core.js';
 export interface PoolOptions {
     /** Override the database name (default: env DB_NAME / DATABASE_URL path). */
     database?: string;
     max?: number;
+    /** Receives idle-client errors (see logPoolErrors). Without one they go to
+     *  stderr — on the Cloud Logging backstop, but not in Axiom. */
+    logger?: Logger;
 }
+/** pg-pool re-emits an idle client's socket error as its own 'error' event —
+ * in production "Connection terminated unexpectedly" when Cloud SQL closes an
+ * idle connection. Unhandled, that event throws and takes the process down
+ * (todo 2026-08-20, notes 2026-08-27 ×2). The failed client has already been
+ * removed from the pool and the next checkout opens a fresh connection, so
+ * the right response is to log it and carry on. createPool attaches this;
+ * exported for apps that build their own pg.Pool. */
+export declare function logPoolErrors(pool: pg.Pool, logger?: Logger): void;
 export declare function createPool(opts?: PoolOptions): Promise<pg.Pool>;
 /** End a pool and resolve only once every client's connection has actually
  * closed. pg-pool's own `end()` resolves as soon as it has *asked* its idle

package/dist/db.js

@@ -6,6 +6,7 @@ import { readdir, readFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import pg from 'pg';
 import { requireEnv } from './config.js';
+import { serializeError } from './log-core.js';
 // DATE columns come back as 'YYYY-MM-DD' strings, not JS Dates: calendar
 // dates are timezone-free by meaning, and a Date object smears them across
 // zones (and JSON-serializes to a full ISO timestamp). Set here — on the
@@ -13,7 +14,27 @@ import { requireEnv } from './config.js';
 // own pg copy miss this one (file:-dependency installs resolve separate pg
 // instances).
 pg.types.setTypeParser(pg.types.builtins.DATE, (v) => v);
+/** pg-pool re-emits an idle client's socket error as its own 'error' event —
+ * in production "Connection terminated unexpectedly" when Cloud SQL closes an
+ * idle connection. Unhandled, that event throws and takes the process down
+ * (todo 2026-08-20, notes 2026-08-27 ×2). The failed client has already been
+ * removed from the pool and the next checkout opens a fresh connection, so
+ * the right response is to log it and carry on. createPool attaches this;
+ * exported for apps that build their own pg.Pool. */
+export function logPoolErrors(pool, logger) {
+    pool.on('error', (err) => {
+        if (logger)
+            logger.error('database pool: idle client error', { error: err });
+        else
+            console.error('database pool: idle client error', serializeError(err));
+    });
+}
 export async function createPool(opts = {}) {
+    const pool = await newPool(opts);
+    logPoolErrors(pool, opts.logger);
+    return pool;
+}
+async function newPool(opts) {
     const instance = process.env.CLOUD_SQL_INSTANCE;
     if (instance) {
         const { Connector, AuthTypes, IpAddressTypes } = await import('@google-cloud/cloud-sql-connector');

package/dist/deploy/config.d.ts

@@ -0,0 +1,5 @@
+import type { DeployConfig, DeployHooks } from './types.js';
+export declare const DEFAULT_ACCOUNT = "coding-agent@plasticine-prod.iam.gserviceaccount.com";
+export declare function validateConfig(raw: unknown): DeployConfig;
+export declare function loadConfig(repoRoot: string): Promise<DeployConfig>;
+export declare function loadHooks(repoRoot: string, config: DeployConfig): Promise<DeployHooks>;

package/dist/deploy/config.js

@@ -0,0 +1,67 @@
+// Loading and validation of an app's checked-in deploy.config.json and its
+// optional hooks module.
+import { readFile } from 'node:fs/promises';
+import { pathToFileURL } from 'node:url';
+import { resolve } from 'node:path';
+export const DEFAULT_ACCOUNT = 'coding-agent@plasticine-prod.iam.gserviceaccount.com';
+const NAME = /^[a-z][a-z0-9-]{1,30}$/;
+const PREFIX = /^[a-z0-9][a-z0-9/_-]*\/$/;
+function fail(detail) {
+    throw new Error(`deploy.config.json invalid: ${detail}`);
+}
+function validateService(raw, index) {
+    const s = raw;
+    if (!s.name || !NAME.test(s.name))
+        fail(`services[${index}].name`);
+    const kinds = ['public-canonical', 'oidc-run-url', 'oidc-audience', 'none'];
+    if (!s.health || !kinds.includes(s.health.kind))
+        fail(`services[${index}].health.kind`);
+    if (s.health.kind === 'oidc-audience' && !s.health.audience)
+        fail(`services[${index}].health.audience`);
+    if (s.health.kind === 'public-canonical' && !s.canonicalUrl)
+        fail(`services[${index}].canonicalUrl`);
+    return s;
+}
+export function validateConfig(raw) {
+    const c = raw;
+    if (!c.app || !NAME.test(c.app))
+        fail('app');
+    if (!c.project)
+        fail('project');
+    if (!c.region)
+        fail('region');
+    if (!c.bucket)
+        fail('bucket');
+    if (!c.image?.arRepo || !c.image?.name || !c.image?.buildServiceAccount)
+        fail('image');
+    if (!c.build?.command || !Array.isArray(c.build.contextFiles) || c.build.contextFiles.length === 0)
+        fail('build');
+    if (!Array.isArray(c.gates))
+        fail('gates');
+    if (!c.releasePrefix || !PREFIX.test(c.releasePrefix))
+        fail('releasePrefix (must end with /)');
+    if (c.assets && (!c.assets.dir || !PREFIX.test(c.assets.prefix ?? '')))
+        fail('assets (prefix must end with /)');
+    if (!Array.isArray(c.services) || c.services.length === 0)
+        fail('services');
+    c.services.forEach((s, i) => validateService(s, i));
+    if (c.updateSecrets) {
+        for (const [key, value] of Object.entries(c.updateSecrets)) {
+            if (!/^[A-Z0-9_]+$/.test(key) || !/^[a-z0-9-]+--[a-z0-9-]+:[0-9]+$/.test(value))
+                fail(`updateSecrets.${key} (want <secret-name>:<numeric version>)`);
+        }
+    }
+    return c;
+}
+export async function loadConfig(repoRoot) {
+    const path = resolve(repoRoot, 'deploy.config.json');
+    const raw = await readFile(path, 'utf8');
+    return validateConfig(JSON.parse(raw));
+}
+export async function loadHooks(repoRoot, config) {
+    if (!config.hooks)
+        return {};
+    const url = pathToFileURL(resolve(repoRoot, config.hooks)).href;
+    const mod = (await import(url));
+    return mod;
+}

package/dist/deploy/deploy.d.ts

@@ -0,0 +1,13 @@
+import type { Runner } from './types.js';
+export interface DeployOptions {
+    repoRoot: string;
+    runner: Runner;
+    log?: (message: string) => void;
+    argv?: string[];
+    /** Test seam: skip real sleeps in retry loops. */
+    sleep?: (ms: number) => Promise<void>;
+    fetchFn?: typeof fetch;
+    /** Test seam: the publish workflow's verify gate runs on Node 24. */
+    nodeVersion?: string;
+}
+export declare function runDeploy(options: DeployOptions): Promise<number>;

package/dist/deploy/deploy.js

@@ -0,0 +1,405 @@
+// The pw-deploy orchestrator. Flow (design 2026-08-31):
+// preflight -> lock -> build -> assets -> seal -> [image build ‖ gates] ->
+// fence -> per-service update + explicit traffic pin -> hooks/smoke/log check
+// -> release record -> unlock. A failure before the traffic switch leaves the
+// old revision serving; after it, the failure is reported with the rollback
+// command printed.
+import { createHash } from 'node:crypto';
+import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { hostname } from 'node:os';
+import { join, resolve } from 'node:path';
+import { DEFAULT_ACCOUNT, loadConfig, loadHooks } from './config.js';
+import { must, mustJson } from './exec.js';
+import { openEvidence, PhaseTimer } from './evidence.js';
+import { acquireLock, clearHold, publishRetainedAssets, readHold, readReleaseRecord, releaseLock, verifyLock, writeReleaseRecord, } from './gcs.js';
+import { consistentBaseSha, servingStateOf, SHA40 } from './plan.js';
+const RELEASE_DEADLINE_MS = 25 * 60_000;
+const IMAGE_DIGEST = /@sha256:[0-9a-f]{64}$/;
+export async function runDeploy(options) {
+    const log = options.log ?? ((m) => console.error(`[pw-deploy] ${m}`));
+    const runner = options.runner;
+    const sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const fetchFn = options.fetchFn ?? fetch;
+    const overrideHold = (options.argv ?? []).includes('--override-hold');
+    const timer = new PhaseTimer(log);
+    const startedAt = Date.now();
+    let trafficMoved = false;
+    let evidenceDir = '';
+    let lock = null;
+    try {
+        // ---- preflight ----
+        const config = await loadConfig(options.repoRoot);
+        const hooks = await loadHooks(options.repoRoot, config);
+        const gcloudBase = ['--project', config.project];
+        await must(runner, 'git', ['fetch', 'origin', 'main'], { cwd: options.repoRoot });
+        const branch = (await must(runner, 'git', ['branch', '--show-current'], { cwd: options.repoRoot })).stdout.trim();
+        if (branch !== 'main')
+            throw new Error('deploys run from main');
+        const dirty = (await must(runner, 'git', ['status', '--porcelain=v1', '--untracked-files=all'], {
+            cwd: options.repoRoot,
+        })).stdout.trim();
+        if (dirty)
+            throw new Error('deploys require a clean worktree');
+        const sha = (await must(runner, 'git', ['rev-parse', 'HEAD'], { cwd: options.repoRoot })).stdout.trim();
+        const originMain = (await must(runner, 'git', ['rev-parse', 'origin/main'], { cwd: options.repoRoot })).stdout.trim();
+        if (sha !== originMain)
+            throw new Error('deploys require pushed main');
+        if (!SHA40.test(sha))
+            throw new Error(`unexpected HEAD sha: ${sha}`);
+        const account = (await must(runner, 'gcloud', ['config', 'get', 'account'], {})).stdout.trim();
+        if (account !== (config.account ?? DEFAULT_ACCOUNT)) {
+            throw new Error(`gcloud account must be ${config.account ?? DEFAULT_ACCOUNT} (is ${account})`);
+        }
+        const project = (await must(runner, 'gcloud', ['config', 'get', 'project'], {})).stdout.trim();
+        if (project !== config.project) {
+            throw new Error(`gcloud project must be ${config.project} (is ${project})`);
+        }
+        const nodeVersion = options.nodeVersion ?? process.version;
+        if (!nodeVersion.startsWith('v22')) {
+            throw new Error(`Node 22 required (running ${nodeVersion})`);
+        }
+        const evidence = await openEvidence(config.app, sha);
+        evidenceDir = evidence.dir;
+        log(`deploying ${config.app} @ ${sha.slice(0, 12)}`);
+        // node_modules freshness: hash the lockfile against our stamp.
+        const lockHash = createHash('sha256')
+            .update(await readFile(join(options.repoRoot, 'package-lock.json')))
+            .digest('hex');
+        const stampPath = join(options.repoRoot, '.deploy', 'npm-ci.stamp');
+        let stamp = '';
+        try {
+            stamp = (await readFile(stampPath, 'utf8')).trim();
+        }
+        catch {
+            /* no stamp yet */
+        }
+        if (stamp !== lockHash) {
+            log('node_modules stamp stale or missing — running npm ci');
+            await must(runner, 'npm', ['ci'], { cwd: options.repoRoot, timeoutMs: 10 * 60_000 });
+            await mkdir(join(options.repoRoot, '.deploy'), { recursive: true });
+            await writeFile(stampPath, `${lockHash}\n`);
+        }
+        const holdUri = `gs://${config.bucket}/${config.app}/deploy.hold`;
+        const hold = await readHold(runner, holdUri);
+        if (hold && hold.heldShas.includes(sha) && !overrideHold) {
+            throw new Error(`SHA ${sha.slice(0, 12)} is held (${hold.reason} at ${hold.at}); ` +
+                'a rollback moved traffic off it. Deploy a fix, or pass --override-hold.');
+        }
+        const serving = [];
+        for (const service of config.services) {
+            const described = await mustJson(runner, 'gcloud', [
+                'run', 'services', 'describe', service.name,
+                ...gcloudBase, '--region', config.region, '--format=json',
+            ]);
+            await evidence.save(`${service.name}-before.json`, JSON.stringify(described, null, 2));
+            serving.push(servingStateOf(service.name, described));
+        }
+        const base = consistentBaseSha(serving, sha);
+        if (!base.ok)
+            throw new Error(base.detail);
+        const baseSha = base.baseSha;
+        if (config.refuseMigrationDiffOnDeploy && baseSha && baseSha !== sha) {
+            await must(runner, 'git', ['cat-file', '-e', `${baseSha}^{commit}`], {
+                cwd: options.repoRoot,
+            });
+            const diff = (await must(runner, 'git', ['diff', '--name-only', '--diff-filter=ACMRD', `${baseSha}..${sha}`, '--', 'migrations'], { cwd: options.repoRoot })).stdout.trim();
+            if (diff) {
+                throw new Error(`migrations changed since serving build ${baseSha.slice(0, 12)}; ` +
+                    'run the migration operator workflow first');
+            }
+        }
+        const priorRecord = await readReleaseRecord(runner, config.bucket, config.releasePrefix, sha);
+        const skipGates = priorRecord !== null;
+        if (skipGates)
+            log('re-release of an already-released SHA — gates skipped');
+        // ---- lock ----
+        timer.enter('lock');
+        const lockUri = `gs://${config.bucket}/${config.app}/deploy.lock`;
+        lock = await acquireLock(runner, lockUri, `host=${hostname()}\nsha=${sha}\nstarted=${new Date().toISOString()}\n`, { log, sleep });
+        // ---- build ----
+        timer.enter('build');
+        const build = await must(runner, 'bash', ['-c', config.build.command], {
+            cwd: options.repoRoot,
+            env: { BUILD_ID: sha },
+            timeoutMs: 10 * 60_000,
+        });
+        await evidence.save('build.log', build.stdout + build.stderr);
+        const ctx = {
+            sha,
+            baseSha,
+            config,
+            repoRoot: options.repoRoot,
+            contextDir: '',
+            evidenceDir: evidence.dir,
+            serving,
+            exec: runner,
+            log,
+        };
+        // ---- assets (before sealing: outputs may belong in the image) ----
+        timer.enter('assets');
+        if (hooks.publishAssets) {
+            await hooks.publishAssets(ctx);
+        }
+        else if (config.assets) {
+            const result = await publishRetainedAssets(runner, {
+                dir: resolve(options.repoRoot, config.assets.dir),
+                bucket: config.bucket,
+                prefix: config.assets.prefix,
+                buildId: sha,
+            });
+            log(`retained assets: ${result.published} published, ${result.reused} reused`);
+        }
+        // ---- seal the build context ----
+        timer.enter('seal');
+        const contextDir = join(options.repoRoot, '.deploy', `context-${sha.slice(0, 12)}`);
+        ctx.contextDir = contextDir;
+        await rm(contextDir, { recursive: true, force: true });
+        await mkdir(contextDir, { recursive: true });
+        for (const name of [
+            'Dockerfile', 'cloudbuild.yaml', 'package.json', 'package-lock.json',
+            ...config.build.contextFiles,
+        ]) {
+            await cp(resolve(options.repoRoot, name), join(contextDir, name), { recursive: true });
+        }
+        // ---- image (async) ‖ gates ----
+        timer.enter('image+gates');
+        const imageUri = `${config.region}-docker.pkg.dev/${config.project}/${config.image.arRepo}/${config.image.name}:${sha}`;
+        const describeImage = () => mustJson(runner, 'gcloud', [
+            'artifacts', 'docker', 'images', 'describe', imageUri, ...gcloudBase, '--format=json',
+        ]);
+        const imageStart = Date.now();
+        const imagePromise = (async () => {
+            const existing = await runner('gcloud', [
+                'artifacts', 'docker', 'images', 'describe', imageUri, ...gcloudBase, '--format=json',
+            ]);
+            if (existing.code === 0) {
+                log('reusing existing immutable commit image');
+            }
+            else {
+                const submit = await must(runner, 'gcloud', [
+                    'builds', 'submit', contextDir,
+                    '--config', join(contextDir, 'cloudbuild.yaml'),
+                    '--region', config.region, ...gcloudBase,
+                    '--substitutions', `_IMAGE=${imageUri}`,
+                    '--suppress-logs', '--format=json',
+                ], { timeoutMs: 15 * 60_000 });
+                await evidence.save('cloud-build.json', submit.stdout);
+            }
+            const digest = (await describeImage()).image_summary?.fully_qualified_digest ?? '';
+            if (!IMAGE_DIGEST.test(digest))
+                throw new Error(`unexpected image digest: ${digest}`);
+            return digest;
+        })().then((digest) => {
+            timer.record('image', Date.now() - imageStart);
+            return digest;
+        });
+        // Surface an image failure even if a gate throws first.
+        let imageFailure = null;
+        const imageGuarded = imagePromise.catch((error) => {
+            imageFailure = error;
+            return null;
+        });
+        if (!skipGates) {
+            for (const [index, gate] of config.gates.entries()) {
+                const gateStart = Date.now();
+                const result = await runner('bash', ['-c', gate], {
+                    cwd: options.repoRoot,
+                    timeoutMs: 15 * 60_000,
+                });
+                await evidence.save(`gate-${index}.log`, result.stdout + result.stderr);
+                timer.record(`gate:${gate}`, Date.now() - gateStart);
+                if (result.code !== 0) {
+                    const tail = (result.stderr.trim() || result.stdout.trim()).slice(-1500);
+                    throw new Error(`gate failed: ${gate}\n${tail}`);
+                }
+                log(`gate passed: ${gate}`);
+            }
+        }
+        const imageDigest = await imageGuarded;
+        if (imageDigest === null)
+            throw imageFailure ?? new Error('image build failed');
+        ctx.imageDigest = imageDigest;
+        // ---- release (the fence, then per-service update + explicit pin) ----
+        timer.enter('release');
+        if (Date.now() - startedAt > RELEASE_DEADLINE_MS) {
+            throw new Error('deploy exceeded its 25-minute deadline; lock may be breakable — aborting before traffic');
+        }
+        if (!(await verifyLock(runner, lock))) {
+            lock = null; // someone broke and superseded us; never touch their lock
+            throw new Error('deploy lock lost (superseded by another deploy or rollback); aborting before traffic');
+        }
+        if (hooks.preRelease)
+            await hooks.preRelease(ctx);
+        const deployed = [];
+        for (const [index, service] of config.services.entries()) {
+            const current = serving[index];
+            if (current.buildSha === sha && current.imageDigest === imageDigest) {
+                log(`${service.name}: already serving ${sha.slice(0, 12)}`);
+                deployed.push({
+                    name: service.name, revision: current.revision, imageDigest, updated: false,
+                });
+                continue;
+            }
+            const updateArgs = [
+                'run', 'services', 'update', service.name,
+                ...gcloudBase, '--region', config.region, '--quiet', '--format=json',
+                '--image', imageDigest,
+                '--update-env-vars', `BUILD_ID=${sha}`,
+                '--update-labels', `pw-build-sha=${sha.slice(0, 40)}`,
+            ];
+            if (config.updateSecrets) {
+                const pairs = Object.entries(config.updateSecrets)
+                    .map(([k, v]) => `${k}=${v}`)
+                    .join(',');
+                updateArgs.push('--update-secrets', pairs);
+            }
+            const updated = await must(runner, 'gcloud', updateArgs, { timeoutMs: 10 * 60_000 });
+            await evidence.save(`${service.name}-update.json`, updated.stdout);
+            const after = await mustJson(runner, 'gcloud', [
+                'run', 'services', 'describe', service.name,
+                ...gcloudBase, '--region', config.region, '--format=json',
+            ]);
+            const newRevision = after.status?.latestCreatedRevisionName ?? '';
+            if (!newRevision || newRevision === current.revision) {
+                throw new Error(`${service.name}: expected a new revision, got ${newRevision || 'none'}`);
+            }
+            if (after.status?.latestReadyRevisionName !== newRevision) {
+                throw new Error(`${service.name}: revision ${newRevision} is not ready`);
+            }
+            await must(runner, 'gcloud', [
+                'run', 'services', 'update-traffic', service.name,
+                ...gcloudBase, '--region', config.region, '--quiet', '--format=json',
+                '--to-revisions', `${newRevision}=100`,
+            ]);
+            trafficMoved = true;
+            log(`${service.name}: serving ${newRevision}`);
+            deployed.push({ name: service.name, revision: newRevision, imageDigest, updated: true });
+        }
+        // ---- post ----
+        timer.enter('post');
+        if (hooks.postTraffic)
+            await hooks.postTraffic(ctx);
+        for (const [index, service] of config.services.entries()) {
+            const target = deployed[index];
+            if (service.health.kind === 'none')
+                continue;
+            const path = service.health.path ?? '/health';
+            let url;
+            let headers = {};
+            if (service.health.kind === 'public-canonical') {
+                url = `${service.canonicalUrl}${path}`;
+            }
+            else {
+                const described = await mustJson(runner, 'gcloud', [
+                    'run', 'services', 'describe', service.name,
+                    ...gcloudBase, '--region', config.region, '--format=json',
+                ]);
+                const runUrl = described.status?.url ?? '';
+                if (!/^https:\/\/[a-z0-9.-]+$/.test(runUrl)) {
+                    throw new Error(`${service.name}: unexpected run URL ${runUrl}`);
+                }
+                const audience = service.health.kind === 'oidc-audience' ? service.health.audience : runUrl;
+                const token = (await must(runner, 'gcloud', ['auth', 'print-identity-token', `--audiences=${audience}`])).stdout.trim();
+                headers = { authorization: `Bearer ${token}` };
+                url = `${runUrl}${path}`;
+            }
+            let lastDetail = '';
+            let passed = false;
+            for (let attempt = 0; attempt < 10; attempt += 1) {
+                try {
+                    const response = await fetchFn(url, { headers });
+                    const body = (await response.json());
+                    const trace = response.headers.get('x-cloud-trace-context') ?? '';
+                    if (!response.ok || body.ok !== true) {
+                        lastDetail = `status ${response.status}`;
+                    }
+                    else if (body.revision !== undefined && body.revision !== target.revision) {
+                        lastDetail = `revision ${body.revision} != ${target.revision}`;
+                    }
+                    else {
+                        target.traceId = trace.split('/')[0] || undefined;
+                        passed = true;
+                        break;
+                    }
+                }
+                catch (error) {
+                    lastDetail = String(error);
+                }
+                await sleep(3000);
+            }
+            if (!passed)
+                throw new Error(`${service.name}: smoke failed at ${url} (${lastDetail})`);
+            log(`${service.name}: smoke passed`);
+        }
+        if (hooks.smoke)
+            await hooks.smoke(ctx);
+        // Log check with an ingestion barrier: wait for the smoke request's own
+        // trace to be queryable, then scan the new revision for ERRORs.
+        for (const target of deployed) {
+            if (!target.updated)
+                continue;
+            if (target.traceId && /^[0-9a-f]{16,32}$/.test(target.traceId)) {
+                let seen = false;
+                for (let attempt = 0; attempt < 10; attempt += 1) {
+                    const read = await mustJson(runner, 'gcloud', [
+                        'logging', 'read',
+                        `trace="projects/${config.project}/traces/${target.traceId}" AND resource.labels.revision_name="${target.revision}"`,
+                        ...gcloudBase, '--limit=1', '--format=json',
+                    ]);
+                    if (Array.isArray(read) && read.length > 0) {
+                        seen = true;
+                        break;
+                    }
+                    await sleep(2000);
+                }
+                if (!seen)
+                    log(`${target.name}: smoke trace not yet ingested — error scan may be early`);
+            }
+            const errors = await mustJson(runner, 'gcloud', [
+                'logging', 'read',
+                `resource.type=cloud_run_revision AND resource.labels.revision_name="${target.revision}" AND severity>=ERROR`,
+                ...gcloudBase, '--limit=5', '--format=json',
+            ]);
+            if (Array.isArray(errors) && errors.length > 0) {
+                await evidence.save(`${target.name}-errors.json`, JSON.stringify(errors, null, 2));
+                throw new Error(`${target.name}: ${errors.length} ERROR log entr${errors.length === 1 ? 'y' : 'ies'} on ${target.revision} (see evidence)`);
+            }
+        }
+        if (hooks.postSuccess)
+            await hooks.postSuccess(ctx);
+        timer.enter('record');
+        timer.close();
+        const record = await writeReleaseRecord(runner, config.bucket, config.releasePrefix, {
+            sha,
+            at: new Date().toISOString(),
+            app: config.app,
+            services: deployed.map(({ name, revision, imageDigest: digest }) => ({
+                name, revision, imageDigest: digest,
+            })),
+            timingsMs: timer.timings,
+        });
+        if (hold && !hold.heldShas.includes(sha)) {
+            await clearHold(runner, holdUri);
+            log('cleared rollback hold (a newer build deployed successfully)');
+        }
+        log(`deployed ${sha.slice(0, 12)} (${record})`);
+        log(`timings:\n${timer.table()}`);
+        log(`rollback: npm run rollback   (or: npm run rollback -- <sha>)`);
+        return 0;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(message);
+        if (trafficMoved) {
+            console.error('NOTE: traffic already moved to the new build; to undo: npm run rollback');
+        }
+        console.error(`deploy failed in phase: ${timer.current}`);
+        if (evidenceDir)
+            console.error(`evidence: ${evidenceDir}`);
+        return 1;
+    }
+    finally {
+        if (lock)
+            await releaseLock(runner, lock);
+    }
+}

package/dist/deploy/evidence.d.ts

@@ -0,0 +1,19 @@
+export interface Evidence {
+    dir: string;
+    save: (name: string, content: string) => Promise<void>;
+}
+export declare function openEvidence(app: string, sha: string): Promise<Evidence>;
+export declare class PhaseTimer {
+    private log;
+    readonly timings: Record<string, number>;
+    private phaseName;
+    private phaseStart;
+    constructor(log: (message: string) => void);
+    get current(): string;
+    /** Close the current phase and open the next. */
+    enter(name: string): void;
+    close(): void;
+    /** Record a concurrently-measured span without switching phases. */
+    record(name: string, ms: number): void;
+    table(): string;
+}

package/dist/deploy/evidence.js

@@ -0,0 +1,60 @@
+// Evidence directories and phase timing for pw-deploy/pw-rollback. The final
+// stderr lines on failure are the one-screen summary the Coder daemon
+// persists (2000-char tail), so keep them short and put detail in files.
+import { mkdir, readdir, rm, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+const KEEP_RUNS = 10;
+export async function openEvidence(app, sha) {
+    const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+    const root = join(homedir(), '.deploy-evidence', app);
+    const dir = join(root, `${stamp}-${sha.slice(0, 7)}`);
+    await mkdir(dir, { recursive: true });
+    try {
+        const runs = (await readdir(root)).sort();
+        for (const old of runs.slice(0, Math.max(0, runs.length - KEEP_RUNS))) {
+            await rm(join(root, old), { recursive: true, force: true });
+        }
+    }
+    catch {
+        /* housekeeping only */
+    }
+    return {
+        dir,
+        save: async (name, content) => {
+            await writeFile(join(dir, name), content);
+        },
+    };
+}
+export class PhaseTimer {
+    log;
+    timings = {};
+    phaseName = 'preflight';
+    phaseStart = Date.now();
+    constructor(log) {
+        this.log = log;
+    }
+    get current() {
+        return this.phaseName;
+    }
+    /** Close the current phase and open the next. */
+    enter(name) {
+        this.close();
+        this.phaseName = name;
+        this.phaseStart = Date.now();
+    }
+    close() {
+        const elapsed = Date.now() - this.phaseStart;
+        this.timings[this.phaseName] = (this.timings[this.phaseName] ?? 0) + elapsed;
+        this.log(`phase ${this.phaseName}: ${(elapsed / 1000).toFixed(1)}s`);
+    }
+    /** Record a concurrently-measured span without switching phases. */
+    record(name, ms) {
+        this.timings[name] = ms;
+    }
+    table() {
+        return Object.entries(this.timings)
+            .map(([name, ms]) => `  ${name}: ${(ms / 1000).toFixed(1)}s`)
+            .join('\n');
+    }
+}