@steve31415/baselib 2.2.2 → 2.4.0

package/dist/deploy/rollback.js

@@ -0,0 +1,176 @@
+// The pw-rollback orchestrator: one-command traffic move to a previously
+// released build. Never rebuilds. Breaks-and-takes the deploy lock (a
+// rollback must not queue behind the stuck deploy that motivated it), places
+// a hold so unattended deploys cannot silently re-ship the rolled-back SHA,
+// and only ever moves backward (release records + ancestry checks).
+import { hostname } from 'node:os';
+import { must, mustJson } from './exec.js';
+import { openEvidence, PhaseTimer } from './evidence.js';
+import { acquireLock, listReleaseRecords, placeHold, releaseLock } from './gcs.js';
+import { DEFAULT_ACCOUNT, loadConfig } from './config.js';
+import { pickRevisionForSha, releaseEntriesFrom, selectRollbackTarget, servingStateOf, SHA40, } from './plan.js';
+export async function runRollback(options) {
+    const log = options.log ?? ((m) => console.error(`[pw-rollback] ${m}`));
+    const runner = options.runner;
+    const sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const fetchFn = options.fetchFn ?? fetch;
+    const argv = options.argv ?? [];
+    const allowMigrationDiff = argv.includes('--allow-migration-diff');
+    const requested = argv.find((a) => SHA40.test(a));
+    const timer = new PhaseTimer(log);
+    let evidenceDir = '';
+    let lock = null;
+    try {
+        const config = await loadConfig(options.repoRoot);
+        const gcloudBase = ['--project', config.project];
+        // A rollback is an incident tool: it needs origin's commits for ancestry
+        // checks but does not demand a clean tree or the main branch.
+        await must(runner, 'git', ['fetch', 'origin', 'main'], { cwd: options.repoRoot });
+        const account = (await must(runner, 'gcloud', ['config', 'get', 'account'])).stdout.trim();
+        if (account !== (config.account ?? DEFAULT_ACCOUNT)) {
+            throw new Error(`gcloud account must be ${config.account ?? DEFAULT_ACCOUNT} (is ${account})`);
+        }
+        const project = (await must(runner, 'gcloud', ['config', 'get', 'project'])).stdout.trim();
+        if (project !== config.project) {
+            throw new Error(`gcloud project must be ${config.project} (is ${project})`);
+        }
+        timer.enter('lock');
+        lock = await acquireLock(runner, `gs://${config.bucket}/${config.app}/deploy.lock`, `host=${hostname()}\nrollback=true\nstarted=${new Date().toISOString()}\n`, { log, sleep, breakExisting: true });
+        timer.enter('target');
+        const serving = [];
+        for (const service of config.services) {
+            const described = await mustJson(runner, 'gcloud', [
+                'run', 'services', 'describe', service.name,
+                ...gcloudBase, '--region', config.region, '--format=json',
+            ]);
+            serving.push(servingStateOf(service.name, described));
+        }
+        const evidence = await openEvidence(config.app, requested ?? serving[0].buildSha ?? 'rollback');
+        evidenceDir = evidence.dir;
+        await evidence.save('serving-before.json', JSON.stringify(serving, null, 2));
+        const rows = await listReleaseRecords(runner, config.bucket, config.releasePrefix);
+        const records = releaseEntriesFrom(rows, config.releasePrefix);
+        const isAncestor = async (maybeAncestor, descendant) => {
+            const exists = await runner('git', ['cat-file', '-e', `${maybeAncestor}^{commit}`], {
+                cwd: options.repoRoot,
+            });
+            if (exists.code !== 0)
+                return false;
+            const result = await runner('git', ['merge-base', '--is-ancestor', maybeAncestor, descendant], { cwd: options.repoRoot });
+            if (result.code > 1)
+                throw new Error(`git merge-base failed: ${result.stderr.trim()}`);
+            return result.code === 0;
+        };
+        const selection = await selectRollbackTarget({ requested, records, serving, isAncestor });
+        if (!selection.ok || !selection.sha) {
+            throw new Error(`no rollback target: ${selection.refusal}`);
+        }
+        const target = selection.sha;
+        log(`rolling back to ${target.slice(0, 12)}`);
+        const servingShas = [...new Set(serving.map((s) => s.buildSha))];
+        if (!allowMigrationDiff) {
+            for (const fromSha of servingShas) {
+                const diff = (await must(runner, 'git', ['diff', '--name-only', '--diff-filter=ACMRD', `${target}..${fromSha}`, '--', 'migrations'], { cwd: options.repoRoot })).stdout.trim();
+                if (diff) {
+                    throw new Error(`migrations changed between ${target.slice(0, 12)} and serving ` +
+                        `${fromSha.slice(0, 12)} — old code would run against a migrated schema. ` +
+                        'Decide deliberately; pass --allow-migration-diff to proceed.');
+                }
+            }
+        }
+        const targets = [];
+        for (const service of config.services) {
+            const revisions = await mustJson(runner, 'gcloud', [
+                'run', 'revisions', 'list', '--service', service.name,
+                ...gcloudBase, '--region', config.region, '--format=json',
+            ]);
+            const picked = pickRevisionForSha(revisions, target);
+            if (!picked) {
+                throw new Error(`${service.name}: no ready revision carries BUILD_ID ${target.slice(0, 12)}; ` +
+                    'use the manual traffic-move recipe in OPERATIONS.md');
+            }
+            targets.push({ name: service.name, ...picked });
+        }
+        if (new Set(targets.map((t) => t.imageDigest)).size > 1) {
+            throw new Error(`services would land on different images for ${target.slice(0, 12)}: ` +
+                targets.map((t) => `${t.name}=${t.imageDigest.slice(-12)}`).join(' '));
+        }
+        timer.enter('traffic');
+        for (const t of targets) {
+            await must(runner, 'gcloud', [
+                'run', 'services', 'update-traffic', t.name,
+                ...gcloudBase, '--region', config.region, '--quiet', '--format=json',
+                '--to-revisions', `${t.revision}=100`,
+            ]);
+            log(`${t.name}: pinned to ${t.revision}`);
+        }
+        timer.enter('hold');
+        await placeHold(runner, `gs://${config.bucket}/${config.app}/deploy.hold`, {
+            heldShas: servingShas.filter((sha) => sha !== target),
+            reason: 'rollback',
+            at: new Date().toISOString(),
+        });
+        timer.enter('smoke');
+        for (const [index, service] of config.services.entries()) {
+            if (service.health.kind === 'none')
+                continue;
+            const path = service.health.path ?? '/health';
+            let url;
+            let headers = {};
+            if (service.health.kind === 'public-canonical') {
+                url = `${service.canonicalUrl}${path}`;
+            }
+            else {
+                const described = await mustJson(runner, 'gcloud', [
+                    'run', 'services', 'describe', service.name,
+                    ...gcloudBase, '--region', config.region, '--format=json',
+                ]);
+                const runUrl = described.status?.url ?? '';
+                const audience = service.health.kind === 'oidc-audience' ? service.health.audience : runUrl;
+                const token = (await must(runner, 'gcloud', ['auth', 'print-identity-token', `--audiences=${audience}`])).stdout.trim();
+                headers = { authorization: `Bearer ${token}` };
+                url = `${runUrl}${path}`;
+            }
+            let passed = false;
+            let lastDetail = '';
+            for (let attempt = 0; attempt < 10; attempt += 1) {
+                try {
+                    const response = await fetchFn(url, { headers });
+                    const body = (await response.json());
+                    if (response.ok &&
+                        body.ok === true &&
+                        (body.revision === undefined || body.revision === targets[index].revision)) {
+                        passed = true;
+                        break;
+                    }
+                    lastDetail = `status ${response.status} revision ${body.revision ?? '?'}`;
+                }
+                catch (error) {
+                    lastDetail = String(error);
+                }
+                await sleep(3000);
+            }
+            if (!passed)
+                throw new Error(`${service.name}: post-rollback smoke failed (${lastDetail})`);
+            log(`${service.name}: smoke passed`);
+        }
+        timer.close();
+        log(`rolled back to ${target.slice(0, 12)}`);
+        log(`hold placed on ${servingShas.map((sha) => sha.slice(0, 12)).join(', ')} — ` +
+            'deploys of held SHAs refuse without --override-hold; the hold clears when a different SHA deploys');
+        log(`timings:\n${timer.table()}`);
+        return 0;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(message);
+        console.error(`rollback failed in phase: ${timer.current}`);
+        if (evidenceDir)
+            console.error(`evidence: ${evidenceDir}`);
+        return 1;
+    }
+    finally {
+        if (lock)
+            await releaseLock(runner, lock);
+    }
+}

package/dist/deploy/types.d.ts

@@ -0,0 +1,119 @@
+export interface ServiceHealth {
+    /** How the shared smoke reaches /health:
+     *  - public-canonical: unauthenticated GET on the canonical URL
+     *  - oidc-run-url: OIDC identity token with the service's run.app URL audience
+     *  - oidc-audience: OIDC identity token with a fixed custom audience
+     *  - none: no shared health check (app hook covers it) */
+    kind: 'public-canonical' | 'oidc-run-url' | 'oidc-audience' | 'none';
+    audience?: string;
+    path?: string;
+}
+export interface ServiceConfig {
+    name: string;
+    /** Canonical public URL (used by public-canonical health and printed). */
+    canonicalUrl?: string;
+    health: ServiceHealth;
+}
+export interface DeployConfig {
+    /** Short app name; prefixes the lock/hold/release objects. */
+    app: string;
+    project: string;
+    region: string;
+    /** GCS bucket holding lock, hold, release records, and retained assets. */
+    bucket: string;
+    /** Expected gcloud account (defaults to the fleet coding agent). */
+    account?: string;
+    image: {
+        arRepo: string;
+        name: string;
+        buildServiceAccount: string;
+    };
+    build: {
+        /** Command producing the deployable outputs; run with BUILD_ID=<sha>. */
+        command: string;
+        /** Repo-relative files/dirs copied into the sealed build context
+         *  alongside Dockerfile, cloudbuild.yaml, package.json, package-lock.json. */
+        contextFiles: string[];
+    };
+    /** Gate commands run while the image builds; all must pass before traffic. */
+    gates: string[];
+    /** Default retained-asset publish (omit when a publishAssets hook or no
+     *  assets exist): dir is repo-relative, prefix is bucket-relative. */
+    assets?: {
+        dir: string;
+        prefix: string;
+    };
+    /** Bucket-relative prefix for shared release records, e.g. "todo/pw-releases/". */
+    releasePrefix: string;
+    /** Secrets re-pinned on every routine deploy (name -> secret:version). */
+    updateSecrets?: Record<string, string>;
+    /** Refuse to deploy across a migrations/ diff (notes2 semantics). */
+    refuseMigrationDiffOnDeploy?: boolean;
+    /** Repo-relative path of an optional ES module exporting hooks. */
+    hooks?: string;
+    services: ServiceConfig[];
+}
+export interface ExecResult {
+    stdout: string;
+    stderr: string;
+    code: number;
+}
+export interface ExecOptions {
+    timeoutMs?: number;
+    cwd?: string;
+    env?: Record<string, string | undefined>;
+    input?: string;
+}
+/** Injected process runner; never throws on nonzero exit (code reports it). */
+export type Runner = (file: string, args: string[], opts?: ExecOptions) => Promise<ExecResult>;
+export interface ReleaseRecord {
+    sha: string;
+    at: string;
+    app: string;
+    services: {
+        name: string;
+        revision: string;
+        imageDigest: string;
+    }[];
+    timingsMs?: Record<string, number>;
+}
+export interface HoldRecord {
+    /** SHAs that must not deploy without an explicit override. */
+    heldShas: string[];
+    reason: string;
+    at: string;
+}
+export interface ServingState {
+    service: string;
+    revision: string;
+    /** BUILD_ID env of the serving revision; null before unification. */
+    buildSha: string | null;
+    imageDigest: string | null;
+}
+export interface HookContext {
+    sha: string;
+    /** Consistent serving base SHA across services, when determinable. */
+    baseSha: string | null;
+    config: DeployConfig;
+    repoRoot: string;
+    contextDir: string;
+    evidenceDir: string;
+    /** Resolved image digest (release phase onward). */
+    imageDigest?: string;
+    serving: ServingState[];
+    exec: Runner;
+    log: (message: string) => void;
+}
+export interface DeployHooks {
+    /** Runs after the local build, before the context is sealed. Overrides the
+     *  default retained-asset publish when present. */
+    publishAssets?: (ctx: HookContext) => Promise<void>;
+    /** Runs after all gates pass, immediately before the first traffic switch. */
+    preRelease?: (ctx: HookContext) => Promise<void>;
+    /** Runs right after traffic moves (e.g. todo2's announce-build). */
+    postTraffic?: (ctx: HookContext) => Promise<void>;
+    /** Extra smoke checks after the shared health checks. */
+    smoke?: (ctx: HookContext) => Promise<void>;
+    /** Runs after everything else succeeded (e.g. notes2's release state). */
+    postSuccess?: (ctx: HookContext) => Promise<void>;
+}

package/dist/deploy/types.js

@@ -0,0 +1,3 @@
+// Shared deploy/rollback types for the fleet-wide `pw-deploy` / `pw-rollback`
+// bins (design: ~/migration/research/deploy-procedure-design.md, 2026-08-31).
+export {};

package/dist/log-core.d.ts

@@ -52,7 +52,8 @@ export interface ShipperOptions {
      *  fire at the next CPU window (next request or shutdown), not on the
      *  wall clock. */
     retryDelaysMs?: number[];
-    /** Per-attempt fetch timeout (default 3s). */
+    /** Per-attempt fetch timeout (default 10s; 3s aborted most of the fleet's
+     *  attempts in the two weeks to 2026-08-28). */
     attemptTimeoutMs?: number;
     /** Caps across ALL undelivered batches; overflow drops oldest with an
      *  onGiveUp('buffer-overflow'). Defaults 5000 events / 5 MB. */

package/dist/log-core.js

@@ -70,7 +70,7 @@ export class LogShipper {
         this.intervalMs = opts.intervalMs ?? 2000;
         this.maxAttempts = opts.maxAttempts ?? 4;
         this.retryDelaysMs = opts.retryDelaysMs ?? [5_000, 15_000, 40_000];
-        this.attemptTimeoutMs = opts.attemptTimeoutMs ?? 3_000;
+        this.attemptTimeoutMs = opts.attemptTimeoutMs ?? 10_000;
         this.maxBufferedEvents = opts.maxBufferedEvents ?? 5_000;
         this.maxBufferedBytes = opts.maxBufferedBytes ?? 5 * 1024 * 1024;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "2.2.2",
+  "version": "2.4.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",
@@ -63,7 +63,9 @@
     }
   },
   "bin": {
-    "check-test-owners": "./dist/bin/check-test-owners.js"
+    "check-test-owners": "./dist/bin/check-test-owners.js",
+    "pw-deploy": "./dist/bin/pw-deploy.js",
+    "pw-rollback": "./dist/bin/pw-rollback.js"
   },
   "engines": {
     "node": "22.x"