@steve31415/baselib 3.2.0 → 3.3.1

package/dist/deploy/deploy.js

@@ -12,7 +12,7 @@ import { DEFAULT_ACCOUNT, loadConfig, loadHooks } from './config.js';
 import { must, mustJson } from './exec.js';
 import { openEvidence, PhaseTimer } from './evidence.js';
 import { acquireLock, clearHold, publishRetainedAssets, readHold, readReleaseRecord, releaseLock, verifyLock, writeReleaseRecord, } from './gcs.js';
-import { consistentBaseSha, servingStateOf, SHA40 } from './plan.js';
+import { consistentBaseSha, readServingState, SHA40 } from './plan.js';
 const RELEASE_DEADLINE_MS = 25 * 60_000;
 const IMAGE_DIGEST = /@sha256:[0-9a-f]{64}$/;
 export async function runDeploy(options) {
@@ -87,12 +87,9 @@ export async function runDeploy(options) {
         }
         const serving = [];
         for (const service of config.services) {
-            const described = await mustJson(runner, 'gcloud', [
-                'run', 'services', 'describe', service.name,
-                ...gcloudBase, '--region', config.region, '--format=json',
-            ]);
+            const { described, state } = await readServingState(runner, gcloudBase, config.region, service.name);
             await evidence.save(`${service.name}-before.json`, JSON.stringify(described, null, 2));
-            serving.push(servingStateOf(service.name, described));
+            serving.push(state);
         }
         const base = consistentBaseSha(serving, sha);
         if (!base.ok)
@@ -228,6 +225,7 @@ export async function runDeploy(options) {
             throw new Error('deploy exceeded its 25-minute deadline; lock may be breakable — aborting before traffic');
         }
         if (!(await verifyLock(runner, lock))) {
+            lock.stopHeartbeat();
             lock = null; // someone broke and superseded us; never touch their lock
             throw new Error('deploy lock lost (superseded by another deploy or rollback); aborting before traffic');
         }

package/dist/deploy/gcs.d.ts

@@ -2,11 +2,16 @@ import type { HoldRecord, ReleaseRecord, Runner } from './types.js';
 export interface LockHandle {
     uri: string;
     generation: string;
+    /** Ends the heartbeat; releaseLock calls it, and so must anyone who gives
+     *  the lock up without releasing it. */
+    stopHeartbeat(): void;
 }
 export interface AcquireOptions {
     waitMs?: number;
     staleMs?: number;
     pollMs?: number;
+    /** Refresh interval; 0 disables the heartbeat (tests). */
+    heartbeatMs?: number;
     now?: () => number;
     sleep?: (ms: number) => Promise<void>;
     log?: (message: string) => void;

package/dist/deploy/gcs.js

@@ -1,12 +1,20 @@
 // GCS-backed coordination for pw-deploy/pw-rollback: the per-app deploy lock
-// (create-only, generation-fenced, server-time staleness), the rollback hold,
-// release records, and the default retained-asset publisher.
+// (create-only, generation-fenced, heartbeat-refreshed, server-time
+// staleness), the rollback hold, release records, and the default
+// retained-asset publisher.
 import { readdir, stat, writeFile } from 'node:fs/promises';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { CommandFailure, must } from './exec.js';
 import { contentTypeFor, lockIsStale, SHA40 } from './plan.js';
-const STALE_MS = 30 * 60_000;
+/** A holder refreshes its lock every HEARTBEAT_MS; a lock not refreshed for
+ *  STALE_MS is a dead holder's and may be broken — well inside the WAIT_MS a
+ *  waiter is prepared to spend, so a killed deploy no longer needs a human
+ *  to delete its lock. (Until 2026-09-04 staleness was judged from creation
+ *  time with a 30-minute threshold, read from a key gcloud does not emit:
+ *  the automatic break never fired.) */
+const HEARTBEAT_MS = 30_000;
+const STALE_MS = 3 * 60_000;
 const WAIT_MS = 5 * 60_000;
 const POLL_MS = 10_000;
 async function describeObject(runner, uri) {
@@ -45,17 +53,23 @@ export async function acquireLock(runner, uri, body, opts = {}) {
             const meta = await describeObject(runner, uri);
             if (!meta?.generation)
                 throw new Error(`lock created but not describable: ${uri}`);
-            return { uri, generation: String(meta.generation) };
+            const generation = String(meta.generation);
+            const stopHeartbeat = startHeartbeat(runner, uri, generation, {
+                everyMs: opts.heartbeatMs ?? HEARTBEAT_MS,
+                now,
+                log,
+            });
+            return { uri, generation, stopHeartbeat };
         }
         const meta = await describeObject(runner, uri);
         if (meta?.generation) {
             const stale = opts.breakExisting ||
-                (meta.timeCreated !== undefined &&
-                    lockIsStale(meta.timeCreated, now(), opts.staleMs ?? STALE_MS));
+                (meta.update_time !== undefined &&
+                    lockIsStale(meta.update_time, now(), opts.staleMs ?? STALE_MS));
             if (stale) {
                 log(opts.breakExisting
                     ? `breaking existing lock (rollback takes priority): ${uri}`
-                    : `breaking stale lock (created ${meta.timeCreated}): ${uri}`);
+                    : `breaking stale lock (last refreshed ${meta.update_time}): ${uri}`);
                 // Conditional on the observed generation: two breakers cannot both
                 // win, and a just-released-and-reacquired lock is not clobbered.
                 await conditionalDelete(runner, uri, String(meta.generation));
@@ -77,12 +91,46 @@ export async function acquireLock(runner, uri, body, opts = {}) {
         await sleep(opts.pollMs ?? POLL_MS);
     }
 }
+/** Refresh the lock's custom metadata, fenced on our generation: a metadata
+ *  update advances the object's server-side update_time (what staleness is
+ *  judged from) without changing its generation (what the fence is). A
+ *  refresh that fails the fence means a successor broke us — stop, and let
+ *  verifyLock report it before traffic. Refreshes run one at a time. */
+function startHeartbeat(runner, uri, generation, opts) {
+    if (opts.everyMs <= 0)
+        return () => { };
+    let stopped = false;
+    let chain = Promise.resolve();
+    const stop = () => {
+        stopped = true;
+        clearInterval(timer);
+    };
+    const refresh = async () => {
+        if (stopped)
+            return;
+        const result = await runner('gcloud', [
+            'storage', 'objects', 'update', uri,
+            `--custom-metadata=heartbeat=${new Date(opts.now()).toISOString()}`,
+            `--if-generation-match=${generation}`, '--quiet',
+        ]);
+        if (result.code !== 0) {
+            opts.log(`lock heartbeat failed (generation ${generation} gone?): ${result.stderr.trim()}`);
+            stop();
+        }
+    };
+    const timer = setInterval(() => {
+        chain = chain.then(refresh);
+    }, opts.everyMs);
+    timer.unref?.();
+    return stop;
+}
 /** The fence: true only while our exact generation still exists. */
 export async function verifyLock(runner, handle) {
     const meta = await describeObject(runner, handle.uri);
     return meta !== null && String(meta.generation) === handle.generation;
 }
 export async function releaseLock(runner, handle) {
+    handle.stopHeartbeat();
     const ok = await conditionalDelete(runner, handle.uri, handle.generation);
     if (!ok) {
         // A successor broke us; their lock must survive. Nothing to clean up.

package/dist/deploy/plan.d.ts

@@ -1,4 +1,4 @@
-import type { ServingState } from './types.js';
+import type { Runner, ServingState } from './types.js';
 export declare const SHA40: RegExp;
 /** Lock staleness is judged from the GCS object's server-side timeCreated,
  *  never from anything the lock writer wrote (its clock may be wrong). */
@@ -41,8 +41,32 @@ export declare function pickRevisionForSha(revisions: unknown[], targetSha: stri
     revision: string;
     imageDigest: string;
 } | null;
-/** Extract serving state from a `gcloud run services describe --format=json`. */
-export declare function servingStateOf(service: string, described: unknown): ServingState;
+/** The one revision carrying 100% of traffic in a `gcloud run services
+ *  describe --format=json`, from *observed* traffic (`status.traffic`, never
+ *  `spec.traffic`, which is only the desired state). */
+export declare function servingRevisionName(service: string, described: unknown): string;
+/**
+ * Extract serving state from a `gcloud run services describe --format=json`.
+ *
+ * The serving revision's BUILD_ID and image are read from the service
+ * template only when that revision IS the latest created one. When it is
+ * not — a previous release created a revision that never took traffic (a
+ * failed secret-access check, a crashed startup, an aborted deploy) — the
+ * template describes the *wrong* revision, so the caller must pass that
+ * revision's own `gcloud run revisions describe --format=json` as
+ * `servingRevision` (readServingState does). Lesson 2026-09-04, watchdog2:
+ * a re-run after a failed traffic switch read the new SHA out of the
+ * template, logged "already serving", wrote the release record, and left
+ * the old revision serving.
+ */
+export declare function servingStateOf(service: string, described: unknown, servingRevision?: unknown): ServingState;
+/** Describe a service and resolve its serving state, describing the serving
+ *  revision too whenever the template no longer belongs to it. Returns the
+ *  raw service description as well, for evidence. */
+export declare function readServingState(runner: Runner, gcloudBase: string[], region: string, service: string): Promise<{
+    described: unknown;
+    state: ServingState;
+}>;
 export declare function contentTypeFor(path: string): string;
 /** Parse release-record listing rows into {sha, at} entries. */
 export declare function releaseEntriesFrom(rows: {

package/dist/deploy/plan.js

@@ -1,6 +1,7 @@
 // Pure decision logic for pw-deploy/pw-rollback: everything here is
 // side-effect-free so the concurrency-sensitive choices (lock staleness,
 // rollback targeting, serving-state consistency) are unit-tested directly.
+import { mustJson } from './exec.js';
 export const SHA40 = /^[0-9a-f]{40}$/;
 /** Lock staleness is judged from the GCS object's server-side timeCreated,
  *  never from anything the lock writer wrote (its clock may be wrong). */
@@ -101,22 +102,71 @@ export function pickRevisionForSha(revisions, targetSha) {
     matches.sort((a, b) => b.created - a.created);
     return matches[0] ? { revision: matches[0].revision, imageDigest: matches[0].imageDigest } : null;
 }
-/** Extract serving state from a `gcloud run services describe --format=json`. */
-export function servingStateOf(service, described) {
+/** The one revision carrying 100% of traffic in a `gcloud run services
+ *  describe --format=json`, from *observed* traffic (`status.traffic`, never
+ *  `spec.traffic`, which is only the desired state). */
+export function servingRevisionName(service, described) {
     const d = described;
     const active = (d.status?.traffic ?? []).filter((t) => (t.percent ?? 0) === 100);
     if (active.length !== 1 || !active[0].revisionName) {
         throw new Error(`${service}: expected exactly one 100% traffic target`);
     }
-    const env = d.spec?.template?.spec?.containers?.[0]?.env ?? [];
+    return active[0].revisionName;
+}
+/**
+ * Extract serving state from a `gcloud run services describe --format=json`.
+ *
+ * The serving revision's BUILD_ID and image are read from the service
+ * template only when that revision IS the latest created one. When it is
+ * not — a previous release created a revision that never took traffic (a
+ * failed secret-access check, a crashed startup, an aborted deploy) — the
+ * template describes the *wrong* revision, so the caller must pass that
+ * revision's own `gcloud run revisions describe --format=json` as
+ * `servingRevision` (readServingState does). Lesson 2026-09-04, watchdog2:
+ * a re-run after a failed traffic switch read the new SHA out of the
+ * template, logged "already serving", wrote the release record, and left
+ * the old revision serving.
+ */
+export function servingStateOf(service, described, servingRevision) {
+    const d = described;
+    const revision = servingRevisionName(service, described);
+    const latest = d.status?.latestCreatedRevisionName;
+    let container;
+    if (latest === undefined || latest === revision) {
+        container = d.spec?.template?.spec?.containers?.[0];
+    }
+    else if (servingRevision !== undefined) {
+        container = servingRevision.spec?.containers?.[0];
+    }
+    else {
+        throw new Error(`${service}: traffic serves ${revision} but the latest revision is ${latest}; ` +
+            'the serving revision must be described separately');
+    }
+    const env = container?.env ?? [];
     const buildSha = env.find((e) => e.name === 'BUILD_ID')?.value ?? null;
     return {
         service,
-        revision: active[0].revisionName,
+        revision,
         buildSha: buildSha !== null && SHA40.test(buildSha) ? buildSha : null,
-        imageDigest: d.spec?.template?.spec?.containers?.[0]?.image ?? null,
+        imageDigest: container?.image ?? null,
     };
 }
+/** Describe a service and resolve its serving state, describing the serving
+ *  revision too whenever the template no longer belongs to it. Returns the
+ *  raw service description as well, for evidence. */
+export async function readServingState(runner, gcloudBase, region, service) {
+    const described = await mustJson(runner, 'gcloud', [
+        'run', 'services', 'describe', service, ...gcloudBase, '--region', region, '--format=json',
+    ]);
+    const revision = servingRevisionName(service, described);
+    const latest = described.status?.latestCreatedRevisionName;
+    const servingRevision = latest !== undefined && latest !== revision
+        ? await mustJson(runner, 'gcloud', [
+            'run', 'revisions', 'describe', revision, ...gcloudBase, '--region', region, '--format=json',
+        ])
+        : undefined;
+    return { described, state: servingStateOf(service, described, servingRevision) };
+}
 const CONTENT_TYPES = {
     '.js': 'text/javascript',
     '.mjs': 'text/javascript',

package/dist/deploy/rollback.js

@@ -8,7 +8,7 @@ import { must, mustJson } from './exec.js';
 import { openEvidence, PhaseTimer } from './evidence.js';
 import { acquireLock, listReleaseRecords, placeHold, releaseLock } from './gcs.js';
 import { DEFAULT_ACCOUNT, loadConfig } from './config.js';
-import { pickRevisionForSha, releaseEntriesFrom, selectRollbackTarget, servingStateOf, SHA40, } from './plan.js';
+import { pickRevisionForSha, releaseEntriesFrom, selectRollbackTarget, readServingState, SHA40, } from './plan.js';
 export async function runRollback(options) {
     const log = options.log ?? ((m) => console.error(`[pw-rollback] ${m}`));
     const runner = options.runner;
@@ -39,11 +39,8 @@ export async function runRollback(options) {
         timer.enter('target');
         const serving = [];
         for (const service of config.services) {
-            const described = await mustJson(runner, 'gcloud', [
-                'run', 'services', 'describe', service.name,
-                ...gcloudBase, '--region', config.region, '--format=json',
-            ]);
-            serving.push(servingStateOf(service.name, described));
+            const { state } = await readServingState(runner, gcloudBase, config.region, service.name);
+            serving.push(state);
         }
         const evidence = await openEvidence(config.app, requested ?? serving[0].buildSha ?? 'rollback');
         evidenceDir = evidence.dir;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.2.0",
+  "version": "3.3.1",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",