@steve31415/baselib 3.30.0 → 3.32.0

package/README.md

@@ -29,7 +29,9 @@ ERROR log (`~/plasticine-way/docs/SECURITY.md`, "Resend API key").
 
 Bins: `check-test-owners` — the fleet's structural test-coverage gate; every
 app runs it from `npm run verify`. `pw-deploy` / `pw-rollback` — the fleet
-deploy (`~/plasticine-way/docs/OPERATIONS.md`, "Deployment safety").
+deploy (`~/plasticine-way/docs/OPERATIONS.md`, "Deployment safety"); an
+app's hooks module (`DeployHooks` in `src/deploy/types.ts`) extends both,
+`postRollback` being the one hook `pw-rollback` runs.
 `pw-sql` / `pw-logs` — read-only production queries (database, Axiom), the
 first step of any production investigation (same doc, "Investigating
 production state").

package/dist/deploy/rollback.js

@@ -7,7 +7,7 @@ import { hostname } from 'node:os';
 import { must, mustJson } from './exec.js';
 import { openEvidence, PhaseTimer } from './evidence.js';
 import { acquireLock, listReleaseRecords, placeHold, releaseLock } from './gcs.js';
-import { DEFAULT_ACCOUNT, loadConfig } from './config.js';
+import { DEFAULT_ACCOUNT, loadConfig, loadHooks } from './config.js';
 import { pickRevisionForSha, releaseEntriesFrom, selectRollbackTarget, readServingState, SHA40, } from './plan.js';
 export async function runRollback(options) {
     const log = options.log ?? ((m) => console.error(`[pw-rollback] ${m}`));
@@ -19,6 +19,7 @@ export async function runRollback(options) {
     const requested = argv.find((a) => SHA40.test(a));
     const timer = new PhaseTimer(log);
     let evidenceDir = '';
+    let rolledBackTo = '';
     let lock = null;
     try {
         const config = await loadConfig(options.repoRoot);
@@ -151,6 +152,26 @@ export async function runRollback(options) {
                 throw new Error(`${service.name}: post-rollback smoke failed (${lastDetail})`);
             log(`${service.name}: smoke passed`);
         }
+        // The hooks module is loaded only now: a broken one must not stand
+        // between an incident and the traffic move.
+        timer.enter('hooks');
+        rolledBackTo = target;
+        const hooks = await loadHooks(options.repoRoot, config);
+        if (hooks.postRollback) {
+            const ctx = {
+                sha: target,
+                baseSha: servingShas.length === 1 ? servingShas[0] : null,
+                config,
+                repoRoot: options.repoRoot,
+                contextDir: '',
+                evidenceDir,
+                imageDigest: targets[0].imageDigest,
+                serving,
+                exec: runner,
+                log,
+            };
+            await hooks.postRollback(ctx);
+        }
         timer.close();
         log(`rolled back to ${target.slice(0, 12)}`);
         log(`hold placed on ${servingShas.map((sha) => sha.slice(0, 12)).join(', ')} — ` +
@@ -162,6 +183,10 @@ export async function runRollback(options) {
         const message = error instanceof Error ? error.message : String(error);
         console.error(message);
         console.error(`rollback failed in phase: ${timer.current}`);
+        if (rolledBackTo) {
+            console.error(`traffic IS on ${rolledBackTo.slice(0, 12)} and the hold is placed; only the app's ` +
+                'postRollback hook failed — finish what it does by hand');
+        }
         if (evidenceDir)
             console.error(`evidence: ${evidenceDir}`);
         return 1;

package/dist/deploy/types.d.ts

@@ -120,4 +120,11 @@ export interface DeployHooks {
     smoke?: (ctx: HookContext) => Promise<void>;
     /** Runs after everything else succeeded (e.g. notes's release state). */
     postSuccess?: (ctx: HookContext) => Promise<void>;
+    /** pw-rollback only: runs after traffic is back on the target build, the
+     *  hold is placed and the smoke passed — for what a traffic move does not
+     *  carry (e.g. podcasts' render Job image). `sha` and `imageDigest` are the
+     *  rollback target's, `serving` is the state rolled back from, and
+     *  `contextDir` is empty (nothing is built). A throw fails the command but
+     *  undoes nothing. */
+    postRollback?: (ctx: HookContext) => Promise<void>;
 }

package/dist/sync-browser/client.d.ts

@@ -32,6 +32,9 @@ export interface CatchupResult<S extends EventBody> {
 }
 export interface SyncBrowserRuntime {
     now(): number;
+    /** A monotonic clock (`performance.now()`), used only to tell a page that
+     * was not running from one that was. Defaults to `performance.now()`. */
+    monotonicNow?(): number;
     randomGuid(): string;
     createSocket(url: string): WebSocket;
     sendBeacon(url: string, body: Blob): boolean;
@@ -171,6 +174,9 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     private lastContactAt;
     private spellCounters;
     private returnWindow;
+    private clockOffset;
+    private stallCoveredUntil;
+    private visibleSinceMono;
     private readonly counters;
     constructor(options: SyncClientOptions<B, C, S, Summary>);
     init(): Promise<void>;
@@ -192,6 +198,17 @@ export declare class SyncClient<B, C extends EventBody, S extends EventBody, Sum
     state(): SyncClientState;
     subscribe(listener: (state: SyncClientState) => void): () => void;
     private installRuntimeListeners;
+    private monoNow;
+    private rebaselineClocks;
+    /** Runs first in every client timer. A device asleep stops the monotonic
+     * clock and the timers with it, so the wall clock's lead over it grows by
+     * the sleep; a frozen page leaves both clocks running and its timers run
+     * late by the monotonic one. Overlapping late timers count once. Only time
+     * the page counted as visible is counted (hidden pages are throttled by
+     * design and their time is `hiddenMs`). */
+    private observeStall;
+    private setTimer;
+    private setRepeating;
     private isHidden;
     private now;
     /** Counters with the running hidden spell's time folded in. */

package/dist/sync-browser/client.js

@@ -35,6 +35,9 @@ const RETURN_QUICK_RETRY_MS = 250;
  * resolves (notes, 2026-10-01: four minutes). Fast failures (offline) are
  * not logged. */
 const SLOW_HANDSHAKE_MS = 3_000;
+/** A timer that runs at least this late, or a wall-clock jump of at least
+ * this much past the monotonic clock, counts as time the page was stalled. */
+const STALL_MIN_MS = 1_000;
 const SOCKET_CONNECTING = 0;
 const SOCKET_OPEN = 1;
 class LifecycleAbort {
@@ -106,8 +109,13 @@ export class SyncClient {
     lastContactAt = null;
     spellCounters = null;
     returnWindow = null;
+    // Stall accounting (Counters.stalledMs).
+    clockOffset = null;
+    stallCoveredUntil = 0;
+    visibleSinceMono = 0;
     counters = {
         hiddenMs: 0,
+        stalledMs: 0,
         freezes: 0,
         resumes: 0,
         watchdogKills: 0,
@@ -277,6 +285,7 @@ export class SyncClient {
     }
     installRuntimeListeners() {
         const runtime = this.options.runtime;
+        this.rebaselineClocks();
         // Loaded hidden (a background tab): the spell starts now; the first
         // connect still runs, and its failure goes dormant rather than retrying.
         if (runtime.visibility() === 'hidden') {
@@ -296,6 +305,48 @@ export class SyncClient {
         if (onResume)
             this.removeListeners.push(onResume);
     }
+    monoNow() {
+        return this.options.runtime.monotonicNow?.() ?? performance.now();
+    }
+    rebaselineClocks() {
+        this.clockOffset = this.now() - this.monoNow();
+    }
+    /** Runs first in every client timer. A device asleep stops the monotonic
+     * clock and the timers with it, so the wall clock's lead over it grows by
+     * the sleep; a frozen page leaves both clocks running and its timers run
+     * late by the monotonic one. Overlapping late timers count once. Only time
+     * the page counted as visible is counted (hidden pages are throttled by
+     * design and their time is `hiddenMs`). */
+    observeStall(dueMono) {
+        const mono = this.monoNow();
+        const offset = this.now() - mono;
+        const visible = !this.isHidden();
+        if (visible && this.clockOffset !== null && offset - this.clockOffset >= STALL_MIN_MS) {
+            this.counters.stalledMs += offset - this.clockOffset;
+        }
+        this.clockOffset = offset;
+        if (visible && mono - dueMono >= STALL_MIN_MS) {
+            const from = Math.max(dueMono, this.stallCoveredUntil, this.visibleSinceMono);
+            if (mono > from)
+                this.counters.stalledMs += mono - from;
+            this.stallCoveredUntil = mono;
+        }
+    }
+    setTimer(callback, delayMs) {
+        const dueMono = this.monoNow() + delayMs;
+        return this.options.runtime.setTimeout(() => {
+            this.observeStall(dueMono);
+            callback();
+        }, delayMs);
+    }
+    setRepeating(callback, periodMs) {
+        let dueMono = this.monoNow() + periodMs;
+        return this.options.runtime.setInterval(() => {
+            this.observeStall(dueMono);
+            dueMono = this.monoNow() + periodMs;
+            callback();
+        }, periodMs);
+    }
     isHidden() {
         return this.hiddenSince !== null;
     }
@@ -310,6 +361,7 @@ export class SyncClient {
     static delta(since, now) {
         return {
             hiddenMs: Math.max(0, now.hiddenMs - since.hiddenMs),
+            stalledMs: now.stalledMs - since.stalledMs,
             freezes: now.freezes - since.freezes,
             resumes: now.resumes - since.resumes,
             watchdogKills: now.watchdogKills - since.watchdogKills,
@@ -323,6 +375,8 @@ export class SyncClient {
         if (this.halted || this.disposed || this.hiddenSince !== null)
             return;
         this.finishReturnWindow();
+        // A device that slept and then hid the page: the sleep counts as hidden.
+        this.rebaselineClocks();
         this.hiddenSince = this.now();
         this.spellCounters = this.snapshot();
         this.hiddenSocketFailed = false;
@@ -339,6 +393,8 @@ export class SyncClient {
             return;
         const hiddenSince = this.hiddenSince;
         if (hiddenSince !== null) {
+            this.rebaselineClocks();
+            this.visibleSinceMono = this.monoNow();
             const hiddenMs = Math.max(0, this.now() - hiddenSince);
             const spell = this.spellCounters ?? this.snapshot();
             this.counters.hiddenMs += hiddenMs;
@@ -382,7 +438,7 @@ export class SyncClient {
             // replaces it.
             if (this.connectReplyTimer !== null)
                 this.options.runtime.clearTimeout(this.connectReplyTimer);
-            this.connectReplyTimer = this.options.runtime.setTimeout(() => {
+            this.connectReplyTimer = this.setTimer(() => {
                 this.connectReplyTimer = null;
                 if (this.socket === socket && !this.isHidden())
                     this.replaceSocket();
@@ -452,7 +508,7 @@ export class SyncClient {
         if (this.socketCreatedHidden)
             this.counters.backgroundSocketAttempts++;
         let killedByWatchdog = false;
-        this.watchdogTimer = this.options.runtime.setTimeout(() => {
+        this.watchdogTimer = this.setTimer(() => {
             if (socket.readyState !== SOCKET_OPEN) {
                 this.counters.watchdogKills++;
                 killedByWatchdog = true;
@@ -479,8 +535,8 @@ export class SyncClient {
             }
             this.cancelBackgroundTick();
             this.sendConnect();
-            this.pingTimer = this.options.runtime.setInterval(() => this.sendRaw({ type: 'ping' }), PING_INTERVAL_MS);
-            this.rolloverTimer = this.options.runtime.setTimeout(() => {
+            this.pingTimer = this.setRepeating(() => this.sendRaw({ type: 'ping' }), PING_INTERVAL_MS);
+            this.rolloverTimer = this.setTimer(() => {
                 this.options.logger.info('ws rollover', {});
                 this.plannedClose = socket;
                 socket.close();
@@ -556,7 +612,7 @@ export class SyncClient {
             return;
         const interval = this.options.backgroundCatchupMs ?? DEFAULT_BACKGROUND_CATCHUP_MS;
         const due = this.lastTickAt === null ? 0 : Math.max(0, this.lastTickAt + interval - this.now());
-        this.backgroundTickTimer = this.options.runtime.setTimeout(() => {
+        this.backgroundTickTimer = this.setTimer(() => {
             this.backgroundTickTimer = null;
             void this.backgroundTick();
         }, Math.min(due, interval));
@@ -633,7 +689,7 @@ export class SyncClient {
         const slot = { kind, controller };
         this.catchupSlot = slot;
         this.abortControllers.add(controller);
-        const timeout = this.options.runtime.setTimeout(() => controller.abort(), BACKGROUND_FETCH_TIMEOUT_MS);
+        const timeout = this.setTimer(() => controller.abort(), BACKGROUND_FETCH_TIMEOUT_MS);
         let ok = false;
         try {
             const result = await this.options.fetchCatchup(this.options.store.currentSeq(), controller.signal);
@@ -678,7 +734,7 @@ export class SyncClient {
     async withTimeout(operation) {
         const controller = new AbortController();
         this.abortControllers.add(controller);
-        const timeout = this.options.runtime.setTimeout(() => controller.abort(), BACKGROUND_FETCH_TIMEOUT_MS);
+        const timeout = this.setTimer(() => controller.abort(), BACKGROUND_FETCH_TIMEOUT_MS);
         try {
             return await operation(controller.signal);
         }
@@ -704,7 +760,7 @@ export class SyncClient {
             failures: 0,
             firstFailure: null,
             quickRetryUsed: false,
-            timer: this.options.runtime.setTimeout(() => this.finishReturnWindow(), RETURN_REPORT_TIMEOUT_MS),
+            timer: this.setTimer(() => this.finishReturnWindow(), RETURN_REPORT_TIMEOUT_MS),
         };
     }
     maybeFinishReturnWindow() {
@@ -785,13 +841,13 @@ export class SyncClient {
         }
         let remaining = delay;
         this.publish({ kind: 'reconnecting', pending: this.pendingCount(), inSeconds: remaining });
-        this.countdownTimer = this.options.runtime.setInterval(() => {
+        this.countdownTimer = this.setRepeating(() => {
             remaining--;
             if (remaining > 0) {
                 this.publish({ kind: 'reconnecting', pending: this.pendingCount(), inSeconds: remaining });
             }
         }, 1000);
-        this.reconnectTimer = this.options.runtime.setTimeout(() => {
+        this.reconnectTimer = this.setTimer(() => {
             if (this.countdownTimer !== null)
                 this.options.runtime.clearInterval(this.countdownTimer);
             this.countdownTimer = null;
@@ -858,7 +914,7 @@ export class SyncClient {
         if (this.escalateTimer !== null)
             this.options.runtime.clearTimeout(this.escalateTimer);
         this.escalateArmedAt = this.options.runtime.now();
-        this.escalateTimer = this.options.runtime.setTimeout(() => {
+        this.escalateTimer = this.setTimer(() => {
             this.escalateTimer = null;
             const late = this.options.runtime.now() - this.escalateArmedAt - OUTAGE_ESCALATE_MS;
             if (this.options.runtime.visibility() === 'hidden' || late > 5000)
@@ -1153,7 +1209,7 @@ export class SyncClient {
     }
     armFailureTimer(eventGuid) {
         this.clearFailureTimer(eventGuid);
-        this.failureTimers.set(eventGuid, this.options.runtime.setTimeout(() => this.failUnconfirmed(eventGuid), FAILURE_TIMEOUT_MS));
+        this.failureTimers.set(eventGuid, this.setTimer(() => this.failUnconfirmed(eventGuid), FAILURE_TIMEOUT_MS));
     }
     /** On return: a verdict needs 60 s of visible time (review B2). */
     rearmFailureTimers() {
@@ -1195,6 +1251,7 @@ export class SyncClient {
                 wasClean: info?.wasClean,
                 visibleAtDrop: info?.visibleAtDrop,
                 hiddenMs: delta?.hiddenMs,
+                stalledMs: delta?.stalledMs,
                 watchdogKills: delta?.watchdogKills,
                 backgroundCatchups: delta?.backgroundCatchups,
                 httpPostedEvents: delta?.httpPostedEvents,
@@ -1332,7 +1389,7 @@ export class SyncClient {
             return;
         // A hidden tab's timer throttling may stretch this well past the nominal
         // delay; that only postpones recovery, it never breaks it.
-        this.resyncBackoffTimer = this.options.runtime.setTimeout(() => {
+        this.resyncBackoffTimer = this.setTimer(() => {
             this.resyncBackoffTimer = null;
             void this.resyncFromServer('retry after resync backoff');
         }, RESYNC_RETRY_MS);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.30.0",
+  "version": "3.32.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",