@steve31415/baselib 3.42.0 → 3.44.0

package/dist/log-core.d.ts

@@ -65,7 +65,13 @@ export interface ShipperOptions {
      *  (uncounted) attempt (default 40 s). */
     offlineRetryMs?: number;
     /** Per-attempt fetch timeout (default 10s; 3s aborted most of the fleet's
-     *  attempts in the two weeks to 2026-08-28). */
+     *  attempts in the two weeks to 2026-08-28). Headroom judgment 2026-10-30
+     *  (watchdog task f4ec3337): steady-state ingest takes ~0.2–2 s (≤ 0.2 of
+     *  the limit). What still times out is a fresh instance's startup line
+     *  shipped while Cloud Run throttles its CPU (no request in flight), which
+     *  a longer timeout would not cure — it waits for CPU, not for Axiom — and
+     *  flush() callers wait up to this + 500 ms, so raising it lengthens the
+     *  worst-case request. Kept at 10 s; the retry delivers those batches. */
     attemptTimeoutMs?: number;
     /** Caps across ALL undelivered batches; overflow drops oldest with an
      *  onGiveUp('buffer-overflow'). Defaults 5000 events / 5 MB. */

package/dist/log.js

@@ -88,6 +88,14 @@ export function stdoutShipMarkers(app, dataset) {
             attempt: batch.attempt,
             max_attempts: batch.maxAttempts,
             error: String(error),
+            // Whether Cloud Run had CPU allocated: request-based billing throttles
+            // an idle instance to almost nothing, and a fresh instance's startup
+            // line ships just after its startup probe passes — with no request in
+            // flight when a deploy or replacement starts it ahead of traffic. Its
+            // 10 s attempt then times out (watchdog task f4ec3337, 2026-10-30:
+            // all 12 batches that failed on 10-29 13:33–16:25Z were such startup lines).
+            uptime_ms: Math.round(process.uptime() * 1000),
+            ...processMeta(),
             message: `axiom ship attempt ${batch.attempt}/${batch.maxAttempts} failed: ${String(error)} ` +
                 `(${batch.events.length} events, ${batch.bytes} bytes; will retry)`,
         }),

package/dist/sync/db-bridge.d.ts

@@ -0,0 +1,89 @@
+import type pg from 'pg';
+import type { Logger } from '../log.js';
+/** How often the bridge proves its LISTEN connection alive with a `select
+ *  1`. Cloud SQL drops idle connections silently, the error surfaces only on
+ *  the next query, and a pure listener never issues one, so without this the
+ *  bridge would go permanently deaf while its logs claim health. */
+export declare const DB_BRIDGE_HEARTBEAT_MS = 60000;
+/** How long a heartbeat (or the `unlisten` in stop()) may take before the
+ *  connection is declared lost: a half-open socket never answers and never
+ *  errors. Generous, because a request-billed Cloud Run instance runs this
+ *  timer with almost no CPU. */
+export declare const DB_BRIDGE_HEARTBEAT_TIMEOUT_MS = 10000;
+export declare const DB_BRIDGE_RECONNECT_DELAYS_MS: readonly number[];
+export interface DbBroadcastBridgeOptions {
+    /** The app's pool. The bridge holds one of its clients for as long as it
+     *  runs, so size `max` as working connections + 1, and stop() the bridge
+     *  before the pool ends. */
+    pool: pg.Pool;
+    /** The NOTIFY channel, per app (`<app>_updates`): a lowercase identifier. */
+    channel: string;
+    logger: Logger;
+    /** Each notification's parsed JSON payload. An invalid payload is logged
+     *  and skipped; a throw is logged and does not reach pg. */
+    onMessage: (message: unknown) => void;
+    /** Called after the listener reconnects (not on the first connect).
+     *  Notifications sent while it was down are lost and unknowable, so the
+     *  app re-announces availability to every local scope. */
+    onRecovered: () => void;
+    /** Skip messages whose top-level `origin` is this bridge's `originId`,
+     *  for an app that delivers its own writes locally and publishes them via
+     *  notify() for the other copies. Default false. */
+    ignoreOwn?: boolean;
+    heartbeatMs?: number;
+    heartbeatTimeoutMs?: number;
+    reconnectDelaysMs?: readonly number[];
+}
+type Queryable = Pick<pg.Pool | pg.PoolClient, 'query'>;
+export declare class DbBroadcastBridge {
+    /** Tags this process's notify() payloads when `ignoreOwn` is on. */
+    readonly originId: `${string}-${string}-${string}-${string}-${string}`;
+    private readonly pool;
+    private readonly channel;
+    private readonly logger;
+    private readonly onMessage;
+    private readonly onRecovered;
+    private readonly ignoreOwn;
+    private readonly heartbeatMs;
+    private readonly heartbeatTimeoutMs;
+    private readonly reconnectDelaysMs;
+    private listener;
+    private stopped;
+    private connecting;
+    private connectedOnce;
+    private reconnectAttempt;
+    private reconnectTimer;
+    private heartbeatTimer;
+    constructor(options: DbBroadcastBridgeOptions);
+    /** Connects the listener; rejects if the first connect fails. */
+    start(): Promise<void>;
+    /** Idempotent. Releases the listener client back to the pool; call it
+     *  before the pool ends. Nothing is delivered after it resolves. */
+    stop(): Promise<void>;
+    /** `pg_notify` on this bridge's channel. Pass the transaction's client to
+     *  announce only on commit. With `ignoreOwn`, tags the payload with
+     *  `originId`. */
+    notify(message: Record<string, unknown>, queryable?: Queryable): Promise<void>;
+    private connectListener;
+    /** pg-pool exempts its first `min` clients, counted over all of them,
+     *  checked out or idle, from the idle timeout, and reads `options.min`
+     *  each time it decides. The listener is one of those clients, so without
+     *  this a pool with `min: 1` (FLEET_POOL_DEFAULTS) would reap its last
+     *  working connection after the idle timeout: the warm connection min
+     *  exists to keep. Raise min while the listener is held, lower it on
+     *  release. */
+    private adjustPoolMin;
+    /** Each heartbeat is scheduled when the previous one ends, so a slow one
+     *  never overlaps the next. */
+    private scheduleHeartbeat;
+    /** Clears the current listener, its timers and handlers, and the raised
+     *  pool min; returns the client for the caller to release. */
+    private detachListener;
+    private onListenerLost;
+    private scheduleReconnect;
+    private onNotification;
+    /** A throw from the app's callback would otherwise escape through pg's
+     *  socket handler and crash the process. */
+    private callApp;
+}
+export {};

package/dist/sync/db-bridge.js

@@ -0,0 +1,256 @@
+// The cross-copy half of live delivery: one dedicated LISTEN connection per
+// process relays a Postgres NOTIFY channel to the app's local fan-out. Every
+// copy of a service shares the database, so a hint published by any writer
+// (another copy, the old revision during a deploy, a job) reaches every
+// copy's sessions. Rules: ~/plasticine-way/docs/DATABASE.md "Cross-instance
+// live delivery: LISTEN/NOTIFY". Moved here from todo's src/db-broadcast.ts.
+import { randomUUID } from 'node:crypto';
+/** How often the bridge proves its LISTEN connection alive with a `select
+ *  1`. Cloud SQL drops idle connections silently, the error surfaces only on
+ *  the next query, and a pure listener never issues one, so without this the
+ *  bridge would go permanently deaf while its logs claim health. */
+export const DB_BRIDGE_HEARTBEAT_MS = 60_000;
+/** How long a heartbeat (or the `unlisten` in stop()) may take before the
+ *  connection is declared lost: a half-open socket never answers and never
+ *  errors. Generous, because a request-billed Cloud Run instance runs this
+ *  timer with almost no CPU. */
+export const DB_BRIDGE_HEARTBEAT_TIMEOUT_MS = 10_000;
+export const DB_BRIDGE_RECONNECT_DELAYS_MS = [250, 1_000, 4_000, 15_000, 30_000];
+export class DbBroadcastBridge {
+    /** Tags this process's notify() payloads when `ignoreOwn` is on. */
+    originId = randomUUID();
+    pool;
+    channel;
+    logger;
+    onMessage;
+    onRecovered;
+    ignoreOwn;
+    heartbeatMs;
+    heartbeatTimeoutMs;
+    reconnectDelaysMs;
+    listener = null;
+    stopped = true;
+    connecting = false;
+    connectedOnce = false;
+    reconnectAttempt = 0;
+    reconnectTimer = null;
+    heartbeatTimer = null;
+    constructor(options) {
+        if (!/^[a-z_][a-z0-9_]*$/.test(options.channel)) {
+            throw new Error(`DbBroadcastBridge: channel must be a lowercase identifier, got ${JSON.stringify(options.channel)}`);
+        }
+        this.pool = options.pool;
+        this.channel = options.channel;
+        this.logger = options.logger;
+        this.onMessage = options.onMessage;
+        this.onRecovered = options.onRecovered;
+        this.ignoreOwn = options.ignoreOwn ?? false;
+        this.heartbeatMs = options.heartbeatMs ?? DB_BRIDGE_HEARTBEAT_MS;
+        this.heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? DB_BRIDGE_HEARTBEAT_TIMEOUT_MS;
+        this.reconnectDelaysMs = options.reconnectDelaysMs ?? DB_BRIDGE_RECONNECT_DELAYS_MS;
+    }
+    /** Connects the listener; rejects if the first connect fails. */
+    async start() {
+        if (!this.stopped)
+            return;
+        this.stopped = false;
+        await this.connectListener(true);
+    }
+    /** Idempotent. Releases the listener client back to the pool; call it
+     *  before the pool ends. Nothing is delivered after it resolves. */
+    async stop() {
+        this.stopped = true;
+        if (this.reconnectTimer)
+            clearTimeout(this.reconnectTimer);
+        this.reconnectTimer = null;
+        const listener = this.detachListener();
+        if (!listener)
+            return;
+        // pg emits 'error' on a client whose socket dies even mid-query, and a
+        // checked-out client has no pool listener: keep one until it is back.
+        const ignore = () => { };
+        listener.on('error', ignore);
+        try {
+            await queryWithin(listener, 'unlisten *', this.heartbeatTimeoutMs);
+            listener.removeListener('error', ignore);
+            listener.release();
+        }
+        catch (error) {
+            listener.release(toError(error));
+        }
+    }
+    /** `pg_notify` on this bridge's channel. Pass the transaction's client to
+     *  announce only on commit. With `ignoreOwn`, tags the payload with
+     *  `originId`. */
+    async notify(message, queryable = this.pool) {
+        const payload = this.ignoreOwn ? { ...message, origin: this.originId } : message;
+        await queryable.query('select pg_notify($1, $2)', [this.channel, JSON.stringify(payload)]);
+    }
+    async connectListener(initial) {
+        if (this.stopped || this.connecting || this.listener)
+            return;
+        this.connecting = true;
+        try {
+            const client = await this.pool.connect();
+            // A freshly checked-out pool client carries no 'error' listener — pg-pool
+            // strips the idle one on checkout. If Cloud SQL has silently reset this
+            // (reused) connection, the very next query makes pg emit an 'error' event
+            // on the client even while that query is in flight; with no listener Node
+            // escalates it to an uncaught exception and the process exits (the Reader
+            // 2026-09-10 crash, which hit here during a reconnect). Attach the
+            // handlers BEFORE issuing `listen`. In this pre-`listen` window
+            // this.listener is still null, so onListenerLost no-ops and the failing
+            // `listen` query below drives the cleanup and retry instead.
+            client.on('error', (error) => this.onListenerLost(client, error));
+            client.on('end', () => this.onListenerLost(client, new Error('database listener ended')));
+            try {
+                await client.query(`listen ${this.channel}`);
+            }
+            catch (error) {
+                client.removeAllListeners('error');
+                client.removeAllListeners('end');
+                client.release(toError(error));
+                throw error;
+            }
+            if (this.stopped) {
+                client.removeAllListeners('error');
+                client.removeAllListeners('end');
+                client.release();
+                return;
+            }
+            this.listener = client;
+            this.adjustPoolMin(1);
+            client.on('notification', (notification) => this.onNotification(notification.payload));
+            this.scheduleHeartbeat(client);
+            const recovered = this.connectedOnce;
+            this.connectedOnce = true;
+            this.reconnectAttempt = 0;
+            if (recovered) {
+                this.logger.info('database broadcast listener recovered', { channel: this.channel });
+                this.callApp('onRecovered', () => this.onRecovered());
+            }
+            else if (!initial) {
+                this.logger.info('database broadcast listener connected', { channel: this.channel });
+            }
+        }
+        finally {
+            this.connecting = false;
+        }
+    }
+    /** pg-pool exempts its first `min` clients, counted over all of them,
+     *  checked out or idle, from the idle timeout, and reads `options.min`
+     *  each time it decides. The listener is one of those clients, so without
+     *  this a pool with `min: 1` (FLEET_POOL_DEFAULTS) would reap its last
+     *  working connection after the idle timeout: the warm connection min
+     *  exists to keep. Raise min while the listener is held, lower it on
+     *  release. */
+    adjustPoolMin(delta) {
+        this.pool.options.min = (this.pool.options.min ?? 0) + delta;
+    }
+    /** Each heartbeat is scheduled when the previous one ends, so a slow one
+     *  never overlaps the next. */
+    scheduleHeartbeat(client) {
+        this.heartbeatTimer = setTimeout(() => {
+            this.heartbeatTimer = null;
+            if (this.listener !== client)
+                return;
+            queryWithin(client, 'select 1', this.heartbeatTimeoutMs).then(() => {
+                if (this.listener === client)
+                    this.scheduleHeartbeat(client);
+            }, (error) => this.onListenerLost(client, toError(error)));
+        }, this.heartbeatMs);
+    }
+    /** Clears the current listener, its timers and handlers, and the raised
+     *  pool min; returns the client for the caller to release. */
+    detachListener() {
+        const listener = this.listener;
+        if (!listener)
+            return null;
+        this.listener = null;
+        if (this.heartbeatTimer)
+            clearTimeout(this.heartbeatTimer);
+        this.heartbeatTimer = null;
+        listener.removeAllListeners('notification');
+        listener.removeAllListeners('error');
+        listener.removeAllListeners('end');
+        this.adjustPoolMin(-1);
+        return listener;
+    }
+    onListenerLost(client, error) {
+        if (this.stopped || this.listener !== client)
+            return;
+        this.detachListener();
+        // The pool re-adds its idle error listener on release, so a late socket
+        // error on the discarded client stays handled.
+        client.release(error);
+        this.logger.warn('database broadcast listener lost', { channel: this.channel, error: String(error) });
+        this.scheduleReconnect();
+    }
+    scheduleReconnect() {
+        if (this.stopped || this.reconnectTimer)
+            return;
+        const delays = this.reconnectDelaysMs;
+        const delay = delays[Math.min(this.reconnectAttempt++, delays.length - 1)];
+        this.reconnectTimer = setTimeout(() => {
+            this.reconnectTimer = null;
+            void this.connectListener(false).catch((error) => {
+                this.logger.warn('database broadcast reconnect failed', { channel: this.channel, error: String(error) });
+                this.scheduleReconnect();
+            });
+        }, delay);
+    }
+    onNotification(payload) {
+        if (this.stopped || !payload)
+            return;
+        let message;
+        try {
+            message = JSON.parse(payload);
+        }
+        catch (error) {
+            this.logger.warn('invalid database broadcast', {
+                channel: this.channel,
+                error: String(error),
+                payload: payload.slice(0, 200),
+            });
+            return;
+        }
+        if (this.ignoreOwn && message?.origin === this.originId)
+            return;
+        this.callApp('onMessage', () => this.onMessage(message));
+    }
+    /** A throw from the app's callback would otherwise escape through pg's
+     *  socket handler and crash the process. */
+    callApp(callback, fn) {
+        try {
+            fn();
+        }
+        catch (error) {
+            this.logger.warn('database broadcast handler failed', { channel: this.channel, callback, error });
+        }
+    }
+}
+/** Runs `sql`, failing if it has not answered within `timeoutMs`. The verdict
+ *  waits one more event-loop turn (setImmediate runs after the poll phase),
+ *  so a reply already buffered when a throttled instance gets CPU back is
+ *  read before the timer, which runs first, can convict it. */
+async function queryWithin(client, sql, timeoutMs) {
+    const query = client.query(sql);
+    // On a timeout the caller's release(error) destroys the socket, which
+    // rejects this query; nothing else awaits it.
+    query.catch(() => { });
+    let timer;
+    try {
+        await Promise.race([
+            query,
+            new Promise((_, reject) => {
+                timer = setTimeout(() => setImmediate(() => reject(new Error(`database listener: ${sql} timed out after ${timeoutMs} ms`))), timeoutMs);
+            }),
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+function toError(error) {
+    return error instanceof Error ? error : new Error(String(error));
+}

package/dist/sync/index.d.ts

@@ -3,3 +3,4 @@ export * from './schema.js';
 export * from './engine.js';
 export * from './broadcaster.js';
 export * from './session.js';
+export * from './db-bridge.js';

package/dist/sync/index.js

@@ -3,3 +3,4 @@ export * from './schema.js';
 export * from './engine.js';
 export * from './broadcaster.js';
 export * from './session.js';
+export * from './db-bridge.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.42.0",
+  "version": "3.44.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",