@proteinjs/db-driver-spanner 1.10.17 → 1.10.18

package/src/SpannerDriver.ts

@@ -8,6 +8,7 @@ import {
   tableByName,
 } from '@proteinjs/db';
 import { SpannerConfig } from './SpannerConfig';
+import { SpannerLivenessMonitor } from './SpannerLivenessMonitor';
 import { Logger } from '@proteinjs/logger';
 import { Statement } from '@proteinjs/db-query';
 import { SpannerSchemaOperations } from './SpannerSchemaOperations';
@@ -21,6 +22,7 @@ export class SpannerDriver implements DbDriver {
   private static SPANNER: Spanner;
   private static SPANNER_INSTANCE: Instance;
   private static SPANNER_DB: Database;
+  private static LIVENESS_MONITOR: SpannerLivenessMonitor;
   private logger = new Logger({ name: this.constructor.name });
   private config: SpannerConfig;
   public getTable: ((name: string) => Table<any>) | undefined;
@@ -55,6 +57,7 @@ export class SpannerDriver implements DbDriver {
   private getSpannerDb(): Database {
     if (!SpannerDriver.SPANNER_DB) {
       SpannerDriver.SPANNER_DB = this.getSpannerInstance().database(this.config.databaseName);
+      SpannerDriver.LIVENESS_MONITOR = new SpannerLivenessMonitor(SpannerDriver.SPANNER_DB).start();
     }
 
     return SpannerDriver.SPANNER_DB;
@@ -169,6 +172,7 @@ export class SpannerDriver implements DbDriver {
         message: `Failed when executing query`,
         obj: { sql, params: namedParams, errorDetails: error.details, durationMs },
       });
+      SpannerDriver.LIVENESS_MONITOR.reportError(error);
       throw error;
     }
   }
@@ -225,6 +229,7 @@ export class SpannerDriver implements DbDriver {
         message: `Failed when executing dml`,
         obj: { sql, params: namedParams, errorDetails: error.details, durationMs },
       });
+      SpannerDriver.LIVENESS_MONITOR.reportError(error);
       throw error;
     }
   }

package/src/SpannerLivenessMonitor.ts

@@ -0,0 +1,81 @@
+import { Database } from '@google-cloud/spanner';
+import { Logger } from '@proteinjs/logger';
+
+/** grpc status codes that indicate connectivity trouble rather than an application error */
+const CONNECTIVITY_GRPC_CODES = [4 /* DEADLINE_EXCEEDED */, 14 /* UNAVAILABLE */];
+
+export class SpannerLivenessMonitor {
+  private static readonly PROBE_SQL = 'SELECT 1';
+  private static readonly PROBE_TIMEOUT_MS = 10_000;
+  /** delay before each attempt; 5 attempts spanning ~2 min (fast failures) to ~4.5 min (30s-deadline failures) */
+  private static readonly PROBE_DELAYS_MS = [0, 5_000, 15_000, 30_000, 60_000];
+  private logger = new Logger({ name: this.constructor.name });
+  private checkInFlight = false;
+
+  constructor(private db: Database) {}
+
+  /** Attach the single 'error' listener; called once when the Database singleton is created. */
+  start(): this {
+    this.db.on('error', (error: any) => {
+      this.logger.warn({
+        message: `Spanner session pool emitted a background error; verifying db connectivity`,
+        obj: { code: error?.code, errorDetails: error?.details ?? String(error) },
+      });
+      void this.verifyLiveness();
+    });
+    return this;
+  }
+
+  /** Called from driver catch blocks; probes only for connectivity-shaped errors. */
+  reportError(error: any): void {
+    if (!CONNECTIVITY_GRPC_CODES.includes(error?.code)) {
+      return;
+    }
+    void this.verifyLiveness();
+  }
+
+  // --- helpers last ---
+
+  private async verifyLiveness(): Promise<void> {
+    if (this.checkInFlight) {
+      return; // coalesce: eviction sweeps emit bursts of _destroy errors (the incident evicted several sessions)
+    }
+    this.checkInFlight = true;
+    try {
+      for (let attempt = 0; attempt < SpannerLivenessMonitor.PROBE_DELAYS_MS.length; attempt++) {
+        await this.sleep(SpannerLivenessMonitor.PROBE_DELAYS_MS[attempt]);
+        try {
+          await this.probe();
+          this.logger.info({ message: `Db connectivity verified`, obj: { attempt: attempt + 1 } });
+          return;
+        } catch (error: any) {
+          this.logger.warn({
+            message: `Db connectivity probe failed`,
+            obj: { attempt: attempt + 1, errorDetails: error?.details ?? String(error) },
+          });
+        }
+      }
+      this.logger.error({
+        message: `Db unreachable after sustained probing; exiting so supervision can restart into a valid state`,
+      });
+      this.exit();
+    } finally {
+      this.checkInFlight = false;
+    }
+  }
+
+  private async probe(): Promise<void> {
+    await this.db.run({
+      sql: SpannerLivenessMonitor.PROBE_SQL,
+      gaxOptions: { timeout: SpannerLivenessMonitor.PROBE_TIMEOUT_MS },
+    });
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  private exit(): void {
+    process.exit(1);
+  }
+}

package/test/Liveness.test.ts

@@ -0,0 +1,49 @@
+import { EventEmitter } from 'events';
+import { Database } from '@google-cloud/spanner';
+import { SpannerDriver, SpannerLivenessMonitor } from '@proteinjs/db-driver-spanner';
+
+const spannerDriver = new SpannerDriver({
+  projectId: 'proteinjs-test',
+  instanceName: 'proteinjs-test',
+  databaseName: 'test',
+});
+
+const waitFor = async (condition: () => boolean, timeoutMs = 15_000): Promise<void> => {
+  const start = Date.now();
+  while (!condition()) {
+    if (Date.now() - start > timeoutMs) {
+      throw new Error(`Timed out after ${timeoutMs}ms waiting for condition`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, 25));
+  }
+};
+
+describe('Liveness', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test('session-pool background error does not crash the process; liveness probe runs and succeeds', async () => {
+    await spannerDriver.createDbIfNotExists();
+    const db = (spannerDriver as unknown as { getSpannerDb(): Database }).getSpannerDb();
+    const monitor = (SpannerDriver as unknown as { LIVENESS_MONITOR: SpannerLivenessMonitor }).LIVENESS_MONITOR;
+    const exitSpy = jest.spyOn(monitor as any, 'exit').mockImplementation(() => {});
+    const probeSpy = jest.spyOn(monitor as any, 'probe');
+
+    // Exercise the real library-side path: SessionPool.emit('error') is forwarded to the
+    // Database (database.js:156), which our monitor listens on. Before this fix, this emit
+    // was an unhandled 'error' event and killed the process — this test completing at all
+    // is the assertion that the crash is fixed.
+    (db as unknown as { pool_: EventEmitter }).pool_.emit(
+      'error',
+      Object.assign(new Error('4 DEADLINE_EXCEEDED: fake'), { code: 4 })
+    );
+
+    await waitFor(() => probeSpy.mock.calls.length > 0);
+    // the probe ran against the live emulator and must resolve
+    await probeSpy.mock.results[0].value;
+
+    expect(probeSpy).toHaveBeenCalledTimes(1);
+    expect(exitSpy).not.toHaveBeenCalled();
+  });
+});

package/test/SpannerLivenessMonitor.test.ts

@@ -0,0 +1,104 @@
+import { Database } from '@google-cloud/spanner';
+import { SpannerLivenessMonitor } from '@proteinjs/db-driver-spanner';
+
+type MonitorInternals = {
+  probe(): Promise<void>;
+  exit(): void;
+  verifyLiveness(): Promise<void>;
+  checkInFlight: boolean;
+  logger: {
+    info(args: { message: string; obj?: any }): void;
+    warn(args: { message: string; obj?: any }): void;
+    error(args: { message: string; obj?: any }): void;
+  };
+};
+
+/** total of PROBE_DELAYS_MS [0, 5_000, 15_000, 30_000, 60_000] */
+const ALL_PROBE_DELAYS_MS = 110_000;
+
+describe('SpannerLivenessMonitor', () => {
+  let monitor: SpannerLivenessMonitor;
+  let internals: MonitorInternals;
+  let probeSpy: jest.SpyInstance;
+  let exitSpy: jest.SpyInstance;
+  let errorLogSpy: jest.SpyInstance;
+
+  beforeEach(() => {
+    jest.useFakeTimers();
+    monitor = new SpannerLivenessMonitor({} as Database);
+    internals = monitor as unknown as MonitorInternals;
+    probeSpy = jest.spyOn(monitor as any, 'probe');
+    exitSpy = jest.spyOn(monitor as any, 'exit').mockImplementation(() => {});
+    jest.spyOn(internals.logger, 'info').mockImplementation(() => {});
+    jest.spyOn(internals.logger, 'warn').mockImplementation(() => {});
+    errorLogSpy = jest.spyOn(internals.logger, 'error').mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    jest.useRealTimers();
+    jest.restoreAllMocks();
+  });
+
+  test('first probe succeeds: no exit, checkInFlight reset', async () => {
+    probeSpy.mockResolvedValue(undefined);
+
+    const check = internals.verifyLiveness();
+    await jest.advanceTimersByTimeAsync(0);
+    await check;
+
+    expect(probeSpy).toHaveBeenCalledTimes(1);
+    expect(exitSpy).not.toHaveBeenCalled();
+    expect(internals.checkInFlight).toBe(false);
+  });
+
+  test('probe fails twice then succeeds: recovers, no exit', async () => {
+    probeSpy
+      .mockRejectedValueOnce(new Error('4 DEADLINE_EXCEEDED: fake'))
+      .mockRejectedValueOnce(new Error('4 DEADLINE_EXCEEDED: fake'))
+      .mockResolvedValue(undefined);
+
+    const check = internals.verifyLiveness();
+    await jest.advanceTimersByTimeAsync(ALL_PROBE_DELAYS_MS);
+    await check;
+
+    expect(probeSpy).toHaveBeenCalledTimes(3);
+    expect(exitSpy).not.toHaveBeenCalled();
+    expect(internals.checkInFlight).toBe(false);
+  });
+
+  test('all 5 probes fail: exit called exactly once, fatal log emitted', async () => {
+    probeSpy.mockRejectedValue(new Error('4 DEADLINE_EXCEEDED: fake'));
+
+    const check = internals.verifyLiveness();
+    await jest.advanceTimersByTimeAsync(ALL_PROBE_DELAYS_MS);
+    await check;
+
+    expect(probeSpy).toHaveBeenCalledTimes(5);
+    expect(exitSpy).toHaveBeenCalledTimes(1);
+    expect(errorLogSpy).toHaveBeenCalledWith(
+      expect.objectContaining({ message: expect.stringContaining('Db unreachable after sustained probing') })
+    );
+  });
+
+  test('burst coalescing: reportError while a check is in flight triggers one probe cycle', async () => {
+    probeSpy.mockResolvedValue(undefined);
+
+    monitor.reportError(Object.assign(new Error('4 DEADLINE_EXCEEDED: fake'), { code: 4 }));
+    monitor.reportError(Object.assign(new Error('14 UNAVAILABLE: fake'), { code: 14 }));
+    await jest.advanceTimersByTimeAsync(0);
+
+    expect(probeSpy).toHaveBeenCalledTimes(1);
+    expect(exitSpy).not.toHaveBeenCalled();
+    expect(internals.checkInFlight).toBe(false);
+  });
+
+  test('non-connectivity error code: no probe', async () => {
+    probeSpy.mockResolvedValue(undefined);
+
+    monitor.reportError(Object.assign(new Error('6 ALREADY_EXISTS: fake'), { code: 6 }));
+    await jest.advanceTimersByTimeAsync(0);
+
+    expect(probeSpy).not.toHaveBeenCalled();
+    expect(exitSpy).not.toHaveBeenCalled();
+  });
+});