@proteinjs/db-driver-spanner 1.13.0 → 1.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/LICENSE +21 -0
- package/dist/generated/index.js +1 -1
- package/dist/generated/index.js.map +1 -1
- package/dist/generated/test/index.js +1 -1
- package/dist/generated/test/index.js.map +1 -1
- package/dist/src/SpannerConfig.d.ts +22 -0
- package/dist/src/SpannerConfig.d.ts.map +1 -1
- package/dist/src/SpannerDriver.d.ts +46 -11
- package/dist/src/SpannerDriver.d.ts.map +1 -1
- package/dist/src/SpannerDriver.js +170 -25
- package/dist/src/SpannerDriver.js.map +1 -1
- package/dist/src/SpannerLivenessMonitor.d.ts +25 -0
- package/dist/src/SpannerLivenessMonitor.d.ts.map +1 -1
- package/dist/src/SpannerLivenessMonitor.js +48 -2
- package/dist/src/SpannerLivenessMonitor.js.map +1 -1
- package/dist/test/SessionPoolExhaustion.test.d.ts +2 -0
- package/dist/test/SessionPoolExhaustion.test.d.ts.map +1 -0
- package/dist/test/SessionPoolExhaustion.test.js +230 -0
- package/dist/test/SessionPoolExhaustion.test.js.map +1 -0
- package/dist/test/SpannerLivenessMonitor.test.js +42 -0
- package/dist/test/SpannerLivenessMonitor.test.js.map +1 -1
- package/dist/test/SpannerOperationDeadline.test.d.ts +2 -0
- package/dist/test/SpannerOperationDeadline.test.d.ts.map +1 -0
- package/dist/test/SpannerOperationDeadline.test.js +227 -0
- package/dist/test/SpannerOperationDeadline.test.js.map +1 -0
- package/dist/test/TransactionSafety.test.d.ts +2 -0
- package/dist/test/TransactionSafety.test.d.ts.map +1 -0
- package/dist/test/TransactionSafety.test.js +325 -0
- package/dist/test/TransactionSafety.test.js.map +1 -0
- package/generated/index.ts +6 -9
- package/generated/test/index.ts +8 -11
- package/package.json +6 -5
- package/src/SpannerConfig.ts +22 -0
- package/src/SpannerDriver.ts +185 -27
- package/src/SpannerLivenessMonitor.ts +61 -3
- package/test/SessionPoolExhaustion.test.ts +127 -0
- package/test/SpannerLivenessMonitor.test.ts +55 -0
- package/test/SpannerOperationDeadline.test.ts +156 -0
- package/test/TransactionSafety.test.ts +157 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Database } from '@google-cloud/spanner';
|
|
1
|
+
import { Database, SessionPool } from '@google-cloud/spanner';
|
|
2
2
|
import { Logger } from '@proteinjs/logger';
|
|
3
3
|
|
|
4
4
|
/** grpc status codes that indicate connectivity trouble rather than an application error */
|
|
@@ -14,36 +14,88 @@ const CONNECTIVITY_GRPC_CODES = [4 /* DEADLINE_EXCEEDED */, 14 /* UNAVAILABLE */
|
|
|
14
14
|
*/
|
|
15
15
|
const RESTART_REQUEST_EXIT_CODE = 86;
|
|
16
16
|
|
|
17
|
+
/** Session pool gauge (P4a): the four numbers that make pool exhaustion observable. */
|
|
18
|
+
export type SpannerSessionPoolStats = {
|
|
19
|
+
size: number;
|
|
20
|
+
available: number;
|
|
21
|
+
borrowed: number;
|
|
22
|
+
totalWaiters: number;
|
|
23
|
+
};
|
|
24
|
+
|
|
17
25
|
export class SpannerLivenessMonitor {
|
|
18
26
|
private static readonly PROBE_SQL = 'SELECT 1';
|
|
19
27
|
private static readonly PROBE_TIMEOUT_MS = 10_000;
|
|
20
28
|
/** delay before each attempt; 5 attempts spanning ~2 min (fast failures) to ~4.5 min (30s-deadline failures) */
|
|
21
29
|
private static readonly PROBE_DELAYS_MS = [0, 5_000, 15_000, 30_000, 60_000];
|
|
30
|
+
/** waiters > 0 is the wedge signature — warn on sight, but at most once per interval per process */
|
|
31
|
+
private static readonly POOL_PRESSURE_WARN_INTERVAL_MS = 10_000;
|
|
22
32
|
private logger = new Logger({ name: this.constructor.name });
|
|
23
33
|
private checkInFlight = false;
|
|
34
|
+
private stopped = false;
|
|
35
|
+
private lastPoolPressureWarnMs = Number.NEGATIVE_INFINITY;
|
|
24
36
|
|
|
25
37
|
constructor(private db: Database) {}
|
|
26
38
|
|
|
27
39
|
/** Attach the single 'error' listener; called once when the Database singleton is created. */
|
|
28
40
|
start(): this {
|
|
29
41
|
this.db.on('error', (error: any) => {
|
|
42
|
+
if (this.stopped) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
30
45
|
this.logger.warn({
|
|
31
46
|
message: `Spanner session pool emitted a background error; verifying db connectivity`,
|
|
32
|
-
obj: { code: error?.code, errorDetails: error?.details ?? String(error) },
|
|
47
|
+
obj: { code: error?.code, errorDetails: error?.details ?? String(error), pool: this.poolStats() },
|
|
33
48
|
});
|
|
34
49
|
void this.verifyLiveness();
|
|
35
50
|
});
|
|
36
51
|
return this;
|
|
37
52
|
}
|
|
38
53
|
|
|
54
|
+
/**
|
|
55
|
+
* Retire this monitor when its client is recycled (channel-death recycle in SpannerDriver):
|
|
56
|
+
* a stopped monitor must never escalate to process exit for a channel the driver has already
|
|
57
|
+
* abandoned — the replacement client gets a fresh monitor.
|
|
58
|
+
*/
|
|
59
|
+
stop(): void {
|
|
60
|
+
this.stopped = true;
|
|
61
|
+
}
|
|
62
|
+
|
|
39
63
|
/** Called from driver catch blocks; probes only for connectivity-shaped errors. */
|
|
40
64
|
reportError(error: any): void {
|
|
41
|
-
if (!CONNECTIVITY_GRPC_CODES.includes(error?.code)) {
|
|
65
|
+
if (this.stopped || !CONNECTIVITY_GRPC_CODES.includes(error?.code)) {
|
|
42
66
|
return;
|
|
43
67
|
}
|
|
44
68
|
void this.verifyLiveness();
|
|
45
69
|
}
|
|
46
70
|
|
|
71
|
+
poolStats(): SpannerSessionPoolStats {
|
|
72
|
+
const pool = (this.db as unknown as { pool_: SessionPool }).pool_;
|
|
73
|
+
return { size: pool.size, available: pool.available, borrowed: pool.borrowed, totalWaiters: pool.totalWaiters };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Warn whenever operations are queued waiting on the session pool (the silent-wedge
|
|
78
|
+
* signature — with an infinite acquireTimeout a wedged pool otherwise produces no signal at
|
|
79
|
+
* all). Called by the driver as ops are issued; throttled so a contended burst emits one
|
|
80
|
+
* line per interval, not one per queued op.
|
|
81
|
+
*/
|
|
82
|
+
logPoolPressure(nowMs = Date.now()): void {
|
|
83
|
+
const stats = this.poolStats();
|
|
84
|
+
if (stats.totalWaiters === 0) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (nowMs - this.lastPoolPressureWarnMs < SpannerLivenessMonitor.POOL_PRESSURE_WARN_INTERVAL_MS) {
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
this.lastPoolPressureWarnMs = nowMs;
|
|
93
|
+
this.logger.warn({
|
|
94
|
+
message: `Spanner session pool under pressure: operations are waiting for a session`,
|
|
95
|
+
obj: stats,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
|
|
47
99
|
// --- helpers last ---
|
|
48
100
|
|
|
49
101
|
private async verifyLiveness(): Promise<void> {
|
|
@@ -54,6 +106,9 @@ export class SpannerLivenessMonitor {
|
|
|
54
106
|
try {
|
|
55
107
|
for (let attempt = 0; attempt < SpannerLivenessMonitor.PROBE_DELAYS_MS.length; attempt++) {
|
|
56
108
|
await this.sleep(SpannerLivenessMonitor.PROBE_DELAYS_MS[attempt]);
|
|
109
|
+
if (this.stopped) {
|
|
110
|
+
return; // client recycled mid-cycle — this channel's fate no longer matters
|
|
111
|
+
}
|
|
57
112
|
try {
|
|
58
113
|
await this.probe();
|
|
59
114
|
this.logger.info({ message: `Db connectivity verified`, obj: { attempt: attempt + 1 } });
|
|
@@ -65,6 +120,9 @@ export class SpannerLivenessMonitor {
|
|
|
65
120
|
});
|
|
66
121
|
}
|
|
67
122
|
}
|
|
123
|
+
if (this.stopped) {
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
68
126
|
this.logger.error({
|
|
69
127
|
message: `Db unreachable after sustained probing; exiting restart-requested (code ${RESTART_REQUEST_EXIT_CODE}) so supervision respawns into a valid state`,
|
|
70
128
|
});
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { Database, SessionPool } from '@google-cloud/spanner';
|
|
2
|
+
import { Db, Record, StringColumn, Table, withRecordColumns } from '@proteinjs/db';
|
|
3
|
+
import { TransactionContext } from '@proteinjs/db-transaction-context';
|
|
4
|
+
import { SpannerDriver } from '@proteinjs/db-driver-spanner';
|
|
5
|
+
import { getDropTestTable } from './util/getDropTestTable';
|
|
6
|
+
import { SpannerEmulatorProvisioner } from './util/SpannerEmulatorProvisioner';
|
|
7
|
+
import '../generated/test/index';
|
|
8
|
+
|
|
9
|
+
interface WedgeEmployee extends Record {
|
|
10
|
+
name: string;
|
|
11
|
+
department?: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
class WedgeEmployeeTestTable extends Table<WedgeEmployee> {
|
|
15
|
+
name = 'db_test_pool_wedge_employee';
|
|
16
|
+
columns = withRecordColumns<WedgeEmployee>({
|
|
17
|
+
name: new StringColumn('name'),
|
|
18
|
+
department: new StringColumn('department'),
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const employeeTable: Table<WedgeEmployee> = new WedgeEmployeeTestTable();
|
|
23
|
+
// Local table — not in any reflection source graph, so thread getTable explicitly.
|
|
24
|
+
const getTable = () => employeeTable;
|
|
25
|
+
const spannerConfig = {
|
|
26
|
+
projectId: 'proteinjs-test',
|
|
27
|
+
instanceName: 'proteinjs-test',
|
|
28
|
+
databaseName: 'test',
|
|
29
|
+
};
|
|
30
|
+
// max: 1 makes pool mechanics observable in miniature. This file must stay its own jest
|
|
31
|
+
// file — SpannerDriver.SPANNER_DB is a process-wide static, so the max-1 pool would leak
|
|
32
|
+
// into other suites.
|
|
33
|
+
const spannerDriver = new SpannerDriver(
|
|
34
|
+
{
|
|
35
|
+
...spannerConfig,
|
|
36
|
+
sessionPoolOptions: { max: 1, min: 1, acquireTimeout: 2000, fail: true },
|
|
37
|
+
},
|
|
38
|
+
getTable
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
const getSessionPool = (): SessionPool => {
|
|
42
|
+
const db = (spannerDriver as unknown as { getSpannerDb(): Database }).getSpannerDb();
|
|
43
|
+
return (db as unknown as { pool_: SessionPool }).pool_;
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Pool mechanics at max=1 (plans/DB_PERF_PLAN.md P2):
|
|
48
|
+
* 1. The historical wedge — an operation inside a transaction acquiring a SECOND session while
|
|
49
|
+
* the transaction holds the only one — is DESIGNED OUT by the stateless transaction
|
|
50
|
+
* contract: every operation inside the body rides the held session, so the shape that
|
|
51
|
+
* bricked the process cannot be expressed. Proven here at max=1.
|
|
52
|
+
* 2. The exhaustion class that remains REAL — more concurrent transactions than sessions —
|
|
53
|
+
* is documented with two parallel transactions.
|
|
54
|
+
*/
|
|
55
|
+
describe('Session pool at max=1 (wedge designed out; real exhaustion documented)', () => {
|
|
56
|
+
const dropTable = getDropTestTable(spannerDriver);
|
|
57
|
+
const preconstructedDb = new Db(spannerDriver, getTable, new TransactionContext());
|
|
58
|
+
const txnDb = new Db(spannerDriver, getTable, new TransactionContext());
|
|
59
|
+
|
|
60
|
+
beforeAll(async () => {
|
|
61
|
+
await SpannerEmulatorProvisioner.ensureProvisioned(spannerConfig);
|
|
62
|
+
await spannerDriver.createDbIfNotExists();
|
|
63
|
+
// Setup issues overlapping schema-metadata queries; with fail: true the second acquisition
|
|
64
|
+
// would error instantly at max=1. Let setup queue on the single session, then restore the
|
|
65
|
+
// constructed fail-fast posture for the tests.
|
|
66
|
+
const pool = getSessionPool();
|
|
67
|
+
pool.options.fail = false;
|
|
68
|
+
await spannerDriver.getTableManager().loadTable(employeeTable);
|
|
69
|
+
pool.options.fail = true;
|
|
70
|
+
}, 60000);
|
|
71
|
+
|
|
72
|
+
afterAll(async () => {
|
|
73
|
+
getSessionPool().options.fail = false; // teardown queries queue like setup's
|
|
74
|
+
await dropTable(employeeTable);
|
|
75
|
+
await SpannerEmulatorProvisioner.release();
|
|
76
|
+
}, 60000);
|
|
77
|
+
|
|
78
|
+
test('SpannerConfig.sessionPoolOptions reach the session pool', () => {
|
|
79
|
+
const pool = getSessionPool();
|
|
80
|
+
expect(pool.options.max).toBe(1);
|
|
81
|
+
expect(pool.options.min).toBe(1);
|
|
82
|
+
expect(pool.options.acquireTimeout).toBe(2000);
|
|
83
|
+
expect(pool.options.fail).toBe(true);
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test('wedge DESIGNED OUT: at max=1, a pre-constructed Db inside a transaction rides the held session and just works', async () => {
|
|
87
|
+
const employee: Omit<WedgeEmployee, keyof Record> = {
|
|
88
|
+
name: 'PoolWedgeDesignedOut',
|
|
89
|
+
department: 'Engineering',
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
// Under construction-time binding this exact shape was the wedge: the inner query needed a
|
|
93
|
+
// second session the pool didn't have. Statelessness makes it ride the transaction's own
|
|
94
|
+
// session — no acquisition, no exhaustion, correct read.
|
|
95
|
+
const inserted = await txnDb.runTransaction(async () => {
|
|
96
|
+
const emp = await txnDb.insert(employeeTable, employee);
|
|
97
|
+
const seen = await preconstructedDb.query(employeeTable, { name: employee.name });
|
|
98
|
+
expect(seen.length).toBe(1);
|
|
99
|
+
return emp;
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
const committed = await preconstructedDb.query(employeeTable, { id: inserted.id });
|
|
103
|
+
expect(committed.length).toBe(1);
|
|
104
|
+
await txnDb.delete(employeeTable, { id: inserted.id });
|
|
105
|
+
}, 30000);
|
|
106
|
+
|
|
107
|
+
test('real exhaustion class: a second PARALLEL transaction errors fast when the pool is spent (fail: true)', async () => {
|
|
108
|
+
let releaseHold!: () => void;
|
|
109
|
+
const holdGate = new Promise<void>((resolve) => (releaseHold = resolve));
|
|
110
|
+
|
|
111
|
+
// First transaction takes the only session and holds it open on the gate.
|
|
112
|
+
const holder = txnDb.runTransaction(async () => {
|
|
113
|
+
await holdGate;
|
|
114
|
+
return 'held';
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Second transaction cannot get a session; with fail: true it errors immediately instead
|
|
118
|
+
// of joining the pool's silent infinite FIFO (the production-default hang this file's
|
|
119
|
+
// options exist to surface).
|
|
120
|
+
await expect(preconstructedDb.runTransaction(async () => 'never-runs')).rejects.toMatchObject({
|
|
121
|
+
name: 'SessionPoolExhaustedError',
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
releaseHold();
|
|
125
|
+
await expect(holder).resolves.toBe('held');
|
|
126
|
+
}, 30000);
|
|
127
|
+
});
|
|
@@ -117,3 +117,58 @@ describe('SpannerLivenessMonitor', () => {
|
|
|
117
117
|
expect(exitSpy).not.toHaveBeenCalled();
|
|
118
118
|
});
|
|
119
119
|
});
|
|
120
|
+
|
|
121
|
+
describe('SpannerLivenessMonitor pool gauge (P4a)', () => {
|
|
122
|
+
const fakePool = { size: 25, available: 20, borrowed: 5, totalWaiters: 0 };
|
|
123
|
+
let monitor: SpannerLivenessMonitor;
|
|
124
|
+
let warnLogSpy: jest.SpyInstance;
|
|
125
|
+
|
|
126
|
+
beforeEach(() => {
|
|
127
|
+
fakePool.size = 25;
|
|
128
|
+
fakePool.available = 20;
|
|
129
|
+
fakePool.borrowed = 5;
|
|
130
|
+
fakePool.totalWaiters = 0;
|
|
131
|
+
monitor = new SpannerLivenessMonitor({ pool_: fakePool } as unknown as Database);
|
|
132
|
+
warnLogSpy = jest.spyOn((monitor as unknown as MonitorInternals).logger, 'warn').mockImplementation(() => {});
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
afterEach(() => {
|
|
136
|
+
jest.restoreAllMocks();
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('poolStats surfaces the four pool numbers', () => {
|
|
140
|
+
expect(monitor.poolStats()).toEqual({ size: 25, available: 20, borrowed: 5, totalWaiters: 0 });
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test('no waiters: no pressure warning', () => {
|
|
144
|
+
monitor.logPoolPressure(1_000);
|
|
145
|
+
expect(warnLogSpy).not.toHaveBeenCalled();
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('waiters > 0: warns with the four pool numbers', () => {
|
|
149
|
+
fakePool.available = 0;
|
|
150
|
+
fakePool.borrowed = 25;
|
|
151
|
+
fakePool.totalWaiters = 3;
|
|
152
|
+
|
|
153
|
+
monitor.logPoolPressure(1_000);
|
|
154
|
+
|
|
155
|
+
expect(warnLogSpy).toHaveBeenCalledTimes(1);
|
|
156
|
+
expect(warnLogSpy).toHaveBeenCalledWith(
|
|
157
|
+
expect.objectContaining({
|
|
158
|
+
message: expect.stringContaining('session pool under pressure'),
|
|
159
|
+
obj: { size: 25, available: 0, borrowed: 25, totalWaiters: 3 },
|
|
160
|
+
})
|
|
161
|
+
);
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
test('pressure warnings are throttled to one per interval', () => {
|
|
165
|
+
fakePool.totalWaiters = 3;
|
|
166
|
+
|
|
167
|
+
monitor.logPoolPressure(1_000);
|
|
168
|
+
monitor.logPoolPressure(2_000); // within the 10s interval — suppressed
|
|
169
|
+
expect(warnLogSpy).toHaveBeenCalledTimes(1);
|
|
170
|
+
|
|
171
|
+
monitor.logPoolPressure(11_000); // interval elapsed — warns again
|
|
172
|
+
expect(warnLogSpy).toHaveBeenCalledTimes(2);
|
|
173
|
+
});
|
|
174
|
+
});
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { SpannerDriver } from '@proteinjs/db-driver-spanner';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Op-level deadline + channel-death recycle (2026-08-06 overnight wedge: dead gRPC channel
|
|
5
|
+
* after Mac sleep → every op hung forever, borrowed sessions never returned, heap OOM).
|
|
6
|
+
* Pure unit tests: fake Database/Transaction/monitor injected into the driver's process-wide
|
|
7
|
+
* statics via typed casts; no emulator involved.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
type DriverStatics = {
|
|
11
|
+
SPANNER?: unknown;
|
|
12
|
+
SPANNER_INSTANCE?: unknown;
|
|
13
|
+
SPANNER_DB?: unknown;
|
|
14
|
+
LIVENESS_MONITOR?: unknown;
|
|
15
|
+
CLIENT_GENERATION: number;
|
|
16
|
+
CONSECUTIVE_DEADLINE_FAILURES: number;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const statics = SpannerDriver as unknown as DriverStatics;
|
|
20
|
+
|
|
21
|
+
const fakeMonitor = {
|
|
22
|
+
logPoolPressure: () => undefined,
|
|
23
|
+
poolStats: () => ({ size: 0, available: 0, borrowed: 0, totalWaiters: 0 }),
|
|
24
|
+
reportError: () => undefined,
|
|
25
|
+
stop: () => undefined,
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const hang = () => new Promise<never>(() => undefined);
|
|
29
|
+
|
|
30
|
+
const generateStatement = (() => ({ sql: 'UPDATE t SET x = 1', namedParams: { params: {} } })) as any;
|
|
31
|
+
|
|
32
|
+
const waitFor = async (condition: () => boolean, timeoutMs: number, label: string) => {
|
|
33
|
+
const deadline = Date.now() + timeoutMs;
|
|
34
|
+
while (Date.now() < deadline) {
|
|
35
|
+
if (condition()) {
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
39
|
+
}
|
|
40
|
+
throw new Error(`Timed out waiting for: ${label}`);
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const makeDriver = (config: { operationDeadlineMs: number; deadlineFailuresBeforeRecycle: number }, db: unknown) => {
|
|
44
|
+
statics.SPANNER_DB = db;
|
|
45
|
+
statics.LIVENESS_MONITOR = fakeMonitor;
|
|
46
|
+
const driver = new SpannerDriver({
|
|
47
|
+
projectId: 'fake',
|
|
48
|
+
instanceName: 'fake',
|
|
49
|
+
databaseName: 'fake',
|
|
50
|
+
...config,
|
|
51
|
+
});
|
|
52
|
+
// deadline/error logs are expected output of these tests — keep the run quiet
|
|
53
|
+
jest.spyOn((driver as any).logger, 'error').mockImplementation(() => undefined);
|
|
54
|
+
jest.spyOn((driver as any).logger, 'warn').mockImplementation(() => undefined);
|
|
55
|
+
return driver;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
describe('Spanner op deadlines', () => {
|
|
59
|
+
beforeEach(() => {
|
|
60
|
+
statics.CLIENT_GENERATION = 0;
|
|
61
|
+
statics.CONSECUTIVE_DEADLINE_FAILURES = 0;
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
afterEach(() => {
|
|
65
|
+
statics.SPANNER = undefined;
|
|
66
|
+
statics.SPANNER_INSTANCE = undefined;
|
|
67
|
+
statics.SPANNER_DB = undefined;
|
|
68
|
+
statics.LIVENESS_MONITOR = undefined;
|
|
69
|
+
statics.CONSECUTIVE_DEADLINE_FAILURES = 0;
|
|
70
|
+
jest.restoreAllMocks();
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test('a hanging query op fails at the deadline with an error naming it, carrying the same gRPC deadline', async () => {
|
|
74
|
+
const run = jest.fn((_request: any) => hang());
|
|
75
|
+
const driver = makeDriver({ operationDeadlineMs: 150, deadlineFailuresBeforeRecycle: 99 }, { run });
|
|
76
|
+
|
|
77
|
+
await expect(driver.runQuery(generateStatement)).rejects.toThrow(/150ms deadline.*spanner query/);
|
|
78
|
+
|
|
79
|
+
// The gRPC deadline on the request is the lever that cancels the RPC on a dead channel —
|
|
80
|
+
// the library's stream error path is what returns the borrowed session to the pool.
|
|
81
|
+
expect(run.mock.calls[0][0]).toMatchObject({ gaxOptions: { timeout: 150 } });
|
|
82
|
+
}, 5000);
|
|
83
|
+
|
|
84
|
+
test('a dml transaction hung on a dead channel (dml AND rollback hang) still RETURNS its session', async () => {
|
|
85
|
+
// Fake Database.runTransactionAsync with the library's real session contract: the session
|
|
86
|
+
// is released only when the run function settles. Every await inside the driver's run
|
|
87
|
+
// function must therefore be deadline-bounded, or the session leaks forever.
|
|
88
|
+
const released: string[] = [];
|
|
89
|
+
const transaction = {
|
|
90
|
+
runUpdate: jest.fn(() => hang()),
|
|
91
|
+
rollback: jest.fn(() => hang()),
|
|
92
|
+
commit: jest.fn(() => hang()),
|
|
93
|
+
};
|
|
94
|
+
const db = {
|
|
95
|
+
runTransactionAsync: async (fn: (transaction: unknown) => Promise<unknown>) => {
|
|
96
|
+
try {
|
|
97
|
+
return await fn(transaction);
|
|
98
|
+
} finally {
|
|
99
|
+
released.push('session');
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
const driver = makeDriver({ operationDeadlineMs: 150, deadlineFailuresBeforeRecycle: 99 }, db);
|
|
104
|
+
|
|
105
|
+
await expect(driver.runDml(generateStatement)).rejects.toThrow(/deadline/);
|
|
106
|
+
|
|
107
|
+
// dml deadline (~150ms) → bounded rollback deadline (~150ms more) → run function settles
|
|
108
|
+
// → the library's finally releases the session. A leak shows up as this wait timing out.
|
|
109
|
+
await waitFor(() => released.length === 1, 2000, 'session returned to the pool');
|
|
110
|
+
expect(transaction.rollback).toHaveBeenCalled();
|
|
111
|
+
}, 5000);
|
|
112
|
+
|
|
113
|
+
test('channel-death recycle: 3 consecutive deadline failures recycle the client once; success resets the count', async () => {
|
|
114
|
+
let hangOps = true;
|
|
115
|
+
const db = { run: jest.fn(() => (hangOps ? hang() : Promise.resolve([[]]))) };
|
|
116
|
+
const driver = makeDriver({ operationDeadlineMs: 100, deadlineFailuresBeforeRecycle: 3 }, db);
|
|
117
|
+
const recycleSpy = jest.spyOn(driver as any, 'recycleClient').mockImplementation(() => {
|
|
118
|
+
statics.CLIENT_GENERATION += 1;
|
|
119
|
+
});
|
|
120
|
+
const failingOp = () => expect(driver.runQuery(generateStatement)).rejects.toThrow(/deadline/);
|
|
121
|
+
|
|
122
|
+
await failingOp();
|
|
123
|
+
await failingOp();
|
|
124
|
+
expect(recycleSpy).not.toHaveBeenCalled();
|
|
125
|
+
|
|
126
|
+
// Any success resets the consecutive count.
|
|
127
|
+
hangOps = false;
|
|
128
|
+
await driver.runQuery(generateStatement);
|
|
129
|
+
expect(statics.CONSECUTIVE_DEADLINE_FAILURES).toBe(0);
|
|
130
|
+
|
|
131
|
+
hangOps = true;
|
|
132
|
+
await failingOp();
|
|
133
|
+
await failingOp();
|
|
134
|
+
expect(recycleSpy).not.toHaveBeenCalled(); // the reset really pushed the threshold out
|
|
135
|
+
|
|
136
|
+
await failingOp();
|
|
137
|
+
expect(recycleSpy).toHaveBeenCalledTimes(1); // 3rd consecutive failure → one recycle
|
|
138
|
+
expect(statics.CONSECUTIVE_DEADLINE_FAILURES).toBe(0);
|
|
139
|
+
}, 10000);
|
|
140
|
+
|
|
141
|
+
test('deadline failures from a recycled (stale-generation) client never count against the fresh channel', async () => {
|
|
142
|
+
const db = { run: jest.fn(() => hang()) };
|
|
143
|
+
const driver = makeDriver({ operationDeadlineMs: 100, deadlineFailuresBeforeRecycle: 1 }, db);
|
|
144
|
+
const recycleSpy = jest.spyOn(driver as any, 'recycleClient').mockImplementation(() => {
|
|
145
|
+
statics.CLIENT_GENERATION += 1;
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
const pending = expect(driver.runQuery(generateStatement)).rejects.toThrow(/deadline/);
|
|
149
|
+
// The client is recycled while the op is in flight — its failure belongs to the old channel.
|
|
150
|
+
statics.CLIENT_GENERATION += 1;
|
|
151
|
+
await pending;
|
|
152
|
+
|
|
153
|
+
expect(recycleSpy).not.toHaveBeenCalled();
|
|
154
|
+
expect(statics.CONSECUTIVE_DEADLINE_FAILURES).toBe(0);
|
|
155
|
+
}, 5000);
|
|
156
|
+
});
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import { Db, QueryBuilderFactory, Record, StringColumn, Table, withRecordColumns } from '@proteinjs/db';
|
|
2
|
+
import { TransactionContext } from '@proteinjs/db-transaction-context';
|
|
3
|
+
import { SpannerDriver } from '@proteinjs/db-driver-spanner';
|
|
4
|
+
import { getDropTestTable } from './util/getDropTestTable';
|
|
5
|
+
import { SpannerEmulatorProvisioner } from './util/SpannerEmulatorProvisioner';
|
|
6
|
+
import '../generated/test/index';
|
|
7
|
+
|
|
8
|
+
interface SafetyEmployee extends Record {
|
|
9
|
+
name: string;
|
|
10
|
+
department?: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
class SafetyEmployeeTestTable extends Table<SafetyEmployee> {
|
|
14
|
+
name = 'db_test_txn_safety_employee';
|
|
15
|
+
columns = withRecordColumns<SafetyEmployee>({
|
|
16
|
+
name: new StringColumn('name'),
|
|
17
|
+
department: new StringColumn('department'),
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const employeeTable: Table<SafetyEmployee> = new SafetyEmployeeTestTable();
|
|
22
|
+
// Local table — not in any reflection source graph, so thread getTable explicitly.
|
|
23
|
+
const getTable = () => employeeTable;
|
|
24
|
+
const spannerConfig = {
|
|
25
|
+
projectId: 'proteinjs-test',
|
|
26
|
+
instanceName: 'proteinjs-test',
|
|
27
|
+
databaseName: 'test',
|
|
28
|
+
};
|
|
29
|
+
const spannerDriver = new SpannerDriver(spannerConfig, getTable);
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The STATELESS transaction contract (plans/DB_PERF_PLAN.md P2, superseding the
|
|
33
|
+
* construction-time-binding guard): Db instances carry no transaction state — every operation
|
|
34
|
+
* resolves the ambient transaction (AsyncLocalStorage) at call time. Inside a transaction body
|
|
35
|
+
* every Db rides the transaction, whenever it was constructed; outside, every Db uses the
|
|
36
|
+
* pool. The one escape shape — work spawned inside a body that outlives the transaction while
|
|
37
|
+
* holding its context — fails loudly by name (the ended-context tombstone).
|
|
38
|
+
*/
|
|
39
|
+
describe('Transaction safety (stateless contract)', () => {
|
|
40
|
+
const dropTable = getDropTestTable(spannerDriver);
|
|
41
|
+
// Constructed at describe scope, BEFORE any transaction — under the old construction-time
|
|
42
|
+
// binding this instance was the "stale Db" hazard; now it must simply ride whatever ambient
|
|
43
|
+
// transaction is active at call time.
|
|
44
|
+
const preconstructedDb = new Db(spannerDriver, getTable, new TransactionContext());
|
|
45
|
+
const txnDb = new Db(spannerDriver, getTable, new TransactionContext());
|
|
46
|
+
|
|
47
|
+
beforeAll(async () => {
|
|
48
|
+
await SpannerEmulatorProvisioner.ensureProvisioned(spannerConfig);
|
|
49
|
+
await spannerDriver.createDbIfNotExists();
|
|
50
|
+
await spannerDriver.getTableManager().loadTable(employeeTable);
|
|
51
|
+
}, 60000);
|
|
52
|
+
|
|
53
|
+
afterAll(async () => {
|
|
54
|
+
await dropTable(employeeTable);
|
|
55
|
+
await SpannerEmulatorProvisioner.release();
|
|
56
|
+
}, 60000);
|
|
57
|
+
|
|
58
|
+
test('a PRE-CONSTRUCTED Db rides the ambient transaction: its reads see the txn write, and the txn commits', async () => {
|
|
59
|
+
const employee: Omit<SafetyEmployee, keyof Record> = {
|
|
60
|
+
name: 'TxnSafetyRides',
|
|
61
|
+
department: 'Engineering',
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const inserted = await txnDb.runTransaction(async () => {
|
|
65
|
+
const emp = await txnDb.insert(employeeTable, employee);
|
|
66
|
+
// The pre-constructed instance resolves the ambient transaction at call time — it sees
|
|
67
|
+
// the uncommitted write (no second session, no stale read: the old hazard shapes).
|
|
68
|
+
const seenInside = await preconstructedDb.query(employeeTable, { id: emp.id });
|
|
69
|
+
expect(seenInside.length).toBe(1);
|
|
70
|
+
return emp;
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
const committed = await preconstructedDb.query(employeeTable, { id: inserted.id });
|
|
74
|
+
expect(committed.length).toBe(1);
|
|
75
|
+
await txnDb.delete(employeeTable, { id: inserted.id });
|
|
76
|
+
}, 30000);
|
|
77
|
+
|
|
78
|
+
test('a Db constructed INSIDE a transaction, used after commit, cleanly uses the pool and sees committed state', async () => {
|
|
79
|
+
let escapedDb!: Db<SafetyEmployee>;
|
|
80
|
+
const inserted = await txnDb.runTransaction(async () => {
|
|
81
|
+
escapedDb = new Db(spannerDriver, getTable, new TransactionContext());
|
|
82
|
+
return await escapedDb.insert(employeeTable, { name: 'TxnSafetyAfterCommit' } as SafetyEmployee);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
// No instance binding survives the transaction — this is a plain pool-path read now.
|
|
86
|
+
const seen = await escapedDb.query(employeeTable, { id: inserted.id });
|
|
87
|
+
expect(seen.length).toBe(1);
|
|
88
|
+
await escapedDb.delete(employeeTable, { id: inserted.id });
|
|
89
|
+
}, 30000);
|
|
90
|
+
|
|
91
|
+
test('atomicity through ANY instance: a rollback takes the pre-constructed Db write with it', async () => {
|
|
92
|
+
await expect(
|
|
93
|
+
txnDb.runTransaction(async () => {
|
|
94
|
+
// If this insert did NOT ride the transaction it would self-commit in its own
|
|
95
|
+
// transaction and survive the rollback below — the exact decay a severed ambient
|
|
96
|
+
// resolution produces (a commit-path assertion can't see it; this one can).
|
|
97
|
+
await preconstructedDb.insert(employeeTable, { name: 'TxnSafetyAtomic' } as SafetyEmployee);
|
|
98
|
+
throw new Error('force-rollback');
|
|
99
|
+
})
|
|
100
|
+
).rejects.toThrow('force-rollback');
|
|
101
|
+
|
|
102
|
+
const after = await preconstructedDb.query(employeeTable, { name: 'TxnSafetyAtomic' });
|
|
103
|
+
expect(after.length).toBe(0);
|
|
104
|
+
}, 30000);
|
|
105
|
+
|
|
106
|
+
test('nested transactions throw, from ANY instance', async () => {
|
|
107
|
+
await expect(
|
|
108
|
+
txnDb.runTransaction(async () => {
|
|
109
|
+
await preconstructedDb.runTransaction(async () => 'never-runs');
|
|
110
|
+
})
|
|
111
|
+
).rejects.toThrow(/already running in this context/);
|
|
112
|
+
}, 30000);
|
|
113
|
+
|
|
114
|
+
test('two concurrent transactions on ONE shared instance commit independently', async () => {
|
|
115
|
+
const [a, b] = await Promise.all([
|
|
116
|
+
txnDb.runTransaction(
|
|
117
|
+
async () => await txnDb.insert(employeeTable, { name: 'TxnSafetyConcurrentA' } as SafetyEmployee)
|
|
118
|
+
),
|
|
119
|
+
txnDb.runTransaction(
|
|
120
|
+
async () => await txnDb.insert(employeeTable, { name: 'TxnSafetyConcurrentB' } as SafetyEmployee)
|
|
121
|
+
),
|
|
122
|
+
]);
|
|
123
|
+
|
|
124
|
+
const seen = await txnDb.query(
|
|
125
|
+
employeeTable,
|
|
126
|
+
new QueryBuilderFactory()
|
|
127
|
+
.createQueryBuilder(employeeTable)
|
|
128
|
+
.condition({ field: 'id', operator: 'IN', value: [a.id, b.id] })
|
|
129
|
+
);
|
|
130
|
+
expect(seen.length).toBe(2);
|
|
131
|
+
await txnDb.delete(employeeTable, { id: a.id });
|
|
132
|
+
await txnDb.delete(employeeTable, { id: b.id });
|
|
133
|
+
}, 30000);
|
|
134
|
+
|
|
135
|
+
test('tombstone: work that escapes a finished transaction fails loudly by name', async () => {
|
|
136
|
+
let releaseEscape!: () => void;
|
|
137
|
+
const escapeGate = new Promise<void>((resolve) => (releaseEscape = resolve));
|
|
138
|
+
let escapedOp!: Promise<unknown>;
|
|
139
|
+
|
|
140
|
+
await txnDb.runTransaction(async () => {
|
|
141
|
+
// Spawned inside the body, NOT awaited by it — it captures the transaction's async
|
|
142
|
+
// context and outlives the transaction.
|
|
143
|
+
escapedOp = (async () => {
|
|
144
|
+
await escapeGate;
|
|
145
|
+
return await txnDb.query(employeeTable, { name: 'TxnSafetyEscapee' });
|
|
146
|
+
})();
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
releaseEscape();
|
|
150
|
+
await expect(escapedOp).rejects.toThrow(/transaction that already ended/);
|
|
151
|
+
}, 30000);
|
|
152
|
+
|
|
153
|
+
test('queries outside any transaction are plain pool reads', async () => {
|
|
154
|
+
const seen = await preconstructedDb.query(employeeTable, { name: 'TxnSafetyNoTxn' });
|
|
155
|
+
expect(seen.length).toBe(0);
|
|
156
|
+
}, 30000);
|
|
157
|
+
});
|