@effectstream/db 0.100.19 → 0.100.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/migrations/system-up-v-0.0.0.sql +0 -2
- package/package.json +7 -7
- package/scripts/start-pglite.ts +2 -9
- package/src/MaterializedViewStrategy.ts +50 -0
- package/src/capabilities.ts +118 -0
- package/src/dynamic-tables.ts +26 -4
- package/src/mod.ts +7 -1
- package/src/pg-connection.ts +27 -26
- package/src/pool-error-tracker.test.ts +0 -65
- package/src/pool-error-tracker.ts +0 -87
- package/src/transient-pg-errors.test.ts +0 -47
- package/src/transient-pg-errors.ts +0 -56
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effectstream/db",
|
|
3
|
-
"version": "0.100.
|
|
3
|
+
"version": "0.100.22",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"homepage": "https://effectstream.github.io/docs/",
|
|
6
6
|
"repository": {
|
|
@@ -21,15 +21,15 @@
|
|
|
21
21
|
"./version": "./migrations/system-version.ts"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@effectstream/coroutine": "0.100.
|
|
25
|
-
"@effectstream/log": "0.100.
|
|
26
|
-
"@effectstream/runtime": "0.100.
|
|
27
|
-
"@effectstream/sync": "0.100.
|
|
28
|
-
"@effectstream/utils": "0.100.
|
|
24
|
+
"@effectstream/coroutine": "0.100.22",
|
|
25
|
+
"@effectstream/log": "0.100.22",
|
|
26
|
+
"@effectstream/runtime": "0.100.22",
|
|
27
|
+
"@effectstream/sync": "0.100.22",
|
|
28
|
+
"@effectstream/utils": "0.100.22",
|
|
29
29
|
"@electric-sql/pglite": "^0.3.14",
|
|
30
30
|
"@pgtyped/runtime": "2.4.2",
|
|
31
31
|
"@pgtyped/parser": "2.4.2",
|
|
32
|
-
"pg": "8.
|
|
32
|
+
"pg": "8.21.0",
|
|
33
33
|
"pg-tx": "^1.0.1",
|
|
34
34
|
"effection": "^3.5.0",
|
|
35
35
|
"pg-gateway": "^0.3.0-beta.4"
|
package/scripts/start-pglite.ts
CHANGED
|
@@ -31,10 +31,6 @@ export async function startPglite(port = 5432): Promise<PgliteHandle> {
|
|
|
31
31
|
},
|
|
32
32
|
);
|
|
33
33
|
|
|
34
|
-
// Serialize every client's execProtocolRaw through one promise chain
|
|
35
|
-
// PGlite allows only one in-flight query, and per-process db_mutex can't coordinate across separate TCP clients.
|
|
36
|
-
let queue: Promise<unknown> = Promise.resolve();
|
|
37
|
-
|
|
38
34
|
// TODO: consider switching to pglite-socket once it works
|
|
39
35
|
// https://discord.com/channels/933657521581858818/1371976702674075780/1371992712076595250
|
|
40
36
|
const server = net.createServer(async (socket) => {
|
|
@@ -58,11 +54,8 @@ export async function startPglite(port = 5432): Promise<PgliteHandle> {
|
|
|
58
54
|
return;
|
|
59
55
|
}
|
|
60
56
|
|
|
61
|
-
// Forward raw message to PGlite and send response to client
|
|
62
|
-
|
|
63
|
-
const run = queue.then(() => db.execProtocolRaw(data));
|
|
64
|
-
queue = run.catch(() => {}); // keep the chain alive even if a query errors
|
|
65
|
-
return await run;
|
|
57
|
+
// Forward raw message to PGlite and send response to client
|
|
58
|
+
return await db.execProtocolRaw(data);
|
|
66
59
|
},
|
|
67
60
|
});
|
|
68
61
|
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strategy for publishing the read-side relation that primitives query
|
|
3
|
+
* (e.g. `primitives.erc20_balances_view_<name>`).
|
|
4
|
+
*
|
|
5
|
+
* The intermediate table and the maintenance trigger are identical in both
|
|
6
|
+
* modes — only how the view itself is created differs. The query API
|
|
7
|
+
* (`SELECT * FROM primitives.<viewName>`) is unchanged across strategies.
|
|
8
|
+
*/
|
|
9
|
+
export interface MaterializedViewStrategy {
|
|
10
|
+
/**
|
|
11
|
+
* Stable identifier used to namespace migration rows per strategy.
|
|
12
|
+
* Empty string for the legacy pg_ivm strategy, so its migration rows keep
|
|
13
|
+
* the same names that existing deployments already have in their
|
|
14
|
+
* `effectstream_migration_history`. Only the new fallback strategy uses a
|
|
15
|
+
* suffixed name. This avoids re-applying dynamic-table DDL (which would
|
|
16
|
+
* fail on already-existing triggers/IMMVs) when an existing app upgrades.
|
|
17
|
+
*/
|
|
18
|
+
readonly migrationSuffix: "" | "-view";
|
|
19
|
+
/** Emit DDL that publishes `viewName` as a queryable relation backed by `selectSql`. */
|
|
20
|
+
createView(viewName: string, selectSql: string): string;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Uses the `pg_ivm` extension to create an incrementally maintained
|
|
25
|
+
* materialized view. Requires the extension to be installed in the database.
|
|
26
|
+
*
|
|
27
|
+
* Migration rows for this strategy use the historical unsuffixed name
|
|
28
|
+
* (`dynamic-tables-<type>-<name>`) so deployments that were already running
|
|
29
|
+
* the engine before the strategy pattern existed keep matching their
|
|
30
|
+
* previously-applied rows on upgrade — no re-application, no DDL conflicts.
|
|
31
|
+
*/
|
|
32
|
+
export const pgIvmStrategy: MaterializedViewStrategy = {
|
|
33
|
+
migrationSuffix: "",
|
|
34
|
+
createView: (viewName, selectSql) =>
|
|
35
|
+
`SELECT pgivm.create_immv('${viewName}', $IVM$${selectSql}$IVM$);`,
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Fallback: a plain SQL view computed on read. The intermediate table is
|
|
40
|
+
* still trigger-maintained, so the only extra cost is re-applying the
|
|
41
|
+
* (typically cheap) WHERE filter at query time.
|
|
42
|
+
*
|
|
43
|
+
* Migration rows use a `-view` suffix to keep this strategy's history
|
|
44
|
+
* distinct from the pg_ivm strategy's.
|
|
45
|
+
*/
|
|
46
|
+
export const plainViewStrategy: MaterializedViewStrategy = {
|
|
47
|
+
migrationSuffix: "-view",
|
|
48
|
+
createView: (viewName, selectSql) =>
|
|
49
|
+
`CREATE OR REPLACE VIEW ${viewName} AS ${selectSql};`,
|
|
50
|
+
};
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import type { PoolClient } from "pg";
|
|
2
|
+
import { ENV } from "@effectstream/utils/node-env";
|
|
3
|
+
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
4
|
+
import {
|
|
5
|
+
type MaterializedViewStrategy,
|
|
6
|
+
pgIvmStrategy,
|
|
7
|
+
plainViewStrategy,
|
|
8
|
+
} from "./MaterializedViewStrategy.ts";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Optional database features the engine probes for at startup.
|
|
12
|
+
*
|
|
13
|
+
* - `pgIvm`: the `pg_ivm` extension that provides incrementally maintained
|
|
14
|
+
* materialized views. When unavailable, the engine can fall back to plain
|
|
15
|
+
* SQL views over the same trigger-maintained intermediate tables — but
|
|
16
|
+
* only if the operator has explicitly opted in via `ENV.ALLOW_NO_PG_IVM`.
|
|
17
|
+
*/
|
|
18
|
+
export interface DatabaseCapabilities {
|
|
19
|
+
pgIvm: boolean;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Probes the connected database for optional features. Pure detection —
|
|
24
|
+
* no policy decision is made here; the caller (typically via
|
|
25
|
+
* {@link selectViewStrategy}) decides whether the resulting capability set
|
|
26
|
+
* is acceptable.
|
|
27
|
+
*
|
|
28
|
+
* - Best-effort `CREATE EXTENSION IF NOT EXISTS pg_ivm` (no-op if already
|
|
29
|
+
* installed, soft-fail if the binary is missing on the server).
|
|
30
|
+
* - Then queries `pg_extension` to confirm the real state.
|
|
31
|
+
*/
|
|
32
|
+
export async function detectCapabilities(
|
|
33
|
+
client: Pick<PoolClient, "query">,
|
|
34
|
+
): Promise<DatabaseCapabilities> {
|
|
35
|
+
try {
|
|
36
|
+
await client.query("CREATE EXTENSION IF NOT EXISTS pg_ivm");
|
|
37
|
+
} catch {
|
|
38
|
+
// Extension binary not installed on this server — fall through and let
|
|
39
|
+
// the pg_extension lookup confirm the absence.
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const { rows } = await client.query<{ exists: boolean }>(
|
|
43
|
+
`SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'pg_ivm') AS exists`,
|
|
44
|
+
);
|
|
45
|
+
return { pgIvm: rows[0]?.exists === true };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Decide which materialized-view strategy to use based on detected
|
|
50
|
+
* capabilities and operator policy.
|
|
51
|
+
*
|
|
52
|
+
* Behavior matrix:
|
|
53
|
+
*
|
|
54
|
+
* | pg_ivm available | ALLOW_NO_PG_IVM | Result |
|
|
55
|
+
* | :--------------- | :-------------- | :--------------------------------------- |
|
|
56
|
+
* | yes | any | pgIvmStrategy (production path) |
|
|
57
|
+
* | no | false / unset | **throws** — startup aborts loudly |
|
|
58
|
+
* | no | true | plainViewStrategy + prominent startup WARN |
|
|
59
|
+
*
|
|
60
|
+
* The throw is intentional: silently degrading to a slower read path in
|
|
61
|
+
* production is much worse than failing early with a clear remediation
|
|
62
|
+
* message. Operators who genuinely need the dev/test fallback must opt in.
|
|
63
|
+
*/
|
|
64
|
+
export function selectViewStrategy(
|
|
65
|
+
capabilities: DatabaseCapabilities,
|
|
66
|
+
): MaterializedViewStrategy {
|
|
67
|
+
if (capabilities.pgIvm) return pgIvmStrategy;
|
|
68
|
+
|
|
69
|
+
if (!ENV.ALLOW_NO_PG_IVM) {
|
|
70
|
+
throw new Error(formatMissingPgIvmError());
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
log.local(
|
|
74
|
+
ComponentNames.EFFECTSTREAM_RUNTIME,
|
|
75
|
+
"pg_ivm-fallback",
|
|
76
|
+
SeverityNumber.WARN,
|
|
77
|
+
(l) =>
|
|
78
|
+
l(
|
|
79
|
+
[
|
|
80
|
+
"─────────────────────────────────────────────────────────────",
|
|
81
|
+
" effectstream: running WITHOUT pg_ivm (fallback mode)",
|
|
82
|
+
" ALLOW_NO_PG_IVM=true is set; using plain SQL views over the",
|
|
83
|
+
" trigger-maintained intermediate tables.",
|
|
84
|
+
"",
|
|
85
|
+
" This path is intended for dev/test and low-volume apps only.",
|
|
86
|
+
" Read-path performance degrades sharply on long-lived contracts",
|
|
87
|
+
" with high address cardinality (intermediate tables grow",
|
|
88
|
+
" monotonically; the plain view does a seq scan + filter per",
|
|
89
|
+
" read). DO NOT run production on this configuration.",
|
|
90
|
+
"─────────────────────────────────────────────────────────────",
|
|
91
|
+
].join("\n"),
|
|
92
|
+
),
|
|
93
|
+
);
|
|
94
|
+
return plainViewStrategy;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function formatMissingPgIvmError(): string {
|
|
98
|
+
return [
|
|
99
|
+
"",
|
|
100
|
+
"Effectstream startup aborted: the `pg_ivm` PostgreSQL extension is not",
|
|
101
|
+
"available on this database.",
|
|
102
|
+
"",
|
|
103
|
+
"The engine uses `pg_ivm` for incrementally maintained materialized views,",
|
|
104
|
+
"which give compact, fast reads on long-lived contracts. Without it the",
|
|
105
|
+
"engine can fall back to plain SQL views over the trigger-maintained",
|
|
106
|
+
"intermediate tables — but that path degrades sharply on high-cardinality",
|
|
107
|
+
"data and is only suitable for dev/test and low-volume apps.",
|
|
108
|
+
"",
|
|
109
|
+
"To proceed:",
|
|
110
|
+
" • Install pg_ivm in your PostgreSQL instance:",
|
|
111
|
+
" https://github.com/sraoss/pg_ivm",
|
|
112
|
+
" • OR explicitly accept the dev-mode fallback by setting:",
|
|
113
|
+
" ALLOW_NO_PG_IVM=true",
|
|
114
|
+
" (NOT recommended for production — see the database docs for the",
|
|
115
|
+
" performance ceiling.)",
|
|
116
|
+
"",
|
|
117
|
+
].join("\n");
|
|
118
|
+
}
|
package/src/dynamic-tables.ts
CHANGED
|
@@ -6,26 +6,42 @@ import type { PoolClient } from "pg";
|
|
|
6
6
|
import type { VersionInfo } from "../migrations/system-version.ts";
|
|
7
7
|
import { applyMigrations } from "../scripts/apply-migrations.ts";
|
|
8
8
|
import { findMigrationByName } from "./sql/system.queries.ts";
|
|
9
|
+
import {
|
|
10
|
+
type MaterializedViewStrategy,
|
|
11
|
+
plainViewStrategy,
|
|
12
|
+
} from "./MaterializedViewStrategy.ts";
|
|
9
13
|
|
|
10
14
|
/**
|
|
11
15
|
* Creates dynamic tables for the given sync protocols.
|
|
12
|
-
* This is used to create the
|
|
16
|
+
* This is used to create the read-side relations (incrementally maintained
|
|
17
|
+
* via pg_ivm when available, otherwise plain SQL views) for each primitive's
|
|
18
|
+
* intermediate state.
|
|
13
19
|
*
|
|
14
20
|
* For example for ERC20 it creates a table that holds the balance for each address.
|
|
15
21
|
* For ERC721 it creates a table that holds the token id for each address.
|
|
16
22
|
*
|
|
17
23
|
* @param dbConn - The database connection.
|
|
18
24
|
* @param syncProtocols - The sync protocols.
|
|
25
|
+
* @param strategy - How to publish the read-side view. Defaults to the plain
|
|
26
|
+
* view fallback so callers that haven't probed capabilities yet stay safe
|
|
27
|
+
* on managed Postgres without pg_ivm. The runtime detects the extension
|
|
28
|
+
* and passes `pgIvmStrategy` explicitly when available.
|
|
19
29
|
*/
|
|
20
30
|
export function* createDynamicTables(
|
|
21
31
|
versionInfo: VersionInfo,
|
|
22
32
|
lastBlockHeight: number,
|
|
23
33
|
dbConn: PoolClient,
|
|
24
34
|
syncProtocols: any[], // AllSyncProtocols[],
|
|
35
|
+
strategy: MaterializedViewStrategy = plainViewStrategy,
|
|
25
36
|
) {
|
|
26
37
|
for (const syncProtocol of syncProtocols) {
|
|
27
38
|
for (const primitive of syncProtocol.config.primitives) {
|
|
28
|
-
yield* createDynamicTableForPrimitive(
|
|
39
|
+
yield* createDynamicTableForPrimitive(
|
|
40
|
+
primitive,
|
|
41
|
+
lastBlockHeight,
|
|
42
|
+
dbConn,
|
|
43
|
+
strategy,
|
|
44
|
+
);
|
|
29
45
|
}
|
|
30
46
|
}
|
|
31
47
|
}
|
|
@@ -39,6 +55,7 @@ function* createDynamicTableForPrimitive(
|
|
|
39
55
|
},
|
|
40
56
|
lastBlockHeight: number,
|
|
41
57
|
dbConn: PoolClient,
|
|
58
|
+
strategy: MaterializedViewStrategy,
|
|
42
59
|
) {
|
|
43
60
|
const type = p.primitive.type;
|
|
44
61
|
const name = p.primitive.name;
|
|
@@ -54,10 +71,15 @@ function* createDynamicTableForPrimitive(
|
|
|
54
71
|
|
|
55
72
|
// This primitive does not have dynamic tables.
|
|
56
73
|
if (!sqlFunction) return;
|
|
57
|
-
const code = sqlFunction(name);
|
|
74
|
+
const code = sqlFunction(name, strategy);
|
|
58
75
|
if (!code) return;
|
|
59
76
|
|
|
60
|
-
|
|
77
|
+
// pgIvmStrategy uses the legacy unsuffixed name so existing deployments
|
|
78
|
+
// upgrading from a pre-strategy build keep matching their previously-
|
|
79
|
+
// applied migration rows. plainViewStrategy uses a `-view` suffix to keep
|
|
80
|
+
// its history distinct, so a deployment that later gains or loses pg_ivm
|
|
81
|
+
// re-applies the correct view-creation DDL.
|
|
82
|
+
const migrationName = `dynamic-tables-${type}-${name}${strategy.migrationSuffix}`;
|
|
61
83
|
const [migration] = yield* until(findMigrationByName.run({
|
|
62
84
|
name: migrationName,
|
|
63
85
|
isSystemMigration: true,
|
package/src/mod.ts
CHANGED
|
@@ -54,7 +54,6 @@ export * from "./sql/tables.queries.ts";
|
|
|
54
54
|
|
|
55
55
|
export * from "./event-indexing.ts";
|
|
56
56
|
export * from "./register-events.ts";
|
|
57
|
-
export * from "./transient-pg-errors.ts";
|
|
58
57
|
export * from "./pg-connection.ts";
|
|
59
58
|
export * from "./scheduled-constructors.ts";
|
|
60
59
|
export * from "./system.ts";
|
|
@@ -64,6 +63,13 @@ export {
|
|
|
64
63
|
getPrimitiveIntermediatePrefix,
|
|
65
64
|
getPrimitivePrefix,
|
|
66
65
|
} from "./dynamic-tables.ts";
|
|
66
|
+
export { detectCapabilities, selectViewStrategy } from "./capabilities.ts";
|
|
67
|
+
export type { DatabaseCapabilities } from "./capabilities.ts";
|
|
68
|
+
export {
|
|
69
|
+
pgIvmStrategy,
|
|
70
|
+
plainViewStrategy,
|
|
71
|
+
} from "./MaterializedViewStrategy.ts";
|
|
72
|
+
export type { MaterializedViewStrategy } from "./MaterializedViewStrategy.ts";
|
|
67
73
|
// export { startPGlite } from "./start-pglite.ts";
|
|
68
74
|
|
|
69
75
|
export { getMigrationsForBlockHeight } from "./migrations.ts";
|
package/src/pg-connection.ts
CHANGED
|
@@ -1,16 +1,10 @@
|
|
|
1
1
|
import type { Client, PoolClient, PoolConfig } from "pg";
|
|
2
2
|
import pg from "pg";
|
|
3
|
-
import { ComponentNames,
|
|
3
|
+
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
4
4
|
import { type Operation, run, sleep } from "effection";
|
|
5
5
|
import { ENV } from "@effectstream/utils/node-env";
|
|
6
|
-
import { isTransientPgError } from "./transient-pg-errors.ts";
|
|
7
|
-
import { poolErrors } from "./pool-error-tracker.ts";
|
|
8
|
-
|
|
9
|
-
export { isTransientPgError } from "./transient-pg-errors.ts";
|
|
10
|
-
export { poolErrors, PoolErrorTracker } from "./pool-error-tracker.ts";
|
|
11
6
|
|
|
12
7
|
let readonlyDBConn: pg.Pool | null;
|
|
13
|
-
let unhandledRejectionLoggerInstalled = false;
|
|
14
8
|
|
|
15
9
|
// PGLite does not support multiple connections, so we need to use a mutex to ensure that only one query is executed at a time.
|
|
16
10
|
// * For transactions use yield* acquireDBMutex(); ...Your Operations... releaseDBMutex();
|
|
@@ -144,8 +138,6 @@ export async function runPreparedQuery<T>(
|
|
|
144
138
|
});
|
|
145
139
|
result = await Promise.race([p, timeout]);
|
|
146
140
|
clearTimeout(t);
|
|
147
|
-
// A successful query proves the DB is reachable end-to-end
|
|
148
|
-
poolErrors.markHealthy();
|
|
149
141
|
} finally {
|
|
150
142
|
releaseDBMutex(`pq:${name}`);
|
|
151
143
|
}
|
|
@@ -156,14 +148,6 @@ export const getConnection = (
|
|
|
156
148
|
creds?: PoolConfig,
|
|
157
149
|
readonly = false,
|
|
158
150
|
): pg.Pool => {
|
|
159
|
-
if (!unhandledRejectionLoggerInstalled) {
|
|
160
|
-
unhandledRejectionLoggerInstalled = true;
|
|
161
|
-
installUnhandledRejectionLogger(
|
|
162
|
-
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
163
|
-
isTransientPgError,
|
|
164
|
-
);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
151
|
if (!creds) {
|
|
168
152
|
creds = {
|
|
169
153
|
host: ENV.DB_HOST,
|
|
@@ -179,21 +163,30 @@ export const getConnection = (
|
|
|
179
163
|
const MAX_CONNECTIONS = ENV.PGLITE ? 1 : 10;
|
|
180
164
|
|
|
181
165
|
const pool = new pg.Pool({ ...creds, max: MAX_CONNECTIONS });
|
|
182
|
-
pool.on("error", (err: unknown) =>
|
|
166
|
+
pool.on("error", (err: unknown) =>
|
|
167
|
+
log.remote(
|
|
168
|
+
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
169
|
+
["query"],
|
|
170
|
+
SeverityNumber.ERROR,
|
|
171
|
+
(log) => log(err),
|
|
172
|
+
));
|
|
183
173
|
pool.on("connect", (_client: PoolClient) => {
|
|
184
|
-
//
|
|
185
|
-
poolErrors.markHealthy();
|
|
186
|
-
// Register a per-client error handler; pg can emit errors on idle clients.
|
|
174
|
+
// On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
|
|
187
175
|
// https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
|
|
188
|
-
_client.on("error", (err: Error) =>
|
|
176
|
+
_client.on("error", (err: Error) => {
|
|
177
|
+
log.remote(
|
|
178
|
+
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
179
|
+
["connect"],
|
|
180
|
+
SeverityNumber.ERROR,
|
|
181
|
+
(log) => log(err),
|
|
182
|
+
);
|
|
183
|
+
});
|
|
189
184
|
});
|
|
190
185
|
|
|
191
186
|
if (readonly) {
|
|
192
187
|
const ensureReadOnly =
|
|
193
188
|
`SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY;`;
|
|
194
|
-
void pool.query(ensureReadOnly)
|
|
195
|
-
poolErrors.log(err, ["readonly-setup"]),
|
|
196
|
-
);
|
|
189
|
+
void pool.query(ensureReadOnly); // note: this query modifies the DB state
|
|
197
190
|
readonlyDBConn = pool;
|
|
198
191
|
}
|
|
199
192
|
|
|
@@ -205,6 +198,14 @@ export const getPersistentConnection = (creds: PoolConfig): Client => {
|
|
|
205
198
|
const client = new pg.Client(creds);
|
|
206
199
|
client.connect(() => {});
|
|
207
200
|
// https://github.com/brianc/node-postgres/issues/2499#issuecomment-805477725
|
|
208
|
-
client
|
|
201
|
+
// On each new client initiated, need to register for error(this is a serious bug on pg, the client throw errors although it should not)
|
|
202
|
+
client.on("error", (err: Error) => {
|
|
203
|
+
log.remote(
|
|
204
|
+
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
205
|
+
["query"],
|
|
206
|
+
SeverityNumber.ERROR,
|
|
207
|
+
(log) => log(err),
|
|
208
|
+
);
|
|
209
|
+
});
|
|
209
210
|
return client;
|
|
210
211
|
};
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import { afterEach, expect, test } from "bun:test";
|
|
2
|
-
|
|
3
|
-
// Thresholds are read from env at module load (and `envMs` rejects 0, so use
|
|
4
|
-
// small positive values). SUSTAINED=5ms → a failure older than 5ms is
|
|
5
|
-
// "sustained"; FATAL=30ms → older than 30ms triggers the process.exit
|
|
6
|
-
// escalation. Dynamic import so these are picked up before the module
|
|
7
|
-
// computes its constants.
|
|
8
|
-
process.env.POOL_SUSTAINED_THRESHOLD_MS = "5";
|
|
9
|
-
process.env.POOL_FATAL_THRESHOLD_MS = "30";
|
|
10
|
-
const { PoolErrorTracker } = await import("./pool-error-tracker.ts");
|
|
11
|
-
|
|
12
|
-
const transient = () => ({
|
|
13
|
-
code: "ECONNRESET",
|
|
14
|
-
message: "Client network socket disconnected before secure TLS connection",
|
|
15
|
-
stack: " at node_modules/pg-pool/index.js:45:11",
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
const realExit = process.exit;
|
|
19
|
-
afterEach(() => {
|
|
20
|
-
process.exit = realExit;
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
test("ignores non-transient errors (auth/schema bugs)", () => {
|
|
24
|
-
const t = new PoolErrorTracker();
|
|
25
|
-
t.log(new Error("column foo does not exist"), ["sync-write"]);
|
|
26
|
-
expect(t.state().consecutive).toBe(0);
|
|
27
|
-
expect(t.state().sustained).toBe(false);
|
|
28
|
-
});
|
|
29
|
-
|
|
30
|
-
test("counts consecutive transient failures and reports sustained", async () => {
|
|
31
|
-
const t = new PoolErrorTracker();
|
|
32
|
-
t.log(transient(), ["sync-write"]);
|
|
33
|
-
expect(t.state().sustained).toBe(false); // fresh failure, < 5ms old
|
|
34
|
-
await Bun.sleep(15); // exceed SUSTAINED_THRESHOLD_MS=5, stay under FATAL=30
|
|
35
|
-
t.log(transient(), ["sync-write"]);
|
|
36
|
-
const s = t.state();
|
|
37
|
-
expect(s.consecutive).toBe(2);
|
|
38
|
-
expect(s.sustained).toBe(true);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
test("markHealthy resets the failure clock", () => {
|
|
42
|
-
const t = new PoolErrorTracker();
|
|
43
|
-
t.log(transient(), ["sync-write"]);
|
|
44
|
-
t.markHealthy();
|
|
45
|
-
const s = t.state();
|
|
46
|
-
expect(s.consecutive).toBe(0);
|
|
47
|
-
expect(s.firstFailureAt).toBe(0);
|
|
48
|
-
expect(s.sustained).toBe(false);
|
|
49
|
-
});
|
|
50
|
-
|
|
51
|
-
test("exits the process once failures persist past the fatal threshold", async () => {
|
|
52
|
-
let exitCode: number | undefined;
|
|
53
|
-
process.exit = ((code?: number) => {
|
|
54
|
-
exitCode = code;
|
|
55
|
-
}) as typeof process.exit;
|
|
56
|
-
|
|
57
|
-
const t = new PoolErrorTracker();
|
|
58
|
-
t.log(transient(), ["sync-write"]); // starts the clock; not yet fatal
|
|
59
|
-
await Bun.sleep(40); // exceed FATAL_THRESHOLD_MS=20
|
|
60
|
-
t.log(transient(), ["sync-write"]); // now duration >= fatal → schedules exit
|
|
61
|
-
|
|
62
|
-
// exit is deferred via setImmediate so the fatal log line can flush first.
|
|
63
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
64
|
-
expect(exitCode).toBe(1);
|
|
65
|
-
});
|
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
2
|
-
import { isTransientPgError } from "./transient-pg-errors.ts";
|
|
3
|
-
|
|
4
|
-
function envMs(name: string, fallback: number): number {
|
|
5
|
-
const raw = process.env[name];
|
|
6
|
-
const parsed = raw != null ? Number(raw) : NaN;
|
|
7
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
/** After this long, treat continuous transient failures as a real outage. */
|
|
11
|
-
const SUSTAINED_THRESHOLD_MS = envMs("POOL_SUSTAINED_THRESHOLD_MS", 60_000);
|
|
12
|
-
/** After this long, exit so the orchestrator restarts the process. */
|
|
13
|
-
const FATAL_THRESHOLD_MS = envMs("POOL_FATAL_THRESHOLD_MS", 5 * 60_000);
|
|
14
|
-
|
|
15
|
-
/**
|
|
16
|
-
* Tracks consecutive transient pool errors. Severity escalates WARN → ERROR
|
|
17
|
-
* once failures persist past `SUSTAINED_THRESHOLD_MS`, and the process exits
|
|
18
|
-
* past `FATAL_THRESHOLD_MS` (relying on the orchestrator's restart policy).
|
|
19
|
-
* Call `markHealthy()` on any positive signal to reset.
|
|
20
|
-
*/
|
|
21
|
-
export class PoolErrorTracker {
|
|
22
|
-
private consecutive = 0;
|
|
23
|
-
private firstFailureAt = 0;
|
|
24
|
-
|
|
25
|
-
log(err: unknown, tags: string[]): void {
|
|
26
|
-
// Non-transient errors are a different failure mode (auth, schema, etc.).
|
|
27
|
-
if (!isTransientPgError(err)) {
|
|
28
|
-
log.remote(
|
|
29
|
-
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
30
|
-
tags,
|
|
31
|
-
SeverityNumber.ERROR,
|
|
32
|
-
(l) => l(err),
|
|
33
|
-
);
|
|
34
|
-
return;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
if (this.consecutive === 0) this.firstFailureAt = Date.now();
|
|
38
|
-
this.consecutive++;
|
|
39
|
-
|
|
40
|
-
const durationMs = Date.now() - this.firstFailureAt;
|
|
41
|
-
const sustained = durationMs >= SUSTAINED_THRESHOLD_MS;
|
|
42
|
-
const fatal = durationMs >= FATAL_THRESHOLD_MS;
|
|
43
|
-
|
|
44
|
-
log.remote(
|
|
45
|
-
ComponentNames.EFFECTSTREAM_PGLITE,
|
|
46
|
-
fatal
|
|
47
|
-
? [...tags, "sustained-failure", "fatal"]
|
|
48
|
-
: sustained
|
|
49
|
-
? [...tags, "sustained-failure"]
|
|
50
|
-
: tags,
|
|
51
|
-
sustained ? SeverityNumber.ERROR : SeverityNumber.WARN,
|
|
52
|
-
(l) => l({ err, consecutive: this.consecutive, durationMs }),
|
|
53
|
-
);
|
|
54
|
-
|
|
55
|
-
if (fatal) {
|
|
56
|
-
// Defer to the next tick so the log line above has a chance to flush.
|
|
57
|
-
setImmediate(() => process.exit(1));
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
/** Call when the DB has proven reachable (e.g. successful pool connect). */
|
|
62
|
-
markHealthy(): void {
|
|
63
|
-
this.consecutive = 0;
|
|
64
|
-
this.firstFailureAt = 0;
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/** Read-only snapshot for health checks, heartbeats, or external probes. */
|
|
68
|
-
state(): {
|
|
69
|
-
consecutive: number;
|
|
70
|
-
firstFailureAt: number;
|
|
71
|
-
sustainedDurationMs: number;
|
|
72
|
-
sustained: boolean;
|
|
73
|
-
} {
|
|
74
|
-
const durationMs = this.firstFailureAt === 0
|
|
75
|
-
? 0
|
|
76
|
-
: Date.now() - this.firstFailureAt;
|
|
77
|
-
return {
|
|
78
|
-
consecutive: this.consecutive,
|
|
79
|
-
firstFailureAt: this.firstFailureAt,
|
|
80
|
-
sustainedDurationMs: durationMs,
|
|
81
|
-
sustained: durationMs >= SUSTAINED_THRESHOLD_MS,
|
|
82
|
-
};
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
/** Process-wide singleton — failures aggregate at DB-health granularity. */
|
|
87
|
-
export const poolErrors = new PoolErrorTracker();
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { test, expect } from "bun:test";
|
|
2
|
-
import { isTransientPgError } from "./transient-pg-errors.ts";
|
|
3
|
-
|
|
4
|
-
test("isTransientPgError - Neon pg-pool TLS reset", () => {
|
|
5
|
-
expect(
|
|
6
|
-
isTransientPgError({
|
|
7
|
-
code: "ECONNRESET",
|
|
8
|
-
message:
|
|
9
|
-
"Client network socket disconnected before secure TLS connection was established",
|
|
10
|
-
stack: " at /app/node_modules/.bun/pg-pool@3.14.0/node_modules/pg-pool/index.js:45:11",
|
|
11
|
-
}),
|
|
12
|
-
).toBe(true);
|
|
13
|
-
});
|
|
14
|
-
|
|
15
|
-
test("isTransientPgError - rejects non-pg ETIMEDOUT", () => {
|
|
16
|
-
expect(
|
|
17
|
-
isTransientPgError({
|
|
18
|
-
code: "ETIMEDOUT",
|
|
19
|
-
message: "fetch failed",
|
|
20
|
-
stack: " at node:internal/deps/undici/undici.js:1234:5",
|
|
21
|
-
}),
|
|
22
|
-
).toBe(false);
|
|
23
|
-
});
|
|
24
|
-
|
|
25
|
-
test("isTransientPgError - walks cause chain", () => {
|
|
26
|
-
const inner = {
|
|
27
|
-
code: "ECONNRESET",
|
|
28
|
-
message:
|
|
29
|
-
"Client network socket disconnected before secure TLS connection was established",
|
|
30
|
-
stack: " at node_modules/pg-pool/index.js:45:11",
|
|
31
|
-
};
|
|
32
|
-
expect(isTransientPgError({ message: "wrapper", cause: inner })).toBe(true);
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
test("isTransientPgError - rejects logic errors", () => {
|
|
36
|
-
expect(isTransientPgError(new Error("column foo does not exist"))).toBe(
|
|
37
|
-
false,
|
|
38
|
-
);
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
test("isTransientPgError - idle pool message without stack", () => {
|
|
42
|
-
expect(
|
|
43
|
-
isTransientPgError({
|
|
44
|
-
message: "Connection terminated unexpectedly",
|
|
45
|
-
}),
|
|
46
|
-
).toBe(true);
|
|
47
|
-
});
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
const PG_STACK_RE = /node_modules\/(pg-pool|pg)\//;
|
|
2
|
-
|
|
3
|
-
const TRANSIENT_PG_MESSAGES = [
|
|
4
|
-
"Connection terminated unexpectedly",
|
|
5
|
-
"Client has encountered a connection error",
|
|
6
|
-
"Client network socket disconnected before secure TLS connection",
|
|
7
|
-
"Query read timeout",
|
|
8
|
-
] as const;
|
|
9
|
-
|
|
10
|
-
function* walkErrors(reason: unknown): Generator<unknown> {
|
|
11
|
-
let current: unknown = reason;
|
|
12
|
-
const seen = new Set<unknown>();
|
|
13
|
-
while (current != null && typeof current === "object" && !seen.has(current)) {
|
|
14
|
-
seen.add(current);
|
|
15
|
-
yield current;
|
|
16
|
-
current = (current as { cause?: unknown }).cause;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
function isFromPgStack(err: unknown): boolean {
|
|
21
|
-
if (err == null || typeof err !== "object" || !("stack" in err)) {
|
|
22
|
-
return false;
|
|
23
|
-
}
|
|
24
|
-
return PG_STACK_RE.test(String((err as Error).stack));
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function hasTransientPgMessage(err: unknown): boolean {
|
|
28
|
-
if (err == null || typeof err !== "object") return false;
|
|
29
|
-
const msg = (err as { message?: string }).message;
|
|
30
|
-
if (typeof msg !== "string") return false;
|
|
31
|
-
return TRANSIENT_PG_MESSAGES.some((fragment) => msg.includes(fragment));
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function isTransientPgErrorSingle(err: unknown): boolean {
|
|
35
|
-
if (err == null || typeof err !== "object") return false;
|
|
36
|
-
if (hasTransientPgMessage(err)) return true;
|
|
37
|
-
if (!isFromPgStack(err)) return false;
|
|
38
|
-
const code = (err as { code?: string }).code;
|
|
39
|
-
return code === "ECONNRESET" || code === "EPIPE" ||
|
|
40
|
-
code === "ECONNREFUSED";
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* Returns true for pg / network errors that are normal on serverless Postgres
|
|
45
|
-
* (Neon, etc.): cold-start RSTs, idle-socket evictions, transient TLS blips.
|
|
46
|
-
* The pool recovers on the next query — use with `installUnhandledRejectionLogger`
|
|
47
|
-
* from `@effectstream/log` so these do not crash the process.
|
|
48
|
-
*
|
|
49
|
-
* Intentionally narrow: known-recoverable conditions only; logic/SQL bugs still crash.
|
|
50
|
-
*/
|
|
51
|
-
export function isTransientPgError(reason: unknown): boolean {
|
|
52
|
-
for (const err of walkErrors(reason)) {
|
|
53
|
-
if (isTransientPgErrorSingle(err)) return true;
|
|
54
|
-
}
|
|
55
|
-
return false;
|
|
56
|
-
}
|