@constructive-io/graphql-server 4.10.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -0
- package/diagnostics/debug-db-snapshot.d.ts +19 -0
- package/diagnostics/debug-db-snapshot.js +196 -0
- package/diagnostics/debug-memory-snapshot.d.ts +53 -0
- package/diagnostics/debug-memory-snapshot.js +77 -0
- package/diagnostics/debug-sampler.d.ts +5 -0
- package/diagnostics/debug-sampler.js +195 -0
- package/diagnostics/observability.d.ts +6 -0
- package/diagnostics/observability.js +62 -0
- package/esm/diagnostics/debug-db-snapshot.js +191 -0
- package/esm/diagnostics/debug-memory-snapshot.js +70 -0
- package/esm/diagnostics/debug-sampler.js +188 -0
- package/esm/diagnostics/observability.js +53 -0
- package/esm/middleware/graphile.js +8 -1
- package/esm/middleware/observability/debug-db.js +23 -0
- package/esm/middleware/observability/debug-memory.js +8 -0
- package/esm/middleware/observability/graphile-build-stats.js +165 -0
- package/esm/middleware/observability/guard.js +14 -0
- package/esm/middleware/observability/request-logger.js +42 -0
- package/esm/server.js +37 -25
- package/middleware/graphile.js +8 -1
- package/middleware/observability/debug-db.d.ts +3 -0
- package/middleware/observability/debug-db.js +27 -0
- package/middleware/observability/debug-memory.d.ts +2 -0
- package/middleware/observability/debug-memory.js +12 -0
- package/middleware/observability/graphile-build-stats.d.ts +45 -0
- package/middleware/observability/graphile-build-stats.js +171 -0
- package/middleware/observability/guard.d.ts +2 -0
- package/middleware/observability/guard.js +18 -0
- package/middleware/observability/request-logger.d.ts +6 -0
- package/middleware/observability/request-logger.js +46 -0
- package/package.json +31 -29
- package/server.d.ts +1 -0
- package/server.js +37 -25
- package/esm/middleware/debug-memory.js +0 -54
- package/middleware/debug-memory.d.ts +0 -15
- package/middleware/debug-memory.js +0 -58
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
const MAX_BUILD_EVENTS = 100;
|
|
2
|
+
const MAX_AGGREGATE_KEYS = 100;
|
|
3
|
+
const buildStats = {
|
|
4
|
+
started: 0,
|
|
5
|
+
succeeded: 0,
|
|
6
|
+
failed: 0,
|
|
7
|
+
totalMs: 0,
|
|
8
|
+
maxMs: 0,
|
|
9
|
+
lastMs: 0,
|
|
10
|
+
lastKey: null,
|
|
11
|
+
lastStartedAt: null,
|
|
12
|
+
lastFinishedAt: null,
|
|
13
|
+
lastError: null,
|
|
14
|
+
lastServiceKey: null,
|
|
15
|
+
lastDatabaseId: null,
|
|
16
|
+
recentEvents: [],
|
|
17
|
+
byServiceKey: new Map(),
|
|
18
|
+
};
|
|
19
|
+
const pushBuildEvent = (event) => {
|
|
20
|
+
buildStats.recentEvents.push(event);
|
|
21
|
+
if (buildStats.recentEvents.length > MAX_BUILD_EVENTS) {
|
|
22
|
+
buildStats.recentEvents.splice(0, buildStats.recentEvents.length - MAX_BUILD_EVENTS);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
const updateAggregate = (map, key, durationMs) => {
|
|
26
|
+
const hasKey = map.has(key);
|
|
27
|
+
if (!hasKey && map.size >= MAX_AGGREGATE_KEYS) {
|
|
28
|
+
const oldestKey = map.keys().next().value;
|
|
29
|
+
if (oldestKey) {
|
|
30
|
+
map.delete(oldestKey);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
const current = map.get(key) ?? {
|
|
34
|
+
count: 0,
|
|
35
|
+
totalMs: 0,
|
|
36
|
+
maxMs: 0,
|
|
37
|
+
lastMs: 0,
|
|
38
|
+
lastAt: null,
|
|
39
|
+
};
|
|
40
|
+
current.count += 1;
|
|
41
|
+
current.totalMs += durationMs;
|
|
42
|
+
current.maxMs = Math.max(current.maxMs, durationMs);
|
|
43
|
+
current.lastMs = durationMs;
|
|
44
|
+
current.lastAt = new Date().toISOString();
|
|
45
|
+
// Keep LRU order by reinserting after each update.
|
|
46
|
+
if (hasKey) {
|
|
47
|
+
map.delete(key);
|
|
48
|
+
}
|
|
49
|
+
map.set(key, current);
|
|
50
|
+
};
|
|
51
|
+
const recordBuildStart = (context, startedAt) => {
|
|
52
|
+
buildStats.started += 1;
|
|
53
|
+
buildStats.lastKey = context.cacheKey;
|
|
54
|
+
buildStats.lastStartedAt = new Date(startedAt).toISOString();
|
|
55
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
56
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
57
|
+
pushBuildEvent({
|
|
58
|
+
type: 'start',
|
|
59
|
+
cacheKey: context.cacheKey,
|
|
60
|
+
serviceKey: context.serviceKey,
|
|
61
|
+
databaseId: context.databaseId,
|
|
62
|
+
timestamp: new Date(startedAt).toISOString(),
|
|
63
|
+
});
|
|
64
|
+
};
|
|
65
|
+
const recordBuildSuccess = (context, startedAt) => {
|
|
66
|
+
const durationMs = Date.now() - startedAt;
|
|
67
|
+
buildStats.succeeded += 1;
|
|
68
|
+
buildStats.totalMs += durationMs;
|
|
69
|
+
buildStats.maxMs = Math.max(buildStats.maxMs, durationMs);
|
|
70
|
+
buildStats.lastMs = durationMs;
|
|
71
|
+
buildStats.lastFinishedAt = new Date().toISOString();
|
|
72
|
+
buildStats.lastError = null;
|
|
73
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
74
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
75
|
+
if (context.serviceKey) {
|
|
76
|
+
updateAggregate(buildStats.byServiceKey, context.serviceKey, durationMs);
|
|
77
|
+
}
|
|
78
|
+
pushBuildEvent({
|
|
79
|
+
type: 'success',
|
|
80
|
+
cacheKey: context.cacheKey,
|
|
81
|
+
serviceKey: context.serviceKey,
|
|
82
|
+
databaseId: context.databaseId,
|
|
83
|
+
durationMs,
|
|
84
|
+
timestamp: new Date().toISOString(),
|
|
85
|
+
});
|
|
86
|
+
};
|
|
87
|
+
const recordBuildFailure = (context, startedAt, error) => {
|
|
88
|
+
const durationMs = Date.now() - startedAt;
|
|
89
|
+
buildStats.failed += 1;
|
|
90
|
+
buildStats.totalMs += durationMs;
|
|
91
|
+
buildStats.maxMs = Math.max(buildStats.maxMs, durationMs);
|
|
92
|
+
buildStats.lastMs = durationMs;
|
|
93
|
+
buildStats.lastFinishedAt = new Date().toISOString();
|
|
94
|
+
buildStats.lastError = error instanceof Error ? error.message : String(error);
|
|
95
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
96
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
97
|
+
pushBuildEvent({
|
|
98
|
+
type: 'failure',
|
|
99
|
+
cacheKey: context.cacheKey,
|
|
100
|
+
serviceKey: context.serviceKey,
|
|
101
|
+
databaseId: context.databaseId,
|
|
102
|
+
durationMs,
|
|
103
|
+
error: error instanceof Error ? error.message : String(error),
|
|
104
|
+
timestamp: new Date().toISOString(),
|
|
105
|
+
});
|
|
106
|
+
};
|
|
107
|
+
export const observeGraphileBuild = async (context, fn, opts) => {
|
|
108
|
+
if (!opts.enabled) {
|
|
109
|
+
return fn();
|
|
110
|
+
}
|
|
111
|
+
const startedAt = Date.now();
|
|
112
|
+
recordBuildStart(context, startedAt);
|
|
113
|
+
try {
|
|
114
|
+
const result = await fn();
|
|
115
|
+
recordBuildSuccess(context, startedAt);
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
recordBuildFailure(context, startedAt, error);
|
|
120
|
+
throw error;
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
export const resetGraphileBuildStats = () => {
|
|
124
|
+
buildStats.started = 0;
|
|
125
|
+
buildStats.succeeded = 0;
|
|
126
|
+
buildStats.failed = 0;
|
|
127
|
+
buildStats.totalMs = 0;
|
|
128
|
+
buildStats.maxMs = 0;
|
|
129
|
+
buildStats.lastMs = 0;
|
|
130
|
+
buildStats.lastKey = null;
|
|
131
|
+
buildStats.lastStartedAt = null;
|
|
132
|
+
buildStats.lastFinishedAt = null;
|
|
133
|
+
buildStats.lastError = null;
|
|
134
|
+
buildStats.lastServiceKey = null;
|
|
135
|
+
buildStats.lastDatabaseId = null;
|
|
136
|
+
buildStats.recentEvents = [];
|
|
137
|
+
buildStats.byServiceKey.clear();
|
|
138
|
+
};
|
|
139
|
+
export function getGraphileBuildStats() {
|
|
140
|
+
const completed = buildStats.succeeded + buildStats.failed;
|
|
141
|
+
const withAverages = (map) => Object.fromEntries([...map.entries()].map(([key, value]) => [
|
|
142
|
+
key,
|
|
143
|
+
{
|
|
144
|
+
...value,
|
|
145
|
+
averageMs: value.count > 0 ? value.totalMs / value.count : 0,
|
|
146
|
+
},
|
|
147
|
+
]));
|
|
148
|
+
return {
|
|
149
|
+
started: buildStats.started,
|
|
150
|
+
succeeded: buildStats.succeeded,
|
|
151
|
+
failed: buildStats.failed,
|
|
152
|
+
totalMs: buildStats.totalMs,
|
|
153
|
+
maxMs: buildStats.maxMs,
|
|
154
|
+
lastMs: buildStats.lastMs,
|
|
155
|
+
averageMs: completed > 0 ? buildStats.totalMs / completed : 0,
|
|
156
|
+
lastKey: buildStats.lastKey,
|
|
157
|
+
lastStartedAt: buildStats.lastStartedAt,
|
|
158
|
+
lastFinishedAt: buildStats.lastFinishedAt,
|
|
159
|
+
lastError: buildStats.lastError,
|
|
160
|
+
lastServiceKey: buildStats.lastServiceKey,
|
|
161
|
+
lastDatabaseId: buildStats.lastDatabaseId,
|
|
162
|
+
recentEvents: [...buildStats.recentEvents],
|
|
163
|
+
byServiceKey: withAverages(buildStats.byServiceKey),
|
|
164
|
+
};
|
|
165
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { isLoopbackAddress, isLoopbackHost } from '../../diagnostics/observability';
|
|
2
|
+
export const localObservabilityOnly = (req, res, next) => {
|
|
3
|
+
const remoteAddress = req.socket.remoteAddress;
|
|
4
|
+
if (isLoopbackAddress(remoteAddress)) {
|
|
5
|
+
next();
|
|
6
|
+
return;
|
|
7
|
+
}
|
|
8
|
+
const hostHeader = req.headers.host;
|
|
9
|
+
if (!remoteAddress && isLoopbackHost(hostHeader)) {
|
|
10
|
+
next();
|
|
11
|
+
return;
|
|
12
|
+
}
|
|
13
|
+
res.status(404).send('Not found');
|
|
14
|
+
};
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
2
|
+
import { Logger } from '@pgpmjs/logger';
|
|
3
|
+
const log = new Logger('server');
|
|
4
|
+
const SAFE_REQUEST_ID = /^[a-zA-Z0-9\-_]{1,128}$/;
|
|
5
|
+
export const createRequestLogger = ({ observabilityEnabled }) => {
|
|
6
|
+
return (req, res, next) => {
|
|
7
|
+
const headerRequestId = req.header('x-request-id');
|
|
8
|
+
const reqId = (headerRequestId && SAFE_REQUEST_ID.test(headerRequestId))
|
|
9
|
+
? headerRequestId
|
|
10
|
+
: randomUUID();
|
|
11
|
+
const start = process.hrtime.bigint();
|
|
12
|
+
let finished = false;
|
|
13
|
+
req.requestId = reqId;
|
|
14
|
+
const host = req.hostname || req.headers.host || 'unknown';
|
|
15
|
+
const ip = req.clientIp ?? req.ip ?? 'unknown';
|
|
16
|
+
log.debug(`[${reqId}] -> ${req.method} ${req.originalUrl} host=${host} ip=${ip}`);
|
|
17
|
+
res.on('finish', () => {
|
|
18
|
+
finished = true;
|
|
19
|
+
const durationMs = Number(process.hrtime.bigint() - start) / 1e6;
|
|
20
|
+
const apiInfo = req.api
|
|
21
|
+
? `db=${req.api.dbname} schemas=${req.api.schema?.join(',') || 'none'}`
|
|
22
|
+
: 'api=unresolved';
|
|
23
|
+
const authInfo = req.token ? 'auth=token' : 'auth=anon';
|
|
24
|
+
const svcInfo = req.svc_key ? `svc=${req.svc_key}` : 'svc=unset';
|
|
25
|
+
log.debug(`[${reqId}] <- ${res.statusCode} ${req.method} ${req.originalUrl} (${durationMs.toFixed(1)} ms) ${apiInfo} ${svcInfo} ${authInfo}`);
|
|
26
|
+
});
|
|
27
|
+
if (observabilityEnabled) {
|
|
28
|
+
res.on('close', () => {
|
|
29
|
+
if (finished) {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const durationMs = Number(process.hrtime.bigint() - start) / 1e6;
|
|
33
|
+
const apiInfo = req.api
|
|
34
|
+
? `db=${req.api.dbname} schemas=${req.api.schema?.join(',') || 'none'}`
|
|
35
|
+
: 'api=unresolved';
|
|
36
|
+
log.warn(`[${reqId}] connection closed before response completed ` +
|
|
37
|
+
`${req.method} ${req.originalUrl} (${durationMs.toFixed(1)} ms) ${apiInfo}`);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
next();
|
|
41
|
+
};
|
|
42
|
+
};
|
package/esm/server.js
CHANGED
|
@@ -2,12 +2,13 @@ import { getEnvOptions } from '@constructive-io/graphql-env';
|
|
|
2
2
|
import { Logger } from '@pgpmjs/logger';
|
|
3
3
|
import { healthz, poweredBy, svcCache, trustProxy } from '@pgpmjs/server-utils';
|
|
4
4
|
import { middleware as parseDomains } from '@constructive-io/url-domains';
|
|
5
|
-
import { randomUUID } from 'crypto';
|
|
6
5
|
import express from 'express';
|
|
7
6
|
import graphqlUpload from 'graphql-upload';
|
|
8
7
|
import { graphileCache, closeAllCaches } from 'graphile-cache';
|
|
9
8
|
import { getPgPool } from 'pg-cache';
|
|
10
9
|
import requestIp from 'request-ip';
|
|
10
|
+
import { closeDebugDatabasePools } from './diagnostics/debug-db-snapshot';
|
|
11
|
+
import { isDevelopmentObservabilityMode, isGraphqlObservabilityEnabled, isGraphqlObservabilityRequested, isLoopbackHost, } from './diagnostics/observability';
|
|
11
12
|
import { createApiMiddleware } from './middleware/api';
|
|
12
13
|
import { createAuthenticateMiddleware } from './middleware/auth';
|
|
13
14
|
import { cors } from './middleware/cors';
|
|
@@ -16,8 +17,12 @@ import { favicon } from './middleware/favicon';
|
|
|
16
17
|
import { flush, flushService } from './middleware/flush';
|
|
17
18
|
import { graphile } from './middleware/graphile';
|
|
18
19
|
import { multipartBridge } from './middleware/multipart-bridge';
|
|
20
|
+
import { createDebugDatabaseMiddleware } from './middleware/observability/debug-db';
|
|
21
|
+
import { debugMemory } from './middleware/observability/debug-memory';
|
|
22
|
+
import { localObservabilityOnly } from './middleware/observability/guard';
|
|
23
|
+
import { createRequestLogger } from './middleware/observability/request-logger';
|
|
19
24
|
import { createUploadAuthenticateMiddleware, uploadRoute } from './middleware/upload';
|
|
20
|
-
import {
|
|
25
|
+
import { startDebugSampler } from './diagnostics/debug-sampler';
|
|
21
26
|
const log = new Logger('server');
|
|
22
27
|
/**
|
|
23
28
|
* Creates and starts a GraphQL server instance
|
|
@@ -54,35 +59,17 @@ class Server {
|
|
|
54
59
|
shuttingDown = false;
|
|
55
60
|
closed = false;
|
|
56
61
|
httpServer = null;
|
|
62
|
+
debugSampler = null;
|
|
57
63
|
constructor(opts) {
|
|
58
64
|
this.opts = getEnvOptions(opts);
|
|
59
65
|
const effectiveOpts = this.opts;
|
|
66
|
+
const observabilityRequested = isGraphqlObservabilityRequested();
|
|
67
|
+
const observabilityEnabled = isGraphqlObservabilityEnabled(effectiveOpts.server?.host);
|
|
60
68
|
const app = express();
|
|
61
69
|
const api = createApiMiddleware(effectiveOpts);
|
|
62
70
|
const authenticate = createAuthenticateMiddleware(effectiveOpts);
|
|
63
71
|
const uploadAuthenticate = createUploadAuthenticateMiddleware(effectiveOpts);
|
|
64
|
-
const
|
|
65
|
-
const requestLogger = (req, res, next) => {
|
|
66
|
-
const headerRequestId = req.header('x-request-id');
|
|
67
|
-
const reqId = (headerRequestId && SAFE_REQUEST_ID.test(headerRequestId))
|
|
68
|
-
? headerRequestId
|
|
69
|
-
: randomUUID();
|
|
70
|
-
const start = process.hrtime.bigint();
|
|
71
|
-
req.requestId = reqId;
|
|
72
|
-
const host = req.hostname || req.headers.host || 'unknown';
|
|
73
|
-
const ip = req.clientIp || req.ip;
|
|
74
|
-
log.debug(`[${reqId}] -> ${req.method} ${req.originalUrl} host=${host} ip=${ip}`);
|
|
75
|
-
res.on('finish', () => {
|
|
76
|
-
const durationMs = Number(process.hrtime.bigint() - start) / 1e6;
|
|
77
|
-
const apiInfo = req.api
|
|
78
|
-
? `db=${req.api.dbname} schemas=${req.api.schema?.join(',') || 'none'}`
|
|
79
|
-
: 'api=unresolved';
|
|
80
|
-
const authInfo = req.token ? 'auth=token' : 'auth=anon';
|
|
81
|
-
const svcInfo = req.svc_key ? `svc=${req.svc_key}` : 'svc=unset';
|
|
82
|
-
log.debug(`[${reqId}] <- ${res.statusCode} ${req.method} ${req.originalUrl} (${durationMs.toFixed(1)} ms) ${apiInfo} ${svcInfo} ${authInfo}`);
|
|
83
|
-
});
|
|
84
|
-
next();
|
|
85
|
-
};
|
|
72
|
+
const requestLogger = createRequestLogger({ observabilityEnabled });
|
|
86
73
|
// Log startup configuration (non-sensitive values only)
|
|
87
74
|
const apiOpts = effectiveOpts.api || {};
|
|
88
75
|
log.info('[server] Starting with config:', {
|
|
@@ -97,9 +84,28 @@ class Server {
|
|
|
97
84
|
exposedSchemas: apiOpts.exposedSchemas?.join(',') || 'none',
|
|
98
85
|
anonRole: apiOpts.anonRole,
|
|
99
86
|
roleName: apiOpts.roleName,
|
|
87
|
+
observabilityEnabled,
|
|
100
88
|
});
|
|
89
|
+
if (observabilityRequested && !observabilityEnabled) {
|
|
90
|
+
const reasons = [];
|
|
91
|
+
if (!isDevelopmentObservabilityMode()) {
|
|
92
|
+
reasons.push('NODE_ENV must be development');
|
|
93
|
+
}
|
|
94
|
+
if (!isLoopbackHost(effectiveOpts.server?.host)) {
|
|
95
|
+
reasons.push('server host must be localhost, 127.0.0.1, or ::1');
|
|
96
|
+
}
|
|
97
|
+
log.warn(`GRAPHQL_OBSERVABILITY_ENABLED was requested but observability remains disabled${reasons.length > 0 ? `: ${reasons.join('; ')}` : ''}`);
|
|
98
|
+
}
|
|
101
99
|
healthz(app);
|
|
102
|
-
|
|
100
|
+
if (observabilityEnabled) {
|
|
101
|
+
app.get('/debug/memory', localObservabilityOnly, debugMemory);
|
|
102
|
+
app.get('/debug/db', localObservabilityOnly, createDebugDatabaseMiddleware(effectiveOpts));
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
105
|
+
app.use('/debug', (_req, res) => {
|
|
106
|
+
res.status(404).send('Not found');
|
|
107
|
+
});
|
|
108
|
+
}
|
|
103
109
|
app.use(favicon);
|
|
104
110
|
trustProxy(app, effectiveOpts.server.trustProxy);
|
|
105
111
|
// Warn if a global CORS override is set in production
|
|
@@ -132,6 +138,7 @@ class Server {
|
|
|
132
138
|
app.use(notFoundHandler); // Catches unmatched routes (404)
|
|
133
139
|
app.use(errorHandler); // Catches all thrown errors
|
|
134
140
|
this.app = app;
|
|
141
|
+
this.debugSampler = observabilityEnabled ? startDebugSampler(effectiveOpts) : null;
|
|
135
142
|
}
|
|
136
143
|
listen() {
|
|
137
144
|
const { server } = this.opts;
|
|
@@ -221,9 +228,14 @@ class Server {
|
|
|
221
228
|
this.closed = true;
|
|
222
229
|
this.shuttingDown = true;
|
|
223
230
|
await this.removeEventListener();
|
|
231
|
+
if (this.debugSampler) {
|
|
232
|
+
await this.debugSampler.stop();
|
|
233
|
+
this.debugSampler = null;
|
|
234
|
+
}
|
|
224
235
|
if (this.httpServer?.listening) {
|
|
225
236
|
await new Promise((resolve) => this.httpServer.close(() => resolve()));
|
|
226
237
|
}
|
|
238
|
+
await closeDebugDatabasePools();
|
|
227
239
|
if (closeCaches) {
|
|
228
240
|
await Server.closeCaches({ closePools: true });
|
|
229
241
|
}
|
package/middleware/graphile.js
CHANGED
|
@@ -15,7 +15,9 @@ const graphile_settings_1 = require("graphile-settings");
|
|
|
15
15
|
const pg_cache_1 = require("pg-cache");
|
|
16
16
|
const pg_env_1 = require("pg-env");
|
|
17
17
|
require("./types"); // for Request type
|
|
18
|
+
const observability_1 = require("../diagnostics/observability");
|
|
18
19
|
const api_errors_1 = require("../errors/api-errors");
|
|
20
|
+
const graphile_build_stats_1 = require("./observability/graphile-build-stats");
|
|
19
21
|
const maskErrorLog = new logger_1.Logger('graphile:maskError');
|
|
20
22
|
const SAFE_ERROR_CODES = new Set([
|
|
21
23
|
// GraphQL standard
|
|
@@ -179,6 +181,7 @@ const buildPreset = (connectionString, schemas, anonRole, roleName) => ({
|
|
|
179
181
|
},
|
|
180
182
|
});
|
|
181
183
|
const graphile = (opts) => {
|
|
184
|
+
const observabilityEnabled = (0, observability_1.isGraphqlObservabilityEnabled)(opts.server?.host);
|
|
182
185
|
return async (req, res, next) => {
|
|
183
186
|
const label = reqLabel(req);
|
|
184
187
|
try {
|
|
@@ -242,7 +245,11 @@ const graphile = (opts) => {
|
|
|
242
245
|
const connectionString = (0, pg_cache_1.buildConnectionString)(pgConfig.user, pgConfig.password, pgConfig.host, pgConfig.port, pgConfig.database);
|
|
243
246
|
// Create promise and store in in-flight map BEFORE try block
|
|
244
247
|
const preset = buildPreset(connectionString, schema || [], anonRole, roleName);
|
|
245
|
-
const creationPromise = (0,
|
|
248
|
+
const creationPromise = (0, graphile_build_stats_1.observeGraphileBuild)({
|
|
249
|
+
cacheKey: key,
|
|
250
|
+
serviceKey: key,
|
|
251
|
+
databaseId: api.databaseId ?? null,
|
|
252
|
+
}, () => (0, graphile_cache_1.createGraphileInstance)({ preset, cacheKey: key }), { enabled: observabilityEnabled });
|
|
246
253
|
creating.set(key, creationPromise);
|
|
247
254
|
try {
|
|
248
255
|
const instance = await creationPromise;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createDebugDatabaseMiddleware = void 0;
|
|
4
|
+
const logger_1 = require("@pgpmjs/logger");
|
|
5
|
+
const debug_db_snapshot_1 = require("../../diagnostics/debug-db-snapshot");
|
|
6
|
+
const log = new logger_1.Logger('debug-db');
|
|
7
|
+
const createDebugDatabaseMiddleware = (opts) => {
|
|
8
|
+
return async (_req, res) => {
|
|
9
|
+
try {
|
|
10
|
+
const response = await (0, debug_db_snapshot_1.getDebugDatabaseSnapshot)(opts);
|
|
11
|
+
log.debug('Database debug snapshot', {
|
|
12
|
+
activeActivity: response.activeActivity.length,
|
|
13
|
+
blockedActivity: response.blockedActivity.length,
|
|
14
|
+
lockSummary: response.lockSummary.length,
|
|
15
|
+
});
|
|
16
|
+
res.json(response);
|
|
17
|
+
}
|
|
18
|
+
catch (error) {
|
|
19
|
+
log.error('Failed to fetch debug DB snapshot', error);
|
|
20
|
+
res.status(500).json({
|
|
21
|
+
error: 'Failed to fetch database debug snapshot',
|
|
22
|
+
message: error instanceof Error ? error.message : String(error),
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
};
|
|
27
|
+
exports.createDebugDatabaseMiddleware = createDebugDatabaseMiddleware;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.debugMemory = void 0;
|
|
4
|
+
const logger_1 = require("@pgpmjs/logger");
|
|
5
|
+
const debug_memory_snapshot_1 = require("../../diagnostics/debug-memory-snapshot");
|
|
6
|
+
const log = new logger_1.Logger('debug-memory');
|
|
7
|
+
const debugMemory = (_req, res) => {
|
|
8
|
+
const response = (0, debug_memory_snapshot_1.getDebugMemorySnapshot)();
|
|
9
|
+
log.debug('Memory snapshot:', response);
|
|
10
|
+
res.json(response);
|
|
11
|
+
};
|
|
12
|
+
exports.debugMemory = debugMemory;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export interface GraphileBuildEvent {
|
|
2
|
+
type: 'start' | 'success' | 'failure';
|
|
3
|
+
cacheKey: string;
|
|
4
|
+
serviceKey: string | null;
|
|
5
|
+
databaseId: string | null;
|
|
6
|
+
durationMs?: number;
|
|
7
|
+
error?: string;
|
|
8
|
+
timestamp: string;
|
|
9
|
+
}
|
|
10
|
+
interface GraphileBuildAggregate {
|
|
11
|
+
count: number;
|
|
12
|
+
totalMs: number;
|
|
13
|
+
maxMs: number;
|
|
14
|
+
lastMs: number;
|
|
15
|
+
lastAt: string | null;
|
|
16
|
+
}
|
|
17
|
+
interface GraphileBuildContext {
|
|
18
|
+
cacheKey: string;
|
|
19
|
+
serviceKey: string | null;
|
|
20
|
+
databaseId: string | null;
|
|
21
|
+
}
|
|
22
|
+
export declare const observeGraphileBuild: <T>(context: GraphileBuildContext, fn: () => Promise<T>, opts: {
|
|
23
|
+
enabled: boolean;
|
|
24
|
+
}) => Promise<T>;
|
|
25
|
+
export declare const resetGraphileBuildStats: () => void;
|
|
26
|
+
export declare function getGraphileBuildStats(): {
|
|
27
|
+
started: number;
|
|
28
|
+
succeeded: number;
|
|
29
|
+
failed: number;
|
|
30
|
+
totalMs: number;
|
|
31
|
+
maxMs: number;
|
|
32
|
+
lastMs: number;
|
|
33
|
+
averageMs: number;
|
|
34
|
+
lastKey: string | null;
|
|
35
|
+
lastStartedAt: string | null;
|
|
36
|
+
lastFinishedAt: string | null;
|
|
37
|
+
lastError: string | null;
|
|
38
|
+
lastServiceKey: string | null;
|
|
39
|
+
lastDatabaseId: string | null;
|
|
40
|
+
recentEvents: GraphileBuildEvent[];
|
|
41
|
+
byServiceKey: Record<string, GraphileBuildAggregate & {
|
|
42
|
+
averageMs: number;
|
|
43
|
+
}>;
|
|
44
|
+
};
|
|
45
|
+
export {};
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.resetGraphileBuildStats = exports.observeGraphileBuild = void 0;
|
|
4
|
+
exports.getGraphileBuildStats = getGraphileBuildStats;
|
|
5
|
+
const MAX_BUILD_EVENTS = 100;
|
|
6
|
+
const MAX_AGGREGATE_KEYS = 100;
|
|
7
|
+
const buildStats = {
|
|
8
|
+
started: 0,
|
|
9
|
+
succeeded: 0,
|
|
10
|
+
failed: 0,
|
|
11
|
+
totalMs: 0,
|
|
12
|
+
maxMs: 0,
|
|
13
|
+
lastMs: 0,
|
|
14
|
+
lastKey: null,
|
|
15
|
+
lastStartedAt: null,
|
|
16
|
+
lastFinishedAt: null,
|
|
17
|
+
lastError: null,
|
|
18
|
+
lastServiceKey: null,
|
|
19
|
+
lastDatabaseId: null,
|
|
20
|
+
recentEvents: [],
|
|
21
|
+
byServiceKey: new Map(),
|
|
22
|
+
};
|
|
23
|
+
const pushBuildEvent = (event) => {
|
|
24
|
+
buildStats.recentEvents.push(event);
|
|
25
|
+
if (buildStats.recentEvents.length > MAX_BUILD_EVENTS) {
|
|
26
|
+
buildStats.recentEvents.splice(0, buildStats.recentEvents.length - MAX_BUILD_EVENTS);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const updateAggregate = (map, key, durationMs) => {
|
|
30
|
+
const hasKey = map.has(key);
|
|
31
|
+
if (!hasKey && map.size >= MAX_AGGREGATE_KEYS) {
|
|
32
|
+
const oldestKey = map.keys().next().value;
|
|
33
|
+
if (oldestKey) {
|
|
34
|
+
map.delete(oldestKey);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
const current = map.get(key) ?? {
|
|
38
|
+
count: 0,
|
|
39
|
+
totalMs: 0,
|
|
40
|
+
maxMs: 0,
|
|
41
|
+
lastMs: 0,
|
|
42
|
+
lastAt: null,
|
|
43
|
+
};
|
|
44
|
+
current.count += 1;
|
|
45
|
+
current.totalMs += durationMs;
|
|
46
|
+
current.maxMs = Math.max(current.maxMs, durationMs);
|
|
47
|
+
current.lastMs = durationMs;
|
|
48
|
+
current.lastAt = new Date().toISOString();
|
|
49
|
+
// Keep LRU order by reinserting after each update.
|
|
50
|
+
if (hasKey) {
|
|
51
|
+
map.delete(key);
|
|
52
|
+
}
|
|
53
|
+
map.set(key, current);
|
|
54
|
+
};
|
|
55
|
+
const recordBuildStart = (context, startedAt) => {
|
|
56
|
+
buildStats.started += 1;
|
|
57
|
+
buildStats.lastKey = context.cacheKey;
|
|
58
|
+
buildStats.lastStartedAt = new Date(startedAt).toISOString();
|
|
59
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
60
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
61
|
+
pushBuildEvent({
|
|
62
|
+
type: 'start',
|
|
63
|
+
cacheKey: context.cacheKey,
|
|
64
|
+
serviceKey: context.serviceKey,
|
|
65
|
+
databaseId: context.databaseId,
|
|
66
|
+
timestamp: new Date(startedAt).toISOString(),
|
|
67
|
+
});
|
|
68
|
+
};
|
|
69
|
+
const recordBuildSuccess = (context, startedAt) => {
|
|
70
|
+
const durationMs = Date.now() - startedAt;
|
|
71
|
+
buildStats.succeeded += 1;
|
|
72
|
+
buildStats.totalMs += durationMs;
|
|
73
|
+
buildStats.maxMs = Math.max(buildStats.maxMs, durationMs);
|
|
74
|
+
buildStats.lastMs = durationMs;
|
|
75
|
+
buildStats.lastFinishedAt = new Date().toISOString();
|
|
76
|
+
buildStats.lastError = null;
|
|
77
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
78
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
79
|
+
if (context.serviceKey) {
|
|
80
|
+
updateAggregate(buildStats.byServiceKey, context.serviceKey, durationMs);
|
|
81
|
+
}
|
|
82
|
+
pushBuildEvent({
|
|
83
|
+
type: 'success',
|
|
84
|
+
cacheKey: context.cacheKey,
|
|
85
|
+
serviceKey: context.serviceKey,
|
|
86
|
+
databaseId: context.databaseId,
|
|
87
|
+
durationMs,
|
|
88
|
+
timestamp: new Date().toISOString(),
|
|
89
|
+
});
|
|
90
|
+
};
|
|
91
|
+
const recordBuildFailure = (context, startedAt, error) => {
|
|
92
|
+
const durationMs = Date.now() - startedAt;
|
|
93
|
+
buildStats.failed += 1;
|
|
94
|
+
buildStats.totalMs += durationMs;
|
|
95
|
+
buildStats.maxMs = Math.max(buildStats.maxMs, durationMs);
|
|
96
|
+
buildStats.lastMs = durationMs;
|
|
97
|
+
buildStats.lastFinishedAt = new Date().toISOString();
|
|
98
|
+
buildStats.lastError = error instanceof Error ? error.message : String(error);
|
|
99
|
+
buildStats.lastServiceKey = context.serviceKey;
|
|
100
|
+
buildStats.lastDatabaseId = context.databaseId;
|
|
101
|
+
pushBuildEvent({
|
|
102
|
+
type: 'failure',
|
|
103
|
+
cacheKey: context.cacheKey,
|
|
104
|
+
serviceKey: context.serviceKey,
|
|
105
|
+
databaseId: context.databaseId,
|
|
106
|
+
durationMs,
|
|
107
|
+
error: error instanceof Error ? error.message : String(error),
|
|
108
|
+
timestamp: new Date().toISOString(),
|
|
109
|
+
});
|
|
110
|
+
};
|
|
111
|
+
const observeGraphileBuild = async (context, fn, opts) => {
|
|
112
|
+
if (!opts.enabled) {
|
|
113
|
+
return fn();
|
|
114
|
+
}
|
|
115
|
+
const startedAt = Date.now();
|
|
116
|
+
recordBuildStart(context, startedAt);
|
|
117
|
+
try {
|
|
118
|
+
const result = await fn();
|
|
119
|
+
recordBuildSuccess(context, startedAt);
|
|
120
|
+
return result;
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
recordBuildFailure(context, startedAt, error);
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
exports.observeGraphileBuild = observeGraphileBuild;
|
|
128
|
+
const resetGraphileBuildStats = () => {
|
|
129
|
+
buildStats.started = 0;
|
|
130
|
+
buildStats.succeeded = 0;
|
|
131
|
+
buildStats.failed = 0;
|
|
132
|
+
buildStats.totalMs = 0;
|
|
133
|
+
buildStats.maxMs = 0;
|
|
134
|
+
buildStats.lastMs = 0;
|
|
135
|
+
buildStats.lastKey = null;
|
|
136
|
+
buildStats.lastStartedAt = null;
|
|
137
|
+
buildStats.lastFinishedAt = null;
|
|
138
|
+
buildStats.lastError = null;
|
|
139
|
+
buildStats.lastServiceKey = null;
|
|
140
|
+
buildStats.lastDatabaseId = null;
|
|
141
|
+
buildStats.recentEvents = [];
|
|
142
|
+
buildStats.byServiceKey.clear();
|
|
143
|
+
};
|
|
144
|
+
exports.resetGraphileBuildStats = resetGraphileBuildStats;
|
|
145
|
+
function getGraphileBuildStats() {
|
|
146
|
+
const completed = buildStats.succeeded + buildStats.failed;
|
|
147
|
+
const withAverages = (map) => Object.fromEntries([...map.entries()].map(([key, value]) => [
|
|
148
|
+
key,
|
|
149
|
+
{
|
|
150
|
+
...value,
|
|
151
|
+
averageMs: value.count > 0 ? value.totalMs / value.count : 0,
|
|
152
|
+
},
|
|
153
|
+
]));
|
|
154
|
+
return {
|
|
155
|
+
started: buildStats.started,
|
|
156
|
+
succeeded: buildStats.succeeded,
|
|
157
|
+
failed: buildStats.failed,
|
|
158
|
+
totalMs: buildStats.totalMs,
|
|
159
|
+
maxMs: buildStats.maxMs,
|
|
160
|
+
lastMs: buildStats.lastMs,
|
|
161
|
+
averageMs: completed > 0 ? buildStats.totalMs / completed : 0,
|
|
162
|
+
lastKey: buildStats.lastKey,
|
|
163
|
+
lastStartedAt: buildStats.lastStartedAt,
|
|
164
|
+
lastFinishedAt: buildStats.lastFinishedAt,
|
|
165
|
+
lastError: buildStats.lastError,
|
|
166
|
+
lastServiceKey: buildStats.lastServiceKey,
|
|
167
|
+
lastDatabaseId: buildStats.lastDatabaseId,
|
|
168
|
+
recentEvents: [...buildStats.recentEvents],
|
|
169
|
+
byServiceKey: withAverages(buildStats.byServiceKey),
|
|
170
|
+
};
|
|
171
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.localObservabilityOnly = void 0;
|
|
4
|
+
const observability_1 = require("../../diagnostics/observability");
|
|
5
|
+
const localObservabilityOnly = (req, res, next) => {
|
|
6
|
+
const remoteAddress = req.socket.remoteAddress;
|
|
7
|
+
if ((0, observability_1.isLoopbackAddress)(remoteAddress)) {
|
|
8
|
+
next();
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const hostHeader = req.headers.host;
|
|
12
|
+
if (!remoteAddress && (0, observability_1.isLoopbackHost)(hostHeader)) {
|
|
13
|
+
next();
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
res.status(404).send('Not found');
|
|
17
|
+
};
|
|
18
|
+
exports.localObservabilityOnly = localObservabilityOnly;
|