@constructive-io/express-context 0.26.1 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/admission.d.ts +146 -0
- package/admission.js +249 -0
- package/esm/admission.js +242 -0
- package/esm/index.js +5 -1
- package/esm/loaders/index.js +4 -0
- package/esm/loaders/request-protection.js +146 -0
- package/esm/refusals.js +214 -0
- package/esm/request-protection.js +106 -0
- package/esm/usage-counter.js +190 -0
- package/index.d.ts +9 -1
- package/index.js +31 -1
- package/loaders/index.d.ts +2 -0
- package/loaders/index.js +6 -1
- package/loaders/request-protection.d.ts +22 -0
- package/loaders/request-protection.js +150 -0
- package/package.json +10 -10
- package/refusals.d.ts +154 -0
- package/refusals.js +223 -0
- package/request-protection.d.ts +117 -0
- package/request-protection.js +111 -0
- package/types.d.ts +2 -0
- package/usage-counter.d.ts +137 -0
- package/usage-counter.js +195 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Request Protection Loader
|
|
3
|
+
*
|
|
4
|
+
* Reads the per-database bounds and the per-API overrides raw, then resolves
|
|
5
|
+
* them through the platform's own defaults and ceilings.
|
|
6
|
+
*
|
|
7
|
+
* Unlike the feature-flag loader next door this cannot `COALESCE` in SQL: an
|
|
8
|
+
* API override is lower-only, so the two scopes are combined with `LEAST` and
|
|
9
|
+
* then clamped in code, where the platform constants live.
|
|
10
|
+
*
|
|
11
|
+
* The columns are also selected by feature detection rather than by name
|
|
12
|
+
* alone. A serving cluster runs routing planes of several ages — the plane is
|
|
13
|
+
* a published package a tenant upgrades on its own schedule — and a plane that
|
|
14
|
+
* predates these columns must resolve to the platform defaults, not fail every
|
|
15
|
+
* request against it with `column ds.… does not exist`.
|
|
16
|
+
*/
|
|
17
|
+
import { resolveRequestProtection } from '../request-protection';
|
|
18
|
+
import { createModuleLoader } from './create-loader';
|
|
19
|
+
import { routingSchemaOf } from './types';
|
|
20
|
+
// ─── SQL ────────────────────────────────────────────────────────────────────
|
|
21
|
+
const PROTECTION_COLUMNS = [
|
|
22
|
+
'statement_timeout_ms',
|
|
23
|
+
'idle_in_transaction_timeout_ms',
|
|
24
|
+
'lock_timeout_ms',
|
|
25
|
+
'max_concurrent_requests',
|
|
26
|
+
'max_queue_wait_ms',
|
|
27
|
+
'rate_limit_rpm',
|
|
28
|
+
'rate_limit_burst',
|
|
29
|
+
'max_query_depth',
|
|
30
|
+
'max_query_cost',
|
|
31
|
+
'max_page_size',
|
|
32
|
+
'max_request_bytes',
|
|
33
|
+
'enable_introspection'
|
|
34
|
+
];
|
|
35
|
+
const availableColumnsSql = `
|
|
36
|
+
SELECT table_name, column_name
|
|
37
|
+
FROM information_schema.columns
|
|
38
|
+
WHERE table_schema = $1
|
|
39
|
+
AND table_name IN ('database_settings', 'api_settings')
|
|
40
|
+
AND column_name = ANY($2::text[])
|
|
41
|
+
`;
|
|
42
|
+
const requestProtectionSql = (schema, columns) => {
|
|
43
|
+
const select = PROTECTION_COLUMNS.flatMap((column) => [
|
|
44
|
+
columns.database.has(column) ? `ds.${column} AS db_${column}` : null,
|
|
45
|
+
columns.api.has(column) ? `aps.${column} AS api_${column}` : null
|
|
46
|
+
])
|
|
47
|
+
.filter((expr) => expr !== null)
|
|
48
|
+
.join(',\n ');
|
|
49
|
+
// The join is dropped when the plane carries no API-scope column, so a plane
|
|
50
|
+
// without an `api_settings` table at all is still readable.
|
|
51
|
+
const join = columns.api.size
|
|
52
|
+
? `\n LEFT JOIN "${schema}".api_settings aps ON ds.database_id = aps.database_id AND aps.api_id = $2`
|
|
53
|
+
: '';
|
|
54
|
+
return `
|
|
55
|
+
SELECT
|
|
56
|
+
ds.database_id${select ? `,\n ${select}` : ''}
|
|
57
|
+
FROM "${schema}".database_settings ds${join}
|
|
58
|
+
WHERE ds.database_id = $1
|
|
59
|
+
LIMIT 1
|
|
60
|
+
`;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* The routing plane's shape changes only when it is redeployed, so it is read
|
|
64
|
+
* once per pool+schema. Keyed by the pool object so a test pool and a serving
|
|
65
|
+
* pool never share an answer.
|
|
66
|
+
*/
|
|
67
|
+
const columnCache = new WeakMap();
|
|
68
|
+
const availableColumns = (pool, schema) => {
|
|
69
|
+
let bySchema = columnCache.get(pool);
|
|
70
|
+
if (!bySchema) {
|
|
71
|
+
bySchema = new Map();
|
|
72
|
+
columnCache.set(pool, bySchema);
|
|
73
|
+
}
|
|
74
|
+
const cached = bySchema.get(schema);
|
|
75
|
+
if (cached)
|
|
76
|
+
return cached;
|
|
77
|
+
const pending = pool
|
|
78
|
+
.query(availableColumnsSql, [
|
|
79
|
+
schema,
|
|
80
|
+
[...PROTECTION_COLUMNS]
|
|
81
|
+
])
|
|
82
|
+
.then((result) => {
|
|
83
|
+
const columns = { database: new Set(), api: new Set() };
|
|
84
|
+
for (const { table_name, column_name } of result.rows) {
|
|
85
|
+
if (table_name === 'database_settings')
|
|
86
|
+
columns.database.add(column_name);
|
|
87
|
+
else
|
|
88
|
+
columns.api.add(column_name);
|
|
89
|
+
}
|
|
90
|
+
return columns;
|
|
91
|
+
})
|
|
92
|
+
.catch((e) => {
|
|
93
|
+
// A failed probe is not an answer: drop it so the next request retries
|
|
94
|
+
// rather than caching "this plane has no protection columns" forever.
|
|
95
|
+
bySchema.delete(schema);
|
|
96
|
+
throw e;
|
|
97
|
+
});
|
|
98
|
+
bySchema.set(schema, pending);
|
|
99
|
+
return pending;
|
|
100
|
+
};
|
|
101
|
+
/** Test seam: forget what a pool's routing plane looked like. */
|
|
102
|
+
export const forgetProtectionColumns = (pool) => {
|
|
103
|
+
columnCache.delete(pool);
|
|
104
|
+
};
|
|
105
|
+
/**
|
|
106
|
+
* `bigint` columns arrive as strings from node-postgres (pg returns int8 as
|
|
107
|
+
* text to avoid the 2^53 cliff), so every numeric bound is normalized here
|
|
108
|
+
* rather than trusted to already be a number.
|
|
109
|
+
*/
|
|
110
|
+
const num = (value) => {
|
|
111
|
+
if (value === null || value === undefined)
|
|
112
|
+
return null;
|
|
113
|
+
const parsed = typeof value === 'number' ? value : Number(value);
|
|
114
|
+
return Number.isFinite(parsed) ? parsed : null;
|
|
115
|
+
};
|
|
116
|
+
const bool = (value) => value === null || value === undefined ? null : Boolean(value);
|
|
117
|
+
const scopeOf = (row, prefix) => ({
|
|
118
|
+
statementTimeoutMs: num(row[`${prefix}_statement_timeout_ms`]),
|
|
119
|
+
idleInTransactionTimeoutMs: num(row[`${prefix}_idle_in_transaction_timeout_ms`]),
|
|
120
|
+
lockTimeoutMs: num(row[`${prefix}_lock_timeout_ms`]),
|
|
121
|
+
maxConcurrentRequests: num(row[`${prefix}_max_concurrent_requests`]),
|
|
122
|
+
maxQueueWaitMs: num(row[`${prefix}_max_queue_wait_ms`]),
|
|
123
|
+
rateLimitRpm: num(row[`${prefix}_rate_limit_rpm`]),
|
|
124
|
+
rateLimitBurst: num(row[`${prefix}_rate_limit_burst`]),
|
|
125
|
+
maxQueryDepth: num(row[`${prefix}_max_query_depth`]),
|
|
126
|
+
maxQueryCost: num(row[`${prefix}_max_query_cost`]),
|
|
127
|
+
maxPageSize: num(row[`${prefix}_max_page_size`]),
|
|
128
|
+
maxRequestBytes: num(row[`${prefix}_max_request_bytes`]),
|
|
129
|
+
enableIntrospection: bool(row[`${prefix}_enable_introspection`])
|
|
130
|
+
});
|
|
131
|
+
// ─── Loader ─────────────────────────────────────────────────────────────────
|
|
132
|
+
export const requestProtectionLoader = createModuleLoader({
|
|
133
|
+
name: 'requestProtection',
|
|
134
|
+
ttlMs: 5 * 60_000,
|
|
135
|
+
async resolve(ctx) {
|
|
136
|
+
const { routingPool, databaseId, apiId } = ctx;
|
|
137
|
+
const schema = routingSchemaOf(ctx);
|
|
138
|
+
const columns = await availableColumns(routingPool, schema);
|
|
139
|
+
const params = columns.api.size ? [databaseId, apiId ?? null] : [databaseId];
|
|
140
|
+
const result = await routingPool.query(requestProtectionSql(schema, columns), params);
|
|
141
|
+
const row = result.rows[0];
|
|
142
|
+
if (!row)
|
|
143
|
+
return undefined;
|
|
144
|
+
return resolveRequestProtection(scopeOf(row, 'db'), scopeOf(row, 'api'));
|
|
145
|
+
}
|
|
146
|
+
});
|
package/esm/refusals.js
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* refusals — the refusal-observability instantiation of `usage-counter`.
|
|
3
|
+
*
|
|
4
|
+
* Every refusal the platform issues (a 429 from admission control, a 413 for an
|
|
5
|
+
* oversized body, a document-gate rejection, a gateway's route or IP window)
|
|
6
|
+
* is one `RefusalRecorder.record(...)` call at the point the response is
|
|
7
|
+
* written. That call is an in-memory increment and nothing else — no promise,
|
|
8
|
+
* no pool, no allocation beyond the key — so a flood costs a counter bump and
|
|
9
|
+
* refusing traffic keeps working when the recorder's store is down.
|
|
10
|
+
*
|
|
11
|
+
* Counts are keyed by `(minute, database, lane, reason, route, source /24)`,
|
|
12
|
+
* which is exactly the set of questions the platform table answers: which
|
|
13
|
+
* tenant, why, on which route, from which network, since when. Raw addresses
|
|
14
|
+
* are never part of the key; `sourceBucket` truncates to a /24 (v4) or /48
|
|
15
|
+
* (v6) before anything is counted.
|
|
16
|
+
*
|
|
17
|
+
* The flush writes the drained batch as one `record_refusals(jsonb)` call —
|
|
18
|
+
* one statement per window regardless of how many keys it holds.
|
|
19
|
+
*
|
|
20
|
+
* @module refusals
|
|
21
|
+
*/
|
|
22
|
+
import { Logger } from '@pgpmjs/logger';
|
|
23
|
+
import { withPgClient } from 'pg-query-context';
|
|
24
|
+
import { BoundedCounter, CounterFlusher } from './usage-counter';
|
|
25
|
+
const log = new Logger('refusals');
|
|
26
|
+
export const REFUSAL_REASONS = [
|
|
27
|
+
'rate_limited',
|
|
28
|
+
'concurrency_saturated',
|
|
29
|
+
'queue_timeout',
|
|
30
|
+
'request_too_large',
|
|
31
|
+
'query_too_deep',
|
|
32
|
+
'query_too_costly',
|
|
33
|
+
'page_size_too_large',
|
|
34
|
+
'anonymous_not_callable',
|
|
35
|
+
'route_rate_limited'
|
|
36
|
+
];
|
|
37
|
+
/** `source_bucket` when the lane could not resolve an address. */
|
|
38
|
+
export const UNKNOWN_SOURCE = 'unknown';
|
|
39
|
+
/** `source_bucket` / `route_key` of the fold-down entry once the counter is full. */
|
|
40
|
+
export const OVERFLOW_SOURCE = 'overflow';
|
|
41
|
+
export const OVERFLOW_ROUTE = '*';
|
|
42
|
+
const MINUTE_MS = 60_000;
|
|
43
|
+
const IPV4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
|
|
44
|
+
/**
|
|
45
|
+
* Reduce a client address to a network prefix: `/24` for IPv4, `/48` for IPv6.
|
|
46
|
+
* Enough to tell "one network hammering an anonymous route" from "the tenant's
|
|
47
|
+
* own users"; not enough to be a person. Anything unparseable is `unknown`.
|
|
48
|
+
*/
|
|
49
|
+
export const sourceBucket = (ip) => {
|
|
50
|
+
if (!ip)
|
|
51
|
+
return UNKNOWN_SOURCE;
|
|
52
|
+
let addr = ip.trim();
|
|
53
|
+
if (addr.startsWith('[')) {
|
|
54
|
+
const end = addr.indexOf(']');
|
|
55
|
+
if (end === -1)
|
|
56
|
+
return UNKNOWN_SOURCE;
|
|
57
|
+
addr = addr.slice(1, end);
|
|
58
|
+
}
|
|
59
|
+
const zone = addr.indexOf('%');
|
|
60
|
+
if (zone !== -1)
|
|
61
|
+
addr = addr.slice(0, zone);
|
|
62
|
+
if (addr.toLowerCase().startsWith('::ffff:')) {
|
|
63
|
+
const mapped = addr.slice('::ffff:'.length);
|
|
64
|
+
if (IPV4.test(mapped))
|
|
65
|
+
addr = mapped;
|
|
66
|
+
}
|
|
67
|
+
const v4 = IPV4.exec(addr);
|
|
68
|
+
if (v4) {
|
|
69
|
+
const octets = v4.slice(1, 5).map(Number);
|
|
70
|
+
if (octets.some((o) => o > 255))
|
|
71
|
+
return UNKNOWN_SOURCE;
|
|
72
|
+
return `${octets[0]}.${octets[1]}.${octets[2]}.0/24`;
|
|
73
|
+
}
|
|
74
|
+
const groups = expandIpv6(addr);
|
|
75
|
+
if (!groups)
|
|
76
|
+
return UNKNOWN_SOURCE;
|
|
77
|
+
return `${groups[0]}:${groups[1]}:${groups[2]}::/48`;
|
|
78
|
+
};
|
|
79
|
+
/** The eight hextets of an IPv6 address, or null when it does not parse. */
|
|
80
|
+
const expandIpv6 = (addr) => {
|
|
81
|
+
if (!/^[0-9a-fA-F:]+$/.test(addr) || addr.indexOf(':::') !== -1)
|
|
82
|
+
return null;
|
|
83
|
+
const halves = addr.split('::');
|
|
84
|
+
if (halves.length > 2)
|
|
85
|
+
return null;
|
|
86
|
+
const head = halves[0] ? halves[0].split(':') : [];
|
|
87
|
+
const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
|
|
88
|
+
const missing = 8 - head.length - tail.length;
|
|
89
|
+
if (halves.length === 2 ? missing < 1 : missing !== 0)
|
|
90
|
+
return null;
|
|
91
|
+
const groups = [...head, ...Array(missing).fill('0'), ...tail];
|
|
92
|
+
if (groups.some((g) => g.length === 0 || g.length > 4))
|
|
93
|
+
return null;
|
|
94
|
+
return groups.map((g) => parseInt(g, 16).toString(16));
|
|
95
|
+
};
|
|
96
|
+
const minuteOf = (at) => Math.floor(at / MINUTE_MS) * MINUTE_MS;
|
|
97
|
+
export const refusalKeyOf = (key) => `${key.minuteBucket}\u0000${key.databaseId ?? ''}\u0000${key.lane}\u0000${key.reason}\u0000${key.routeKey}\u0000${key.sourceBucket}`;
|
|
98
|
+
/**
|
|
99
|
+
* Where a key folds to once the counter is full: same minute, tenant, lane and
|
|
100
|
+
* reason; route and source collapsed. A source-spraying flood therefore costs
|
|
101
|
+
* `maxKeys` entries plus one per `(database, lane, reason)`, the tenant/reason
|
|
102
|
+
* attribution is never lost, and the presence of an `overflow` row is itself
|
|
103
|
+
* the signal that a spray happened.
|
|
104
|
+
*/
|
|
105
|
+
export const refusalOverflowKey = (key) => ({
|
|
106
|
+
...key,
|
|
107
|
+
routeKey: OVERFLOW_ROUTE,
|
|
108
|
+
sourceBucket: OVERFLOW_SOURCE
|
|
109
|
+
});
|
|
110
|
+
/** Shape a drained batch for `record_refusals(jsonb)`. */
|
|
111
|
+
export const refusalRows = (entries) => entries.map(({ key, count, firstAt, lastAt }) => ({
|
|
112
|
+
minute_bucket: new Date(key.minuteBucket).toISOString(),
|
|
113
|
+
database_id: key.databaseId,
|
|
114
|
+
lane: key.lane,
|
|
115
|
+
reason: key.reason,
|
|
116
|
+
route_key: key.routeKey,
|
|
117
|
+
source_bucket: key.sourceBucket,
|
|
118
|
+
count,
|
|
119
|
+
first_seen_at: new Date(firstAt).toISOString(),
|
|
120
|
+
last_seen_at: new Date(lastAt).toISOString()
|
|
121
|
+
}));
|
|
122
|
+
export const DEFAULT_REFUSAL_FLUSH_INTERVAL_MS = 10_000;
|
|
123
|
+
export const DEFAULT_REFUSAL_FLUSH_JITTER_MS = 2_000;
|
|
124
|
+
export const DEFAULT_REFUSAL_MAX_KEYS = 2_000;
|
|
125
|
+
const defaultOnError = (err, dropped) => {
|
|
126
|
+
const refusals = dropped.reduce((sum, e) => sum + e.count, 0);
|
|
127
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
128
|
+
log.error(`[refusals] flush failed; dropped ${refusals} refusals across ${dropped.length} keys: ${message}`);
|
|
129
|
+
};
|
|
130
|
+
/**
|
|
131
|
+
* `BoundedCounter<RefusalKey>` + `CounterFlusher` + a `record_refusals` sink.
|
|
132
|
+
* Construct one per process, `start()` it, `record()` from every refusal site,
|
|
133
|
+
* `stop()` on shutdown.
|
|
134
|
+
*/
|
|
135
|
+
export class RefusalRecorder {
|
|
136
|
+
counter;
|
|
137
|
+
flusher;
|
|
138
|
+
recordFailures = 0;
|
|
139
|
+
constructor(opts) {
|
|
140
|
+
this.counter = new BoundedCounter({
|
|
141
|
+
maxKeys: opts.maxKeys ?? DEFAULT_REFUSAL_MAX_KEYS,
|
|
142
|
+
keyOf: refusalKeyOf,
|
|
143
|
+
overflowKey: refusalOverflowKey
|
|
144
|
+
});
|
|
145
|
+
this.flusher = new CounterFlusher(this.counter, (entries) => opts.sink(refusalRows(entries)), {
|
|
146
|
+
intervalMs: opts.intervalMs ?? DEFAULT_REFUSAL_FLUSH_INTERVAL_MS,
|
|
147
|
+
jitterMs: opts.jitterMs ?? DEFAULT_REFUSAL_FLUSH_JITTER_MS,
|
|
148
|
+
onError: opts.onError ?? defaultOnError
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Count one refusal. Synchronous and never throws: the request that is
|
|
153
|
+
* being refused must finish being refused whatever state the recorder is in.
|
|
154
|
+
*/
|
|
155
|
+
record(refusal) {
|
|
156
|
+
try {
|
|
157
|
+
const at = refusal.at ?? Date.now();
|
|
158
|
+
this.counter.increment({
|
|
159
|
+
minuteBucket: minuteOf(at),
|
|
160
|
+
databaseId: refusal.databaseId,
|
|
161
|
+
lane: refusal.lane,
|
|
162
|
+
reason: refusal.reason,
|
|
163
|
+
routeKey: refusal.routeKey,
|
|
164
|
+
sourceBucket: sourceBucket(refusal.sourceIp)
|
|
165
|
+
}, 1, at);
|
|
166
|
+
}
|
|
167
|
+
catch (err) {
|
|
168
|
+
this.recordFailures += 1;
|
|
169
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
170
|
+
log.error(`[refusals] record failed (refusal still served): ${message}`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
start() {
|
|
174
|
+
this.flusher.start();
|
|
175
|
+
}
|
|
176
|
+
/** Final flush, then stop. */
|
|
177
|
+
stop() {
|
|
178
|
+
return this.flusher.stop();
|
|
179
|
+
}
|
|
180
|
+
/** Drain and write now; a sink failure is reported, never thrown. */
|
|
181
|
+
flush() {
|
|
182
|
+
return this.flusher.flush();
|
|
183
|
+
}
|
|
184
|
+
stats() {
|
|
185
|
+
return {
|
|
186
|
+
keys: this.counter.size,
|
|
187
|
+
overflowed: this.counter.overflowed,
|
|
188
|
+
recordFailures: this.recordFailures,
|
|
189
|
+
flusher: this.flusher.stats()
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
const FUNCTION_NAME = /^[a-z_][a-z0-9_]*\.[a-z_][a-z0-9_]*$/;
|
|
194
|
+
/**
|
|
195
|
+
* A sink that writes a batch as one `record_refusals(jsonb)` call inside a
|
|
196
|
+
* transaction carrying the caller's claims. Claims are resolved per flush so a
|
|
197
|
+
* pool that was unreachable at startup does not leave the recorder wedged.
|
|
198
|
+
*/
|
|
199
|
+
export const createRecordRefusalsSink = (opts) => {
|
|
200
|
+
const fn = opts.functionName ?? 'constructive_usage_private.record_refusals';
|
|
201
|
+
if (!FUNCTION_NAME.test(fn)) {
|
|
202
|
+
throw new Error(`createRecordRefusalsSink: invalid function name ${JSON.stringify(fn)}`);
|
|
203
|
+
}
|
|
204
|
+
const [schema, name] = fn.split('.');
|
|
205
|
+
const sql = `SELECT "${schema}"."${name}"($1::jsonb) AS recorded`;
|
|
206
|
+
return async (rows) => {
|
|
207
|
+
if (rows.length === 0)
|
|
208
|
+
return;
|
|
209
|
+
const claims = await opts.claims();
|
|
210
|
+
await withPgClient(opts.pool, claims, async (client) => {
|
|
211
|
+
await client.query(sql, [JSON.stringify(rows)]);
|
|
212
|
+
});
|
|
213
|
+
};
|
|
214
|
+
};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* request-protection — platform bounds for the per-request protection settings.
|
|
3
|
+
*
|
|
4
|
+
* `database_settings` / `api_settings` carry a tenant's *preferences*; this
|
|
5
|
+
* module owns the numbers the platform will actually honour. The platform
|
|
6
|
+
* default and the hard ceiling are code constants rather than rows, so a
|
|
7
|
+
* tenant can never widen a bound by writing metadata: a setting is only ever
|
|
8
|
+
* read as a request to be *stricter* than the platform is.
|
|
9
|
+
*
|
|
10
|
+
* effective = clamp(LEAST(api_override, database_setting) ?? default, floor, max)
|
|
11
|
+
*
|
|
12
|
+
* `LEAST` (not `COALESCE`) is what makes an API override lower-only: an API
|
|
13
|
+
* that asks for a 60s statement timeout under a database that says 10s gets
|
|
14
|
+
* 10s. The floor exists so a tenant cannot write a value that makes its own
|
|
15
|
+
* API unusable (a 0ms statement timeout), and the ceiling so it cannot raise
|
|
16
|
+
* one past what the shared cluster will absorb.
|
|
17
|
+
*/
|
|
18
|
+
// ─── Platform bounds ────────────────────────────────────────────────────────
|
|
19
|
+
/**
|
|
20
|
+
* The platform's own numbers. Deliberately constants: a ceiling stored next to
|
|
21
|
+
* the value it caps is a ceiling a tenant can edit.
|
|
22
|
+
*/
|
|
23
|
+
export const PROTECTION_BOUNDS = {
|
|
24
|
+
statementTimeoutMs: { default: 30_000, floor: 1_000, max: 120_000 },
|
|
25
|
+
idleInTransactionTimeoutMs: { default: 60_000, floor: 1_000, max: 300_000 },
|
|
26
|
+
lockTimeoutMs: { default: 5_000, floor: 100, max: 60_000 },
|
|
27
|
+
maxConcurrentRequests: { default: 50, floor: 1, max: 500 },
|
|
28
|
+
maxQueueWaitMs: { default: 5_000, floor: 0, max: 60_000 },
|
|
29
|
+
rateLimitRpm: { default: 600, floor: 1, max: 60_000 },
|
|
30
|
+
rateLimitBurst: { default: 60, floor: 1, max: 10_000 },
|
|
31
|
+
maxQueryDepth: { default: 12, floor: 1, max: 50 },
|
|
32
|
+
// Cost is measured in rows a document can pull (see ASSUMED_PAGE_SIZE), so
|
|
33
|
+
// the default has to leave room for the nesting Graphile encourages while
|
|
34
|
+
// still refusing the millions-of-rows shapes.
|
|
35
|
+
maxQueryCost: { default: 1_000_000, floor: 100, max: 100_000_000 },
|
|
36
|
+
maxPageSize: { default: 1_000, floor: 1, max: 10_000 },
|
|
37
|
+
maxRequestBytes: { default: 1_000_000, floor: 1_024, max: 10_000_000 }
|
|
38
|
+
};
|
|
39
|
+
/**
|
|
40
|
+
* What a connection with no `first`/`last` is charged.
|
|
41
|
+
*
|
|
42
|
+
* Such a field is unbounded in principle, so charging it the tenant's
|
|
43
|
+
* `maxPageSize` would make a large ceiling self-defeating: raising the page
|
|
44
|
+
* size a client *may* ask for would shrink the query it can write. A fixed
|
|
45
|
+
* estimate keeps the two bounds independent.
|
|
46
|
+
*/
|
|
47
|
+
export const ASSUMED_PAGE_SIZE = 100;
|
|
48
|
+
/**
|
|
49
|
+
* Introspection stays on until metadata says otherwise, and `platformAllows`
|
|
50
|
+
* is the kill switch that takes it away cluster-wide regardless.
|
|
51
|
+
*
|
|
52
|
+
* The platform default is permissive here where the metadata default is not
|
|
53
|
+
* (`database_settings.enable_introspection` is `NOT NULL DEFAULT false`) on
|
|
54
|
+
* purpose: this value applies to a database that has expressed *no*
|
|
55
|
+
* preference — typically one whose routing plane predates the column — and
|
|
56
|
+
* silently disabling introspection for it would break GraphiQL and codegen for
|
|
57
|
+
* every tenant the moment this code deployed, without anyone opting in.
|
|
58
|
+
*/
|
|
59
|
+
export const INTROSPECTION_BOUND = {
|
|
60
|
+
default: true,
|
|
61
|
+
platformAllows: true
|
|
62
|
+
};
|
|
63
|
+
/** What a database with no settings row gets. */
|
|
64
|
+
export const DEFAULT_REQUEST_PROTECTION = resolveDefaults();
|
|
65
|
+
function resolveDefaults() {
|
|
66
|
+
const numeric = Object.fromEntries(Object.entries(PROTECTION_BOUNDS).map(([key, bound]) => [key, bound.default]));
|
|
67
|
+
return { ...numeric, enableIntrospection: INTROSPECTION_BOUND.default };
|
|
68
|
+
}
|
|
69
|
+
// ─── Resolution ─────────────────────────────────────────────────────────────
|
|
70
|
+
const clamp = (value, bound) => Math.min(Math.max(value, bound.floor), bound.max);
|
|
71
|
+
/** The stricter of two preferences, ignoring the scopes that expressed none. */
|
|
72
|
+
const strictest = (a, b) => {
|
|
73
|
+
const values = [a, b].filter((v) => typeof v === 'number' && Number.isFinite(v));
|
|
74
|
+
return values.length > 0 ? Math.min(...values) : null;
|
|
75
|
+
};
|
|
76
|
+
/**
|
|
77
|
+
* Resolve the bounds a request runs under.
|
|
78
|
+
*
|
|
79
|
+
* @param database - the `database_settings` row for the tenant, if any
|
|
80
|
+
* @param api - the `api_settings` overrides for the API the request arrived on
|
|
81
|
+
*/
|
|
82
|
+
export function resolveRequestProtection(database, api) {
|
|
83
|
+
const resolved = {};
|
|
84
|
+
for (const [name, bound] of Object.entries(PROTECTION_BOUNDS)) {
|
|
85
|
+
const preferred = strictest(database?.[name], api?.[name]);
|
|
86
|
+
resolved[name] = clamp(preferred ?? bound.default, bound);
|
|
87
|
+
}
|
|
88
|
+
// Lower-only applies to the flag too: an API may switch introspection off
|
|
89
|
+
// for itself, never back on for a database that turned it off, and the
|
|
90
|
+
// platform switch can only ever take it away.
|
|
91
|
+
const enableIntrospection = (database?.enableIntrospection ?? INTROSPECTION_BOUND.default) &&
|
|
92
|
+
(api?.enableIntrospection ?? true) &&
|
|
93
|
+
INTROSPECTION_BOUND.platformAllows;
|
|
94
|
+
return { ...resolved, enableIntrospection };
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The timeout GUCs as `SET LOCAL` pairs, for merging into a request's
|
|
98
|
+
* pgSettings. PostgreSQL reads a bare integer for all three as milliseconds.
|
|
99
|
+
*/
|
|
100
|
+
export function protectionPgSettings(protection) {
|
|
101
|
+
return {
|
|
102
|
+
statement_timeout: String(protection.statementTimeoutMs),
|
|
103
|
+
idle_in_transaction_session_timeout: String(protection.idleInTransactionTimeoutMs),
|
|
104
|
+
lock_timeout: String(protection.lockTimeoutMs)
|
|
105
|
+
};
|
|
106
|
+
}
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* usage-counter — the shared in-memory count + batch-flush mechanism.
|
|
3
|
+
*
|
|
4
|
+
* Two classes, neither of which knows what is being counted:
|
|
5
|
+
*
|
|
6
|
+
* - `BoundedCounter<K>` is the map. Consumers pass the key type, how a key
|
|
7
|
+
* serialises (`keyOf`) and where a key folds to once the map is full
|
|
8
|
+
* (`overflowKey`). `increment` is O(1), synchronous and never throws, so a
|
|
9
|
+
* request path can call it while refusing traffic without ever waiting on,
|
|
10
|
+
* or being affected by, whatever eventually stores the counts.
|
|
11
|
+
* - `CounterFlusher<K>` is the loop. It drains the counter on a jittered
|
|
12
|
+
* interval, hands the batch to a `sink`, and owns the failure policy: a
|
|
13
|
+
* batch the sink cannot write is reported through `onError` *and dropped*.
|
|
14
|
+
* Re-queuing under a broken sink is how a bounded counter becomes an
|
|
15
|
+
* unbounded one, and the counter is already accumulating the next window.
|
|
16
|
+
*
|
|
17
|
+
* Refusal observability instantiates these with a refusal key and a sink that
|
|
18
|
+
* calls `constructive_usage_private.record_refusals` (see `refusals.ts`); the
|
|
19
|
+
* page/request metering lane instantiates the same two classes with its own
|
|
20
|
+
* `K`, `keyOf`, `overflowKey`, bound, interval and sink. The seam is the
|
|
21
|
+
* constructor arguments — nothing in this file is specific to either.
|
|
22
|
+
*
|
|
23
|
+
* Counters live in process memory only. A crash loses at most one interval's
|
|
24
|
+
* worth of counts; `stop()` flushes on graceful shutdown. That is deliberate:
|
|
25
|
+
* the alternative is a durable write per event, which is the thing counting in
|
|
26
|
+
* memory exists to avoid.
|
|
27
|
+
*
|
|
28
|
+
* @module usage-counter
|
|
29
|
+
*/
|
|
30
|
+
export class BoundedCounter {
|
|
31
|
+
entries = new Map();
|
|
32
|
+
overflowedSinceDrain = 0;
|
|
33
|
+
maxKeys;
|
|
34
|
+
keyOf;
|
|
35
|
+
overflowKey;
|
|
36
|
+
constructor(opts) {
|
|
37
|
+
if (!Number.isInteger(opts.maxKeys) || opts.maxKeys < 1) {
|
|
38
|
+
throw new Error(`BoundedCounter: maxKeys must be a positive integer, got ${opts.maxKeys}`);
|
|
39
|
+
}
|
|
40
|
+
this.maxKeys = opts.maxKeys;
|
|
41
|
+
this.keyOf = opts.keyOf;
|
|
42
|
+
this.overflowKey = opts.overflowKey;
|
|
43
|
+
}
|
|
44
|
+
/** Distinct keys currently held. */
|
|
45
|
+
get size() {
|
|
46
|
+
return this.entries.size;
|
|
47
|
+
}
|
|
48
|
+
/** Increments folded into an overflow key since the last drain. */
|
|
49
|
+
get overflowed() {
|
|
50
|
+
return this.overflowedSinceDrain;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Add `by` to `key`. Synchronous, O(1), never throws for a well-formed
|
|
54
|
+
* `keyOf`/`overflowKey`; the caller's request path must not depend on it in
|
|
55
|
+
* any other way.
|
|
56
|
+
*/
|
|
57
|
+
increment(key, by = 1, at = Date.now()) {
|
|
58
|
+
let id = this.keyOf(key);
|
|
59
|
+
let entry = this.entries.get(id);
|
|
60
|
+
if (!entry && this.entries.size >= this.maxKeys) {
|
|
61
|
+
key = this.overflowKey(key);
|
|
62
|
+
id = this.keyOf(key);
|
|
63
|
+
entry = this.entries.get(id);
|
|
64
|
+
this.overflowedSinceDrain += by;
|
|
65
|
+
}
|
|
66
|
+
if (!entry) {
|
|
67
|
+
this.entries.set(id, { key, count: by, firstAt: at, lastAt: at });
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
entry.count += by;
|
|
71
|
+
if (at < entry.firstAt)
|
|
72
|
+
entry.firstAt = at;
|
|
73
|
+
if (at > entry.lastAt)
|
|
74
|
+
entry.lastAt = at;
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Return every entry and start a fresh window. The returned array is the
|
|
78
|
+
* only reference to those entries, so a slow sink never contends with the
|
|
79
|
+
* hot path.
|
|
80
|
+
*/
|
|
81
|
+
drain() {
|
|
82
|
+
const drained = Array.from(this.entries.values());
|
|
83
|
+
this.entries.clear();
|
|
84
|
+
this.overflowedSinceDrain = 0;
|
|
85
|
+
return drained;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
export class CounterFlusher {
|
|
89
|
+
counter;
|
|
90
|
+
sink;
|
|
91
|
+
opts;
|
|
92
|
+
timer = null;
|
|
93
|
+
inFlight = null;
|
|
94
|
+
running = false;
|
|
95
|
+
lastFlushAt = null;
|
|
96
|
+
lastErrorAt = null;
|
|
97
|
+
lastError = null;
|
|
98
|
+
consecutiveFailures = 0;
|
|
99
|
+
droppedEntries = 0;
|
|
100
|
+
constructor(counter, sink, opts) {
|
|
101
|
+
this.counter = counter;
|
|
102
|
+
this.sink = sink;
|
|
103
|
+
this.opts = opts;
|
|
104
|
+
if (!(opts.intervalMs > 0)) {
|
|
105
|
+
throw new Error(`CounterFlusher: intervalMs must be positive, got ${opts.intervalMs}`);
|
|
106
|
+
}
|
|
107
|
+
if (!(opts.jitterMs >= 0)) {
|
|
108
|
+
throw new Error(`CounterFlusher: jitterMs must be non-negative, got ${opts.jitterMs}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/** Arm the timer. Idempotent. The timer is unref'd so it never pins the process. */
|
|
112
|
+
start() {
|
|
113
|
+
if (this.running)
|
|
114
|
+
return;
|
|
115
|
+
this.running = true;
|
|
116
|
+
this.arm();
|
|
117
|
+
}
|
|
118
|
+
/** Disarm the timer and flush whatever is pending. Idempotent. */
|
|
119
|
+
async stop() {
|
|
120
|
+
this.running = false;
|
|
121
|
+
if (this.timer) {
|
|
122
|
+
clearTimeout(this.timer);
|
|
123
|
+
this.timer = null;
|
|
124
|
+
}
|
|
125
|
+
await this.flush();
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Drain and write now. Never rejects: a sink failure goes to `onError`. If a
|
|
129
|
+
* flush is already in flight the call waits for it and then runs its own, so
|
|
130
|
+
* two overlapping timers cannot send batches out of order.
|
|
131
|
+
*/
|
|
132
|
+
async flush() {
|
|
133
|
+
if (this.inFlight)
|
|
134
|
+
await this.inFlight;
|
|
135
|
+
const run = this.flushOnce();
|
|
136
|
+
this.inFlight = run;
|
|
137
|
+
try {
|
|
138
|
+
await run;
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
if (this.inFlight === run)
|
|
142
|
+
this.inFlight = null;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
stats() {
|
|
146
|
+
return {
|
|
147
|
+
lastFlushAt: this.lastFlushAt,
|
|
148
|
+
lastErrorAt: this.lastErrorAt,
|
|
149
|
+
lastError: this.lastError,
|
|
150
|
+
consecutiveFailures: this.consecutiveFailures,
|
|
151
|
+
droppedEntries: this.droppedEntries,
|
|
152
|
+
running: this.running
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
arm() {
|
|
156
|
+
const delay = this.opts.intervalMs + Math.floor(Math.random() * (this.opts.jitterMs + 1));
|
|
157
|
+
this.timer = setTimeout(() => {
|
|
158
|
+
this.timer = null;
|
|
159
|
+
void this.flush().finally(() => {
|
|
160
|
+
if (this.running)
|
|
161
|
+
this.arm();
|
|
162
|
+
});
|
|
163
|
+
}, delay);
|
|
164
|
+
this.timer.unref?.();
|
|
165
|
+
}
|
|
166
|
+
async flushOnce() {
|
|
167
|
+
const entries = this.counter.drain();
|
|
168
|
+
if (entries.length === 0)
|
|
169
|
+
return;
|
|
170
|
+
try {
|
|
171
|
+
await this.sink(entries);
|
|
172
|
+
this.lastFlushAt = Date.now();
|
|
173
|
+
this.consecutiveFailures = 0;
|
|
174
|
+
}
|
|
175
|
+
catch (err) {
|
|
176
|
+
this.lastErrorAt = Date.now();
|
|
177
|
+
this.lastError = err instanceof Error ? err.message : String(err);
|
|
178
|
+
this.consecutiveFailures += 1;
|
|
179
|
+
this.droppedEntries += entries.length;
|
|
180
|
+
// The batch is gone whether or not onError itself misbehaves; a
|
|
181
|
+
// reporter that throws must not turn one lost window into a stuck loop.
|
|
182
|
+
try {
|
|
183
|
+
this.opts.onError(err, entries);
|
|
184
|
+
}
|
|
185
|
+
catch {
|
|
186
|
+
// Fire-and-forget by design: the failure is already recorded in stats().
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
package/index.d.ts
CHANGED
|
@@ -40,9 +40,17 @@ export { createBillingClient } from './billing-client';
|
|
|
40
40
|
export type { PgSettingsInput } from './pg-settings';
|
|
41
41
|
export { buildPgSettings } from './pg-settings';
|
|
42
42
|
export { withPgClient } from './pg-client';
|
|
43
|
+
export type { NumericProtectionKey, ProtectionBound, RequestProtection, RequestProtectionInput, } from './request-protection';
|
|
44
|
+
export { ASSUMED_PAGE_SIZE, DEFAULT_REQUEST_PROTECTION, INTROSPECTION_BOUND, PROTECTION_BOUNDS, protectionPgSettings, resolveRequestProtection, } from './request-protection';
|
|
45
|
+
export type { AdmissionLease, AdmissionRefusal, AdmissionRequest } from './admission';
|
|
46
|
+
export { clientIpFrom, ConcurrencyLimiter, RateWindow, trustedProxyHops } from './admission';
|
|
47
|
+
export type { RecordRefusalsSinkOptions, Refusal, RefusalKey, RefusalLane, RefusalReason, RefusalRecorderOptions, RefusalRecorderStats, RefusalRow, RefusalSink } from './refusals';
|
|
48
|
+
export { createRecordRefusalsSink, DEFAULT_REFUSAL_FLUSH_INTERVAL_MS, DEFAULT_REFUSAL_FLUSH_JITTER_MS, DEFAULT_REFUSAL_MAX_KEYS, OVERFLOW_ROUTE, OVERFLOW_SOURCE, REFUSAL_REASONS, refusalKeyOf, refusalOverflowKey, RefusalRecorder, refusalRows, sourceBucket, UNKNOWN_SOURCE } from './refusals';
|
|
49
|
+
export type { BoundedCounterOptions, CounterEntry, CounterFlusherOptions, CounterFlusherStats, CounterSink } from './usage-counter';
|
|
50
|
+
export { BoundedCounter, CounterFlusher } from './usage-counter';
|
|
43
51
|
export { requestIdMiddleware } from './request-id';
|
|
44
52
|
export type { ContextMiddlewareOptions } from './context';
|
|
45
53
|
export { buildContext, createContextMiddleware } from './context';
|
|
46
54
|
export type { CreateLoaderOptions, LoaderContext, LoaderRegistry, ModuleLoader, } from './loaders';
|
|
47
|
-
export { agentChatLoader, authSettingsLoader, authSurfaceLoader, billingLoader, computeLoader, corsLoader, createDefaultRegistry, createLoaderRegistry, createModuleLoader, databaseSettingsLoader, identityProvidersLoader, inferenceLogLoader, llmLoader, pubkeyLoader, requireDatabaseId, requireIdentityProvider, rlsLoader, webauthnLoader, } from './loaders';
|
|
55
|
+
export { agentChatLoader, authSettingsLoader, authSurfaceLoader, billingLoader, computeLoader, corsLoader, createDefaultRegistry, createLoaderRegistry, createModuleLoader, databaseSettingsLoader, identityProvidersLoader, inferenceLogLoader, llmLoader, pubkeyLoader, requestProtectionLoader, requireDatabaseId, requireIdentityProvider, rlsLoader, webauthnLoader, } from './loaders';
|
|
48
56
|
import './types';
|