@mistriwork/engine 1.0.0-rc.34 → 1.0.0-rc.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/cache/cacheStore.d.ts +7 -6
- package/dist/adapters/cache/cacheStore.js +13 -7
- package/dist/adapters/scoped/sqliteScopeCatalog.js +13 -18
- package/dist/adapters/sqlite.d.ts +1 -0
- package/dist/adapters/sqlite.js +59 -6
- package/dist/adapters/sqliteHandles.d.ts +2 -0
- package/dist/adapters/sqliteHandles.js +18 -1
- package/dist/adapters/telemetry/drivers.js +5 -0
- package/dist/adapters/telemetry/trace.d.ts +2 -0
- package/dist/adapters/telemetry/trace.js +2 -0
- package/dist/server/http/clock.js +21 -9
- package/dist/server/http/exportRoute.d.ts +14 -2
- package/dist/server/http/exportRoute.js +44 -3
- package/dist/server/http/rateLimit.d.ts +7 -3
- package/dist/server/http/rateLimit.js +47 -21
- package/dist/shared/scopeCatalog.js +35 -1
- package/package.json +1 -1
|
@@ -18,11 +18,12 @@ export interface CacheStore {
|
|
|
18
18
|
set(key: string, value: string, ttlMs: number): Promise<void>;
|
|
19
19
|
delete(key: string): Promise<void>;
|
|
20
20
|
/**
|
|
21
|
-
* Count one under `key
|
|
22
|
-
*
|
|
23
|
-
*
|
|
21
|
+
* Count one under `key` — or `by`, a positive whole number, where what is
|
|
22
|
+
* counted is a size rather than a request. A new key starts a window of
|
|
23
|
+
* `ttlMs`; the answer is the count within it and the milliseconds it has
|
|
24
|
+
* left. Throws CacheUnavailable when the count cannot be kept.
|
|
24
25
|
*/
|
|
25
|
-
incr(key: string, ttlMs: number): Promise<{
|
|
26
|
+
incr(key: string, ttlMs: number, by?: number): Promise<{
|
|
26
27
|
count: number;
|
|
27
28
|
ttlMs: number;
|
|
28
29
|
}>;
|
|
@@ -42,7 +43,7 @@ export declare class MemoryCacheStore implements CacheStore {
|
|
|
42
43
|
get(key: string): Promise<string | undefined>;
|
|
43
44
|
set(key: string, value: string, ttlMs: number): Promise<void>;
|
|
44
45
|
delete(key: string): Promise<void>;
|
|
45
|
-
incr(key: string, ttlMs: number): Promise<{
|
|
46
|
+
incr(key: string, ttlMs: number, by?: number): Promise<{
|
|
46
47
|
count: number;
|
|
47
48
|
ttlMs: number;
|
|
48
49
|
}>;
|
|
@@ -64,7 +65,7 @@ export declare class ValkeyCacheStore implements CacheStore {
|
|
|
64
65
|
get(key: string): Promise<string | undefined>;
|
|
65
66
|
set(key: string, value: string, ttlMs: number): Promise<void>;
|
|
66
67
|
delete(key: string): Promise<void>;
|
|
67
|
-
incr(key: string, ttlMs: number): Promise<{
|
|
68
|
+
incr(key: string, ttlMs: number, by?: number): Promise<{
|
|
68
69
|
count: number;
|
|
69
70
|
ttlMs: number;
|
|
70
71
|
}>;
|
|
@@ -14,6 +14,10 @@ import { CacheUnavailable, RespReplyError } from './resp.js';
|
|
|
14
14
|
export { CacheUnavailable } from './resp.js';
|
|
15
15
|
/* Thirty days: a copy kept longer than that is a copy nobody meant to keep. */
|
|
16
16
|
const MAX_TTL_MS = 30 * 24 * 3_600_000;
|
|
17
|
+
function assertBy(by) {
|
|
18
|
+
if (!Number.isSafeInteger(by) || by < 1)
|
|
19
|
+
throw new Error(`A count goes up by a positive whole number; ${by} is not one.`);
|
|
20
|
+
}
|
|
17
21
|
function assertTtl(ttlMs) {
|
|
18
22
|
if (!Number.isSafeInteger(ttlMs) || ttlMs < 1 || ttlMs > MAX_TTL_MS) {
|
|
19
23
|
throw new Error(`A cached value needs a time to live between 1 ms and 30 days; ${ttlMs} is not one.`);
|
|
@@ -50,11 +54,12 @@ export class MemoryCacheStore {
|
|
|
50
54
|
async delete(key) {
|
|
51
55
|
this.entries.delete(key);
|
|
52
56
|
}
|
|
53
|
-
async incr(key, ttlMs) {
|
|
57
|
+
async incr(key, ttlMs, by = 1) {
|
|
54
58
|
assertTtl(ttlMs);
|
|
59
|
+
assertBy(by);
|
|
55
60
|
const entry = this.live(key);
|
|
56
61
|
const now = this.now();
|
|
57
|
-
const next = entry ? { value: String(Number(entry.value) +
|
|
62
|
+
const next = entry ? { value: String(Number(entry.value) + by), expires: entry.expires } : { value: String(by), expires: now + ttlMs };
|
|
58
63
|
this.entries.delete(key);
|
|
59
64
|
this.entries.set(key, next);
|
|
60
65
|
this.trim();
|
|
@@ -82,8 +87,8 @@ export class MemoryCacheStore {
|
|
|
82
87
|
}
|
|
83
88
|
/* One round trip, and atomic: a window that is counted is a window that
|
|
84
89
|
expires, even if this process stops between the two steps. */
|
|
85
|
-
const COUNT_IN_WINDOW = `local n = redis.call('
|
|
86
|
-
if n ==
|
|
90
|
+
const COUNT_IN_WINDOW = `local n = redis.call('INCRBY', KEYS[1], ARGV[2])
|
|
91
|
+
if n == tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[1]) end
|
|
87
92
|
local t = redis.call('PTTL', KEYS[1])
|
|
88
93
|
if t < 0 then redis.call('PEXPIRE', KEYS[1], ARGV[1]) t = tonumber(ARGV[1]) end
|
|
89
94
|
return {n, t}`;
|
|
@@ -126,11 +131,12 @@ export class ValkeyCacheStore {
|
|
|
126
131
|
this.unavailable(error);
|
|
127
132
|
}
|
|
128
133
|
}
|
|
129
|
-
async incr(key, ttlMs) {
|
|
134
|
+
async incr(key, ttlMs, by = 1) {
|
|
130
135
|
assertTtl(ttlMs);
|
|
136
|
+
assertBy(by);
|
|
131
137
|
let reply;
|
|
132
138
|
try {
|
|
133
|
-
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs);
|
|
139
|
+
reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
|
|
134
140
|
}
|
|
135
141
|
catch (error) {
|
|
136
142
|
this.unavailable(error);
|
|
@@ -154,7 +160,7 @@ export function scopedCache(store, prefix) {
|
|
|
154
160
|
get: (key) => store.get(prefix + key),
|
|
155
161
|
set: (key, value, ttlMs) => store.set(prefix + key, value, ttlMs),
|
|
156
162
|
delete: (key) => store.delete(prefix + key),
|
|
157
|
-
incr: (key, ttlMs) => store.incr(prefix + key, ttlMs),
|
|
163
|
+
incr: (key, ttlMs, by) => store.incr(prefix + key, ttlMs, by),
|
|
158
164
|
close: () => store.close(),
|
|
159
165
|
};
|
|
160
166
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ANSI_STYLE, Refusal } from '../adapter.js';
|
|
2
|
+
import { cachedStatement as prepared } from '../sqliteHandles.js';
|
|
2
3
|
import { chooseColumnReference, deleteRuleOf, } from '../../shared/schema.js';
|
|
3
4
|
import { mapScopeCatalog } from '../../shared/scopeCatalog.js';
|
|
4
5
|
import { referenceKey } from './scopedTransaction.js';
|
|
@@ -127,7 +128,7 @@ export function sqliteKeyFacts(db) {
|
|
|
127
128
|
const definitions = new Map();
|
|
128
129
|
const definitionOf = (table) => {
|
|
129
130
|
if (!definitions.has(table.id)) {
|
|
130
|
-
const row = db
|
|
131
|
+
const row = prepared(db, "SELECT sql FROM main.sqlite_schema WHERE type = 'table' AND name = ?").get(table.name);
|
|
131
132
|
definitions.set(table.id, typeof row?.sql === 'string' ? row.sql : undefined);
|
|
132
133
|
}
|
|
133
134
|
return definitions.get(table.id);
|
|
@@ -170,14 +171,12 @@ export function sqliteKeyFacts(db) {
|
|
|
170
171
|
// A rowid alias holds integers alone, compared by no collation; it has no index of its own.
|
|
171
172
|
if (primary.length === 1 && table.columns.find((column) => column.name === primary[0])?.identity)
|
|
172
173
|
return [primary];
|
|
173
|
-
const index = db
|
|
174
|
-
.prepare(`PRAGMA main.index_list(${ANSI_STYLE.quote(table.name)})`)
|
|
174
|
+
const index = prepared(db, `PRAGMA main.index_list(${ANSI_STYLE.quote(table.name)})`)
|
|
175
175
|
.all()
|
|
176
176
|
.find((row) => row.origin === 'pk');
|
|
177
177
|
if (typeof index?.name !== 'string')
|
|
178
178
|
return [];
|
|
179
|
-
const indexed = db
|
|
180
|
-
.prepare(`PRAGMA main.index_xinfo(${ANSI_STYLE.quote(index.name)})`)
|
|
179
|
+
const indexed = prepared(db, `PRAGMA main.index_xinfo(${ANSI_STYLE.quote(index.name)})`)
|
|
181
180
|
.all()
|
|
182
181
|
.filter((row) => row.key === 1);
|
|
183
182
|
const same = primary.every((name) => {
|
|
@@ -196,7 +195,7 @@ export function sqliteKeyFacts(db) {
|
|
|
196
195
|
*/
|
|
197
196
|
export function sqliteBlockedTables(db) {
|
|
198
197
|
const blocked = new Map();
|
|
199
|
-
for (const table of db
|
|
198
|
+
for (const table of prepared(db, 'PRAGMA main.table_list').all()) {
|
|
200
199
|
if (table.schema !== 'main' || typeof table.name !== 'string')
|
|
201
200
|
continue;
|
|
202
201
|
if (table.type === 'virtual' || table.type === 'shadow')
|
|
@@ -216,12 +215,10 @@ export function readSqliteReadCatalog(db) {
|
|
|
216
215
|
}
|
|
217
216
|
function scanSqliteScopeCatalog(db) {
|
|
218
217
|
try {
|
|
219
|
-
if (db
|
|
218
|
+
if (prepared(db, 'PRAGMA foreign_keys').get()?.foreign_keys !== 1)
|
|
220
219
|
return refuse();
|
|
221
|
-
const entries = db
|
|
222
|
-
|
|
223
|
-
.all();
|
|
224
|
-
const listed = db.prepare('PRAGMA main.table_list').all();
|
|
220
|
+
const entries = prepared(db, "SELECT name, type FROM main.sqlite_schema WHERE type IN ('table', 'view') ORDER BY name").all();
|
|
221
|
+
const listed = prepared(db, 'PRAGMA main.table_list').all();
|
|
225
222
|
// Unknown PRAGMAs can return nothing without throwing. Never interpret
|
|
226
223
|
// unavailable table-kind metadata as proof that a table is ordinary.
|
|
227
224
|
if (!listed.some((table) => table.schema === 'main' && table.name === 'sqlite_schema' && table.type === 'table'))
|
|
@@ -231,9 +228,7 @@ function scanSqliteScopeCatalog(db) {
|
|
|
231
228
|
it only validates its row, is read from that text per table below (sqliteTriggers.ts, scopeTriggers.ts). */
|
|
232
229
|
const definitions = new Map();
|
|
233
230
|
for (const namespace of ['main', 'temp']) {
|
|
234
|
-
const triggers = db
|
|
235
|
-
.prepare(`SELECT tbl_name AS name, name AS trigger, sql FROM ${namespace}.sqlite_schema WHERE type = 'trigger' ORDER BY name`)
|
|
236
|
-
.all();
|
|
231
|
+
const triggers = prepared(db, `SELECT tbl_name AS name, name AS trigger, sql FROM ${namespace}.sqlite_schema WHERE type = 'trigger' ORDER BY name`).all();
|
|
237
232
|
for (const trigger of triggers) {
|
|
238
233
|
if (typeof trigger.name !== 'string' || typeof trigger.trigger !== 'string')
|
|
239
234
|
return refuse();
|
|
@@ -245,7 +240,7 @@ function scanSqliteScopeCatalog(db) {
|
|
|
245
240
|
}
|
|
246
241
|
const triggered = new Map();
|
|
247
242
|
const shadowed = new Set();
|
|
248
|
-
for (const table of db
|
|
243
|
+
for (const table of prepared(db, "SELECT name FROM temp.sqlite_schema WHERE type IN ('table', 'view')").all()) {
|
|
249
244
|
if (typeof table.name !== 'string')
|
|
250
245
|
return refuse();
|
|
251
246
|
shadowed.add(folded(table.name));
|
|
@@ -262,7 +257,7 @@ function scanSqliteScopeCatalog(db) {
|
|
|
262
257
|
!['table', 'view', 'virtual', 'shadow'].includes(kind.type) ||
|
|
263
258
|
(entry.type === 'view') !== (kind.type === 'view'))
|
|
264
259
|
return refuse();
|
|
265
|
-
const columns = db
|
|
260
|
+
const columns = prepared(db, `PRAGMA main.table_xinfo(${ANSI_STYLE.quote(entry.name)})`).all();
|
|
266
261
|
identities(columns);
|
|
267
262
|
if (!columns.length || columns.length !== kind.ncol)
|
|
268
263
|
return refuse();
|
|
@@ -279,7 +274,7 @@ function scanSqliteScopeCatalog(db) {
|
|
|
279
274
|
const primary = columns.filter((column) => column.pk > 0).sort((a, b) => a.pk - b.pk);
|
|
280
275
|
if (primary.some((column, i) => column.pk !== i + 1))
|
|
281
276
|
return refuse();
|
|
282
|
-
const indexes = db
|
|
277
|
+
const indexes = prepared(db, `PRAGMA main.index_list(${ANSI_STYLE.quote(entry.name)})`).all();
|
|
283
278
|
// INTEGER PRIMARY KEY DESC has a PK index and is not a rowid alias.
|
|
284
279
|
const identity = kind.type === 'table' &&
|
|
285
280
|
kind.wr === 0 &&
|
|
@@ -329,7 +324,7 @@ function scanSqliteScopeCatalog(db) {
|
|
|
329
324
|
const byName = identities(tables);
|
|
330
325
|
const foreignKeys = [];
|
|
331
326
|
for (const table of tables) {
|
|
332
|
-
const rows = db
|
|
327
|
+
const rows = prepared(db, `PRAGMA main.foreign_key_list(${ANSI_STYLE.quote(table.name)})`).all();
|
|
333
328
|
const groups = new Map();
|
|
334
329
|
for (const row of rows) {
|
|
335
330
|
if (!Number.isInteger(row.id) ||
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { cachedStatement, schemaFingerprint } from './sqliteHandles.js';
|
|
1
2
|
import { DatabaseSync, type SQLInputValue } from 'node:sqlite';
|
|
2
3
|
import { type Adapter, type ScopedReads, type Filter, type QueryRequest, type QueryResult, type SqlRequest, type SqlResult, type DeleteRequest, type DeleteResult, type UpdateRequest, type UpdateResult, type ViewResult, type BreakdownResult, TransactionOperations, type WriteTransaction, type InsertRequest, type InsertResult, type ExplainRequest, type ExplainResult, type ProfileLimits, type ProfileResult, type CountItem, type AggregateRequest, type AggregateResult } from './adapter.js';
|
|
3
4
|
import type { TriggerAcceptance } from './scoped/triggerProof.js';
|
package/dist/adapters/sqlite.js
CHANGED
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { statSync } from 'node:fs';
|
|
11
11
|
import { waitForLocks, assertIndependentWrite, writeToFile, prepareGuarded, transactionEpoch, storeEpoch, schemaFingerprint, } from './sqliteHandles.js';
|
|
12
|
+
/* The tenant sources keep their catalog and store-layout checks by these (tenantReadSource.ts, tenantStore.ts). */
|
|
13
|
+
export { cachedStatement, schemaFingerprint } from './sqliteHandles.js';
|
|
12
14
|
import { captureScopeCatalog, mapScopeCatalog } from '../shared/scopeCatalog.js';
|
|
13
15
|
import { inlineRows } from '../shared/anyRows.js';
|
|
14
16
|
import { asksAboutRows } from '../shared/filter.js';
|
|
@@ -41,6 +43,49 @@ import { flagsFromDDL, vocabularyFromDDL } from '../shared/vocabulary.js';
|
|
|
41
43
|
import { coerceForColumn, columnKind, deleteRuleOf, } from '../shared/schema.js';
|
|
42
44
|
/** A generated column, virtual (2) or stored (3): the database writes it from its expression. */
|
|
43
45
|
const generated = (column) => column.hidden === 2 || column.hidden === 3;
|
|
46
|
+
/* The catalog answers a write transaction trusts outlive the adapter that read them: a writer reopened, or a second
|
|
47
|
+
one on the file, starts warm. Kept per file, by the identity of the file the handle opened (device, inode and birth
|
|
48
|
+
time, so a new file at a reused inode is a new key) and never across files; a file with no birth time to tell it
|
|
49
|
+
from its inode's next tenant, and an in-memory database, keep their own. Every entry is still served only under the
|
|
50
|
+
epoch the reader holds (epochCache.ts), and frozen, since another adapter reads the same value. */
|
|
51
|
+
const sharedCatalogs = new Map();
|
|
52
|
+
function catalogCacheOf(identity) {
|
|
53
|
+
if (identity === undefined)
|
|
54
|
+
return new EpochCache();
|
|
55
|
+
const cache = sharedCatalogs.get(identity) ?? new EpochCache();
|
|
56
|
+
sharedCatalogs.delete(identity);
|
|
57
|
+
sharedCatalogs.set(identity, cache);
|
|
58
|
+
while (sharedCatalogs.size > 32)
|
|
59
|
+
sharedCatalogs.delete(sharedCatalogs.keys().next().value);
|
|
60
|
+
return cache;
|
|
61
|
+
}
|
|
62
|
+
function frozen(value) {
|
|
63
|
+
if (!value || typeof value !== 'object' || Object.isFrozen(value))
|
|
64
|
+
return value;
|
|
65
|
+
if (value instanceof Map)
|
|
66
|
+
for (const [key, each] of value) {
|
|
67
|
+
frozen(key);
|
|
68
|
+
frozen(each);
|
|
69
|
+
}
|
|
70
|
+
else if (value instanceof Set)
|
|
71
|
+
for (const each of value)
|
|
72
|
+
frozen(each);
|
|
73
|
+
else
|
|
74
|
+
for (const each of Object.values(value))
|
|
75
|
+
frozen(each);
|
|
76
|
+
return Object.freeze(value);
|
|
77
|
+
}
|
|
78
|
+
/* What the scope catalog was resolved with is part of its key: a caller judging trigger effects reads another
|
|
79
|
+
catalog than one that does not. */
|
|
80
|
+
const resolverIds = new WeakMap();
|
|
81
|
+
let resolvers = 0;
|
|
82
|
+
const resolverKey = (resolve) => {
|
|
83
|
+
if (!resolve)
|
|
84
|
+
return 'scope';
|
|
85
|
+
if (!resolverIds.has(resolve))
|
|
86
|
+
resolverIds.set(resolve, ++resolvers);
|
|
87
|
+
return `scope:${resolverIds.get(resolve)}`;
|
|
88
|
+
};
|
|
44
89
|
const verifiedCommandAdapters = new WeakSet();
|
|
45
90
|
/** Identity-only native provenance; no caller can register a lookalike. */
|
|
46
91
|
export const isSqliteVerifiedCommandSource = (value) => !!value && typeof value === 'object' && !isProxy(value) && verifiedCommandAdapters.has(value);
|
|
@@ -59,8 +104,8 @@ export class SqliteAdapter {
|
|
|
59
104
|
/* Catalog answers a write transaction trusts — the journal's layout, the
|
|
60
105
|
scope catalog, a table's existence and its columns — kept under the
|
|
61
106
|
schema epoch the transaction read (transactionEpoch), so a warm command
|
|
62
|
-
asks the catalog nothing (epochCache.ts). */
|
|
63
|
-
catalogCache
|
|
107
|
+
asks the catalog nothing (epochCache.ts). Shared by every adapter on the file (catalogCacheOf). */
|
|
108
|
+
catalogCache;
|
|
64
109
|
/** The epoch the open write transaction read, on the handle it read it on. */
|
|
65
110
|
txEpoch;
|
|
66
111
|
constructor(file, label = file) {
|
|
@@ -76,9 +121,12 @@ export class SqliteAdapter {
|
|
|
76
121
|
if (typeof filename === 'string' && filename) {
|
|
77
122
|
const info = statSync(filename, { bigint: true });
|
|
78
123
|
this.writeIdentity = `${info.dev}:${info.ino}`;
|
|
124
|
+
this.catalogCache = catalogCacheOf(info.birthtimeNs > 0n ? `${info.dev}:${info.ino}:${info.birthtimeNs}` : undefined);
|
|
79
125
|
}
|
|
80
|
-
else
|
|
126
|
+
else {
|
|
81
127
|
this.writeIdentity = Symbol('sqlite-memory');
|
|
128
|
+
this.catalogCache = catalogCacheOf(undefined);
|
|
129
|
+
}
|
|
82
130
|
waitForLocks(this.db);
|
|
83
131
|
}
|
|
84
132
|
catch (error) {
|
|
@@ -1445,7 +1493,12 @@ export class SqliteAdapter {
|
|
|
1445
1493
|
verified();
|
|
1446
1494
|
};
|
|
1447
1495
|
if (scope !== undefined) {
|
|
1448
|
-
|
|
1496
|
+
/* Its schema captured once here, so the evidence mapping below is the same object every write and is
|
|
1497
|
+
made once per schema rather than copied and checked twice a command (mapScopeCatalog). */
|
|
1498
|
+
const catalog = this.catalogCache.throughSync(epoch, resolverKey(effects?.resolveTriggerEffects), () => {
|
|
1499
|
+
const read = readSqliteScopeCatalog(handle, effects?.resolveTriggerEffects);
|
|
1500
|
+
return frozen({ ...read, schema: captureScopeCatalog(read.schema) });
|
|
1501
|
+
});
|
|
1449
1502
|
physicalCatalog = catalog;
|
|
1450
1503
|
journal?.admit?.(handle, catalog, scope);
|
|
1451
1504
|
// Keep the complete physical graph for row/FK enforcement. Runtime
|
|
@@ -1655,7 +1708,7 @@ export class SqliteAdapter {
|
|
|
1655
1708
|
the first write of an epoch reads the list, and no later one asks. */
|
|
1656
1709
|
const epoch = this.epochOf(db);
|
|
1657
1710
|
if (epoch !== undefined) {
|
|
1658
|
-
const names = this.catalogCache.throughSync(epoch, 'tables', () => new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type IN ('table','view')`).all().map((row) => row.name)));
|
|
1711
|
+
const names = this.catalogCache.throughSync(epoch, 'tables', () => frozen(new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type IN ('table','view')`).all().map((row) => row.name))));
|
|
1659
1712
|
if (!names.has(name))
|
|
1660
1713
|
throw new Error(`No table or view called "${name}" in this database.`);
|
|
1661
1714
|
return name;
|
|
@@ -1668,7 +1721,7 @@ export class SqliteAdapter {
|
|
|
1668
1721
|
/** A table's columns, as a write reads them: from the catalog, or inside a
|
|
1669
1722
|
* write transaction from what was read under the epoch it holds. */
|
|
1670
1723
|
tableInfo(table, db) {
|
|
1671
|
-
return this.catalogCache.throughSync(this.epochOf(db), `columns:${table}`, () => db.prepare(`PRAGMA main.table_xinfo(${ANSI_STYLE.quote(table)})`).all().filter((c) => c.hidden !== 1));
|
|
1724
|
+
return this.catalogCache.throughSync(this.epochOf(db), `columns:${table}`, () => frozen(db.prepare(`PRAGMA main.table_xinfo(${ANSI_STYLE.quote(table)})`).all().filter((c) => c.hidden !== 1)));
|
|
1672
1725
|
}
|
|
1673
1726
|
/** Whether a relation is a view, as a write reads it: kept under the epoch, as its columns are. */
|
|
1674
1727
|
isView(table, db) {
|
|
@@ -44,6 +44,8 @@ export declare function transactionEpoch(db: DatabaseSync, journal?: {
|
|
|
44
44
|
* this contract, as it is for captureSchemaLifetime: no business operation can send one.
|
|
45
45
|
*/
|
|
46
46
|
export declare function schemaFingerprint(db: DatabaseSync): () => string;
|
|
47
|
+
/** `db.prepare(sql)`, kept per handle: for the catalog's and a store layout's questions, asked again and again. */
|
|
48
|
+
export declare function cachedStatement(db: DatabaseSync, sql: string): StatementSync;
|
|
47
49
|
/** The schema and whether a store's meta row says its version, in one statement; undefined when the store is not there. */
|
|
48
50
|
export declare function storeEpoch(db: DatabaseSync, meta: {
|
|
49
51
|
table: string;
|
|
@@ -136,9 +136,26 @@ export function transactionEpoch(db, journal) {
|
|
|
136
136
|
* this contract, as it is for captureSchemaLifetime: no business operation can send one.
|
|
137
137
|
*/
|
|
138
138
|
export function schemaFingerprint(db) {
|
|
139
|
-
const main = db
|
|
139
|
+
const main = cachedStatement(db, 'PRAGMA main.schema_version'), temp = cachedStatement(db, 'PRAGMA temp.schema_version');
|
|
140
140
|
return () => `${main.get()?.schema_version}.${temp.get()?.schema_version}`;
|
|
141
141
|
}
|
|
142
|
+
/* A catalog scan asks the same few hundred questions every time it runs, and preparing them was most of its cost (a
|
|
143
|
+
field report); a store's layout check asks its few on every command. Each handle keeps its own, by text: a statement is bound to its connection, and SQLite prepares one
|
|
144
|
+
again by itself when the schema under it has moved. A catalog of more tables than the bound starts the map over. */
|
|
145
|
+
const statements = new WeakMap();
|
|
146
|
+
/** `db.prepare(sql)`, kept per handle: for the catalog's and a store layout's questions, asked again and again. */
|
|
147
|
+
export function cachedStatement(db, sql) {
|
|
148
|
+
let kept = statements.get(db);
|
|
149
|
+
if (!kept)
|
|
150
|
+
statements.set(db, (kept = new Map()));
|
|
151
|
+
let statement = kept.get(sql);
|
|
152
|
+
if (!statement) {
|
|
153
|
+
if (kept.size >= 2048)
|
|
154
|
+
kept.clear();
|
|
155
|
+
kept.set(sql, (statement = db.prepare(sql)));
|
|
156
|
+
}
|
|
157
|
+
return statement;
|
|
158
|
+
}
|
|
142
159
|
/** The schema and whether a store's meta row says its version, in one statement; undefined when the store is not there. */
|
|
143
160
|
export function storeEpoch(db, meta) {
|
|
144
161
|
try {
|
|
@@ -38,12 +38,17 @@ export async function explainStatement(spanId) {
|
|
|
38
38
|
return explain ? explain() : undefined;
|
|
39
39
|
}
|
|
40
40
|
const EXPLAINABLE = new Set(['SELECT', 'WITH']);
|
|
41
|
+
/* A catalog read or a PRAGMA: the engine's own checks, which run by the dozen
|
|
42
|
+
around every command, and each span cost two random ids even in a trace the
|
|
43
|
+
exporter drops (a field report). They are spans only in a sampled trace. */
|
|
44
|
+
const INTERNAL = /^\s*PRAGMA\b|\b(?:sqlite_schema|sqlite_master|sqlite_temp_schema|sqlite_temp_master|pg_catalog|information_schema)\b/i;
|
|
41
45
|
/** A statement's span, begun but not made current: a statement has no children. The caller ends it. */
|
|
42
46
|
function beginStatement(dialect, text, params, explain) {
|
|
43
47
|
const { operation, table, summary } = summarizeStatement(text);
|
|
44
48
|
const span = startSpan(summary, {
|
|
45
49
|
kind: 'client',
|
|
46
50
|
childOnly: true,
|
|
51
|
+
sampledOnly: INTERNAL.test(text),
|
|
47
52
|
attributes: {
|
|
48
53
|
'db.system.name': dialect,
|
|
49
54
|
'db.operation.name': operation,
|
|
@@ -95,6 +95,8 @@ interface SpanOptions {
|
|
|
95
95
|
links?: readonly SpanLink[];
|
|
96
96
|
/** Start only inside an existing span: a statement outside any request, job or command is not worth a trace. */
|
|
97
97
|
childOnly?: boolean;
|
|
98
|
+
/** Start only beneath a sampled trace: the internal reads a statement span would otherwise open by the dozen (drivers.ts). */
|
|
99
|
+
sampledOnly?: boolean;
|
|
98
100
|
}
|
|
99
101
|
/** Begin a span; the caller ends it. Prefer `inSpan`, which also makes it current. */
|
|
100
102
|
export declare function startSpan(name: string, options?: SpanOptions): Span;
|
|
@@ -151,6 +151,8 @@ export function startSpan(name, options = {}) {
|
|
|
151
151
|
// Beneath a dropped span nothing is traced: the observer's own EXPLAIN is not a trace.
|
|
152
152
|
if (parent?.discarded)
|
|
153
153
|
return NOOP;
|
|
154
|
+
if (options.sampledOnly && !parent?.record.sampled)
|
|
155
|
+
return NOOP;
|
|
154
156
|
const remote = parent ? undefined : options.remote;
|
|
155
157
|
const traceId = parent?.traceId ?? remote?.traceId ?? hex(16);
|
|
156
158
|
const record = {
|
|
@@ -15,14 +15,16 @@
|
|
|
15
15
|
* expires a session or holds a lease past its time.
|
|
16
16
|
*/
|
|
17
17
|
import { appClockOffset, setAppClockOffset } from '../../shared/appClock.js';
|
|
18
|
-
import { productEnv } from '../product.js';
|
|
18
|
+
import { envName, productEnv } from '../product.js';
|
|
19
19
|
/**
|
|
20
20
|
* The pin wins over an as-of moment: a browser suite that pins the clock asks for the same instant on every run, and
|
|
21
21
|
* an offset from the real time would undo that.
|
|
22
22
|
*/
|
|
23
23
|
export function clockNow() {
|
|
24
24
|
const pinned = productEnv('CLOCK');
|
|
25
|
-
|
|
25
|
+
/* configureAsOf refuses the pin at start; this is for a process that never asked it (a worker, an embed), so a
|
|
26
|
+
stray MISTRI_CLOCK in a deployed environment cannot freeze App logic's day either. */
|
|
27
|
+
if (!pinned || process.env.NODE_ENV === 'production')
|
|
26
28
|
return appTime();
|
|
27
29
|
const at = new Date(pinned);
|
|
28
30
|
return Number.isNaN(at.getTime()) ? appTime() : at;
|
|
@@ -102,6 +104,16 @@ export function parseAsOf(given, zone) {
|
|
|
102
104
|
const digits = offset.slice(1).replace(':', '');
|
|
103
105
|
return new Date(wall - sign * (Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2))) * 60_000);
|
|
104
106
|
}
|
|
107
|
+
/** Why this process may not move App logic's clock, or undefined where it may: mistri dev, or a test process. */
|
|
108
|
+
function clockRefusal(asked, env) {
|
|
109
|
+
if (env.NODE_ENV === 'production')
|
|
110
|
+
return 'NODE_ENV is production';
|
|
111
|
+
if (asked.build)
|
|
112
|
+
return 'this serves a build, which is what is deployed';
|
|
113
|
+
if (!asked.dev && env.NODE_ENV !== 'test')
|
|
114
|
+
return 'this is neither mistri dev nor a test process (NODE_ENV=test)';
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
105
117
|
/**
|
|
106
118
|
* Read the as-of moment the flag or MISTRI_AS_OF asks for (the flag wins), and set App logic's clock to start at
|
|
107
119
|
* it. Development and tests only: refused, with a message naming what asked, under NODE_ENV=production, when
|
|
@@ -111,17 +123,17 @@ export function parseAsOf(given, zone) {
|
|
|
111
123
|
*/
|
|
112
124
|
export function configureAsOf(flag, asked) {
|
|
113
125
|
const env = asked.env ?? process.env;
|
|
126
|
+
const refused = clockRefusal(asked, env);
|
|
127
|
+
/* The pin is held to the same rule: it froze App logic's day for every visitor of a deployment it leaked into, and
|
|
128
|
+
clockNow read it without asking where it ran. */
|
|
129
|
+
const pin = productEnv('CLOCK', env);
|
|
130
|
+
if (pin && pin.trim() !== '' && refused)
|
|
131
|
+
throw new Error(`${envName('CLOCK')} pins App logic's clock for development and tests (mistri dev, or NODE_ENV=test), and ` +
|
|
132
|
+
`${refused}. Remove ${envName('CLOCK')} to serve.`);
|
|
114
133
|
const given = flag ?? env.MISTRI_AS_OF;
|
|
115
134
|
if (given === undefined || (flag === undefined && given.trim() === ''))
|
|
116
135
|
return undefined;
|
|
117
136
|
const source = flag !== undefined ? '--as-of' : 'MISTRI_AS_OF';
|
|
118
|
-
const refused = env.NODE_ENV === 'production'
|
|
119
|
-
? 'NODE_ENV is production'
|
|
120
|
-
: asked.build
|
|
121
|
-
? 'this serves a build, which is what is deployed'
|
|
122
|
-
: !asked.dev && env.NODE_ENV !== 'test'
|
|
123
|
-
? 'this is neither mistri dev nor a test process (NODE_ENV=test)'
|
|
124
|
-
: undefined;
|
|
125
137
|
if (refused)
|
|
126
138
|
throw new Error(`${source} moves App logic's clock for development and tests (mistri dev, or NODE_ENV=test), and ${refused}. ` +
|
|
127
139
|
`Remove ${source} to serve.`);
|
|
@@ -8,8 +8,12 @@
|
|
|
8
8
|
* and 175 ms of a blocked event loop at the ceiling), with a total counted
|
|
9
9
|
* beside it that the file never used. Here a batch is read, written, and let
|
|
10
10
|
* go, so memory stays flat whatever the size, the first byte leaves after
|
|
11
|
-
* the first batch, and nothing is counted.
|
|
12
|
-
* holds every row the question matches, or as many as its own
|
|
11
|
+
* the first batch, and nothing is counted. A file paged by key has no row
|
|
12
|
+
* ceiling: it holds every row the question matches, or as many as its own
|
|
13
|
+
* `limit` says. A file paged by offset does (`EXPORT_OFFSET_ROWS`): each of
|
|
14
|
+
* its batches reads past every row before it, so its cost grows with the
|
|
15
|
+
* square of its length, and one past the ceiling is refused before a byte is
|
|
16
|
+
* written. A person has at most `EXPORTS_IN_FLIGHT` files streaming at once.
|
|
13
17
|
*
|
|
14
18
|
* The next batch is read only once the last has been handed on: a write the
|
|
15
19
|
* connection cannot take yet waits for it to drain (backpressure), so a slow
|
|
@@ -51,6 +55,12 @@ import type { PostContext } from './context.js';
|
|
|
51
55
|
* large enough that a million-row file is a thousand reads, small enough that
|
|
52
56
|
* one batch is never a memory event. */
|
|
53
57
|
export declare const EXPORT_BATCH = 1000;
|
|
58
|
+
/** The most rows a file paged by offset holds, as many as the old one-read export did: past it the database reads
|
|
59
|
+
* every row before each batch again, so a deeper file is refused (`export_too_large`) rather than ground out. */
|
|
60
|
+
export declare const EXPORT_OFFSET_ROWS = 100000;
|
|
61
|
+
/** Files one person streams at once. Each holds a connection and a statement at a time for as long as its client
|
|
62
|
+
* reads, so a few clients that read slowly could otherwise hold every connection the pool has. */
|
|
63
|
+
export declare const EXPORTS_IN_FLIGHT = 2;
|
|
54
64
|
/** Who `me` is in a question, where its words name `me` (parseQuery's `me`). */
|
|
55
65
|
type Person = NonNullable<Parameters<typeof parseQuery>[1]>['me'];
|
|
56
66
|
/** What a product decides of an export. */
|
|
@@ -78,6 +88,8 @@ interface ExportAsked {
|
|
|
78
88
|
headers?: Record<string, string>;
|
|
79
89
|
/** A reply that is not the file (a refusal, the question's errors), sent the host's way. */
|
|
80
90
|
answer?: (status: number, body: unknown) => void;
|
|
91
|
+
/** Who is exporting, counted against `EXPORTS_IN_FLIGHT`; the client's address where no one is named. */
|
|
92
|
+
who?: string;
|
|
81
93
|
}
|
|
82
94
|
export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
|
|
83
95
|
/**
|
|
@@ -14,6 +14,13 @@ const EXPORT_PATH = '/api/export';
|
|
|
14
14
|
* large enough that a million-row file is a thousand reads, small enough that
|
|
15
15
|
* one batch is never a memory event. */
|
|
16
16
|
export const EXPORT_BATCH = 1000;
|
|
17
|
+
/** The most rows a file paged by offset holds, as many as the old one-read export did: past it the database reads
|
|
18
|
+
* every row before each batch again, so a deeper file is refused (`export_too_large`) rather than ground out. */
|
|
19
|
+
export const EXPORT_OFFSET_ROWS = 100_000;
|
|
20
|
+
/** Files one person streams at once. Each holds a connection and a statement at a time for as long as its client
|
|
21
|
+
* reads, so a few clients that read slowly could otherwise hold every connection the pool has. */
|
|
22
|
+
export const EXPORTS_IN_FLIGHT = 2;
|
|
23
|
+
const inFlight = new Map();
|
|
17
24
|
const FORMATS = {
|
|
18
25
|
csv: { type: 'text/csv', extension: 'csv' },
|
|
19
26
|
json: { type: 'application/json', extension: 'json' },
|
|
@@ -30,6 +37,7 @@ export async function exportRoute(post, hooks) {
|
|
|
30
37
|
schema: post.schema,
|
|
31
38
|
now: post.appNow,
|
|
32
39
|
may: hooks.may(post),
|
|
40
|
+
...(post.who ? { who: post.who } : {}),
|
|
33
41
|
...(person ? { person: (text) => person(post, text) } : {}),
|
|
34
42
|
...(amounts ? { amounts: (tableId) => amounts(post, tableId) } : {}),
|
|
35
43
|
});
|
|
@@ -57,6 +65,21 @@ export async function exportFile(asked) {
|
|
|
57
65
|
answer(403, { error: 'Export is not available to this person.' });
|
|
58
66
|
return undefined;
|
|
59
67
|
}
|
|
68
|
+
/* Counted until the response closes, whichever way it ends: a refusal, the file, or a client that left. */
|
|
69
|
+
const who = asked.who ?? `address:${res.req?.socket?.remoteAddress ?? ''}`;
|
|
70
|
+
const running = inFlight.get(who) ?? 0;
|
|
71
|
+
if (running >= EXPORTS_IN_FLIGHT) {
|
|
72
|
+
answer(429, { code: 'too_many_exports', error: `At most ${EXPORTS_IN_FLIGHT} exports run at once. Wait for one to finish.` });
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
inFlight.set(who, running + 1);
|
|
76
|
+
res.once('close', () => {
|
|
77
|
+
const left = (inFlight.get(who) ?? 1) - 1;
|
|
78
|
+
if (left > 0)
|
|
79
|
+
inFlight.set(who, left);
|
|
80
|
+
else
|
|
81
|
+
inFlight.delete(who);
|
|
82
|
+
});
|
|
60
83
|
const format = body.format ?? 'csv';
|
|
61
84
|
if (typeof format !== 'string' || !Object.hasOwn(FORMATS, format)) {
|
|
62
85
|
answer(400, { error: 'format must be "csv", "json" or "ndjson".' });
|
|
@@ -88,6 +111,7 @@ export async function exportFile(asked) {
|
|
|
88
111
|
const base = findTable(schema, view.base);
|
|
89
112
|
/* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
|
|
90
113
|
const keyed = base && !base.union ? base : undefined;
|
|
114
|
+
const byOffset = !keyed || !viewPaging(schema, keyed, view, undefined);
|
|
91
115
|
let step;
|
|
92
116
|
const read = async (offset, size) => {
|
|
93
117
|
const paging = keyed && viewPaging(schema, keyed, view, step);
|
|
@@ -100,7 +124,7 @@ export async function exportFile(asked) {
|
|
|
100
124
|
step = settled.bounds ? { after: settled.bounds.last } : undefined;
|
|
101
125
|
return { columns: settled.columns, rows: settled.rows };
|
|
102
126
|
};
|
|
103
|
-
return stream(res, format, view.base, rowsWanted(view.limit), read, requested(body) ?? undefined, amountsOf(view.base), asked.headers);
|
|
127
|
+
return stream(res, format, view.base, rowsWanted(view.limit), read, requested(body) ?? undefined, amountsOf(view.base), asked.headers, byOffset ? answer : undefined);
|
|
104
128
|
}
|
|
105
129
|
}
|
|
106
130
|
if (!parsed.query || parsed.errors.length) {
|
|
@@ -121,6 +145,7 @@ export async function exportFile(asked) {
|
|
|
121
145
|
return refuseTable(answer, reached === UNKNOWN_TABLE ? table.id : reached);
|
|
122
146
|
/* The order the list reads in, made total with the key; an offset batch reads in it too. */
|
|
123
147
|
const order = tableDialect(schema, table) === 'http' && !query.orderBy?.length ? undefined : resolveOrder(table, query.orderBy).order;
|
|
148
|
+
const byOffset = table.union || !recordsPaging(table, tableDialect(schema, table), query.orderBy, wanted, query.filter, undefined);
|
|
124
149
|
let step;
|
|
125
150
|
const read = async (offset, size) => {
|
|
126
151
|
/* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
|
|
@@ -143,7 +168,7 @@ export async function exportFile(asked) {
|
|
|
143
168
|
step = settled.bounds ? { after: settled.bounds.last } : undefined;
|
|
144
169
|
return { columns: settled.columns, rows: settled.rows };
|
|
145
170
|
};
|
|
146
|
-
return stream(res, format, table.id, rowsWanted(query.limit), read, wanted, amountsOf(table.id), asked.headers);
|
|
171
|
+
return stream(res, format, table.id, rowsWanted(query.limit), read, wanted, amountsOf(table.id), asked.headers, byOffset ? answer : undefined);
|
|
147
172
|
}
|
|
148
173
|
function refuseTable(answer, tableId) {
|
|
149
174
|
answer(403, {
|
|
@@ -251,7 +276,9 @@ class Abandoned extends Error {
|
|
|
251
276
|
this.name = 'Abandoned';
|
|
252
277
|
}
|
|
253
278
|
}
|
|
254
|
-
async function stream(res, format, table, total, read, wanted, amounts, headers = {}
|
|
279
|
+
async function stream(res, format, table, total, read, wanted, amounts, headers = {},
|
|
280
|
+
/** Where the file pages by offset: how a file past `EXPORT_OFFSET_ROWS` is refused. */
|
|
281
|
+
byOffset) {
|
|
255
282
|
/* A client that leaves stops the read in flight (the adapters watch this signal) and every one after it. */
|
|
256
283
|
const stop = new AbortController();
|
|
257
284
|
const left = () => {
|
|
@@ -261,6 +288,20 @@ async function stream(res, format, table, total, read, wanted, amounts, headers
|
|
|
261
288
|
res.once('close', left);
|
|
262
289
|
const reading = (offset) => withCancellation(stop.signal, () => read(offset, Math.min(EXPORT_BATCH, total - offset)));
|
|
263
290
|
try {
|
|
291
|
+
/* One row read past the ceiling says, before a byte is written, whether the file would go deeper than an offset
|
|
292
|
+
read should: one read as deep as the ceiling, rather than a file that reads deeper with every batch. */
|
|
293
|
+
if (byOffset && total > EXPORT_OFFSET_ROWS) {
|
|
294
|
+
const beyond = await withCancellation(stop.signal, () => read(EXPORT_OFFSET_ROWS, 1));
|
|
295
|
+
if (stop.signal.aborted)
|
|
296
|
+
return undefined;
|
|
297
|
+
if (beyond.rows.length) {
|
|
298
|
+
byOffset(422, {
|
|
299
|
+
code: 'export_too_large',
|
|
300
|
+
error: `This export is more than ${EXPORT_OFFSET_ROWS.toLocaleString('en-US')} rows, and its rows are read by position. Narrow the question, or give it a limit.`,
|
|
301
|
+
});
|
|
302
|
+
return undefined;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
264
305
|
/* The first batch is read before a byte is written: a refusal, a column
|
|
265
306
|
that does not exist, a database that is down all still answer as the
|
|
266
307
|
ordinary error they are, through the handler's own mapping. */
|
|
@@ -45,7 +45,8 @@ type RateDecision = {
|
|
|
45
45
|
export type RatePart = 'both' | 'client' | 'all';
|
|
46
46
|
/** A limiter that decides now (in memory) or once the shared cache has counted. */
|
|
47
47
|
interface WindowLimiter {
|
|
48
|
-
|
|
48
|
+
/** `weight` is what this request counts for: one, or its size where a window counts bytes. */
|
|
49
|
+
take(address: string, part?: RatePart, weight?: number): RateDecision | Promise<RateDecision>;
|
|
49
50
|
/** Forget this process's own counts; tests and a policy reload use it. A shared cache's windows expire on their own. */
|
|
50
51
|
reset(): void;
|
|
51
52
|
}
|
|
@@ -80,7 +81,7 @@ export declare class FixedWindowLimiter implements WindowLimiter {
|
|
|
80
81
|
private total;
|
|
81
82
|
constructor(windows: readonly RateWindow[], now?: () => number, options?: LimiterOptions);
|
|
82
83
|
/** Count one request for the client, or say how long it waits. */
|
|
83
|
-
take(address: string, part?: RatePart): RateDecision;
|
|
84
|
+
take(address: string, part?: RatePart, weight?: number): RateDecision;
|
|
84
85
|
private takeShared;
|
|
85
86
|
/** How many clients are tracked; tests read it. */
|
|
86
87
|
get size(): number;
|
|
@@ -106,7 +107,7 @@ export declare class SharedWindowLimiter implements WindowLimiter {
|
|
|
106
107
|
private readonly fallback;
|
|
107
108
|
private readonly totalWindows?;
|
|
108
109
|
constructor(name: string, windows: readonly RateWindow[], cache: CacheStore, fallback: FixedWindowLimiter, totalWindows?: readonly RateWindow[] | undefined);
|
|
109
|
-
take(address: string, part?: RatePart): Promise<RateDecision>;
|
|
110
|
+
take(address: string, part?: RatePart, weight?: number): Promise<RateDecision>;
|
|
110
111
|
reset(): void;
|
|
111
112
|
private count;
|
|
112
113
|
}
|
|
@@ -117,7 +118,10 @@ export declare const PUBLIC_FORM_SUBMIT_TOTAL: readonly RateWindow[];
|
|
|
117
118
|
export declare const PUBLIC_FORM_READ_WINDOWS: readonly RateWindow[];
|
|
118
119
|
export declare const INVITATION_CLAIM_WINDOWS: readonly RateWindow[];
|
|
119
120
|
export declare const INVITATION_CLAIM_TOTAL: readonly RateWindow[];
|
|
121
|
+
export declare const PUBLIC_FILE_BYTE_WINDOWS: readonly RateWindow[];
|
|
120
122
|
export declare function publicFormLimiter(kind: 'submit' | 'read' | 'start' | 'status' | 'link' | 'code' | 'check' | 'file'): FixedWindowLimiter | SharedWindowLimiter;
|
|
123
|
+
/** The bytes one client may send a public form's files in a day; `take(address, 'client', size)`. */
|
|
124
|
+
export declare function publicFileByteLimiter(): FixedWindowLimiter | SharedWindowLimiter;
|
|
121
125
|
/** The invitation claims' own budget (tenantInvitationHttp.ts), never the forms'. */
|
|
122
126
|
export declare function invitationClaimLimiter(): FixedWindowLimiter | SharedWindowLimiter;
|
|
123
127
|
/**
|
|
@@ -2,6 +2,10 @@ import { CacheUnavailable, scopedCache } from '../../adapters/cache/cacheStore.j
|
|
|
2
2
|
import { forwardedClient, ipv6Groups } from './trust.js';
|
|
3
3
|
import { product } from '../product.js';
|
|
4
4
|
const MAX_CLIENTS = 10_000;
|
|
5
|
+
function assertWeight(weight) {
|
|
6
|
+
if (!Number.isSafeInteger(weight) || weight < 1)
|
|
7
|
+
throw new Error('A request counts for a positive whole number.');
|
|
8
|
+
}
|
|
5
9
|
function assertWindows(windows) {
|
|
6
10
|
if (!windows.length || windows.some((w) => !Number.isSafeInteger(w.limit) || w.limit < 1 || !Number.isFinite(w.ms) || w.ms <= 0)) {
|
|
7
11
|
throw new Error('A rate limiter needs at least one window of a positive limit over a positive span.');
|
|
@@ -17,24 +21,24 @@ class Budget {
|
|
|
17
21
|
this.counts = windows.map(() => 0);
|
|
18
22
|
this.seen = now;
|
|
19
23
|
}
|
|
20
|
-
/** Roll over the windows that have elapsed; the seconds until the longest
|
|
21
|
-
wait(windows, now) {
|
|
24
|
+
/** Roll over the windows that have elapsed; the seconds until the longest one `weight` more would exceed reopens, or 0. */
|
|
25
|
+
wait(windows, now, weight = 1) {
|
|
22
26
|
let wait = 0;
|
|
23
27
|
windows.forEach((window, i) => {
|
|
24
28
|
if (now - this.starts[i] >= window.ms) {
|
|
25
29
|
this.starts[i] = now;
|
|
26
30
|
this.counts[i] = 0;
|
|
27
31
|
}
|
|
28
|
-
if (this.counts[i]
|
|
32
|
+
if (this.counts[i] + weight > window.limit)
|
|
29
33
|
wait = Math.max(wait, Math.ceil((this.starts[i] + window.ms - now) / 1000), 1);
|
|
30
34
|
});
|
|
31
35
|
return wait;
|
|
32
36
|
}
|
|
33
|
-
/** Count one request
|
|
34
|
-
count(windows) {
|
|
37
|
+
/** Count one request, for `weight`; what is left in the tightest window. */
|
|
38
|
+
count(windows, weight = 1) {
|
|
35
39
|
let remaining = Infinity;
|
|
36
40
|
windows.forEach((window, i) => {
|
|
37
|
-
this.counts[i]
|
|
41
|
+
this.counts[i] += weight;
|
|
38
42
|
remaining = Math.min(remaining, window.limit - this.counts[i]);
|
|
39
43
|
});
|
|
40
44
|
return remaining;
|
|
@@ -87,11 +91,12 @@ export class FixedWindowLimiter {
|
|
|
87
91
|
this.totalWindows = options.total;
|
|
88
92
|
}
|
|
89
93
|
/** Count one request for the client, or say how long it waits. */
|
|
90
|
-
take(address, part = 'both') {
|
|
94
|
+
take(address, part = 'both', weight = 1) {
|
|
95
|
+
assertWeight(weight);
|
|
91
96
|
const now = this.now();
|
|
92
97
|
this.expire(now);
|
|
93
98
|
if (part === 'all')
|
|
94
|
-
return this.takeShared(now) ?? { ok: true, remaining: Infinity };
|
|
99
|
+
return this.takeShared(now, weight) ?? { ok: true, remaining: Infinity };
|
|
95
100
|
const key = limitKey(address);
|
|
96
101
|
const entry = this.hits.get(key);
|
|
97
102
|
if (entry) {
|
|
@@ -99,12 +104,18 @@ export class FixedWindowLimiter {
|
|
|
99
104
|
this.hits.delete(key);
|
|
100
105
|
this.hits.set(key, entry);
|
|
101
106
|
entry.seen = now;
|
|
102
|
-
const wait = entry.wait(this.windows, now);
|
|
107
|
+
const wait = entry.wait(this.windows, now, weight);
|
|
108
|
+
if (wait > 0)
|
|
109
|
+
return { ok: false, retryAfterSeconds: wait, scope: 'client' };
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
// A first request heavier than a whole window could never be counted: refused, and nothing tracked for it.
|
|
113
|
+
const wait = new Budget(this.windows, now).wait(this.windows, now, weight);
|
|
103
114
|
if (wait > 0)
|
|
104
115
|
return { ok: false, retryAfterSeconds: wait, scope: 'client' };
|
|
105
116
|
}
|
|
106
117
|
const shared = part === 'both' && this.totalWindows ? (this.total ??= new Budget(this.totalWindows, now)) : undefined;
|
|
107
|
-
const sharedWait = shared ? shared.wait(this.totalWindows, now) : 0;
|
|
118
|
+
const sharedWait = shared ? shared.wait(this.totalWindows, now, weight) : 0;
|
|
108
119
|
if (sharedWait > 0)
|
|
109
120
|
return { ok: false, retryAfterSeconds: sharedWait, scope: 'all' };
|
|
110
121
|
let budget = entry;
|
|
@@ -131,19 +142,19 @@ export class FixedWindowLimiter {
|
|
|
131
142
|
budget = new Budget(this.windows, now);
|
|
132
143
|
this.hits.set(key, budget);
|
|
133
144
|
}
|
|
134
|
-
const remaining = budget.count(this.windows);
|
|
135
|
-
shared?.count(this.totalWindows);
|
|
145
|
+
const remaining = budget.count(this.windows, weight);
|
|
146
|
+
shared?.count(this.totalWindows, weight);
|
|
136
147
|
return { ok: true, remaining };
|
|
137
148
|
}
|
|
138
149
|
/* The shared ceiling alone: a refusal, or undefined once counted (or where there is none). */
|
|
139
|
-
takeShared(now) {
|
|
150
|
+
takeShared(now, weight) {
|
|
140
151
|
if (!this.totalWindows)
|
|
141
152
|
return undefined;
|
|
142
153
|
const shared = (this.total ??= new Budget(this.totalWindows, now));
|
|
143
|
-
const wait = shared.wait(this.totalWindows, now);
|
|
154
|
+
const wait = shared.wait(this.totalWindows, now, weight);
|
|
144
155
|
if (wait > 0)
|
|
145
156
|
return { ok: false, retryAfterSeconds: wait, scope: 'all' };
|
|
146
|
-
shared.count(this.totalWindows);
|
|
157
|
+
shared.count(this.totalWindows, weight);
|
|
147
158
|
return undefined;
|
|
148
159
|
}
|
|
149
160
|
/** How many clients are tracked; tests read it. */
|
|
@@ -194,17 +205,18 @@ export class SharedWindowLimiter {
|
|
|
194
205
|
if (totalWindows)
|
|
195
206
|
assertWindows(totalWindows);
|
|
196
207
|
}
|
|
197
|
-
async take(address, part = 'both') {
|
|
208
|
+
async take(address, part = 'both', weight = 1) {
|
|
209
|
+
assertWeight(weight);
|
|
198
210
|
try {
|
|
199
211
|
let remaining = Infinity;
|
|
200
212
|
if (part !== 'all') {
|
|
201
|
-
const client = await this.count(this.windows, `${this.name}:${limitKey(address)}
|
|
213
|
+
const client = await this.count(this.windows, `${this.name}:${limitKey(address)}`, weight);
|
|
202
214
|
if (client.wait > 0)
|
|
203
215
|
return { ok: false, retryAfterSeconds: client.wait, scope: 'client' };
|
|
204
216
|
remaining = client.remaining;
|
|
205
217
|
}
|
|
206
218
|
if (part !== 'client' && this.totalWindows) {
|
|
207
|
-
const all = await this.count(this.totalWindows, `${this.name}
|
|
219
|
+
const all = await this.count(this.totalWindows, `${this.name}:*`, weight);
|
|
208
220
|
if (all.wait > 0)
|
|
209
221
|
return { ok: false, retryAfterSeconds: all.wait, scope: 'all' };
|
|
210
222
|
}
|
|
@@ -213,17 +225,17 @@ export class SharedWindowLimiter {
|
|
|
213
225
|
catch (error) {
|
|
214
226
|
if (!(error instanceof CacheUnavailable))
|
|
215
227
|
throw error;
|
|
216
|
-
return this.fallback.take(address, part);
|
|
228
|
+
return this.fallback.take(address, part, weight);
|
|
217
229
|
}
|
|
218
230
|
}
|
|
219
231
|
reset() {
|
|
220
232
|
this.fallback.reset();
|
|
221
233
|
}
|
|
222
|
-
async count(windows, key) {
|
|
234
|
+
async count(windows, key, weight) {
|
|
223
235
|
let wait = 0;
|
|
224
236
|
let remaining = Infinity;
|
|
225
237
|
for (const [i, window] of windows.entries()) {
|
|
226
|
-
const { count, ttlMs } = await this.cache.incr(`${key}:${i}`, window.ms);
|
|
238
|
+
const { count, ttlMs } = await this.cache.incr(`${key}:${i}`, window.ms, weight);
|
|
227
239
|
if (count > window.limit)
|
|
228
240
|
wait = Math.max(wait, Math.max(1, Math.ceil(ttlMs / 1000)));
|
|
229
241
|
else
|
|
@@ -294,8 +306,16 @@ const FILE_WINDOWS = Object.freeze([
|
|
|
294
306
|
{ limit: 60, ms: 3_600_000 },
|
|
295
307
|
]);
|
|
296
308
|
const FILE_TOTAL = Object.freeze([{ limit: 1_200, ms: 3_600_000 }]);
|
|
309
|
+
/* And the bytes those files carry, counted per client across a day, charged before a file reaches the share every
|
|
310
|
+
visitor of the App's forms holds together (stagedFiles.ts): without it one client (one cookie per request) could fill
|
|
311
|
+
that share alone and refuse every other visitor's files. Fifty megabytes is a few applications' worth of documents
|
|
312
|
+
from one address; a filled share then needs several addresses, each of which the windows above also count. */
|
|
313
|
+
export const PUBLIC_FILE_BYTE_WINDOWS = Object.freeze([{ limit: 50 * 1024 * 1024, ms: 86_400_000 }]);
|
|
297
314
|
const check = new FixedWindowLimiter(CHECK_WINDOWS, Date.now, { total: CHECK_TOTAL, whenFull: 'evict' });
|
|
298
315
|
const file = new FixedWindowLimiter(FILE_WINDOWS, Date.now, { total: FILE_TOTAL, whenFull: 'refuse' });
|
|
316
|
+
/* `evict`: a day's window, so refusing newcomers once every slot is taken would close the files to everyone for up to a
|
|
317
|
+
day; a forgotten budget is bounded by the file windows' shared ceiling and the share's own cap. */
|
|
318
|
+
const fileBytes = new FixedWindowLimiter(PUBLIC_FILE_BYTE_WINDOWS, Date.now, { whenFull: 'evict' });
|
|
299
319
|
const submit = new FixedWindowLimiter(PUBLIC_FORM_SUBMIT_WINDOWS, Date.now, { total: PUBLIC_FORM_SUBMIT_TOTAL, whenFull: 'refuse' });
|
|
300
320
|
const read = new FixedWindowLimiter(PUBLIC_FORM_READ_WINDOWS, Date.now, { whenFull: 'evict' });
|
|
301
321
|
const claim = new FixedWindowLimiter(INVITATION_CLAIM_WINDOWS, Date.now, { total: INVITATION_CLAIM_TOTAL, whenFull: 'refuse' });
|
|
@@ -323,6 +343,10 @@ export function publicFormLimiter(kind) {
|
|
|
323
343
|
? code
|
|
324
344
|
: status;
|
|
325
345
|
}
|
|
346
|
+
/** The bytes one client may send a public form's files in a day; `take(address, 'client', size)`. */
|
|
347
|
+
export function publicFileByteLimiter() {
|
|
348
|
+
return shared ? shared.fileBytes : fileBytes;
|
|
349
|
+
}
|
|
326
350
|
/** The invitation claims' own budget (tenantInvitationHttp.ts), never the forms'. */
|
|
327
351
|
export function invitationClaimLimiter() {
|
|
328
352
|
return shared ? shared.claim : claim;
|
|
@@ -349,6 +373,7 @@ export function configurePublicFormLimits(cache, namespace) {
|
|
|
349
373
|
code: new SharedWindowLimiter('code', CODE_WINDOWS, scoped, code, CODE_TOTAL),
|
|
350
374
|
check: new SharedWindowLimiter('check', CHECK_WINDOWS, scoped, check, CHECK_TOTAL),
|
|
351
375
|
file: new SharedWindowLimiter('file', FILE_WINDOWS, scoped, file, FILE_TOTAL),
|
|
376
|
+
fileBytes: new SharedWindowLimiter('file-bytes', PUBLIC_FILE_BYTE_WINDOWS, scoped, fileBytes),
|
|
352
377
|
};
|
|
353
378
|
}
|
|
354
379
|
/** Forgets the forms' budgets and the claims'; tests and a policy reload use it. */
|
|
@@ -362,4 +387,5 @@ export function resetPublicFormLimits() {
|
|
|
362
387
|
code.reset();
|
|
363
388
|
check.reset();
|
|
364
389
|
file.reset();
|
|
390
|
+
fileBytes.reset();
|
|
365
391
|
}
|
|
@@ -19,6 +19,13 @@ const frozenDerived = new WeakSet();
|
|
|
19
19
|
const derivedCaptures = new WeakMap();
|
|
20
20
|
/** Test-only: how many catalogs were copied and checked in full. */
|
|
21
21
|
export const scopeCatalogCounters = { captured: 0 };
|
|
22
|
+
/* Every object inside a catalog captured below, with what copying it would
|
|
23
|
+
count. A catalog made from a captured one (a mapping, a pruning) shares its
|
|
24
|
+
frozen parts, and copying those again for each write was most of a capture's
|
|
25
|
+
work (a field report). They are detached plain data that nothing can change,
|
|
26
|
+
so copyData keeps them; the checks after it still read every one. Only seal()
|
|
27
|
+
adds to it, so a caller's object, frozen or not, is always copied. */
|
|
28
|
+
const sealed = new WeakMap();
|
|
22
29
|
/** Validate descriptors before reading values, including non-authoritative metadata. */
|
|
23
30
|
function copyData(input) {
|
|
24
31
|
let nodes = 0;
|
|
@@ -38,6 +45,12 @@ function copyData(input) {
|
|
|
38
45
|
return value;
|
|
39
46
|
if (typeof value !== 'object' || ancestors.has(value))
|
|
40
47
|
return refuse();
|
|
48
|
+
const kept = sealed.get(value);
|
|
49
|
+
if (kept) {
|
|
50
|
+
if ((nodes += kept.nodes - 1) > 262_144 || depth + kept.height > 16 || (characters += kept.characters) > 16_777_216)
|
|
51
|
+
return refuse();
|
|
52
|
+
return value;
|
|
53
|
+
}
|
|
41
54
|
const array = Array.isArray(value);
|
|
42
55
|
const proto = Object.getPrototypeOf(value);
|
|
43
56
|
if (array ? proto !== Array.prototype : proto !== Object.prototype && proto !== null)
|
|
@@ -89,6 +102,26 @@ function freeze(value) {
|
|
|
89
102
|
}
|
|
90
103
|
return value;
|
|
91
104
|
}
|
|
105
|
+
/** Freeze a checked catalog all the way down and remember each part's size for copyData. */
|
|
106
|
+
function seal(value) {
|
|
107
|
+
if (typeof value === 'string')
|
|
108
|
+
return { nodes: 1, characters: value.length, height: 0 };
|
|
109
|
+
if (!value || typeof value !== 'object')
|
|
110
|
+
return { nodes: 1, characters: 0, height: 0 };
|
|
111
|
+
const known = sealed.get(value);
|
|
112
|
+
if (known)
|
|
113
|
+
return known;
|
|
114
|
+
const size = { nodes: 1, characters: 0, height: 0 };
|
|
115
|
+
for (const [key, child] of Object.entries(value)) {
|
|
116
|
+
const inner = seal(child);
|
|
117
|
+
size.nodes += inner.nodes;
|
|
118
|
+
size.characters += inner.characters + (Array.isArray(value) ? 0 : key.length);
|
|
119
|
+
size.height = Math.max(size.height, inner.height + 1);
|
|
120
|
+
}
|
|
121
|
+
Object.freeze(value);
|
|
122
|
+
sealed.set(value, size);
|
|
123
|
+
return size;
|
|
124
|
+
}
|
|
92
125
|
const fkOrder = (a, b) => order(a.from.table, b.from.table) || order(a.name, b.name);
|
|
93
126
|
/** Whether `schema` is a catalog this module captured: detached, validated
|
|
94
127
|
* and deeply frozen, so nothing can change it after a reader binds to it. */
|
|
@@ -268,7 +301,8 @@ export function captureScopeCatalog(schema) {
|
|
|
268
301
|
endpoint(column.references.table, [column.references.column]);
|
|
269
302
|
}
|
|
270
303
|
}
|
|
271
|
-
|
|
304
|
+
seal(captured);
|
|
305
|
+
const frozen = captured;
|
|
272
306
|
capturedCatalogs.add(frozen);
|
|
273
307
|
if (frozenDerived.has(schema))
|
|
274
308
|
derivedCaptures.set(schema, frozen);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mistriwork/engine",
|
|
3
|
-
"version": "1.0.0-rc.
|
|
3
|
+
"version": "1.0.0-rc.36",
|
|
4
4
|
"description": "The engine both products stand on: the adapters (SQLite, Postgres), the catalog, the query language, keyset pages, the HTTP kit and the shared rules. No React.",
|
|
5
5
|
"homepage": "https://mistriwork.dev",
|
|
6
6
|
"bugs": {
|