@mistriwork/engine 1.0.0-rc.35 → 1.0.0-rc.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,11 +18,12 @@ export interface CacheStore {
18
18
  set(key: string, value: string, ttlMs: number): Promise<void>;
19
19
  delete(key: string): Promise<void>;
20
20
  /**
21
- * Count one under `key`. A new key starts a window of `ttlMs`; the answer is
22
- * the count within it and the milliseconds it has left. Throws
23
- * CacheUnavailable when the count cannot be kept.
21
+ * Count one under `key` — or `by`, a positive whole number, where what is
22
+ * counted is a size rather than a request. A new key starts a window of
23
+ * `ttlMs`; the answer is the count within it and the milliseconds it has
24
+ * left. Throws CacheUnavailable when the count cannot be kept.
24
25
  */
25
- incr(key: string, ttlMs: number): Promise<{
26
+ incr(key: string, ttlMs: number, by?: number): Promise<{
26
27
  count: number;
27
28
  ttlMs: number;
28
29
  }>;
@@ -42,7 +43,7 @@ export declare class MemoryCacheStore implements CacheStore {
42
43
  get(key: string): Promise<string | undefined>;
43
44
  set(key: string, value: string, ttlMs: number): Promise<void>;
44
45
  delete(key: string): Promise<void>;
45
- incr(key: string, ttlMs: number): Promise<{
46
+ incr(key: string, ttlMs: number, by?: number): Promise<{
46
47
  count: number;
47
48
  ttlMs: number;
48
49
  }>;
@@ -64,7 +65,7 @@ export declare class ValkeyCacheStore implements CacheStore {
64
65
  get(key: string): Promise<string | undefined>;
65
66
  set(key: string, value: string, ttlMs: number): Promise<void>;
66
67
  delete(key: string): Promise<void>;
67
- incr(key: string, ttlMs: number): Promise<{
68
+ incr(key: string, ttlMs: number, by?: number): Promise<{
68
69
  count: number;
69
70
  ttlMs: number;
70
71
  }>;
@@ -14,6 +14,10 @@ import { CacheUnavailable, RespReplyError } from './resp.js';
14
14
  export { CacheUnavailable } from './resp.js';
15
15
  /* Thirty days: a copy kept longer than that is a copy nobody meant to keep. */
16
16
  const MAX_TTL_MS = 30 * 24 * 3_600_000;
17
+ function assertBy(by) {
18
+ if (!Number.isSafeInteger(by) || by < 1)
19
+ throw new Error(`A count goes up by a positive whole number; ${by} is not one.`);
20
+ }
17
21
  function assertTtl(ttlMs) {
18
22
  if (!Number.isSafeInteger(ttlMs) || ttlMs < 1 || ttlMs > MAX_TTL_MS) {
19
23
  throw new Error(`A cached value needs a time to live between 1 ms and 30 days; ${ttlMs} is not one.`);
@@ -50,11 +54,12 @@ export class MemoryCacheStore {
50
54
  async delete(key) {
51
55
  this.entries.delete(key);
52
56
  }
53
- async incr(key, ttlMs) {
57
+ async incr(key, ttlMs, by = 1) {
54
58
  assertTtl(ttlMs);
59
+ assertBy(by);
55
60
  const entry = this.live(key);
56
61
  const now = this.now();
57
- const next = entry ? { value: String(Number(entry.value) + 1), expires: entry.expires } : { value: '1', expires: now + ttlMs };
62
+ const next = entry ? { value: String(Number(entry.value) + by), expires: entry.expires } : { value: String(by), expires: now + ttlMs };
58
63
  this.entries.delete(key);
59
64
  this.entries.set(key, next);
60
65
  this.trim();
@@ -82,8 +87,8 @@ export class MemoryCacheStore {
82
87
  }
83
88
  /* One round trip, and atomic: a window that is counted is a window that
84
89
  expires, even if this process stops between the two steps. */
85
- const COUNT_IN_WINDOW = `local n = redis.call('INCR', KEYS[1])
86
- if n == 1 then redis.call('PEXPIRE', KEYS[1], ARGV[1]) end
90
+ const COUNT_IN_WINDOW = `local n = redis.call('INCRBY', KEYS[1], ARGV[2])
91
+ if n == tonumber(ARGV[2]) then redis.call('PEXPIRE', KEYS[1], ARGV[1]) end
87
92
  local t = redis.call('PTTL', KEYS[1])
88
93
  if t < 0 then redis.call('PEXPIRE', KEYS[1], ARGV[1]) t = tonumber(ARGV[1]) end
89
94
  return {n, t}`;
@@ -126,11 +131,12 @@ export class ValkeyCacheStore {
126
131
  this.unavailable(error);
127
132
  }
128
133
  }
129
- async incr(key, ttlMs) {
134
+ async incr(key, ttlMs, by = 1) {
130
135
  assertTtl(ttlMs);
136
+ assertBy(by);
131
137
  let reply;
132
138
  try {
133
- reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs);
139
+ reply = await this.client.command('EVAL', COUNT_IN_WINDOW, 1, key, ttlMs, by);
134
140
  }
135
141
  catch (error) {
136
142
  this.unavailable(error);
@@ -154,7 +160,7 @@ export function scopedCache(store, prefix) {
154
160
  get: (key) => store.get(prefix + key),
155
161
  set: (key, value, ttlMs) => store.set(prefix + key, value, ttlMs),
156
162
  delete: (key) => store.delete(prefix + key),
157
- incr: (key, ttlMs) => store.incr(prefix + key, ttlMs),
163
+ incr: (key, ttlMs, by) => store.incr(prefix + key, ttlMs, by),
158
164
  close: () => store.close(),
159
165
  };
160
166
  }
@@ -15,14 +15,16 @@
15
15
  * expires a session or holds a lease past its time.
16
16
  */
17
17
  import { appClockOffset, setAppClockOffset } from '../../shared/appClock.js';
18
- import { productEnv } from '../product.js';
18
+ import { envName, productEnv } from '../product.js';
19
19
  /**
20
20
  * The pin wins over an as-of moment: a browser suite that pins the clock asks for the same instant on every run, and
21
21
  * an offset from the real time would undo that.
22
22
  */
23
23
  export function clockNow() {
24
24
  const pinned = productEnv('CLOCK');
25
- if (!pinned)
25
+ /* configureAsOf refuses the pin at start; this is for a process that never asked it (a worker, an embed), so a
26
+ stray MISTRI_CLOCK in a deployed environment cannot freeze App logic's day either. */
27
+ if (!pinned || process.env.NODE_ENV === 'production')
26
28
  return appTime();
27
29
  const at = new Date(pinned);
28
30
  return Number.isNaN(at.getTime()) ? appTime() : at;
@@ -102,6 +104,16 @@ export function parseAsOf(given, zone) {
102
104
  const digits = offset.slice(1).replace(':', '');
103
105
  return new Date(wall - sign * (Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2))) * 60_000);
104
106
  }
107
+ /** Why this process may not move App logic's clock, or undefined where it may: mistri dev, or a test process. */
108
+ function clockRefusal(asked, env) {
109
+ if (env.NODE_ENV === 'production')
110
+ return 'NODE_ENV is production';
111
+ if (asked.build)
112
+ return 'this serves a build, which is what is deployed';
113
+ if (!asked.dev && env.NODE_ENV !== 'test')
114
+ return 'this is neither mistri dev nor a test process (NODE_ENV=test)';
115
+ return undefined;
116
+ }
105
117
  /**
106
118
  * Read the as-of moment the flag or MISTRI_AS_OF asks for (the flag wins), and set App logic's clock to start at
107
119
  * it. Development and tests only: refused, with a message naming what asked, under NODE_ENV=production, when
@@ -111,17 +123,17 @@ export function parseAsOf(given, zone) {
111
123
  */
112
124
  export function configureAsOf(flag, asked) {
113
125
  const env = asked.env ?? process.env;
126
+ const refused = clockRefusal(asked, env);
127
+ /* The pin is held to the same rule: it froze App logic's day for every visitor of a deployment it leaked into, and
128
+ clockNow read it without asking where it ran. */
129
+ const pin = productEnv('CLOCK', env);
130
+ if (pin && pin.trim() !== '' && refused)
131
+ throw new Error(`${envName('CLOCK')} pins App logic's clock for development and tests (mistri dev, or NODE_ENV=test), and ` +
132
+ `${refused}. Remove ${envName('CLOCK')} to serve.`);
114
133
  const given = flag ?? env.MISTRI_AS_OF;
115
134
  if (given === undefined || (flag === undefined && given.trim() === ''))
116
135
  return undefined;
117
136
  const source = flag !== undefined ? '--as-of' : 'MISTRI_AS_OF';
118
- const refused = env.NODE_ENV === 'production'
119
- ? 'NODE_ENV is production'
120
- : asked.build
121
- ? 'this serves a build, which is what is deployed'
122
- : !asked.dev && env.NODE_ENV !== 'test'
123
- ? 'this is neither mistri dev nor a test process (NODE_ENV=test)'
124
- : undefined;
125
137
  if (refused)
126
138
  throw new Error(`${source} moves App logic's clock for development and tests (mistri dev, or NODE_ENV=test), and ${refused}. ` +
127
139
  `Remove ${source} to serve.`);
@@ -8,8 +8,12 @@
8
8
  * and 175 ms of a blocked event loop at the ceiling), with a total counted
9
9
  * beside it that the file never used. Here a batch is read, written, and let
10
10
  * go, so memory stays flat whatever the size, the first byte leaves after
11
- * the first batch, and nothing is counted. There is no row ceiling: a file
12
- * holds every row the question matches, or as many as its own `limit` says.
11
+ * the first batch, and nothing is counted. A file paged by key has no row
12
+ * ceiling: it holds every row the question matches, or as many as its own
13
+ * `limit` says. A file paged by offset does (`EXPORT_OFFSET_ROWS`): each of
14
+ * its batches reads past every row before it, so its cost grows with the
15
+ * square of its length, and one past the ceiling is refused before a byte is
16
+ * written. A person has at most `EXPORTS_IN_FLIGHT` files streaming at once.
13
17
  *
14
18
  * The next batch is read only once the last has been handed on: a write the
15
19
  * connection cannot take yet waits for it to drain (backpressure), so a slow
@@ -51,6 +55,12 @@ import type { PostContext } from './context.js';
51
55
  * large enough that a million-row file is a thousand reads, small enough that
52
56
  * one batch is never a memory event. */
53
57
  export declare const EXPORT_BATCH = 1000;
58
+ /** The most rows a file paged by offset holds, as many as the old one-read export did: past it the database reads
59
+ * every row before each batch again, so a deeper file is refused (`export_too_large`) rather than ground out. */
60
+ export declare const EXPORT_OFFSET_ROWS = 100000;
61
+ /** Files one person streams at once. Each holds a connection and a statement at a time for as long as its client
62
+ * reads, so a few clients that read slowly could otherwise hold every connection the pool has. */
63
+ export declare const EXPORTS_IN_FLIGHT = 2;
54
64
  /** Who `me` is in a question, where its words name `me` (parseQuery's `me`). */
55
65
  type Person = NonNullable<Parameters<typeof parseQuery>[1]>['me'];
56
66
  /** What a product decides of an export. */
@@ -78,6 +88,8 @@ interface ExportAsked {
78
88
  headers?: Record<string, string>;
79
89
  /** A reply that is not the file (a refusal, the question's errors), sent the host's way. */
80
90
  answer?: (status: number, body: unknown) => void;
91
+ /** Who is exporting, counted against `EXPORTS_IN_FLIGHT`; the client's address where no one is named. */
92
+ who?: string;
81
93
  }
82
94
  export declare function exportRoute(post: PostContext, hooks: ExportHooks): Promise<boolean>;
83
95
  /**
@@ -14,6 +14,13 @@ const EXPORT_PATH = '/api/export';
14
14
  * large enough that a million-row file is a thousand reads, small enough that
15
15
  * one batch is never a memory event. */
16
16
  export const EXPORT_BATCH = 1000;
17
+ /** The most rows a file paged by offset holds, as many as the old one-read export did: past it the database reads
18
+ * every row before each batch again, so a deeper file is refused (`export_too_large`) rather than ground out. */
19
+ export const EXPORT_OFFSET_ROWS = 100_000;
20
+ /** Files one person streams at once. Each holds a connection and a statement at a time for as long as its client
21
+ * reads, so a few clients that read slowly could otherwise hold every connection the pool has. */
22
+ export const EXPORTS_IN_FLIGHT = 2;
23
+ const inFlight = new Map();
17
24
  const FORMATS = {
18
25
  csv: { type: 'text/csv', extension: 'csv' },
19
26
  json: { type: 'application/json', extension: 'json' },
@@ -30,6 +37,7 @@ export async function exportRoute(post, hooks) {
30
37
  schema: post.schema,
31
38
  now: post.appNow,
32
39
  may: hooks.may(post),
40
+ ...(post.who ? { who: post.who } : {}),
33
41
  ...(person ? { person: (text) => person(post, text) } : {}),
34
42
  ...(amounts ? { amounts: (tableId) => amounts(post, tableId) } : {}),
35
43
  });
@@ -57,6 +65,21 @@ export async function exportFile(asked) {
57
65
  answer(403, { error: 'Export is not available to this person.' });
58
66
  return undefined;
59
67
  }
68
+ /* Counted until the response closes, whichever way it ends: a refusal, the file, or a client that left. */
69
+ const who = asked.who ?? `address:${res.req?.socket?.remoteAddress ?? ''}`;
70
+ const running = inFlight.get(who) ?? 0;
71
+ if (running >= EXPORTS_IN_FLIGHT) {
72
+ answer(429, { code: 'too_many_exports', error: `At most ${EXPORTS_IN_FLIGHT} exports run at once. Wait for one to finish.` });
73
+ return undefined;
74
+ }
75
+ inFlight.set(who, running + 1);
76
+ res.once('close', () => {
77
+ const left = (inFlight.get(who) ?? 1) - 1;
78
+ if (left > 0)
79
+ inFlight.set(who, left);
80
+ else
81
+ inFlight.delete(who);
82
+ });
60
83
  const format = body.format ?? 'csv';
61
84
  if (typeof format !== 'string' || !Object.hasOwn(FORMATS, format)) {
62
85
  answer(400, { error: 'format must be "csv", "json" or "ndjson".' });
@@ -88,6 +111,7 @@ export async function exportFile(asked) {
88
111
  const base = findTable(schema, view.base);
89
112
  /* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
90
113
  const keyed = base && !base.union ? base : undefined;
114
+ const byOffset = !keyed || !viewPaging(schema, keyed, view, undefined);
91
115
  let step;
92
116
  const read = async (offset, size) => {
93
117
  const paging = keyed && viewPaging(schema, keyed, view, step);
@@ -100,7 +124,7 @@ export async function exportFile(asked) {
100
124
  step = settled.bounds ? { after: settled.bounds.last } : undefined;
101
125
  return { columns: settled.columns, rows: settled.rows };
102
126
  };
103
- return stream(res, format, view.base, rowsWanted(view.limit), read, requested(body) ?? undefined, amountsOf(view.base), asked.headers);
127
+ return stream(res, format, view.base, rowsWanted(view.limit), read, requested(body) ?? undefined, amountsOf(view.base), asked.headers, byOffset ? answer : undefined);
104
128
  }
105
129
  }
106
130
  if (!parsed.query || parsed.errors.length) {
@@ -121,6 +145,7 @@ export async function exportFile(asked) {
121
145
  return refuseTable(answer, reached === UNKNOWN_TABLE ? table.id : reached);
122
146
  /* The order the list reads in, made total with the key; an offset batch reads in it too. */
123
147
  const order = tableDialect(schema, table) === 'http' && !query.orderBy?.length ? undefined : resolveOrder(table, query.orderBy).order;
148
+ const byOffset = table.union || !recordsPaging(table, tableDialect(schema, table), query.orderBy, wanted, query.filter, undefined);
124
149
  let step;
125
150
  const read = async (offset, size) => {
126
151
  /* A union pages by offset: its rows have no one key order across branches (unionReads.ts), as the list's. */
@@ -143,7 +168,7 @@ export async function exportFile(asked) {
143
168
  step = settled.bounds ? { after: settled.bounds.last } : undefined;
144
169
  return { columns: settled.columns, rows: settled.rows };
145
170
  };
146
- return stream(res, format, table.id, rowsWanted(query.limit), read, wanted, amountsOf(table.id), asked.headers);
171
+ return stream(res, format, table.id, rowsWanted(query.limit), read, wanted, amountsOf(table.id), asked.headers, byOffset ? answer : undefined);
147
172
  }
148
173
  function refuseTable(answer, tableId) {
149
174
  answer(403, {
@@ -251,7 +276,9 @@ class Abandoned extends Error {
251
276
  this.name = 'Abandoned';
252
277
  }
253
278
  }
254
- async function stream(res, format, table, total, read, wanted, amounts, headers = {}) {
279
+ async function stream(res, format, table, total, read, wanted, amounts, headers = {},
280
+ /** Where the file pages by offset: how a file past `EXPORT_OFFSET_ROWS` is refused. */
281
+ byOffset) {
255
282
  /* A client that leaves stops the read in flight (the adapters watch this signal) and every one after it. */
256
283
  const stop = new AbortController();
257
284
  const left = () => {
@@ -261,6 +288,20 @@ async function stream(res, format, table, total, read, wanted, amounts, headers
261
288
  res.once('close', left);
262
289
  const reading = (offset) => withCancellation(stop.signal, () => read(offset, Math.min(EXPORT_BATCH, total - offset)));
263
290
  try {
291
+ /* One row read past the ceiling says, before a byte is written, whether the file would go deeper than an offset
292
+ read should: one read as deep as the ceiling, rather than a file that reads deeper with every batch. */
293
+ if (byOffset && total > EXPORT_OFFSET_ROWS) {
294
+ const beyond = await withCancellation(stop.signal, () => read(EXPORT_OFFSET_ROWS, 1));
295
+ if (stop.signal.aborted)
296
+ return undefined;
297
+ if (beyond.rows.length) {
298
+ byOffset(422, {
299
+ code: 'export_too_large',
300
+ error: `This export is more than ${EXPORT_OFFSET_ROWS.toLocaleString('en-US')} rows, and its rows are read by position. Narrow the question, or give it a limit.`,
301
+ });
302
+ return undefined;
303
+ }
304
+ }
264
305
  /* The first batch is read before a byte is written: a refusal, a column
265
306
  that does not exist, a database that is down all still answer as the
266
307
  ordinary error they are, through the handler's own mapping. */
@@ -45,7 +45,8 @@ type RateDecision = {
45
45
  export type RatePart = 'both' | 'client' | 'all';
46
46
  /** A limiter that decides now (in memory) or once the shared cache has counted. */
47
47
  interface WindowLimiter {
48
- take(address: string, part?: RatePart): RateDecision | Promise<RateDecision>;
48
+ /** `weight` is what this request counts for: one, or its size where a window counts bytes. */
49
+ take(address: string, part?: RatePart, weight?: number): RateDecision | Promise<RateDecision>;
49
50
  /** Forget this process's own counts; tests and a policy reload use it. A shared cache's windows expire on their own. */
50
51
  reset(): void;
51
52
  }
@@ -80,7 +81,7 @@ export declare class FixedWindowLimiter implements WindowLimiter {
80
81
  private total;
81
82
  constructor(windows: readonly RateWindow[], now?: () => number, options?: LimiterOptions);
82
83
  /** Count one request for the client, or say how long it waits. */
83
- take(address: string, part?: RatePart): RateDecision;
84
+ take(address: string, part?: RatePart, weight?: number): RateDecision;
84
85
  private takeShared;
85
86
  /** How many clients are tracked; tests read it. */
86
87
  get size(): number;
@@ -106,7 +107,7 @@ export declare class SharedWindowLimiter implements WindowLimiter {
106
107
  private readonly fallback;
107
108
  private readonly totalWindows?;
108
109
  constructor(name: string, windows: readonly RateWindow[], cache: CacheStore, fallback: FixedWindowLimiter, totalWindows?: readonly RateWindow[] | undefined);
109
- take(address: string, part?: RatePart): Promise<RateDecision>;
110
+ take(address: string, part?: RatePart, weight?: number): Promise<RateDecision>;
110
111
  reset(): void;
111
112
  private count;
112
113
  }
@@ -117,7 +118,10 @@ export declare const PUBLIC_FORM_SUBMIT_TOTAL: readonly RateWindow[];
117
118
  export declare const PUBLIC_FORM_READ_WINDOWS: readonly RateWindow[];
118
119
  export declare const INVITATION_CLAIM_WINDOWS: readonly RateWindow[];
119
120
  export declare const INVITATION_CLAIM_TOTAL: readonly RateWindow[];
121
+ export declare const PUBLIC_FILE_BYTE_WINDOWS: readonly RateWindow[];
120
122
  export declare function publicFormLimiter(kind: 'submit' | 'read' | 'start' | 'status' | 'link' | 'code' | 'check' | 'file'): FixedWindowLimiter | SharedWindowLimiter;
123
+ /** The bytes one client may send a public form's files in a day; `take(address, 'client', size)`. */
124
+ export declare function publicFileByteLimiter(): FixedWindowLimiter | SharedWindowLimiter;
121
125
  /** The invitation claims' own budget (tenantInvitationHttp.ts), never the forms'. */
122
126
  export declare function invitationClaimLimiter(): FixedWindowLimiter | SharedWindowLimiter;
123
127
  /**
@@ -2,6 +2,10 @@ import { CacheUnavailable, scopedCache } from '../../adapters/cache/cacheStore.j
2
2
  import { forwardedClient, ipv6Groups } from './trust.js';
3
3
  import { product } from '../product.js';
4
4
  const MAX_CLIENTS = 10_000;
5
+ function assertWeight(weight) {
6
+ if (!Number.isSafeInteger(weight) || weight < 1)
7
+ throw new Error('A request counts for a positive whole number.');
8
+ }
5
9
  function assertWindows(windows) {
6
10
  if (!windows.length || windows.some((w) => !Number.isSafeInteger(w.limit) || w.limit < 1 || !Number.isFinite(w.ms) || w.ms <= 0)) {
7
11
  throw new Error('A rate limiter needs at least one window of a positive limit over a positive span.');
@@ -17,24 +21,24 @@ class Budget {
17
21
  this.counts = windows.map(() => 0);
18
22
  this.seen = now;
19
23
  }
20
- /** Roll over the windows that have elapsed; the seconds until the longest exceeded one reopens, or 0. */
21
- wait(windows, now) {
24
+ /** Roll over the windows that have elapsed; the seconds until the longest one `weight` more would exceed reopens, or 0. */
25
+ wait(windows, now, weight = 1) {
22
26
  let wait = 0;
23
27
  windows.forEach((window, i) => {
24
28
  if (now - this.starts[i] >= window.ms) {
25
29
  this.starts[i] = now;
26
30
  this.counts[i] = 0;
27
31
  }
28
- if (this.counts[i] >= window.limit)
32
+ if (this.counts[i] + weight > window.limit)
29
33
  wait = Math.max(wait, Math.ceil((this.starts[i] + window.ms - now) / 1000), 1);
30
34
  });
31
35
  return wait;
32
36
  }
33
- /** Count one request; what is left in the tightest window. */
34
- count(windows) {
37
+ /** Count one request, for `weight`; what is left in the tightest window. */
38
+ count(windows, weight = 1) {
35
39
  let remaining = Infinity;
36
40
  windows.forEach((window, i) => {
37
- this.counts[i]++;
41
+ this.counts[i] += weight;
38
42
  remaining = Math.min(remaining, window.limit - this.counts[i]);
39
43
  });
40
44
  return remaining;
@@ -87,11 +91,12 @@ export class FixedWindowLimiter {
87
91
  this.totalWindows = options.total;
88
92
  }
89
93
  /** Count one request for the client, or say how long it waits. */
90
- take(address, part = 'both') {
94
+ take(address, part = 'both', weight = 1) {
95
+ assertWeight(weight);
91
96
  const now = this.now();
92
97
  this.expire(now);
93
98
  if (part === 'all')
94
- return this.takeShared(now) ?? { ok: true, remaining: Infinity };
99
+ return this.takeShared(now, weight) ?? { ok: true, remaining: Infinity };
95
100
  const key = limitKey(address);
96
101
  const entry = this.hits.get(key);
97
102
  if (entry) {
@@ -99,12 +104,18 @@ export class FixedWindowLimiter {
99
104
  this.hits.delete(key);
100
105
  this.hits.set(key, entry);
101
106
  entry.seen = now;
102
- const wait = entry.wait(this.windows, now);
107
+ const wait = entry.wait(this.windows, now, weight);
108
+ if (wait > 0)
109
+ return { ok: false, retryAfterSeconds: wait, scope: 'client' };
110
+ }
111
+ else {
112
+ // A first request heavier than a whole window could never be counted: refused, and nothing tracked for it.
113
+ const wait = new Budget(this.windows, now).wait(this.windows, now, weight);
103
114
  if (wait > 0)
104
115
  return { ok: false, retryAfterSeconds: wait, scope: 'client' };
105
116
  }
106
117
  const shared = part === 'both' && this.totalWindows ? (this.total ??= new Budget(this.totalWindows, now)) : undefined;
107
- const sharedWait = shared ? shared.wait(this.totalWindows, now) : 0;
118
+ const sharedWait = shared ? shared.wait(this.totalWindows, now, weight) : 0;
108
119
  if (sharedWait > 0)
109
120
  return { ok: false, retryAfterSeconds: sharedWait, scope: 'all' };
110
121
  let budget = entry;
@@ -131,19 +142,19 @@ export class FixedWindowLimiter {
131
142
  budget = new Budget(this.windows, now);
132
143
  this.hits.set(key, budget);
133
144
  }
134
- const remaining = budget.count(this.windows);
135
- shared?.count(this.totalWindows);
145
+ const remaining = budget.count(this.windows, weight);
146
+ shared?.count(this.totalWindows, weight);
136
147
  return { ok: true, remaining };
137
148
  }
138
149
  /* The shared ceiling alone: a refusal, or undefined once counted (or where there is none). */
139
- takeShared(now) {
150
+ takeShared(now, weight) {
140
151
  if (!this.totalWindows)
141
152
  return undefined;
142
153
  const shared = (this.total ??= new Budget(this.totalWindows, now));
143
- const wait = shared.wait(this.totalWindows, now);
154
+ const wait = shared.wait(this.totalWindows, now, weight);
144
155
  if (wait > 0)
145
156
  return { ok: false, retryAfterSeconds: wait, scope: 'all' };
146
- shared.count(this.totalWindows);
157
+ shared.count(this.totalWindows, weight);
147
158
  return undefined;
148
159
  }
149
160
  /** How many clients are tracked; tests read it. */
@@ -194,17 +205,18 @@ export class SharedWindowLimiter {
194
205
  if (totalWindows)
195
206
  assertWindows(totalWindows);
196
207
  }
197
- async take(address, part = 'both') {
208
+ async take(address, part = 'both', weight = 1) {
209
+ assertWeight(weight);
198
210
  try {
199
211
  let remaining = Infinity;
200
212
  if (part !== 'all') {
201
- const client = await this.count(this.windows, `${this.name}:${limitKey(address)}`);
213
+ const client = await this.count(this.windows, `${this.name}:${limitKey(address)}`, weight);
202
214
  if (client.wait > 0)
203
215
  return { ok: false, retryAfterSeconds: client.wait, scope: 'client' };
204
216
  remaining = client.remaining;
205
217
  }
206
218
  if (part !== 'client' && this.totalWindows) {
207
- const all = await this.count(this.totalWindows, `${this.name}:*`);
219
+ const all = await this.count(this.totalWindows, `${this.name}:*`, weight);
208
220
  if (all.wait > 0)
209
221
  return { ok: false, retryAfterSeconds: all.wait, scope: 'all' };
210
222
  }
@@ -213,17 +225,17 @@ export class SharedWindowLimiter {
213
225
  catch (error) {
214
226
  if (!(error instanceof CacheUnavailable))
215
227
  throw error;
216
- return this.fallback.take(address, part);
228
+ return this.fallback.take(address, part, weight);
217
229
  }
218
230
  }
219
231
  reset() {
220
232
  this.fallback.reset();
221
233
  }
222
- async count(windows, key) {
234
+ async count(windows, key, weight) {
223
235
  let wait = 0;
224
236
  let remaining = Infinity;
225
237
  for (const [i, window] of windows.entries()) {
226
- const { count, ttlMs } = await this.cache.incr(`${key}:${i}`, window.ms);
238
+ const { count, ttlMs } = await this.cache.incr(`${key}:${i}`, window.ms, weight);
227
239
  if (count > window.limit)
228
240
  wait = Math.max(wait, Math.max(1, Math.ceil(ttlMs / 1000)));
229
241
  else
@@ -294,8 +306,16 @@ const FILE_WINDOWS = Object.freeze([
294
306
  { limit: 60, ms: 3_600_000 },
295
307
  ]);
296
308
  const FILE_TOTAL = Object.freeze([{ limit: 1_200, ms: 3_600_000 }]);
309
+ /* And the bytes those files carry, counted per client across a day, charged before a file reaches the share every
310
+ visitor of the App's forms holds together (stagedFiles.ts): without it one client (one cookie per request) could fill
311
+ that share alone and refuse every other visitor's files. Fifty megabytes is a few applications' worth of documents
312
+ from one address; a filled share then needs several addresses, each of which the windows above also count. */
313
+ export const PUBLIC_FILE_BYTE_WINDOWS = Object.freeze([{ limit: 50 * 1024 * 1024, ms: 86_400_000 }]);
297
314
  const check = new FixedWindowLimiter(CHECK_WINDOWS, Date.now, { total: CHECK_TOTAL, whenFull: 'evict' });
298
315
  const file = new FixedWindowLimiter(FILE_WINDOWS, Date.now, { total: FILE_TOTAL, whenFull: 'refuse' });
316
+ /* `evict`: a day's window, so refusing newcomers once every slot is taken would close the files to everyone for up to a
317
+ day; a forgotten budget is bounded by the file windows' shared ceiling and the share's own cap. */
318
+ const fileBytes = new FixedWindowLimiter(PUBLIC_FILE_BYTE_WINDOWS, Date.now, { whenFull: 'evict' });
299
319
  const submit = new FixedWindowLimiter(PUBLIC_FORM_SUBMIT_WINDOWS, Date.now, { total: PUBLIC_FORM_SUBMIT_TOTAL, whenFull: 'refuse' });
300
320
  const read = new FixedWindowLimiter(PUBLIC_FORM_READ_WINDOWS, Date.now, { whenFull: 'evict' });
301
321
  const claim = new FixedWindowLimiter(INVITATION_CLAIM_WINDOWS, Date.now, { total: INVITATION_CLAIM_TOTAL, whenFull: 'refuse' });
@@ -323,6 +343,10 @@ export function publicFormLimiter(kind) {
323
343
  ? code
324
344
  : status;
325
345
  }
346
+ /** The bytes one client may send a public form's files in a day; `take(address, 'client', size)`. */
347
+ export function publicFileByteLimiter() {
348
+ return shared ? shared.fileBytes : fileBytes;
349
+ }
326
350
  /** The invitation claims' own budget (tenantInvitationHttp.ts), never the forms'. */
327
351
  export function invitationClaimLimiter() {
328
352
  return shared ? shared.claim : claim;
@@ -349,6 +373,7 @@ export function configurePublicFormLimits(cache, namespace) {
349
373
  code: new SharedWindowLimiter('code', CODE_WINDOWS, scoped, code, CODE_TOTAL),
350
374
  check: new SharedWindowLimiter('check', CHECK_WINDOWS, scoped, check, CHECK_TOTAL),
351
375
  file: new SharedWindowLimiter('file', FILE_WINDOWS, scoped, file, FILE_TOTAL),
376
+ fileBytes: new SharedWindowLimiter('file-bytes', PUBLIC_FILE_BYTE_WINDOWS, scoped, fileBytes),
352
377
  };
353
378
  }
354
379
  /** Forgets the forms' budgets and the claims'; tests and a policy reload use it. */
@@ -362,4 +387,5 @@ export function resetPublicFormLimits() {
362
387
  code.reset();
363
388
  check.reset();
364
389
  file.reset();
390
+ fileBytes.reset();
365
391
  }
@@ -35,7 +35,7 @@ import { claimContext, contextJson, contextLines, fromProfile, projectContext }
35
35
  const migrateUsage = () => product() +
36
36
  ' migrate <new <name> | diff <name> | check | squash | status | up | validate | baseline | repair> [--app <dir>] ' +
37
37
  '[<db> | --db <target> | --config <path>] [--connection <name>] [--schema <name>] [--migrations <dir>] [--history-table <name>] ' +
38
- '[--schema-file <path>] [--dev-url <url>] [--dry-run] [--tool flyway] [--repeatable]';
38
+ '[--schema-file <path>] [--dev-url <url>] [--dry-run] [--tool flyway] [--repeatable] [--number <n>]';
39
39
  /** The exit codes: what went wrong, for CI. */
40
40
  const EXIT = { ok: 0, couldNot: 1, invalid: 2, failed: 3, app: 4, stores: 5 };
41
41
  const COMMANDS = ['new', 'diff', 'check', 'squash', 'status', 'up', 'validate', 'baseline', 'repair'];
@@ -72,7 +72,7 @@ export async function runMigrate(argv, write = (line) => process.stdout.write(`$
72
72
  write(`${migrateUsage()}\n\n` +
73
73
  "Versioned SQL files in Flyway's format (V3__add_payoff_quote.sql, R__views.sql), in migrations/ inside the App\n" +
74
74
  'by default, run forward only, each in its own transaction.\n' +
75
- ' new <name> write the next V file (--repeatable: an R__ file)\n' +
75
+ ' new <name> write the next V file (--repeatable: an R__ file; --number 0042: that version, refused when taken)\n' +
76
76
  ' diff <name> write the next V file from schema.sql: what the migrations still lack to build it (Atlas;\n' +
77
77
  ' the first is schema.sql itself, with no download)\n' +
78
78
  ' check the migrations, applied to a scratch database, build schema.sql (exit 2 if not)\n' +
@@ -108,6 +108,7 @@ export async function runMigrate(argv, write = (line) => process.stdout.write(`$
108
108
  let tool;
109
109
  let dryRun = false;
110
110
  let repeatable = false;
111
+ let number;
111
112
  let json = false;
112
113
  let schemaFile;
113
114
  let devUrl;
@@ -173,6 +174,10 @@ export async function runMigrate(argv, write = (line) => process.stdout.write(`$
173
174
  repeatable = true;
174
175
  continue;
175
176
  }
177
+ if (arg === '--number') {
178
+ number = value(i++, arg);
179
+ continue;
180
+ }
176
181
  if (arg === '--json') {
177
182
  json = true;
178
183
  continue;
@@ -209,6 +214,10 @@ export async function runMigrate(argv, write = (line) => process.stdout.write(`$
209
214
  }
210
215
  if (repeatable && command !== 'new')
211
216
  return fail('--repeatable is for new: it names the file R__<name>.sql.');
217
+ if (number !== undefined && command !== 'new')
218
+ return fail('--number is for new: it is the version the V file is written with.');
219
+ if (number !== undefined && repeatable)
220
+ return fail('--number is for a V file; an R__ file has no version.');
212
221
  if (json && (app === undefined || !['status', 'up', 'validate'].includes(command)))
213
222
  return fail('--json is for status, up and validate with --app: it prints the bootstrap plan as data.');
214
223
  if (json && tool)
@@ -275,7 +284,7 @@ export async function runMigrate(argv, write = (line) => process.stdout.write(`$
275
284
  }, write);
276
285
  }
277
286
  if (command === 'new') {
278
- const path = newMigration(targetDir(registry, config, flags, appDir, loaded?.binding), name, repeatable);
287
+ const path = newMigration(targetDir(registry, config, flags, appDir, loaded?.binding), name, repeatable, new Date(), undefined, number);
279
288
  write(`wrote ${shown(path)}`);
280
289
  return EXIT.ok;
281
290
  }
@@ -29,6 +29,8 @@ export declare function readMigrations(dir: string): MigrationDirectory;
29
29
  * runner will do with it. The version is one past the highest the directory
30
30
  * holds, in the directory's own style: a team that numbers by timestamp gets a
31
31
  * timestamp. `body` is the file's text when something already wrote it
32
- * (`migrate diff`).
32
+ * (`migrate diff`). `number` is the version a person chose (`--number 0042`),
33
+ * spelled as given: a team that zero-pads keeps its padding. One the directory
34
+ * already holds is refused, since two files of one version stop every run.
33
35
  */
34
- export declare function newMigration(dir: string, name: string, repeatable: boolean, now?: Date, body?: string): string;
36
+ export declare function newMigration(dir: string, name: string, repeatable: boolean, now?: Date, body?: string, number?: string): string;
@@ -107,9 +107,11 @@ export function readMigrations(dir) {
107
107
  * runner will do with it. The version is one past the highest the directory
108
108
  * holds, in the directory's own style: a team that numbers by timestamp gets a
109
109
  * timestamp. `body` is the file's text when something already wrote it
110
- * (`migrate diff`).
110
+ * (`migrate diff`). `number` is the version a person chose (`--number 0042`),
111
+ * spelled as given: a team that zero-pads keeps its padding. One the directory
112
+ * already holds is refused, since two files of one version stop every run.
111
113
  */
112
- export function newMigration(dir, name, repeatable, now = new Date(), body) {
114
+ export function newMigration(dir, name, repeatable, now = new Date(), body, number) {
113
115
  const slug = name
114
116
  .trim()
115
117
  .replace(/[^A-Za-z0-9]+/g, '_')
@@ -122,6 +124,15 @@ export function newMigration(dir, name, repeatable, now = new Date(), body) {
122
124
  if (repeatable) {
123
125
  file = `R__${slug}.sql`;
124
126
  }
127
+ else if (number !== undefined) {
128
+ if (!/^\d+(?:[._]\d+)*$/.test(number))
129
+ throw new Error(`--number takes a version of digits, like 0042 or 2.1; "${number}" is not one.`);
130
+ const version = number.replace(/_/g, '.');
131
+ const taken = files.find((f) => f.version !== undefined && compareVersions(f.version, version) === 0);
132
+ if (taken)
133
+ throw new Error(`version ${number} is taken: ${taken.script} has it. Choose another --number, or leave it out for the next.`);
134
+ file = `V${number}__${slug}.sql`;
135
+ }
125
136
  else {
126
137
  const majors = files.filter((f) => f.version !== undefined).map((f) => BigInt(f.version.split('.')[0]));
127
138
  const highest = majors.reduce((a, b) => (b > a ? b : a), 0n);
@@ -89,6 +89,17 @@ export function migrateUp(target, options = {}) {
89
89
  }
90
90
  if (initial.plan.problems.length)
91
91
  return { status: initial, applied: [] };
92
+ /* A V file `migrate new` wrote and nobody filled in would be recorded as applied, and the statements written
93
+ into it afterwards never run: its checksum then disagrees and every later run stops on it. An R__ file runs
94
+ again whenever its text changes, so an empty one (a placeholder for views to come) does no harm. */
95
+ const blank = initial.plan.pending.filter((file) => file.kind === 'versioned' && !splitPostgres(file.sql).length);
96
+ if (blank.length) {
97
+ return {
98
+ status: initial,
99
+ applied: [],
100
+ refused: `${blank.map((file) => file.script).join(', ')} ${blank.length === 1 ? 'holds' : 'hold'} no statement, only comments or blank lines. Write the migration into it, or delete it; nothing was applied.`,
101
+ };
102
+ }
92
103
  if (options.dryRun)
93
104
  return { status: initial, applied: [], wouldRun: initial.plan.pending };
94
105
  const expected = initial.snapshot?.objects;
@@ -383,8 +383,11 @@ const words = (raw) => Array.isArray(raw) && raw.every((name) => typeof name ===
383
383
  export function parsePage(raw, at) {
384
384
  const component = parseComponentDecl(raw.component, `${at}, "component"`, 'page');
385
385
  const reads = raw.reads;
386
- if (!words(reads) || !reads.length) {
387
- throw new Error(`[page-reads] ${at} has no "reads": the resources the page reads — reads: ['pick_list', 'order_line']. A page reads nothing it does not declare.`);
386
+ /* `reads: []` is a page that says it reads nothing (a welcome page, one
387
+ that only runs commands): what it declares is still checked against what
388
+ its module reads. Only a missing or malformed list is refused. */
389
+ if (!words(reads)) {
390
+ throw new Error(`[page-reads] ${at} has no "reads": the resources the page reads — reads: ['pick_list', 'order_line'], or reads: [] for none. A page reads nothing it does not declare.`);
388
391
  }
389
392
  const commands = raw.commands === undefined ? [] : raw.commands;
390
393
  if (!words(commands))
@@ -280,7 +280,8 @@ export const THEME_PRESETS = [
280
280
  },
281
281
  "radius": {
282
282
  "control": "8px",
283
- "field": "8px"
283
+ "field": "8px",
284
+ "pill": "999px"
284
285
  },
285
286
  "recipes": {
286
287
  "nav": "pill",
@@ -419,7 +420,8 @@ export const THEME_PRESETS = [
419
420
  },
420
421
  "radius": {
421
422
  "control": "0px",
422
- "field": "0px"
423
+ "field": "0px",
424
+ "pill": "999px"
423
425
  },
424
426
  "recipes": {
425
427
  "nav": "inverted",
@@ -560,7 +562,8 @@ export const THEME_PRESETS = [
560
562
  },
561
563
  "radius": {
562
564
  "control": "8px",
563
- "field": "8px"
565
+ "field": "8px",
566
+ "pill": "999px"
564
567
  },
565
568
  "recipes": {
566
569
  "nav": "quiet",
@@ -287,7 +287,9 @@ const HOUSE = [
287
287
  chart: ['#3559e6', '#9a78ec', '#12a678', '#c88317', '#e0559b', '#1597b9', '#d26b3b', '#6b7694'],
288
288
  },
289
289
  },
290
- shape: { radius: { control: '8px', field: '8px' }, elevation: 'raised', border: { rule: '1px' } },
290
+ /* `pill` is said in every look: an App's stylesheet may read var(--pill-radius) (check accepts it), and a look that left it
291
+ unsaid gave that rule no value and square corners. Round is what the sheets draw a tag as already. */
292
+ shape: { radius: { control: '8px', field: '8px', pill: 'round' }, elevation: 'raised', border: { rule: '1px' } },
291
293
  /* The page's name anchors it: 28px, tight; a list's name a step under, and a section's heading 16px, so the
292
294
  page reads title, section, row at a glance. On a flat page the type is the hierarchy the boxes used to be. */
293
295
  type: {
@@ -380,7 +382,7 @@ const HOUSE = [
380
382
  chart: ['#ffffff', '#8a8a8a', '#cccccc', '#6a6a6a', '#a8a8a8', '#4a4a4a', '#e6e6e6', '#333333'],
381
383
  },
382
384
  },
383
- shape: { radius: { control: '0px', field: '0px' }, elevation: 'flat' },
385
+ shape: { radius: { control: '0px', field: '0px', pill: 'round' }, elevation: 'flat' },
384
386
  /* Figures and the labels that make the look in the platform's monospace, the labels and the bar's entries in
385
387
  capitals: not data, which a reader who wants monospace picks in Preferences. */
386
388
  type: {
@@ -464,7 +466,7 @@ const HOUSE = [
464
466
  chart: ['#7fd1a0', '#c4b1f2', '#f2a08a', '#f08cc4', '#ecc36e', '#6fd1c8', '#92b8f2', '#8c887f'],
465
467
  },
466
468
  },
467
- shape: { radius: { control: '8px', field: '8px' }, elevation: 'soft' },
469
+ shape: { radius: { control: '8px', field: '8px', pill: 'round' }, elevation: 'soft' },
468
470
  type: { font: { ui: SANS, data: SANS, heading: SANS, figures: FIGURES }, roles: { title: { weight: 600 } } },
469
471
  // Compact, as Mono and Lumen: see Mono's density.
470
472
  density: 'compact',
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mistriwork/engine",
3
- "version": "1.0.0-rc.35",
3
+ "version": "1.0.0-rc.37",
4
4
  "description": "The engine both products stand on: the adapters (SQLite, Postgres), the catalog, the query language, keyset pages, the HTTP kit and the shared rules. No React.",
5
5
  "homepage": "https://mistriwork.dev",
6
6
  "bugs": {
package/schema/model.d.ts CHANGED
@@ -122,6 +122,8 @@ export interface CommandChoiceSource {
122
122
  from: string;
123
123
  by: string;
124
124
  labelColumn: string;
125
+ /** The order the rows are offered in: a column of `from` and a direction. Absent, the rows are in the order of `labelColumn`. */
126
+ sort?: { column: string; direction: 'asc' | 'desc' };
125
127
  /** Set by the server on a public form it served this field's rows with: `options` is the whole answer, an empty list
126
128
  meaning none to offer, and the page reads nothing of its own. Never written in an App. */
127
129
  served?: true;