@runeditrun/rer 0.2.110 → 0.2.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -2
- package/cli/README.md +56 -0
- package/cli/agent-door.mjs +12 -45
- package/cli/agent-history-database.mjs +13 -6
- package/cli/bounded.mjs +3 -3
- package/cli/contract.mjs +5 -1
- package/cli/deploy-command.mjs +4 -2
- package/cli/deploy-policy.mjs +17 -0
- package/cli/hosted-output.mjs +30 -10
- package/cli/hosted-owner.mjs +85 -0
- package/cli/inspect.mjs +6 -2
- package/cli/lifecycle.mjs +5 -1
- package/cli/main.mjs +13 -0
- package/cli/parse.mjs +4 -4
- package/cli/portability.mjs +3 -3
- package/cli/size/build-facts.mjs +58 -0
- package/cli/size/build-paths.mjs +3 -0
- package/cli/size/check.mjs +16 -0
- package/cli/size/clauses.mjs +21 -0
- package/cli/size/clock.mjs +21 -0
- package/cli/size/command.mjs +64 -0
- package/cli/size/declaration.mjs +87 -0
- package/cli/size/dml-parts.mjs +108 -0
- package/cli/size/examples/happytohelp.json +68 -0
- package/cli/size/expression-depth.mjs +45 -0
- package/cli/size/expression.mjs +33 -0
- package/cli/size/fixture.mjs +139 -0
- package/cli/size/flows.mjs +81 -0
- package/cli/size/from-items.mjs +124 -0
- package/cli/size/grow.mjs +83 -0
- package/cli/size/install.mjs +136 -0
- package/cli/size/json-duplicates.mjs +45 -0
- package/cli/size/limits.mjs +9 -0
- package/cli/size/measure.mjs +9 -0
- package/cli/size/meter.mjs +167 -0
- package/cli/size/personas.mjs +138 -0
- package/cli/size/report.mjs +23 -0
- package/cli/size/schema-objects.mjs +80 -0
- package/cli/size/schema.mjs +56 -0
- package/cli/size/statements.mjs +53 -0
- package/cli/size/validate-flows.mjs +109 -0
- package/cli/size/validate-kit.mjs +44 -0
- package/cli/size/validate-shape.mjs +75 -0
- package/cli/size/verdict.mjs +131 -0
- package/cli/size/verify-step.mjs +35 -0
- package/global/CHANGELOG.md +50 -0
- package/global/README.md +39 -2
- package/global/cli/install.mjs +1 -1
- package/global/main.mjs +3 -0
- package/global/run.mjs +15 -2
- package/install.sh +3 -0
- package/modules/agent-door/conformance/audit.mjs +3 -0
- package/modules/d1-transport/limits.mjs +5 -1
- package/modules/deploy-kit/sql-tokens.mjs +5 -4
- package/modules/operator-auth/accounts/one-click.mjs +2 -0
- package/modules/operator-auth/dev/index.mjs +7 -1
- package/modules/operator-auth/dev/invited.mjs +11 -0
- package/modules/operator-auth/dev-gate.mjs +3 -2
- package/modules/operator-auth/routes.mjs +4 -0
- package/modules/portability/progress.mjs +12 -10
- package/modules/release/README.md +8 -0
- package/modules/release/archive-guard.mjs +22 -0
- package/modules/release/archive-members.mjs +49 -0
- package/modules/release/export.mjs +7 -1
- package/onboarding/commands.mjs +17 -8
- package/onboarding/migration-progress.mjs +69 -0
- package/onboarding/operator-migrations.mjs +1 -0
- package/package.json +2 -1
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// A declared SQLite expression (or `after` statement) is one expression: a `;` outside quotes and comments would
|
|
2
|
+
// start a second statement inside the INSERT … SELECT the later clauses build around it.
|
|
3
|
+
export const maxExpressionLength = 2000;
|
|
4
|
+
|
|
5
|
+
export function topLevelSemicolon(sql) {
|
|
6
|
+
const closing = { "'": "'", '"': '"', '`': '`', '[': ']' };
|
|
7
|
+
for (let i = 0; i < sql.length; i++) {
|
|
8
|
+
const char = sql[i];
|
|
9
|
+
if (closing[char]) {
|
|
10
|
+
const end = closing[char];
|
|
11
|
+
// A doubled quote inside a quoted string is a literal quote; brackets have no escape.
|
|
12
|
+
for (i++; i < sql.length; i++) {
|
|
13
|
+
if (sql[i] !== end) continue;
|
|
14
|
+
if (end !== ']' && sql[i + 1] === end) i++;
|
|
15
|
+
else break;
|
|
16
|
+
}
|
|
17
|
+
} else if (char === '-' && sql[i + 1] === '-') {
|
|
18
|
+
while (i < sql.length && sql[i] !== '\n') i++;
|
|
19
|
+
} else if (char === '/' && sql[i + 1] === '*') {
|
|
20
|
+
const end = sql.indexOf('*/', i + 2);
|
|
21
|
+
if (end < 0) return false;
|
|
22
|
+
i = end + 1;
|
|
23
|
+
} else if (char === ';') return true;
|
|
24
|
+
}
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// undefined when valid, else what is wrong.
|
|
29
|
+
export function expressionProblem(sql) {
|
|
30
|
+
if (typeof sql !== 'string' || !sql.trim()) return 'must be one SQLite expression in a string';
|
|
31
|
+
if (sql.length > maxExpressionLength) return `is ${sql.length} characters; keep it to ${maxExpressionLength} or fewer`;
|
|
32
|
+
if (topLevelSemicolon(sql)) return "has a ';' outside quotes; write one expression, not several statements";
|
|
33
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { sqlTokens } from '../../modules/deploy-kit/sql-tokens.mjs';
|
|
2
|
+
|
|
3
|
+
// Compiles a size-check.json shape and one size into the ordered statements that build it: each table is chunked
|
|
4
|
+
// INSERT … SELECT over a recursive row sequence, so a database of millions of rows is built inside SQLite with no
|
|
5
|
+
// row crossing the process boundary, and the schema's own triggers derive the rest. Pure: no file, no database.
|
|
6
|
+
// Exported as @runeditrun/rer/size-fixture so the fleet's real-D1 runner seeds exactly the statements rer runs.
|
|
7
|
+
//
|
|
8
|
+
// Names an expression may use: n (row number from 1), p (the parent's row number from 1), k (the index from 0
|
|
9
|
+
// inside a jsonArray item), and the parameters :now, :owner_email and :size. D1 binds only ? and ?NNN, so each
|
|
10
|
+
// parameter is found as a token outside quotes and comments and rewritten to ?NNN.
|
|
11
|
+
|
|
12
|
+
export const sizes = Object.freeze(['small', 'large']);
|
|
13
|
+
const known = new Set(['now', 'owner_email', 'size']);
|
|
14
|
+
|
|
15
|
+
/** A declaration that cannot be compiled, naming the table and, when it is one expression, the column. */
|
|
16
|
+
export class FixtureError extends Error {
|
|
17
|
+
constructor(message, { table, column } = {}) {
|
|
18
|
+
super(`${table ? `${table}${column ? `.${column}` : ''}: ` : ''}${message}`);
|
|
19
|
+
this.name = 'FixtureError';
|
|
20
|
+
this.table = table;
|
|
21
|
+
this.column = column;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Rewrites each :now, :owner_email and :size outside quotes and comments to ?NNN. `order` lists the parameter names
|
|
26
|
+
// in the order of their numbers (?1 is order[0]) and is shared by every expression of one statement.
|
|
27
|
+
function bind(sql, order, where) {
|
|
28
|
+
let tokens;
|
|
29
|
+
try { tokens = sqlTokens(sql); }
|
|
30
|
+
catch { throw new FixtureError('has an unterminated quote or comment', where); }
|
|
31
|
+
let out = '', from = 0;
|
|
32
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
33
|
+
const token = tokens[i];
|
|
34
|
+
if (token.kind !== 'symbol' || !'?:@$'.includes(token.value)) continue;
|
|
35
|
+
const next = tokens[i + 1], name = next?.kind === 'word' && next.start === token.end ? next.value : undefined;
|
|
36
|
+
if (token.value !== ':' || !known.has(name)) throw new FixtureError(`uses ${name ? token.value + name : `"${token.value}"`}, which is not a parameter; the parameters are :now, :owner_email and :size`, where);
|
|
37
|
+
if (!order.includes(name)) order.push(name);
|
|
38
|
+
out += sql.slice(from, token.start) + `?${order.indexOf(name) + 1}`;
|
|
39
|
+
from = next.end;
|
|
40
|
+
i++;
|
|
41
|
+
}
|
|
42
|
+
return out + sql.slice(from);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// A column's SQL. A jsonArray is a correlated recursive-CTE subquery: k runs from 0 to length - 1, and an empty
|
|
46
|
+
// length gives [].
|
|
47
|
+
function columnSql(value, order, where) {
|
|
48
|
+
if (typeof value === 'string') return bind(value, order, where);
|
|
49
|
+
const length = bind(value.jsonArray.length, order, where), item = bind(value.jsonArray.item, order, where);
|
|
50
|
+
return `(WITH RECURSIVE rer_k(k) AS (SELECT 0 WHERE 0 < (${length}) UNION ALL SELECT k + 1 FROM rer_k WHERE k + 1 < (${length})) SELECT json_group_array(${item}) FROM rer_k)`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const plainRows = (rows, size) => typeof rows === 'number' ? rows : rows[size];
|
|
54
|
+
|
|
55
|
+
// The runs of row numbers a table is built in, each with the SQL of its parent row number p. A histogram takes its
|
|
56
|
+
// parents from row 1, those with the most children first.
|
|
57
|
+
function runs(table, total, size) {
|
|
58
|
+
const chunk = table.chunk ?? 10_000, split = (first, last, p) => {
|
|
59
|
+
const found = [];
|
|
60
|
+
for (let a = first; a <= last; a += chunk) found.push({ first: a, last: Math.min(last, a + chunk - 1), p });
|
|
61
|
+
return found;
|
|
62
|
+
};
|
|
63
|
+
const parent = table.parent;
|
|
64
|
+
if (parent?.histogram) {
|
|
65
|
+
const buckets = Object.entries(parent.histogram).map(([children, parents]) => [Number(children), parents]).sort((a, b) => b[0] - a[0]);
|
|
66
|
+
const sum = buckets.reduce((all, [children, parents]) => all + children * parents, 0);
|
|
67
|
+
if (sum !== total) throw new FixtureError(`its histogram builds ${sum.toLocaleString('en-US')} rows but rows is ${total.toLocaleString('en-US')} at the ${size} size; make the sums equal`, { table: table.table });
|
|
68
|
+
const found = [];
|
|
69
|
+
let rows = 0, parents = 0;
|
|
70
|
+
for (const [children, count] of buckets) {
|
|
71
|
+
found.push(...split(rows + 1, rows + children * count, `${parents + 1} + (n - ${rows + 1}) / ${children}`));
|
|
72
|
+
rows += children * count;
|
|
73
|
+
parents += count;
|
|
74
|
+
}
|
|
75
|
+
return { runs: found, parents };
|
|
76
|
+
}
|
|
77
|
+
if (parent) return { runs: split(1, total, `(n - 1) / ${parent.per} + 1`), parents: Math.ceil(total / parent.per) };
|
|
78
|
+
return { runs: split(1, total, 'NULL'), parents: 0 };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The statements that build `size` ('small' or 'large') of a declaration's shape: `{ statements, parts, tables,
|
|
83
|
+
* parameters }`. `statements` is `[{ name, sql, params }]` in the order to run them (the fleet runner's format);
|
|
84
|
+
* `parts[i]` says what statements[i] builds ({ kind: 'rows', table, first, last } or { kind: 'after', index });
|
|
85
|
+
* `tables` is each table's declared row count at this size; `parameters` lists the names its expressions use.
|
|
86
|
+
* `now` is the run's start in Unix milliseconds and `ownerEmail` the owner persona's email.
|
|
87
|
+
*/
|
|
88
|
+
export function compileFixture(declaration, size, { now, ownerEmail } = {}) {
|
|
89
|
+
if (!sizes.includes(size)) throw new FixtureError(`size must be ${sizes.join(' or ')}, not ${JSON.stringify(size)}`);
|
|
90
|
+
const values = { now, owner_email: ownerEmail, size }, statements = [], parts = [], tables = [], used = new Set();
|
|
91
|
+
const rowsOf = new Map();
|
|
92
|
+
const bound = order => order.map(name => {
|
|
93
|
+
used.add(name);
|
|
94
|
+
if (values[name] === undefined) throw new FixtureError(`needs :${name}, which this run has no value for`);
|
|
95
|
+
return values[name];
|
|
96
|
+
});
|
|
97
|
+
for (const table of declaration.shape.tables) {
|
|
98
|
+
const total = plainRows(table.rows, size), where = { table: table.table }, order = [];
|
|
99
|
+
if (table.parent && !rowsOf.has(table.parent.table)) throw new FixtureError(`its parent table ${table.parent.table} must be listed earlier in shape.tables`, where);
|
|
100
|
+
const columns = Object.keys(table.values), expressions = columns.map(column => columnSql(table.values[column], order, { table: table.table, column }));
|
|
101
|
+
const params = bound(order), { runs: chunks, parents } = runs(table, total, size);
|
|
102
|
+
if (table.parent && parents > rowsOf.get(table.parent.table)) throw new FixtureError(`its rows reach parent row ${parents.toLocaleString('en-US')} of ${table.parent.table}, which has ${rowsOf.get(table.parent.table).toLocaleString('en-US')} rows at the ${size} size`, where);
|
|
103
|
+
for (const { first, last, p } of chunks) {
|
|
104
|
+
statements.push({
|
|
105
|
+
name: `${size}:${table.table}:${first}-${last}`,
|
|
106
|
+
sql: `INSERT INTO ${table.table}(${columns.map(column => `"${column}"`).join(',')}) WITH RECURSIVE rer_seq(n) AS (SELECT ${first} UNION ALL SELECT n + 1 FROM rer_seq WHERE n < ${last}), rer_rows(n, p) AS (SELECT n, ${p} FROM rer_seq)\nSELECT ${expressions.join(',\n')}\nFROM rer_rows`,
|
|
107
|
+
params: [...params],
|
|
108
|
+
});
|
|
109
|
+
parts.push({ kind: 'rows', table: table.table, first, last });
|
|
110
|
+
}
|
|
111
|
+
rowsOf.set(table.table, total);
|
|
112
|
+
tables.push({ table: table.table, rows: total });
|
|
113
|
+
}
|
|
114
|
+
(declaration.shape.after ?? []).forEach((statement, index) => {
|
|
115
|
+
const order = [], sql = bind(statement, order, {});
|
|
116
|
+
statements.push({ name: `${size}:after:${index + 1}`, sql, params: bound(order) });
|
|
117
|
+
parts.push({ kind: 'after', index: index + 1 });
|
|
118
|
+
});
|
|
119
|
+
return { statements, parts, tables, parameters: [...used] };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The parameter names a declaration's expressions use, for deciding what a run must prepare (the owner, for :owner_email). */
|
|
123
|
+
export function fixtureParameters(declaration) {
|
|
124
|
+
const names = new Set(), order = [];
|
|
125
|
+
const scan = (sql, where) => { order.length = 0; bind(sql, order, where); order.forEach(name => names.add(name)); };
|
|
126
|
+
for (const table of declaration.shape.tables) for (const [column, value] of Object.entries(table.values)) {
|
|
127
|
+
const where = { table: table.table, column };
|
|
128
|
+
if (typeof value === 'string') scan(value, where); else { scan(value.jsonArray.length, where); scan(value.jsonArray.item, where); }
|
|
129
|
+
}
|
|
130
|
+
for (const statement of declaration.shape.after ?? []) scan(statement, {});
|
|
131
|
+
return names;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The column(s) SQLite names in a failure message ("NOT NULL constraint failed: items.label"), or undefined. */
|
|
135
|
+
export function failureColumn(message) {
|
|
136
|
+
const failed = /constraint failed: ([^:]+)/.exec(message)?.[1];
|
|
137
|
+
const named = failed ? [...failed.matchAll(/\w+\.(\w+)/g)].map(match => match[1]) : [];
|
|
138
|
+
return named.length ? named.join(', ') : /has no column named (\w+)/.exec(message)?.[1] ?? /no such column: (?:\w+\.)?(\w+)/.exec(message)?.[1];
|
|
139
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { RerError } from '../errors.mjs';
|
|
2
|
+
import { pathFor, sendAs, upgradeHeaders } from './personas.mjs';
|
|
3
|
+
|
|
4
|
+
// Runs the declared flows on one installed size, each through the rows-read meter: a request flow makes its request
|
|
5
|
+
// as its persona and reads the whole response; a websocket flow upgrades, waits for the first server message or 2 s
|
|
6
|
+
// and closes; a scheduled flow runs the Worker's scheduled handler with its cron and the run's :now. A flow's window
|
|
7
|
+
// closes when the response is read, its waitUntil work has settled and no statement has been recorded for 250 ms.
|
|
8
|
+
|
|
9
|
+
const socketWaitMs = 2000;
|
|
10
|
+
|
|
11
|
+
const sum = (statements, key) => statements.reduce((all, statement) => all + statement[key], 0);
|
|
12
|
+
|
|
13
|
+
// The text a person reads for what a flow does: "GET /path", "WS /path" or "cron * * * * *".
|
|
14
|
+
export function flowLabel(flow) {
|
|
15
|
+
return flow.scheduled !== undefined ? `cron ${flow.scheduled}` : `${flow.request.websocket ? 'WS' : flow.request.method} ${flow.request.path}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const kindOf = flow => flow.scheduled !== undefined ? 'scheduled' : flow.request.websocket ? 'websocket' : 'request';
|
|
19
|
+
|
|
20
|
+
// Waits for the first message of an accepted socket, its close or an error, or the time, then closes it.
|
|
21
|
+
async function firstMessage(socket) {
|
|
22
|
+
socket.accept();
|
|
23
|
+
await new Promise(resolve => {
|
|
24
|
+
const timer = setTimeout(resolve, socketWaitMs);
|
|
25
|
+
for (const event of ['message', 'close', 'error']) socket.addEventListener(event, () => { clearTimeout(timer); resolve(); }, { once: true });
|
|
26
|
+
});
|
|
27
|
+
try { socket.close(1000, 'size check'); } catch {}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// One run of one flow inside a window: { status, error?, ms }.
|
|
31
|
+
async function once(flow, { hosted, personas, now }) {
|
|
32
|
+
const started = performance.now();
|
|
33
|
+
try {
|
|
34
|
+
if (flow.scheduled !== undefined) {
|
|
35
|
+
await (await hosted.preparation()).scheduled({ scheduledTime: now, cron: flow.scheduled });
|
|
36
|
+
return { status: 'ok', ms: performance.now() - started };
|
|
37
|
+
}
|
|
38
|
+
if (flow.request.websocket) {
|
|
39
|
+
const response = await hosted.upgrade(pathFor(personas, flow).replace(/^\//, ''), { headers: upgradeHeaders(personas, flow, hosted.origin) });
|
|
40
|
+
if (response.status === 101 && response.webSocket) await firstMessage(response.webSocket);
|
|
41
|
+
else await response.arrayBuffer();
|
|
42
|
+
return { status: response.status, ms: performance.now() - started };
|
|
43
|
+
}
|
|
44
|
+
const response = await sendAs(personas, flow);
|
|
45
|
+
const body = await response.text();
|
|
46
|
+
// What an error answered, so a 404 or 500 says why.
|
|
47
|
+
return { status: response.status, ...(response.status >= 400 ? { detail: body.replace(/\s+/g, ' ').trim().slice(0, 240) } : {}), ms: performance.now() - started };
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error instanceof RerError && ['size_flow_failed', 'size_persona_failed'].includes(error.code)) throw error;
|
|
50
|
+
return { status: null, error: String(error?.message ?? error).slice(0, 300), ms: performance.now() - started };
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Runs `flow` `repeat` times on the install and resolves its measurement from the last run (earlier runs warm what
|
|
56
|
+
* the app caches): { name, kind, persona, label, runs, rowsRead, rowsWritten, statements, status, error?, ms,
|
|
57
|
+
* unsettled, executions }. `executions` lists each statement the last run executed, in order.
|
|
58
|
+
*/
|
|
59
|
+
export async function runFlow(flow, install) {
|
|
60
|
+
let last;
|
|
61
|
+
for (let run = 0; run < flow.repeat; run++) {
|
|
62
|
+
const window = install.meter.open();
|
|
63
|
+
let outcome;
|
|
64
|
+
try { outcome = await once(flow, install); }
|
|
65
|
+
catch (error) { await window.close({ quietMs: 0, maxMs: 0 }); throw error; }
|
|
66
|
+
const { statements, unsettled } = await window.close();
|
|
67
|
+
last = { ...outcome, unsettled, executions: statements, rowsRead: sum(statements, 'rows_read'), rowsWritten: sum(statements, 'rows_written'), statements: statements.length };
|
|
68
|
+
}
|
|
69
|
+
return { name: flow.name, kind: kindOf(flow), persona: flow.persona ?? null, label: flowLabel(flow), runs: flow.repeat, ...last };
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Every declared flow, in order, on one install: resolves the list of runFlow results. `each(flow, work)` runs one
|
|
73
|
+
* flow's work (the run's clock names the flow it stopped in). */
|
|
74
|
+
export async function runFlows(flows, install, each = (flow, work) => work()) {
|
|
75
|
+
const results = [];
|
|
76
|
+
for (const flow of flows) {
|
|
77
|
+
const result = await each(flow, () => runFlow(flow, install));
|
|
78
|
+
results.push(result);
|
|
79
|
+
}
|
|
80
|
+
return results;
|
|
81
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { sqlTokens } from '../../modules/deploy-kit/sql-tokens.mjs';
|
|
2
|
+
|
|
3
|
+
// A static model of D1's "VIEWs and/or subqueries nested too deep" counter: the
|
|
4
|
+
// FROM-clause items of a statement (tables, CTE and view references, FROM
|
|
5
|
+
// subqueries, table-valued functions) with every CTE and view reference
|
|
6
|
+
// expanded in place. A recursive CTE's reference to itself is not expanded.
|
|
7
|
+
// It matched real D1 on every K21 probe and product statement.
|
|
8
|
+
const CLAUSE_END = new Set(['WHERE', 'GROUP', 'HAVING', 'ORDER', 'LIMIT', 'UNION', 'EXCEPT', 'INTERSECT', 'WINDOW', 'RETURNING', 'SET', 'VALUES', 'SELECT']);
|
|
9
|
+
|
|
10
|
+
export const tokenize = sql => sqlTokens(sql).map(token => token.kind === 'word' ? { ...token, word: token.value.toUpperCase() } : token);
|
|
11
|
+
const sym = (token, char) => token?.kind === 'symbol' && token.value === char;
|
|
12
|
+
const is = (token, word) => token?.word === word;
|
|
13
|
+
const isSubquery = (t, i) => sym(t[i], '(') && ['SELECT', 'WITH', 'VALUES'].includes(t[i + 1]?.word);
|
|
14
|
+
|
|
15
|
+
function matching(t, open) {
|
|
16
|
+
let depth = 0;
|
|
17
|
+
for (let i = open; i < t.length; i++) {
|
|
18
|
+
if (sym(t[i], '(')) depth++;
|
|
19
|
+
else if (sym(t[i], ')') && !--depth) return i;
|
|
20
|
+
}
|
|
21
|
+
throw new Error('unbalanced parentheses');
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// t[i] is WITH. Returns the scope its definitions make and where the statement resumes.
|
|
25
|
+
function readWith(t, i, scope) {
|
|
26
|
+
i++;
|
|
27
|
+
if (is(t[i], 'RECURSIVE')) i++;
|
|
28
|
+
const local = { ...scope };
|
|
29
|
+
for (;;) {
|
|
30
|
+
const name = t[i]?.value?.toLowerCase();
|
|
31
|
+
if (name === undefined) throw new Error('WITH without a name');
|
|
32
|
+
i++;
|
|
33
|
+
if (sym(t[i], '(')) i = matching(t, i) + 1;
|
|
34
|
+
if (!is(t[i], 'AS')) throw new Error(`expected AS after the CTE name ${name}`);
|
|
35
|
+
i++;
|
|
36
|
+
if (is(t[i], 'NOT')) i++;
|
|
37
|
+
if (is(t[i], 'MATERIALIZED')) i++;
|
|
38
|
+
const end = matching(t, i);
|
|
39
|
+
local[name] = { t, a: i + 1, b: end, scope: local };
|
|
40
|
+
i = end + 1;
|
|
41
|
+
if (!sym(t[i], ',')) return { i, scope: local };
|
|
42
|
+
i++;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// One entry's count does not depend on the path to it unless it ran into a
|
|
47
|
+
// reference on that path, so counts are kept for the entries that did not.
|
|
48
|
+
const memo = new WeakMap();
|
|
49
|
+
function expand(entry, ctx, stack) {
|
|
50
|
+
if (stack.includes(entry)) { ctx.blocked.add(entry); return 0; }
|
|
51
|
+
if (memo.has(entry)) return memo.get(entry);
|
|
52
|
+
const outer = ctx.blocked;
|
|
53
|
+
ctx.blocked = new Set();
|
|
54
|
+
const n = count(ctx, entry.t, entry.a, entry.b, entry.scope, [...stack, entry]);
|
|
55
|
+
ctx.blocked.delete(entry);
|
|
56
|
+
if (!ctx.blocked.size) memo.set(entry, n);
|
|
57
|
+
for (const blocker of outer) ctx.blocked.add(blocker);
|
|
58
|
+
return n;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Items in t[a, b). Each paren level keeps its own FROM state: a parenthesised
|
|
62
|
+
// join group is its own list, and a function call's commas are not items.
|
|
63
|
+
function count(ctx, t, a, b, scope, stack) {
|
|
64
|
+
let total = 0;
|
|
65
|
+
const levels = [{ inFrom: false, expectItem: false }];
|
|
66
|
+
for (let i = a; i < b; i++) {
|
|
67
|
+
const token = t[i], level = levels[levels.length - 1];
|
|
68
|
+
if (is(token, 'WITH') && !sym(t[i - 1], '.')) {
|
|
69
|
+
const read = readWith(t, i, scope);
|
|
70
|
+
scope = read.scope;
|
|
71
|
+
i = read.i - 1;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
if (isSubquery(t, i)) {
|
|
75
|
+
const end = matching(t, i);
|
|
76
|
+
total += count(ctx, t, i + 1, end, scope, stack);
|
|
77
|
+
if (level.expectItem) { total++; level.expectItem = false; }
|
|
78
|
+
i = end;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (sym(token, '(')) {
|
|
82
|
+
levels.push({ inFrom: level.expectItem, expectItem: level.expectItem });
|
|
83
|
+
level.expectItem = false;
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (sym(token, ')')) { if (levels.length > 1) levels.pop(); continue; }
|
|
87
|
+
if ((is(token, 'FROM') && !is(t[i - 1], 'DISTINCT')) || is(token, 'JOIN')) {
|
|
88
|
+
level.expectItem = true;
|
|
89
|
+
if (is(token, 'FROM')) level.inFrom = true;
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
if (sym(token, ',') && level.inFrom) { level.expectItem = true; continue; }
|
|
93
|
+
if (token.word && CLAUSE_END.has(token.word)) level.inFrom = false;
|
|
94
|
+
if (level.expectItem && (token.word || token.kind === 'identifier')) {
|
|
95
|
+
level.expectItem = false;
|
|
96
|
+
total++;
|
|
97
|
+
const entry = scope[token.value.toLowerCase()];
|
|
98
|
+
if (entry) total += expand(entry, ctx, stack);
|
|
99
|
+
// A table-valued function: its arguments can hold subqueries.
|
|
100
|
+
if (sym(t[i + 1], '(')) {
|
|
101
|
+
const end = matching(t, i + 1);
|
|
102
|
+
total += count(ctx, t, i + 2, end, scope, stack);
|
|
103
|
+
i = end;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return total;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** The view scope for fromItems: `views` is [{ name, select }], each select the
|
|
111
|
+
* view's own SELECT text. A view expands wherever a statement reads it. */
|
|
112
|
+
export function viewScope(views) {
|
|
113
|
+
const scope = {};
|
|
114
|
+
for (const { name, select } of views) {
|
|
115
|
+
const t = tokenize(select);
|
|
116
|
+
scope[name.toLowerCase()] = { t, a: 0, b: t.length, scope };
|
|
117
|
+
}
|
|
118
|
+
return scope;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** The FROM items of one statement, with CTEs and the given views expanded. */
|
|
122
|
+
export function fromItems(sql, scope = {}, tokens = tokenize(sql)) {
|
|
123
|
+
return count({ blocked: new Set() }, tokens, 0, tokens.length, scope, []);
|
|
124
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
3
|
+
import { basename, join, resolve } from 'node:path';
|
|
4
|
+
import { RerError, invalid } from '../errors.mjs';
|
|
5
|
+
import { readSizeDeclaration, declarationFile } from './declaration.mjs';
|
|
6
|
+
import { resolveProject } from '../project.mjs';
|
|
7
|
+
import { sizes } from './fixture.mjs';
|
|
8
|
+
import { prepareRun, buildInstall } from './install.mjs';
|
|
9
|
+
import { runClock } from './clock.mjs';
|
|
10
|
+
import { measureInstall } from './measure.mjs';
|
|
11
|
+
import { judgeFlows, statementsFile, verdictLines } from './verdict.mjs';
|
|
12
|
+
import { schemasFromBuild } from './schema.mjs';
|
|
13
|
+
|
|
14
|
+
const seconds = ms => `${(ms / 1000).toFixed(ms < 10_000 ? 2 : 1)} s`;
|
|
15
|
+
const count = number => number.toLocaleString('en-US');
|
|
16
|
+
|
|
17
|
+
/** Whether `rer size` also builds fixtures: a size-check.json at the checkout root, or --declaration FILE. */
|
|
18
|
+
export const growthRequested = options => !options.statements && Boolean(options.declaration || existsSync(join(options.cwd, declarationFile)));
|
|
19
|
+
|
|
20
|
+
/** The checkout's size declaration, read and validated: { declaration, project }. Refuses a declaration with problems. */
|
|
21
|
+
export async function declared(options) {
|
|
22
|
+
const file = options.declaration ? resolve(process.cwd(), options.declaration) : join(options.cwd, declarationFile);
|
|
23
|
+
// Without OPERATIONS.json the checkout has no adapter: crons and deploy files come from the declaration's build paths.
|
|
24
|
+
const project = await resolveProject(options.cwd).catch(() => undefined);
|
|
25
|
+
const result = await readSizeDeclaration(options.cwd, project?.adapter, { file });
|
|
26
|
+
if (!result.present) throw invalid(`No size declaration at ${file}.`);
|
|
27
|
+
if (result.problems.length) throw new RerError('size_declaration_invalid', ['The size declaration has problems:', ...result.problems.map(({ path, message }) => ` SC-15.size.${path || 'declaration'}: ${message}`)].join('\n'), 1);
|
|
28
|
+
return { declaration: result.declaration, project };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function tableLines(built) {
|
|
32
|
+
return built.tables.map(({ table, declared, readBack, before, ms }) => ` ${table.padEnd(26)} declared ${count(declared).padStart(9)} read back ${count(readBack).padStart(9)}${readBack === declared ? '' : ' DIFFERS'}${before ? ` (+${count(before)} already there from the owner bootstrap)` : ''} ${seconds(ms)}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const fixturePhases = ['serve', 'owner', 'generate', 'after', 'readback'];
|
|
36
|
+
const phaseText = phases => Object.entries(phases).map(([phase, ms]) => `${phase} ${seconds(ms)}`).join(', ');
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The growth check: a small and then a large disposable install built from the declared shape, every declared flow
|
|
40
|
+
* run on each through the rows-read meter, and the verdicts. Resolves { ok, small, large, flows } and prints each
|
|
41
|
+
* size's tables, phase times and then the flows; the first failing size stops the run, is printed with no "built"
|
|
42
|
+
* line, and fails it. `--write-statements DIR` writes the statements each size ran.
|
|
43
|
+
*/
|
|
44
|
+
export async function buildFixtures(options, out, loaded) {
|
|
45
|
+
const { declaration, project } = loaded ?? await declared(options), run = await prepareRun(options.cwd, declaration, { project }), clock = runClock(declaration.budget.timeoutSeconds);
|
|
46
|
+
const result = { ok: true }, measured = {};
|
|
47
|
+
if (!options.json) out.human('Size check: queues are held while the flows run; queue consumer work is not measured.');
|
|
48
|
+
try {
|
|
49
|
+
for (const size of sizes) {
|
|
50
|
+
if (!options.json) out.human(`Size check (${options.declaration ? basename(options.declaration) : declarationFile}): building the ${size} fixture on local D1.`);
|
|
51
|
+
try {
|
|
52
|
+
const built = await buildInstall(run, size, clock, measureInstall(run));
|
|
53
|
+
measured[size] = built.measured;
|
|
54
|
+
const phasesMs = Object.fromEntries(Object.entries(built.phases).map(([phase, ms]) => [phase, Math.round(ms)]));
|
|
55
|
+
result[size] = { tables: built.tables.map(table => ({ ...table, ms: Math.round(table.ms) })), totalRows: built.totalRows, phasesMs, statements: built.statements };
|
|
56
|
+
if (options.json) continue;
|
|
57
|
+
tableLines(built).forEach(line => out.human(line));
|
|
58
|
+
const fixture = Object.fromEntries(fixturePhases.map(phase => [phase, built.phases[phase]])), total = Object.values(fixture).reduce((all, ms) => all + ms, 0);
|
|
59
|
+
out.human(` ${size} built in ${seconds(total)} (${phaseText(fixture)}): ${count(built.tables.length)} tables declared, ${count(built.totalRows)} rows in the database with triggers.`);
|
|
60
|
+
out.human(` ${size} measured in ${seconds(built.phases.personas + built.phases.flows)} (personas ${seconds(built.phases.personas)}, flows ${seconds(built.phases.flows)}): ${count(measured[size].length)} flows run.`);
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (!(error instanceof RerError)) throw error;
|
|
63
|
+
result.ok = false;
|
|
64
|
+
result[size] = { error: error.message, code: error.code };
|
|
65
|
+
if (!options.json) out.human(` FAIL ${error.message}`);
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
} finally { clock.stop(); }
|
|
70
|
+
if (!result.ok) return result;
|
|
71
|
+
const verdict = judgeFlows(declaration, measured.small, measured.large, await schemasFromBuild(options.cwd, { wrangler: declaration.build.wrangler }));
|
|
72
|
+
result.ok = verdict.ok;
|
|
73
|
+
result.flows = verdict.flows;
|
|
74
|
+
result.queueConsumersMeasured = false;
|
|
75
|
+
if (!options.json) verdictLines(verdict, declaration).forEach(line => out.human(line));
|
|
76
|
+
if (options.writeStatements) {
|
|
77
|
+
const directory = resolve(process.cwd(), options.writeStatements);
|
|
78
|
+
await mkdir(directory, { recursive: true });
|
|
79
|
+
for (const size of sizes) await writeFile(join(directory, `${size}.json`), JSON.stringify(statementsFile(size, measured[size]), null, 1) + '\n');
|
|
80
|
+
if (!options.json) out.human(` Wrote the statements each size ran to ${join(directory, 'small.json')} and ${join(directory, 'large.json')}.`);
|
|
81
|
+
}
|
|
82
|
+
return result;
|
|
83
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { RerError } from '../errors.mjs';
|
|
4
|
+
import { serveHosted } from '../hosted-output.mjs';
|
|
5
|
+
import { bootstrapOwner, hostedRequest, ownerBindings, ownerClaim, ownerKeys, ownerSession, reviewerDoor } from '../hosted-owner.mjs';
|
|
6
|
+
import { verificationReviewers } from '../agent-reviewers.mjs';
|
|
7
|
+
import { readAgentSurface } from '../agent-surface.mjs';
|
|
8
|
+
import { verificationSponsors } from '../agent-sponsors.mjs';
|
|
9
|
+
import { FixtureError, compileFixture, failureColumn, fixtureParameters, sizes } from './fixture.mjs';
|
|
10
|
+
import { createMeter } from './meter.mjs';
|
|
11
|
+
|
|
12
|
+
// One disposable install per size: the build served as verify's hosted check serves it (a fresh local D1 with the
|
|
13
|
+
// build's migrations), the owner signed in with the bootstrap agent-door conformance uses, the shape generated
|
|
14
|
+
// inside D1 by the compiled statements, then the declared tables read back and the install closed.
|
|
15
|
+
|
|
16
|
+
// A declaration that cannot be compiled is a declaration problem, not an unexpected failure.
|
|
17
|
+
const compiling = work => { try { return work(); } catch (error) { throw error instanceof FixtureError ? new RerError('size_declaration_invalid', `The size declaration cannot be built: ${error.message}`, 1) : error; } };
|
|
18
|
+
const readJson = async path => JSON.parse(await readFile(path, 'utf8'));
|
|
19
|
+
const ownerClaimModes = ['setup-token', 'signed'];
|
|
20
|
+
const refuseNetwork = async () => new Response('rer size installs refuse outbound network.', { status: 502 });
|
|
21
|
+
|
|
22
|
+
/** What every install of one run shares: the declaration, who the owner is and how to sign them in, the database
|
|
23
|
+
* binding and the run's start time. Refuses a declaration that needs the owner when the seed declares no way to
|
|
24
|
+
* create one. */
|
|
25
|
+
export async function prepareRun(root, declaration, { project, now = Date.now() } = {}) {
|
|
26
|
+
const verification = (await readAgentSurface(root))?.surface.verification ?? {};
|
|
27
|
+
const wantsOwner = Object.values(declaration.personas ?? {}).some(persona => persona.kind === 'owner') || compiling(() => fixtureParameters(declaration)).has('owner_email');
|
|
28
|
+
const run = { root, declaration, verification, now, project, wrangler: declaration.build.wrangler };
|
|
29
|
+
const built = await readJson(join(root, declaration.build.wrangler)).catch(() => ({}));
|
|
30
|
+
const bindings = (built.d1_databases ?? []).map(item => item.binding);
|
|
31
|
+
run.binding = declaration.database ?? (bindings.length === 1 ? bindings[0] : undefined);
|
|
32
|
+
if (!run.binding) throw new RerError('size_database_missing', bindings.length ? `The build declares several D1 bindings (${bindings.join(', ')}); set "database" in size-check.json.` : 'This build has no D1 database to generate the shape in.', 1);
|
|
33
|
+
const adapter = project?.adapter;
|
|
34
|
+
run.deployDeclaration = adapter ? adapter.deployDeclaration ?? adapter.deploy?.declaration ?? declaration.build.deployDeclaration : declaration.build.deployDeclaration;
|
|
35
|
+
if (wantsOwner) await prepareOwner(run);
|
|
36
|
+
// Both sizes compile before the first is built, so a mistake in the large size does not wait for the small one.
|
|
37
|
+
run.fixtures = Object.fromEntries(sizes.map(size => [size, compiling(() => compileFixture(declaration, size, { now, ownerEmail: run.owner?.email }))]));
|
|
38
|
+
return run;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function prepareOwner(run) {
|
|
42
|
+
const { root, verification, project } = run, adapter = project?.adapter;
|
|
43
|
+
const manifest = project?.setupManifest ?? await readJson(join(root, 'rer-setup.json')).catch(() => ({}));
|
|
44
|
+
const claim = manifest.steps?.find(step => step.type === 'owner-claim');
|
|
45
|
+
if (!claim && verification.owner === undefined) throw new RerError('size_owner_missing', 'The declaration needs the owner (an owner persona or :owner_email) and this seed declares no way to create one: add an owner-claim step to rer-setup.json steps, or verification.owner to agent-surface.json.', 1);
|
|
46
|
+
// Only the first principal is the owner; extra sponsors belong to agent-door conformance.
|
|
47
|
+
const sponsors = Array.isArray(verification.sponsors) ? verification.sponsors.slice(0, 1).map(({ negativeOnly, ...rest }) => rest) : verification.sponsors;
|
|
48
|
+
const [native] = await verificationSponsors(root, { ...verification, sponsors });
|
|
49
|
+
if (claim?.mode && !ownerClaimModes.includes(claim.mode) && !native.initialize) throw new RerError('agent_owner_invalid', `Owner claim mode ${claim.mode} requires verification.owner.initializer using the seed's production lifecycle.`, 1);
|
|
50
|
+
if (verification.reviewers !== undefined && !native.initialize) throw new RerError('agent_reviewer_invalid', 'verification.reviewers requires a native owner initializer to drive invitation and independent review.', 1);
|
|
51
|
+
run.owner = { native, claim, hook: adapter?.hooks?.owner, email: native.email, sub: native.sub };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The error a failed statement becomes: the size, the table, the row range, SQLite's message and its column. */
|
|
55
|
+
function generationError(size, part, error) {
|
|
56
|
+
if (error instanceof RerError && error.code === 'size_timeout') return error;
|
|
57
|
+
const message = String(error?.diagnosis ?? error?.message ?? error).replace(/^D1_ERROR:\s*/, '').replace(/:\s*SQLITE_\w+(?: \(extended: \w+\))?\s*$/, '');
|
|
58
|
+
const column = failureColumn(message);
|
|
59
|
+
const where = part.kind === 'rows' ? `table ${part.table}, rows ${part.first.toLocaleString('en-US')}-${part.last.toLocaleString('en-US')}` : `after statement ${part.index}`;
|
|
60
|
+
return new RerError('size_generation_failed', `Generating the ${size} size failed in ${where}${column ? `, column ${column}` : ''}: ${message}`, 1, { size, table: part.table, first: part.first, last: part.last, column, sqlite: message });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// The bootstrap agent-door uses: the seed's owner initializer when it declares one, else its owner claim. The
|
|
64
|
+
// initializer gets the declared native reviewers, as in conformance; it gets no mail or fixtures, which only the
|
|
65
|
+
// conformance network provides.
|
|
66
|
+
async function signOwnerIn(run, hosted, keys) {
|
|
67
|
+
const { native, claim, hook, email, sub } = run.owner, request = hostedRequest(hosted);
|
|
68
|
+
const reviewers = await verificationReviewers(run.root, run.verification.reviewers, email);
|
|
69
|
+
try {
|
|
70
|
+
const { path, claimToken } = await ownerClaim({ claim, owner: hook, hosted, keys, email });
|
|
71
|
+
let response;
|
|
72
|
+
try { response = await bootstrapOwner({ hosted, request, native, claim, owner: hook, keys, email, sub, path, claimToken, extra: { reviewers: reviewerDoor(reviewers, { request, hosted, owner: hook }) } }); }
|
|
73
|
+
catch (error) { throw new RerError('size_owner_failed', `rer size could not sign the owner in through the hosted origin: ${String(error?.message ?? error).slice(0, 300)}`, 1); }
|
|
74
|
+
const { cookie, failure } = await ownerSession(response);
|
|
75
|
+
if (failure) throw new RerError('size_owner_failed', `rer size could not sign the owner in: ${failure}`, 1);
|
|
76
|
+
return { email, sub, cookie };
|
|
77
|
+
} finally { reviewers.close(); }
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const count = async (database, table) => Number(await database.prepare(`SELECT count(*) AS c FROM "${table}"`).first('c'));
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Builds one install at `size` and reads back what it holds. Resolves
|
|
84
|
+
* `{ size, tables: [{ table, declared, before, readBack, total, ms }], totalRows, phases, measured }` (phases in ms)
|
|
85
|
+
* and has closed the install; rejects with the first failure, naming where it stopped. With `measure`, the install
|
|
86
|
+
* is served with the rows-read meter and, once built, `measure({ hosted, owner, meter, timed })` runs on it before it
|
|
87
|
+
* closes; what it resolves is `measured`. `timed(phase, flow, work)` runs work under the run's clock and adds its
|
|
88
|
+
* time to that phase.
|
|
89
|
+
*/
|
|
90
|
+
export async function buildInstall(run, size, clock, measure) {
|
|
91
|
+
const { declaration } = run, phases = { serve: 0, owner: 0, generate: 0, after: 0, readback: 0, personas: 0, flows: 0 };
|
|
92
|
+
const fixture = run.fixtures[size];
|
|
93
|
+
const timed = async (phase, table, work, flow) => {
|
|
94
|
+
clock.enter(size, phase, table, flow);
|
|
95
|
+
const started = performance.now();
|
|
96
|
+
try { return await clock.within(work()); } finally { phases[phase] += performance.now() - started; }
|
|
97
|
+
};
|
|
98
|
+
let keys, serving, hosted, owner, measured;
|
|
99
|
+
const meter = measure ? createMeter() : undefined;
|
|
100
|
+
try {
|
|
101
|
+
if (run.owner) keys = await ownerKeys(run.root, run.owner.native, { email: run.owner.email, sub: run.owner.sub });
|
|
102
|
+
serving = serveHosted(run.root, { declaration: run.deployDeclaration, wrangler: run.wrangler, publicHosts: run.verification.publicHosts, rpc: run.verification.rpc,
|
|
103
|
+
fixturePreparation: true, fixtureDiagnostics: true, outboundService: refuseNetwork, meter,
|
|
104
|
+
bindings: keys ? ownerBindings(keys, { claim: run.owner.claim, native: run.owner.native, email: run.owner.email }) : {} });
|
|
105
|
+
hosted = await timed('serve', undefined, () => serving);
|
|
106
|
+
if (keys) owner = await timed('owner', undefined, () => signOwnerIn(run, hosted, keys));
|
|
107
|
+
const database = await (await hosted.preparation()).database(run.binding);
|
|
108
|
+
const before = new Map();
|
|
109
|
+
await timed('readback', undefined, async () => { for (const { table } of fixture.tables) before.set(table, await count(database, table)); });
|
|
110
|
+
const ms = new Map(fixture.tables.map(({ table }) => [table, 0]));
|
|
111
|
+
for (const [index, { sql, params }] of fixture.statements.entries()) {
|
|
112
|
+
const part = fixture.parts[index], phase = part.kind === 'rows' ? 'generate' : 'after', started = performance.now();
|
|
113
|
+
try { await timed(phase, part.table, () => database.prepare(sql).bind(...params).run()); }
|
|
114
|
+
catch (error) { throw generationError(size, part, error); }
|
|
115
|
+
if (part.kind === 'rows') ms.set(part.table, ms.get(part.table) + performance.now() - started);
|
|
116
|
+
}
|
|
117
|
+
const tables = [], names = [];
|
|
118
|
+
let totalRows = 0;
|
|
119
|
+
await timed('readback', undefined, async () => {
|
|
120
|
+
for (const { table, rows } of fixture.tables) {
|
|
121
|
+
const total = await count(database, table);
|
|
122
|
+
tables.push({ table, declared: rows, before: before.get(table), readBack: total - before.get(table), total, ms: ms.get(table) });
|
|
123
|
+
}
|
|
124
|
+
const found = await database.prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name").all();
|
|
125
|
+
names.push(...found.results.map(row => row.name));
|
|
126
|
+
// A virtual table's shadow tables or a table SQLite cannot count are not rows the check made.
|
|
127
|
+
for (const name of names) totalRows += await count(database, name).catch(() => 0);
|
|
128
|
+
});
|
|
129
|
+
if (measure) measured = await measure({ hosted, owner, meter, timed: (phase, flow, work) => timed(phase, undefined, work, flow) });
|
|
130
|
+
return { size, tables, totalRows, phases, statements: fixture.statements.length, measured };
|
|
131
|
+
} finally {
|
|
132
|
+
clock.enter(size, 'closing the install');
|
|
133
|
+
// A timeout can leave the install still starting: close it as soon as it exists.
|
|
134
|
+
if (hosted) await hosted.close(); else serving?.then(late => late.close(), () => {});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// JSON.parse keeps the last of two equal keys, so a repeated persona name would vanish before any rule ran.
|
|
2
|
+
// This reads text that JSON.parse already accepted and returns the path of every object key that repeats.
|
|
3
|
+
export function duplicateKeys(text) {
|
|
4
|
+
const duplicates = [];
|
|
5
|
+
let at = 0;
|
|
6
|
+
const space = () => { while (/\s/.test(text[at])) at++; };
|
|
7
|
+
const string = () => {
|
|
8
|
+
const start = at++;
|
|
9
|
+
while (text[at] !== '"') at += text[at] === '\\' ? 2 : 1;
|
|
10
|
+
return JSON.parse(text.slice(start, ++at));
|
|
11
|
+
};
|
|
12
|
+
function value(path) {
|
|
13
|
+
space();
|
|
14
|
+
if (text[at] === '{') {
|
|
15
|
+
at++;
|
|
16
|
+
const seen = new Set();
|
|
17
|
+
space();
|
|
18
|
+
if (text[at] === '}') { at++; return; }
|
|
19
|
+
for (;;) {
|
|
20
|
+
space();
|
|
21
|
+
const key = string();
|
|
22
|
+
if (seen.has(key)) duplicates.push([...path, key]);
|
|
23
|
+
seen.add(key);
|
|
24
|
+
space(); at++;
|
|
25
|
+
value([...path, key]);
|
|
26
|
+
space();
|
|
27
|
+
if (text[at++] === '}') return;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
if (text[at] === '[') {
|
|
31
|
+
at++;
|
|
32
|
+
space();
|
|
33
|
+
if (text[at] === ']') { at++; return; }
|
|
34
|
+
for (let index = 0; ; index++) {
|
|
35
|
+
value([...path, index]);
|
|
36
|
+
space();
|
|
37
|
+
if (text[at++] === ']') return;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (text[at] === '"') { string(); return; }
|
|
41
|
+
while (at < text.length && !/[\s,\]}]/.test(text[at])) at++;
|
|
42
|
+
}
|
|
43
|
+
value([]);
|
|
44
|
+
return duplicates;
|
|
45
|
+
}
|