@runeditrun/rer 0.2.110 → 0.2.111
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -0
- package/cli/agent-door.mjs +12 -45
- package/cli/agent-history-database.mjs +13 -6
- package/cli/bounded.mjs +3 -3
- package/cli/contract.mjs +5 -1
- package/cli/deploy-command.mjs +2 -2
- package/cli/hosted-output.mjs +30 -10
- package/cli/hosted-owner.mjs +85 -0
- package/cli/inspect.mjs +6 -2
- package/cli/lifecycle.mjs +3 -1
- package/cli/main.mjs +13 -0
- package/cli/parse.mjs +4 -4
- package/cli/size/build-facts.mjs +58 -0
- package/cli/size/build-paths.mjs +3 -0
- package/cli/size/check.mjs +16 -0
- package/cli/size/clauses.mjs +21 -0
- package/cli/size/clock.mjs +21 -0
- package/cli/size/command.mjs +64 -0
- package/cli/size/declaration.mjs +87 -0
- package/cli/size/dml-parts.mjs +108 -0
- package/cli/size/examples/happytohelp.json +68 -0
- package/cli/size/expression-depth.mjs +45 -0
- package/cli/size/expression.mjs +33 -0
- package/cli/size/fixture.mjs +139 -0
- package/cli/size/flows.mjs +81 -0
- package/cli/size/from-items.mjs +124 -0
- package/cli/size/grow.mjs +83 -0
- package/cli/size/install.mjs +136 -0
- package/cli/size/json-duplicates.mjs +45 -0
- package/cli/size/limits.mjs +9 -0
- package/cli/size/measure.mjs +9 -0
- package/cli/size/meter.mjs +167 -0
- package/cli/size/personas.mjs +138 -0
- package/cli/size/report.mjs +23 -0
- package/cli/size/schema-objects.mjs +80 -0
- package/cli/size/schema.mjs +56 -0
- package/cli/size/statements.mjs +53 -0
- package/cli/size/validate-flows.mjs +109 -0
- package/cli/size/validate-kit.mjs +44 -0
- package/cli/size/validate-shape.mjs +75 -0
- package/cli/size/verdict.mjs +131 -0
- package/cli/size/verify-step.mjs +35 -0
- package/global/CHANGELOG.md +31 -0
- package/global/README.md +24 -0
- package/modules/agent-door/conformance/audit.mjs +3 -0
- package/modules/d1-transport/limits.mjs +5 -1
- package/modules/deploy-kit/sql-tokens.mjs +5 -4
- package/modules/operator-auth/accounts/one-click.mjs +2 -0
- package/modules/operator-auth/dev/index.mjs +7 -1
- package/modules/operator-auth/dev/invited.mjs +11 -0
- package/modules/operator-auth/dev-gate.mjs +3 -2
- package/modules/operator-auth/routes.mjs +4 -0
- package/modules/release/README.md +8 -0
- package/modules/release/archive-guard.mjs +22 -0
- package/modules/release/archive-members.mjs +49 -0
- package/modules/release/export.mjs +7 -1
- package/package.json +2 -1
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { measureStatements } from './statements.mjs';
|
|
2
|
+
import { schemaObjects } from './schema-objects.mjs';
|
|
3
|
+
|
|
4
|
+
/** Every view and trigger statement of each schema ({ binding, db }), measured:
|
|
5
|
+
* { records, sql } where sql maps a record to its statement text. A binding
|
|
6
|
+
* names the statements of a build with several databases. */
|
|
7
|
+
export function checkSchemaObjects(schemas, { exact = false } = {}) {
|
|
8
|
+
const records = [], text = new Map();
|
|
9
|
+
for (const { binding, db } of schemas) {
|
|
10
|
+
const { statements } = schemaObjects(db);
|
|
11
|
+
const named = statements.map(statement => ({ ...statement, name: binding ? `${statement.name} [${binding}]` : statement.name }));
|
|
12
|
+
for (const statement of named) text.set(statement.name, statement.sql);
|
|
13
|
+
records.push(...measureStatements(db, named, { exact }));
|
|
14
|
+
}
|
|
15
|
+
return { records, sqlOf: record => text.get(record.name) ?? '' };
|
|
16
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { RerError } from '../errors.mjs';
|
|
2
|
+
import { readSizeDeclaration } from './declaration.mjs';
|
|
3
|
+
|
|
4
|
+
/** `rer check` clauses for a seed's size-check.json: SC-15.size.<field path>. */
|
|
5
|
+
export async function sizeCheckClauses(root, adapter) {
|
|
6
|
+
let result;
|
|
7
|
+
try { result = await readSizeDeclaration(root, adapter); }
|
|
8
|
+
catch (error) {
|
|
9
|
+
if (!(error instanceof RerError)) throw error;
|
|
10
|
+
return [{ id: 'SC-15.size.declaration', ok: false, message: error.message }];
|
|
11
|
+
}
|
|
12
|
+
if (!result.present) return [{ id: 'SC-15.size.undeclared', ok: true }];
|
|
13
|
+
if (!result.problems.length) return [{ id: 'SC-15.size.declaration', ok: true }];
|
|
14
|
+
// One clause per field path; two problems on one field share its clause.
|
|
15
|
+
const byPath = new Map();
|
|
16
|
+
for (const { path, message } of result.problems) byPath.set(path, [...(byPath.get(path) ?? []), message]);
|
|
17
|
+
return [...byPath].map(([path, messages]) => ({
|
|
18
|
+
id: path ? `SC-15.size.${path}` : 'SC-15.size.declaration', ok: false,
|
|
19
|
+
message: `size-check.json ${path || '(file)'}: ${messages.join(' ')}`,
|
|
20
|
+
}));
|
|
21
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { RerError } from '../errors.mjs';
|
|
2
|
+
|
|
3
|
+
/** One deadline for the whole growth run (both sizes). `enter` names the phase work is in; `within` rejects when the
|
|
4
|
+
* deadline passes while that work is still running, naming the size, phase, table and flow it stopped in. */
|
|
5
|
+
export function runClock(timeoutSeconds) {
|
|
6
|
+
const state = { size: undefined, phase: 'starting', table: undefined, flow: undefined };
|
|
7
|
+
let timer;
|
|
8
|
+
const expired = new Promise((_, reject) => {
|
|
9
|
+
timer = setTimeout(() => {
|
|
10
|
+
const where = [state.size && `the ${state.size} size`, state.phase, state.table && `table ${state.table}`, state.flow && `flow ${state.flow}`].filter(Boolean).join(', ');
|
|
11
|
+
reject(new RerError('size_timeout', `rer size stopped after ${timeoutSeconds} s (budget.timeoutSeconds) in ${where}. Raise budget.timeoutSeconds, or declare fewer rows.`, 1));
|
|
12
|
+
}, timeoutSeconds * 1000);
|
|
13
|
+
timer.unref?.();
|
|
14
|
+
});
|
|
15
|
+
expired.catch(() => {});
|
|
16
|
+
return {
|
|
17
|
+
enter(size, phase, table, flow) { Object.assign(state, { size, phase, table, flow }); },
|
|
18
|
+
within: work => Promise.race([work, expired]),
|
|
19
|
+
stop: () => clearTimeout(timer),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { invalid, missing } from '../errors.mjs';
|
|
2
|
+
import { schemasFromBuild, schemaFromMigrations } from './schema.mjs';
|
|
3
|
+
import { checkSchemaObjects } from './check.mjs';
|
|
4
|
+
import { measureStatements, readStatementsFile } from './statements.mjs';
|
|
5
|
+
import { sizeReport } from './report.mjs';
|
|
6
|
+
import { buildFixtures, declared, growthRequested } from './grow.mjs';
|
|
7
|
+
|
|
8
|
+
// The one schema a statements file runs against: --migrations DIR, else the build's only D1 database.
|
|
9
|
+
async function statementsSchema(options) {
|
|
10
|
+
if (options.migrations) return (await schemaFromMigrations(options.migrations)).db;
|
|
11
|
+
const schemas = await schemasFromBuild(options.cwd);
|
|
12
|
+
if (!schemas.length) throw missing('This build has no D1 database to check statements against. Pass --migrations DIR.');
|
|
13
|
+
if (schemas.length > 1) throw invalid(`This build declares several D1 databases (${schemas.map(schema => schema.binding).join(', ')}); pass --migrations DIR for the one the statements run against.`);
|
|
14
|
+
return schemas[0].db;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const undeclaredLine = 'Size check: undeclared (no size-check.json).';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The whole size check, the one code `rer size` and the last step of `rer verify` both run: statement size, then, with
|
|
21
|
+
* a size-check.json (or --declaration FILE), the declared shape built at a small and a large size and every declared
|
|
22
|
+
* flow run on each through a rows-read meter. `options` are rer size's (`cwd`, `json`, `declaration`, `statements`,
|
|
23
|
+
* `migrations`, `writeStatements`); lines go to `out.human`. Resolves { ok, details }, ok being false for a statement
|
|
24
|
+
* at or over the margin, a size that fails to build or a failing flow. It does not take a job-queue slot: the caller holds one.
|
|
25
|
+
*/
|
|
26
|
+
export async function measureSize(options, out) {
|
|
27
|
+
let records, sqlOf, unit;
|
|
28
|
+
if (options.writeStatements && !growthRequested(options)) throw invalid('--write-statements writes what the declared flows ran: add a size-check.json at the checkout root, or --declaration FILE.');
|
|
29
|
+
// A declaration says where the build is, so it is read before the build is.
|
|
30
|
+
const loaded = growthRequested(options) ? await declared(options) : undefined;
|
|
31
|
+
if (options.statements) {
|
|
32
|
+
const statements = await readStatementsFile(options.statements), sql = new Map(statements.map(statement => [statement.name, statement.sql]));
|
|
33
|
+
records = measureStatements(await statementsSchema(options), statements, { exact: options.json });
|
|
34
|
+
sqlOf = record => sql.get(record.name) ?? '';
|
|
35
|
+
unit = 'statements';
|
|
36
|
+
} else {
|
|
37
|
+
const schemas = options.migrations ? [await schemaFromMigrations(options.migrations)] : await schemasFromBuild(options.cwd, { wrangler: loaded?.declaration.build.wrangler });
|
|
38
|
+
if (!schemas.length) {
|
|
39
|
+
out.human('Statement size: this build has no D1 database; nothing to check.');
|
|
40
|
+
records = [];
|
|
41
|
+
} else {
|
|
42
|
+
({ records, sqlOf } = checkSchemaObjects(schemas, { exact: options.json }));
|
|
43
|
+
unit = 'view and trigger statements';
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
let ok = records.every(record => record.ok);
|
|
47
|
+
if (!options.json && records.length) for (const line of sizeReport(records, sqlOf, unit)) out.human(line);
|
|
48
|
+
const details = { statements: records.map(({ name, fromItems, depth, ok, error }) => ({ name, fromItems: fromItems ?? null, depth, ok, ...(error ? { error } : {}) })) };
|
|
49
|
+
if (loaded) {
|
|
50
|
+
const { ok: built, flows, queueConsumersMeasured, ...sizes } = await buildFixtures(options, out, loaded);
|
|
51
|
+
ok = ok && built;
|
|
52
|
+
details.size = sizes;
|
|
53
|
+
if (flows) Object.assign(details, { flows, queueConsumersMeasured });
|
|
54
|
+
} else if (!options.statements && !options.migrations) out.human(undeclaredLine);
|
|
55
|
+
return { ok, details };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** rer size: exit 1 if any statement is at or over the margin, a size fails to build or a flow fails, 0 if none, 3 with
|
|
59
|
+
* no build and no --migrations. */
|
|
60
|
+
export async function sizeCommand(options, out) {
|
|
61
|
+
const { ok, details } = await measureSize(options, out);
|
|
62
|
+
out.result(ok, ok ? 0 : 1, details);
|
|
63
|
+
return ok ? 0 : 1;
|
|
64
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { RerError } from '../errors.mjs';
|
|
4
|
+
import { buildFacts } from './build-facts.mjs';
|
|
5
|
+
import { defaultBuild } from './build-paths.mjs';
|
|
6
|
+
import { duplicateKeys } from './json-duplicates.mjs';
|
|
7
|
+
import { validateFlows, validatePersonas } from './validate-flows.mjs';
|
|
8
|
+
import { validateShape } from './validate-shape.mjs';
|
|
9
|
+
import { integer, isInt, isObject, keyed, pathProblem, problems } from './validate-kit.mjs';
|
|
10
|
+
|
|
11
|
+
export const declarationFile = 'size-check.json';
|
|
12
|
+
export const budgetDefaults = { growthPercent: 10, growthRows: 200, timeoutSeconds: 600 };
|
|
13
|
+
|
|
14
|
+
function validateBudget(found, budget, path) {
|
|
15
|
+
if (!keyed(found, budget, path, ['windowRows', 'growthPercent', 'growthRows', 'timeoutSeconds'], 'budget')) return;
|
|
16
|
+
if (budget.windowRows === undefined) found.add([...path, 'windowRows'], 'is required; give the rows this seed allows in D1\'s 10-second window.');
|
|
17
|
+
else integer(found, budget.windowRows, [...path, 'windowRows'], 1);
|
|
18
|
+
if (budget.growthPercent !== undefined && !(typeof budget.growthPercent === 'number' && budget.growthPercent >= 0)) found.add([...path, 'growthPercent'], 'must be a number of 0 or more (the share of the small size a flow may grow by).');
|
|
19
|
+
if (budget.growthRows !== undefined) integer(found, budget.growthRows, [...path, 'growthRows'], 0);
|
|
20
|
+
if (budget.timeoutSeconds !== undefined) integer(found, budget.timeoutSeconds, [...path, 'timeoutSeconds'], 1);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function validateBuild(found, build, path) {
|
|
24
|
+
if (!keyed(found, build, path, ['wrangler', 'deployDeclaration'], 'build')) return;
|
|
25
|
+
for (const key of ['wrangler', 'deployDeclaration']) if (build[key] !== undefined && pathProblem(build[key])) found.add([...path, key], 'must be a path inside the checkout (no leading "/" and no "..").');
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function validateDatabase(found, database, bindings) {
|
|
29
|
+
if (database === undefined) {
|
|
30
|
+
if (bindings.length > 1) found.add(['database'], `is required: the build declares several D1 bindings (${bindings.join(', ')}); name the one this check measures.`);
|
|
31
|
+
} else if (typeof database !== 'string' || !database) found.add(['database'], 'must name a D1 binding.');
|
|
32
|
+
else if (bindings.length && !bindings.includes(database)) found.add(['database'], `${database} is not a D1 binding the checkout declares (${bindings.join(', ')}); use one of them.`);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Validates a parsed size-check.json against what the checkout declares ({ crons, bindings }, see build-facts.mjs).
|
|
37
|
+
* Pure: no files, no network. Returns { problems: [{ path, message }], declaration }, where declaration is the
|
|
38
|
+
* normalized form with defaults filled (undefined while there are problems).
|
|
39
|
+
*/
|
|
40
|
+
export function validateSizeDeclaration(input, { crons = [], bindings = [], duplicates = [], extra = [] } = {}) {
|
|
41
|
+
const found = problems();
|
|
42
|
+
for (const path of duplicates) found.add(path, `"${path.at(-1)}" appears twice; names must be unique, so rename or delete one.`);
|
|
43
|
+
for (const problem of extra) found.list.push(problem);
|
|
44
|
+
const top = ['schema', 'about', 'database', 'build', 'budget', 'personas', 'shape', 'flows'];
|
|
45
|
+
if (!keyed(found, input, [], top, 'the file')) return { problems: found.list };
|
|
46
|
+
if (input.schema !== 1) found.add(['schema'], 'must be 1.');
|
|
47
|
+
if (input.about !== undefined && typeof input.about !== 'string') found.add(['about'], 'must be text saying where the counts came from.');
|
|
48
|
+
if (input.build !== undefined) validateBuild(found, input.build, ['build']);
|
|
49
|
+
validateDatabase(found, input.database, bindings);
|
|
50
|
+
if (input.budget === undefined) found.add(['budget'], 'is required; give at least { "windowRows": N }.');
|
|
51
|
+
else validateBudget(found, input.budget, ['budget']);
|
|
52
|
+
const personas = validatePersonas(found, input.personas, ['personas']);
|
|
53
|
+
if (input.shape === undefined) found.add(['shape'], 'is required; list the tables to generate.');
|
|
54
|
+
const shape = input.shape === undefined ? undefined : validateShape(found, input.shape, ['shape']);
|
|
55
|
+
const windowRows = isInt(input.budget?.windowRows, 1) ? input.budget.windowRows : undefined;
|
|
56
|
+
const flows = validateFlows(found, input.flows, ['flows'], { personas, crons, windowRows });
|
|
57
|
+
if (found.list.length) return { problems: found.list };
|
|
58
|
+
return {
|
|
59
|
+
problems: [],
|
|
60
|
+
declaration: {
|
|
61
|
+
schema: 1, ...(input.about === undefined ? {} : { about: input.about }),
|
|
62
|
+
database: input.database ?? bindings[0] ?? null,
|
|
63
|
+
build: { ...defaultBuild, ...input.build },
|
|
64
|
+
budget: { ...budgetDefaults, ...input.budget },
|
|
65
|
+
personas: input.personas ?? {}, shape, flows,
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Reads and validates a seed's size-check.json (or the file `file` names). { present: false } when there is none. */
|
|
71
|
+
export async function readSizeDeclaration(root, adapter, { file = join(root, declarationFile) } = {}) {
|
|
72
|
+
let text;
|
|
73
|
+
try { text = await readFile(file, 'utf8'); }
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error.code === 'ENOENT') return { present: false };
|
|
76
|
+
throw new RerError('size_declaration_invalid', `Cannot read ${declarationFile}. Restore it or delete it.`, 1);
|
|
77
|
+
}
|
|
78
|
+
let input;
|
|
79
|
+
try { input = JSON.parse(text); }
|
|
80
|
+
catch (error) { return { present: true, problems: [{ path: '', message: `${declarationFile} must contain valid JSON (${error.message}).` }] }; }
|
|
81
|
+
const build = isObject(input) && isObject(input.build) && !pathProblem(input.build.wrangler ?? 'x') && !pathProblem(input.build.deployDeclaration ?? 'x') ? input.build : {};
|
|
82
|
+
const facts = await buildFacts(root, adapter, build);
|
|
83
|
+
// Unreadable crons matter only when a flow names one; an unreadable build file always does.
|
|
84
|
+
const scheduled = Array.isArray(input?.flows) && input.flows.some(flow => flow?.scheduled !== undefined);
|
|
85
|
+
const extra = facts.problems.filter(problem => problem.path === 'build' || scheduled);
|
|
86
|
+
return { present: true, ...validateSizeDeclaration(input, { ...facts, duplicates: duplicateKeys(text), extra }) };
|
|
87
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { tokenize } from './from-items.mjs';
|
|
2
|
+
|
|
3
|
+
// SQLite's expression depth is measured through SELECT-shaped parts. A SELECT or
|
|
4
|
+
// VALUES is its own part. A DML statement is read as the SELECTs its
|
|
5
|
+
// expressions would form, each over its target table:
|
|
6
|
+
// INSERT its SELECT or VALUES; an upsert's DO UPDATE SET right-hand sides
|
|
7
|
+
// and WHERE, with `excluded.` read as the target table itself
|
|
8
|
+
// UPDATE the SET right-hand sides, FROM, WHERE and RETURNING
|
|
9
|
+
// DELETE its WHERE and RETURNING
|
|
10
|
+
const sym = (token, char) => token?.kind === 'symbol' && token.value === char;
|
|
11
|
+
const is = (token, word) => token?.word === word;
|
|
12
|
+
|
|
13
|
+
// Index of the first top-level `word` in t[from, until), or -1.
|
|
14
|
+
function topLevel(t, from, word, until = t.length) {
|
|
15
|
+
let depth = 0;
|
|
16
|
+
for (let i = from; i < until; i++) {
|
|
17
|
+
if (sym(t[i], '(')) depth++;
|
|
18
|
+
else if (sym(t[i], ')')) depth--;
|
|
19
|
+
else if (!depth && is(t[i], word)) return i;
|
|
20
|
+
}
|
|
21
|
+
return -1;
|
|
22
|
+
}
|
|
23
|
+
function splitTop(t, a, b) {
|
|
24
|
+
const parts = [];
|
|
25
|
+
let depth = 0, start = a;
|
|
26
|
+
for (let i = a; i < b; i++) {
|
|
27
|
+
if (sym(t[i], '(')) depth++;
|
|
28
|
+
else if (sym(t[i], ')')) depth--;
|
|
29
|
+
else if (!depth && sym(t[i], ',')) { parts.push(t.slice(start, i)); start = i + 1; }
|
|
30
|
+
}
|
|
31
|
+
parts.push(t.slice(start, b));
|
|
32
|
+
return parts;
|
|
33
|
+
}
|
|
34
|
+
const skipParens = (t, i) => {
|
|
35
|
+
let depth = 0;
|
|
36
|
+
for (; i < t.length; i++) {
|
|
37
|
+
if (sym(t[i], '(')) depth++;
|
|
38
|
+
else if (sym(t[i], ')') && !--depth) return i + 1;
|
|
39
|
+
}
|
|
40
|
+
return i;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
export function dmlParts(sql, tokens = tokenize(sql)) {
|
|
44
|
+
const t = tokens, text = list => list.length ? sql.slice(list[0].start, list[list.length - 1].end) : '';
|
|
45
|
+
// `[schema.]name [AS alias]`: the target table as a FROM item, and where it ends.
|
|
46
|
+
const target = i => {
|
|
47
|
+
let name = text([t[i]]);
|
|
48
|
+
i++;
|
|
49
|
+
if (sym(t[i], '.')) { name += `.${text([t[i + 1]])}`; i += 2; }
|
|
50
|
+
let alias = '';
|
|
51
|
+
if (is(t[i], 'AS')) { alias = text([t[i + 1]]); i += 2; }
|
|
52
|
+
return { item: alias ? `${name} AS ${alias}` : name, alias: alias || name, next: i };
|
|
53
|
+
};
|
|
54
|
+
let start = 0;
|
|
55
|
+
if (is(t[0], 'WITH')) {
|
|
56
|
+
let depth = 0;
|
|
57
|
+
for (start = 1; start < t.length; start++) {
|
|
58
|
+
if (sym(t[start], '(')) depth++;
|
|
59
|
+
else if (sym(t[start], ')')) depth--;
|
|
60
|
+
else if (!depth && ['SELECT', 'VALUES', 'INSERT', 'UPDATE', 'DELETE', 'REPLACE'].includes(t[start].word) && sym(t[start - 1], ')')) break;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const prefix = text(t.slice(0, start)), main = t[start]?.word;
|
|
64
|
+
const selectOver = (list, item, where, returning = []) => `${prefix} SELECT ${list.join(', ')}${returning.length ? `, ${text(returning)}` : ''} FROM ${item}${where.length ? ` WHERE ${text(where)}` : ''}`;
|
|
65
|
+
if (main === 'INSERT' || main === 'REPLACE') {
|
|
66
|
+
const into = topLevel(t, start, 'INTO'), into1 = target(into + 1);
|
|
67
|
+
let j = into1.next;
|
|
68
|
+
if (sym(t[j], '(')) j = skipParens(t, j);
|
|
69
|
+
const conflicts = [];
|
|
70
|
+
for (let x = j; x < t.length - 1; x++) if (!sym(t[x], '(') && is(t[x], 'ON') && is(t[x + 1], 'CONFLICT')) conflicts.push(x);
|
|
71
|
+
const returning = topLevel(t, j, 'RETURNING'), end = returning >= 0 ? returning : t.length;
|
|
72
|
+
const parts = [];
|
|
73
|
+
const bodyEnd = conflicts[0] ?? end;
|
|
74
|
+
if (!is(t[j], 'DEFAULT')) parts.push(`${prefix} ${text(t.slice(j, bodyEnd))}`);
|
|
75
|
+
const excluded = part => part.replace(/\bexcluded\s*\./gi, `${into1.alias}.`);
|
|
76
|
+
conflicts.forEach((at, n) => {
|
|
77
|
+
const until = conflicts[n + 1] ?? end, doAt = topLevel(t, at, 'DO', until);
|
|
78
|
+
// The conflict target's own WHERE is an expression too.
|
|
79
|
+
const targetWhere = topLevel(t, at, 'WHERE', doAt >= 0 ? doAt : until);
|
|
80
|
+
if (targetWhere >= 0) parts.push(excluded(selectOver(['1'], into1.item, t.slice(targetWhere + 1, doAt >= 0 ? doAt : until))));
|
|
81
|
+
const set = doAt >= 0 && is(t[doAt + 1], 'UPDATE') ? topLevel(t, doAt, 'SET', until) : -1;
|
|
82
|
+
if (set < 0) return;
|
|
83
|
+
const where = topLevel(t, set, 'WHERE', until);
|
|
84
|
+
const right = splitTop(t, set + 1, where >= 0 ? where : until).map(assignment => text(assignment.slice(assignment.findIndex(x => sym(x, '=')) + 1)));
|
|
85
|
+
parts.push(excluded(selectOver(right, into1.item, where >= 0 ? t.slice(where + 1, until) : [])));
|
|
86
|
+
});
|
|
87
|
+
if (returning >= 0) parts.push(`SELECT ${text(t.slice(returning + 1))} FROM ${into1.item}`);
|
|
88
|
+
return parts;
|
|
89
|
+
}
|
|
90
|
+
if (main === 'UPDATE') {
|
|
91
|
+
let j = start + 1;
|
|
92
|
+
if (is(t[j], 'OR')) j += 2;
|
|
93
|
+
const table = target(j), set = topLevel(t, table.next, 'SET');
|
|
94
|
+
const returning = topLevel(t, set, 'RETURNING'), end = returning >= 0 ? returning : t.length;
|
|
95
|
+
const where = topLevel(t, set, 'WHERE', end), from = topLevel(t, set, 'FROM', where >= 0 ? where : end);
|
|
96
|
+
const assignments = from >= 0 ? from : where >= 0 ? where : end;
|
|
97
|
+
const right = splitTop(t, set + 1, assignments).map(assignment => text(assignment.slice(assignment.findIndex(x => sym(x, '=')) + 1)));
|
|
98
|
+
const items = from >= 0 ? `${table.item}, ${text(t.slice(from + 1, where >= 0 ? where : end))}` : table.item;
|
|
99
|
+
return [selectOver(right, items, where >= 0 ? t.slice(where + 1, end) : [], returning >= 0 ? t.slice(returning + 1) : [])];
|
|
100
|
+
}
|
|
101
|
+
if (main === 'DELETE') {
|
|
102
|
+
const from = topLevel(t, start, 'FROM'), table = target(from + 1);
|
|
103
|
+
const returning = topLevel(t, from, 'RETURNING'), end = returning >= 0 ? returning : t.length;
|
|
104
|
+
const where = topLevel(t, from, 'WHERE', end);
|
|
105
|
+
return [selectOver(['1'], table.item, where >= 0 ? t.slice(where + 1, end) : [], returning >= 0 ? t.slice(returning + 1) : [])];
|
|
106
|
+
}
|
|
107
|
+
return [sql];
|
|
108
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": 1,
|
|
3
|
+
"about": "HappyToHelp's chat workspace at ChargeDesk's measured shape, counts only, no contents. Contacts, conversations, solved share and pending events: K26/shape.json 1x (M2 instance reads, 11 Nov 2026 07:53-08:40Z). Reply blocks per conversation (histogram), sources per block and documents: K24/shape.json (instance reads, 11 Nov 2026 03:17-03:23Z). Media: 2,835 rows, K26 INSIGHTS (rows read per transcript open). Messages and outbox: 9,000 chats x 3 messages, K26 shape 1x (45 days). Small keeps every multiplicity and 1% of what grows. Window budget: M2's 3.9M-rows-in-10-s working line (not a measured limit).",
|
|
4
|
+
"database": "DB",
|
|
5
|
+
"budget": { "windowRows": 3900000, "growthPercent": 10, "growthRows": 200, "timeoutSeconds": 600 },
|
|
6
|
+
"personas": {
|
|
7
|
+
"owner": { "kind": "owner" },
|
|
8
|
+
"visitor": {
|
|
9
|
+
"kind": "signin",
|
|
10
|
+
"requests": [
|
|
11
|
+
{ "method": "POST", "path": "/api/identity/widget/session", "body": { "widgetKey": "size-widget" }, "capture": { "token": "/token" } },
|
|
12
|
+
{ "method": "POST", "path": "/api/conversations", "headers": { "Authorization": "Bearer {{token}}" }, "body": { "projectId": "size-p1" }, "capture": { "conversation": "/id" } }
|
|
13
|
+
],
|
|
14
|
+
"headers": { "Authorization": "Bearer {{token}}" }
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"shape": {
|
|
18
|
+
"tables": [
|
|
19
|
+
{ "table": "projects", "rows": 1, "values": { "id": "'size-p1'", "name": "'Size check'", "widget_key": "'size-widget'", "allowed_origin": "'https://rer-hosted-check.invalid'", "created_at": ":now" } },
|
|
20
|
+
{ "table": "memberships", "rows": 1, "values": { "project_id": "'size-p1'", "user_id": "(SELECT id FROM users WHERE email = :owner_email)", "role": "'owner'" } },
|
|
21
|
+
{ "table": "contacts", "rows": { "small": 1620, "large": 162000 }, "chunk": 10000, "values": {
|
|
22
|
+
"id": "printf('c%06d', n)", "project_id": "'size-p1'", "name": "printf('Contact %d', n)",
|
|
23
|
+
"email": "printf('c%06d@customer.example.test', n)", "metadata_json": "'{}'", "created_at": ":now - 7776000000", "is_verified": "0" } },
|
|
24
|
+
{ "table": "conversations", "rows": { "small": 1619, "large": 161893 }, "chunk": 10000, "parent": { "table": "contacts", "per": 1 }, "values": {
|
|
25
|
+
"id": "printf('v%06d', n)", "project_id": "'size-p1'", "contact_id": "printf('c%06d', p)", "requester_id": "printf('c%06d', p)",
|
|
26
|
+
"subject": "'Synthetic question'", "channel": "'widget'",
|
|
27
|
+
"status_id": "CASE WHEN n % 52 = 0 THEN 'solved' ELSE 'new' END", "status_category": "CASE WHEN n % 52 = 0 THEN 'solved' ELSE 'new' END",
|
|
28
|
+
"resolved_at": "CASE WHEN n % 52 = 0 THEN :now - 7200000 END", "created_at": ":now - 3600000 - n * 1000", "updated_at": ":now - 3600000 - n * 1000", "last_message_id": "NULL" } },
|
|
29
|
+
{ "table": "knowledge_datasets", "rows": 1, "values": { "id": "'size-dataset'", "project_id": "'size-p1'", "name": "'Help center'", "status": "'published'", "created_at": ":now" } },
|
|
30
|
+
{ "table": "knowledge_versions", "rows": 1, "values": { "id": "'size-version'", "dataset_id": "'size-dataset'", "fingerprint": "'size'", "created_at": ":now" } },
|
|
31
|
+
{ "table": "knowledge_documents", "rows": 644, "values": { "id": "printf('size-doc-%04d', n - 1)", "version_id": "'size-version'", "url": "printf('https://help.example.test/doc-%04d', n - 1)", "title": "printf('Document %d', n)", "content": "'Synthetic document text.'" } },
|
|
32
|
+
{ "table": "knowledge_reply_blocks", "rows": 373,
|
|
33
|
+
"parent": { "table": "conversations", "histogram": { "16": 1, "13": 1, "12": 1, "11": 1, "10": 3, "9": 8, "8": 7, "7": 19, "6": 1, "5": 3, "3": 3 } },
|
|
34
|
+
"values": {
|
|
35
|
+
"id": "printf('a1a1%028x', n)", "project_id": "'size-p1'", "kind": "'copy'", "payload_json": "json_object('text', printf('Synthetic copied passage %d', n))",
|
|
36
|
+
"sources_json": { "jsonArray": { "length": "CASE WHEN n <= 16 THEN 255 WHEN n <= 246 THEN 354 ELSE 53 END", "item": "printf('knowledge:size-doc-%04d', (((n % 9) * 19 + k) % 410) * 644 / 410)" } },
|
|
37
|
+
"generation_id": "'ticket:' || printf('v%06d', p)", "author_contact_id": "NULL", "created_at": ":now - 3600000", "fingerprint": "printf('size-block-%d', n)" } },
|
|
38
|
+
{ "table": "messages", "rows": { "small": 270, "large": 27000 }, "chunk": 1500, "parent": { "table": "conversations", "per": 3 }, "values": {
|
|
39
|
+
"id": "printf('m%07d', n)", "project_id": "'size-p1'", "conversation_id": "printf('v%06d', p)", "local_id": "printf('size-local-%07d', n)",
|
|
40
|
+
"actor_id": "CASE (n - 1) % 3 WHEN 0 THEN printf('c%06d', p) WHEN 1 THEN 'system' ELSE (SELECT id FROM users WHERE email = :owner_email) END",
|
|
41
|
+
"actor_role": "CASE (n - 1) % 3 WHEN 0 THEN 'contact' WHEN 1 THEN 'system' ELSE 'operator' END",
|
|
42
|
+
"content": "CASE WHEN (n - 1) % 3 = 2 THEN 'See ' || coalesce((SELECT group_concat('⟦k:' || id || '⟧', ' ') FROM knowledge_reply_blocks WHERE generation_id = 'ticket:' || printf('v%06d', p)), 'our help center') ELSE 'Synthetic message' END",
|
|
43
|
+
"revision": "(n - 1) % 3 + 1", "created_at": ":now - 3600000 - p * 1000 + (n - 1) % 3" } },
|
|
44
|
+
{ "table": "conversation_outbox", "rows": { "small": 270, "large": 27000 }, "chunk": 3000, "parent": { "table": "messages", "per": 1 }, "values": {
|
|
45
|
+
"project_id": "'size-p1'", "conversation_id": "printf('v%06d', (p - 1) / 3 + 1)", "message_id": "printf('m%07d', p)",
|
|
46
|
+
"created_at": ":now - 3600000 - ((p - 1) / 3 + 1) * 1000", "delivered_at": ":now - 3500000" } },
|
|
47
|
+
{ "table": "media", "rows": { "small": 28, "large": 2835 }, "parent": { "table": "messages", "per": 1 }, "values": {
|
|
48
|
+
"id": "printf('media-%06d', n)", "project_id": "'size-p1'", "conversation_id": "printf('v%06d', (p - 1) / 3 + 1)", "uploaded_by": "printf('c%06d', (p - 1) / 3 + 1)",
|
|
49
|
+
"object_key": "printf('size/media-%06d', n)", "filename": "printf('file-%d.png', n)", "content_type": "'image/png'", "size": "2048",
|
|
50
|
+
"message_id": "printf('m%07d', p)", "created_at": ":now - 3600000" } },
|
|
51
|
+
{ "table": "workflow_events", "rows": { "small": 30, "large": 3034 }, "parent": { "table": "conversations", "per": 1 }, "values": {
|
|
52
|
+
"event_key": "printf('size-event-%06d', n)", "project_id": "'size-p1'", "conversation_id": "printf('v%06d', p)", "kind": "'conversation.updated'",
|
|
53
|
+
"at": ":now - 3600000", "source": "'size'", "revision": "1", "ticket_version": "1", "after_json": "'{}'", "state": "'pending'", "next_attempt_at": ":now + 86400000" } }
|
|
54
|
+
],
|
|
55
|
+
"after": [ "DELETE FROM knowledge_index_dirty" ]
|
|
56
|
+
},
|
|
57
|
+
"flows": [
|
|
58
|
+
{ "name": "inbox-list", "persona": "owner", "request": { "method": "GET", "path": "/api/dashboard/projects/size-p1/conversations" }, "ceiling": 5000, "callsInRow": 3 },
|
|
59
|
+
{ "name": "inbox-counts", "persona": "owner", "request": { "method": "GET", "path": "/api/dashboard/projects/size-p1/conversations/counts" }, "ceiling": 1000, "callsInRow": 3 },
|
|
60
|
+
{ "name": "open-worst-conversation", "persona": "owner", "request": { "method": "GET", "path": "/api/dashboard/projects/size-p1/conversations/v000001" }, "ceiling": 25000, "callsInRow": 3 },
|
|
61
|
+
{ "name": "transcript-worst", "persona": "owner", "request": { "method": "GET", "path": "/api/conversations/v000001/messages" }, "ceiling": 6000, "callsInRow": 3 },
|
|
62
|
+
{ "name": "owner-reply", "persona": "owner", "request": { "method": "POST", "path": "/api/conversations/v000002/messages", "body": { "content": "Thanks for waiting.", "localId": "size-owner-reply" } }, "ceiling": 15000 },
|
|
63
|
+
{ "name": "visitor-message", "persona": "visitor", "request": { "method": "POST", "path": "/api/conversations/{{visitor.conversation}}/messages", "body": { "content": "Hello", "localId": "size-visitor-1" } }, "ceiling": 12000 },
|
|
64
|
+
{ "name": "visitor-socket-open", "persona": "visitor", "request": { "method": "GET", "path": "/api/conversations/{{visitor.conversation}}/socket?token={{visitor.token}}", "websocket": true }, "ceiling": 2000, "callsInRow": 2 },
|
|
65
|
+
{ "name": "dashboard-socket", "persona": "owner", "request": { "method": "GET", "path": "/api/projects/size-p1/socket", "websocket": true }, "ceiling": 2000, "callsInRow": 2 },
|
|
66
|
+
{ "name": "minute-tick", "scheduled": "* * * * *", "ceiling": 40000 }
|
|
67
|
+
]
|
|
68
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { dmlParts } from './dml-parts.mjs';
|
|
2
|
+
import { tokenize } from './from-items.mjs';
|
|
3
|
+
|
|
4
|
+
// SQLite adds the heights of nested expressions along its name-resolution walk,
|
|
5
|
+
// so a statement's depth can be measured with the compiler it already runs on.
|
|
6
|
+
// node:sqlite (Node 22.20) has no limit setter and keeps SQLite's default of
|
|
7
|
+
// 1000, so each part is wrapped under a chain of k additions:
|
|
8
|
+
// SELECT (1+1+…+1) + EXISTS (SELECT 1 FROM (<part>))
|
|
9
|
+
// It compiles while k + depth(part) fits the limit. `SELECT 1` has depth 1, so
|
|
10
|
+
// its largest k calibrates the sum, and a part's depth is calibration + 1 - k.
|
|
11
|
+
const TOO_LARGE = /Expression tree is too large/;
|
|
12
|
+
const chain = k => k <= 1 ? '1' : Array(k).fill('1').join('+');
|
|
13
|
+
const wrapped = (part, k) => `SELECT (${chain(k)}) + EXISTS (SELECT 1 FROM (${part}))`;
|
|
14
|
+
|
|
15
|
+
export class MeasureError extends Error {}
|
|
16
|
+
|
|
17
|
+
export function depthMeter(db) {
|
|
18
|
+
// null when it compiles; the message when it does not.
|
|
19
|
+
const failure = sql => { try { db.prepare(sql); return null; } catch (error) { return error.message; } };
|
|
20
|
+
const fits = (part, k) => {
|
|
21
|
+
const message = failure(wrapped(part, k));
|
|
22
|
+
if (message && !TOO_LARGE.test(message)) throw new MeasureError(message);
|
|
23
|
+
return !message;
|
|
24
|
+
};
|
|
25
|
+
// The largest k that fits, by binary search; below the first k tried, there is none.
|
|
26
|
+
const largestPadding = (part, high) => {
|
|
27
|
+
let low = 2;
|
|
28
|
+
if (!fits(part, low)) return 1;
|
|
29
|
+
if (fits(part, high)) throw new MeasureError(`the part compiles with ${high} additions above it`);
|
|
30
|
+
while (high - low > 1) {
|
|
31
|
+
const mid = (low + high) >> 1;
|
|
32
|
+
if (fits(part, mid)) low = mid; else high = mid;
|
|
33
|
+
}
|
|
34
|
+
return low;
|
|
35
|
+
};
|
|
36
|
+
const calibration = largestPadding('SELECT 1', 4000);
|
|
37
|
+
const partsOf = sql => dmlParts(sql, tokenize(sql));
|
|
38
|
+
return {
|
|
39
|
+
parts: partsOf,
|
|
40
|
+
// One compile per part: is the statement's depth at most `limit`?
|
|
41
|
+
atMost: (parts, limit) => parts.every(part => fits(part, calibration + 1 - limit)),
|
|
42
|
+
// The exact depth: a search per part. A part too deep to compile at all reports the calibration, the most it can show.
|
|
43
|
+
exact: parts => Math.max(...parts.map(part => calibration + 1 - largestPadding(part, calibration + 2))),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// A declared SQLite expression (or `after` statement) is one expression: a `;` outside quotes and comments would
|
|
2
|
+
// start a second statement inside the INSERT … SELECT the later clauses build around it.
|
|
3
|
+
export const maxExpressionLength = 2000;
|
|
4
|
+
|
|
5
|
+
export function topLevelSemicolon(sql) {
|
|
6
|
+
const closing = { "'": "'", '"': '"', '`': '`', '[': ']' };
|
|
7
|
+
for (let i = 0; i < sql.length; i++) {
|
|
8
|
+
const char = sql[i];
|
|
9
|
+
if (closing[char]) {
|
|
10
|
+
const end = closing[char];
|
|
11
|
+
// A doubled quote inside a quoted string is a literal quote; brackets have no escape.
|
|
12
|
+
for (i++; i < sql.length; i++) {
|
|
13
|
+
if (sql[i] !== end) continue;
|
|
14
|
+
if (end !== ']' && sql[i + 1] === end) i++;
|
|
15
|
+
else break;
|
|
16
|
+
}
|
|
17
|
+
} else if (char === '-' && sql[i + 1] === '-') {
|
|
18
|
+
while (i < sql.length && sql[i] !== '\n') i++;
|
|
19
|
+
} else if (char === '/' && sql[i + 1] === '*') {
|
|
20
|
+
const end = sql.indexOf('*/', i + 2);
|
|
21
|
+
if (end < 0) return false;
|
|
22
|
+
i = end + 1;
|
|
23
|
+
} else if (char === ';') return true;
|
|
24
|
+
}
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// undefined when valid, else what is wrong.
|
|
29
|
+
export function expressionProblem(sql) {
|
|
30
|
+
if (typeof sql !== 'string' || !sql.trim()) return 'must be one SQLite expression in a string';
|
|
31
|
+
if (sql.length > maxExpressionLength) return `is ${sql.length} characters; keep it to ${maxExpressionLength} or fewer`;
|
|
32
|
+
if (topLevelSemicolon(sql)) return "has a ';' outside quotes; write one expression, not several statements";
|
|
33
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { sqlTokens } from '../../modules/deploy-kit/sql-tokens.mjs';
|
|
2
|
+
|
|
3
|
+
// Compiles a size-check.json shape and one size into the ordered statements that build it: each table is chunked
|
|
4
|
+
// INSERT … SELECT over a recursive row sequence, so a database of millions of rows is built inside SQLite with no
|
|
5
|
+
// row crossing the process boundary, and the schema's own triggers derive the rest. Pure: no file, no database.
|
|
6
|
+
// Exported as @runeditrun/rer/size-fixture so the fleet's real-D1 runner seeds exactly the statements rer runs.
|
|
7
|
+
//
|
|
8
|
+
// Names an expression may use: n (row number from 1), p (the parent's row number from 1), k (the index from 0
|
|
9
|
+
// inside a jsonArray item), and the parameters :now, :owner_email and :size. D1 binds only ? and ?NNN, so each
|
|
10
|
+
// parameter is found as a token outside quotes and comments and rewritten to ?NNN.
|
|
11
|
+
|
|
12
|
+
export const sizes = Object.freeze(['small', 'large']);
|
|
13
|
+
const known = new Set(['now', 'owner_email', 'size']);
|
|
14
|
+
|
|
15
|
+
/** A declaration that cannot be compiled, naming the table and, when it is one expression, the column. */
|
|
16
|
+
export class FixtureError extends Error {
|
|
17
|
+
constructor(message, { table, column } = {}) {
|
|
18
|
+
super(`${table ? `${table}${column ? `.${column}` : ''}: ` : ''}${message}`);
|
|
19
|
+
this.name = 'FixtureError';
|
|
20
|
+
this.table = table;
|
|
21
|
+
this.column = column;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Rewrites each :now, :owner_email and :size outside quotes and comments to ?NNN. `order` lists the parameter names
|
|
26
|
+
// in the order of their numbers (?1 is order[0]) and is shared by every expression of one statement.
|
|
27
|
+
function bind(sql, order, where) {
|
|
28
|
+
let tokens;
|
|
29
|
+
try { tokens = sqlTokens(sql); }
|
|
30
|
+
catch { throw new FixtureError('has an unterminated quote or comment', where); }
|
|
31
|
+
let out = '', from = 0;
|
|
32
|
+
for (let i = 0; i < tokens.length; i++) {
|
|
33
|
+
const token = tokens[i];
|
|
34
|
+
if (token.kind !== 'symbol' || !'?:@$'.includes(token.value)) continue;
|
|
35
|
+
const next = tokens[i + 1], name = next?.kind === 'word' && next.start === token.end ? next.value : undefined;
|
|
36
|
+
if (token.value !== ':' || !known.has(name)) throw new FixtureError(`uses ${name ? token.value + name : `"${token.value}"`}, which is not a parameter; the parameters are :now, :owner_email and :size`, where);
|
|
37
|
+
if (!order.includes(name)) order.push(name);
|
|
38
|
+
out += sql.slice(from, token.start) + `?${order.indexOf(name) + 1}`;
|
|
39
|
+
from = next.end;
|
|
40
|
+
i++;
|
|
41
|
+
}
|
|
42
|
+
return out + sql.slice(from);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// A column's SQL. A jsonArray is a correlated recursive-CTE subquery: k runs from 0 to length - 1, and an empty
|
|
46
|
+
// length gives [].
|
|
47
|
+
function columnSql(value, order, where) {
|
|
48
|
+
if (typeof value === 'string') return bind(value, order, where);
|
|
49
|
+
const length = bind(value.jsonArray.length, order, where), item = bind(value.jsonArray.item, order, where);
|
|
50
|
+
return `(WITH RECURSIVE rer_k(k) AS (SELECT 0 WHERE 0 < (${length}) UNION ALL SELECT k + 1 FROM rer_k WHERE k + 1 < (${length})) SELECT json_group_array(${item}) FROM rer_k)`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const plainRows = (rows, size) => typeof rows === 'number' ? rows : rows[size];
|
|
54
|
+
|
|
55
|
+
// The runs of row numbers a table is built in, each with the SQL of its parent row number p. A histogram takes its
|
|
56
|
+
// parents from row 1, those with the most children first.
|
|
57
|
+
function runs(table, total, size) {
|
|
58
|
+
const chunk = table.chunk ?? 10_000, split = (first, last, p) => {
|
|
59
|
+
const found = [];
|
|
60
|
+
for (let a = first; a <= last; a += chunk) found.push({ first: a, last: Math.min(last, a + chunk - 1), p });
|
|
61
|
+
return found;
|
|
62
|
+
};
|
|
63
|
+
const parent = table.parent;
|
|
64
|
+
if (parent?.histogram) {
|
|
65
|
+
const buckets = Object.entries(parent.histogram).map(([children, parents]) => [Number(children), parents]).sort((a, b) => b[0] - a[0]);
|
|
66
|
+
const sum = buckets.reduce((all, [children, parents]) => all + children * parents, 0);
|
|
67
|
+
if (sum !== total) throw new FixtureError(`its histogram builds ${sum.toLocaleString('en-US')} rows but rows is ${total.toLocaleString('en-US')} at the ${size} size; make the sums equal`, { table: table.table });
|
|
68
|
+
const found = [];
|
|
69
|
+
let rows = 0, parents = 0;
|
|
70
|
+
for (const [children, count] of buckets) {
|
|
71
|
+
found.push(...split(rows + 1, rows + children * count, `${parents + 1} + (n - ${rows + 1}) / ${children}`));
|
|
72
|
+
rows += children * count;
|
|
73
|
+
parents += count;
|
|
74
|
+
}
|
|
75
|
+
return { runs: found, parents };
|
|
76
|
+
}
|
|
77
|
+
if (parent) return { runs: split(1, total, `(n - 1) / ${parent.per} + 1`), parents: Math.ceil(total / parent.per) };
|
|
78
|
+
return { runs: split(1, total, 'NULL'), parents: 0 };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* The statements that build `size` ('small' or 'large') of a declaration's shape: `{ statements, parts, tables,
|
|
83
|
+
* parameters }`. `statements` is `[{ name, sql, params }]` in the order to run them (the fleet runner's format);
|
|
84
|
+
* `parts[i]` says what statements[i] builds ({ kind: 'rows', table, first, last } or { kind: 'after', index });
|
|
85
|
+
* `tables` is each table's declared row count at this size; `parameters` lists the names its expressions use.
|
|
86
|
+
* `now` is the run's start in Unix milliseconds and `ownerEmail` the owner persona's email.
|
|
87
|
+
*/
|
|
88
|
+
export function compileFixture(declaration, size, { now, ownerEmail } = {}) {
|
|
89
|
+
if (!sizes.includes(size)) throw new FixtureError(`size must be ${sizes.join(' or ')}, not ${JSON.stringify(size)}`);
|
|
90
|
+
const values = { now, owner_email: ownerEmail, size }, statements = [], parts = [], tables = [], used = new Set();
|
|
91
|
+
const rowsOf = new Map();
|
|
92
|
+
const bound = order => order.map(name => {
|
|
93
|
+
used.add(name);
|
|
94
|
+
if (values[name] === undefined) throw new FixtureError(`needs :${name}, which this run has no value for`);
|
|
95
|
+
return values[name];
|
|
96
|
+
});
|
|
97
|
+
for (const table of declaration.shape.tables) {
|
|
98
|
+
const total = plainRows(table.rows, size), where = { table: table.table }, order = [];
|
|
99
|
+
if (table.parent && !rowsOf.has(table.parent.table)) throw new FixtureError(`its parent table ${table.parent.table} must be listed earlier in shape.tables`, where);
|
|
100
|
+
const columns = Object.keys(table.values), expressions = columns.map(column => columnSql(table.values[column], order, { table: table.table, column }));
|
|
101
|
+
const params = bound(order), { runs: chunks, parents } = runs(table, total, size);
|
|
102
|
+
if (table.parent && parents > rowsOf.get(table.parent.table)) throw new FixtureError(`its rows reach parent row ${parents.toLocaleString('en-US')} of ${table.parent.table}, which has ${rowsOf.get(table.parent.table).toLocaleString('en-US')} rows at the ${size} size`, where);
|
|
103
|
+
for (const { first, last, p } of chunks) {
|
|
104
|
+
statements.push({
|
|
105
|
+
name: `${size}:${table.table}:${first}-${last}`,
|
|
106
|
+
sql: `INSERT INTO ${table.table}(${columns.map(column => `"${column}"`).join(',')}) WITH RECURSIVE rer_seq(n) AS (SELECT ${first} UNION ALL SELECT n + 1 FROM rer_seq WHERE n < ${last}), rer_rows(n, p) AS (SELECT n, ${p} FROM rer_seq)\nSELECT ${expressions.join(',\n')}\nFROM rer_rows`,
|
|
107
|
+
params: [...params],
|
|
108
|
+
});
|
|
109
|
+
parts.push({ kind: 'rows', table: table.table, first, last });
|
|
110
|
+
}
|
|
111
|
+
rowsOf.set(table.table, total);
|
|
112
|
+
tables.push({ table: table.table, rows: total });
|
|
113
|
+
}
|
|
114
|
+
(declaration.shape.after ?? []).forEach((statement, index) => {
|
|
115
|
+
const order = [], sql = bind(statement, order, {});
|
|
116
|
+
statements.push({ name: `${size}:after:${index + 1}`, sql, params: bound(order) });
|
|
117
|
+
parts.push({ kind: 'after', index: index + 1 });
|
|
118
|
+
});
|
|
119
|
+
return { statements, parts, tables, parameters: [...used] };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The parameter names a declaration's expressions use, for deciding what a run must prepare (the owner, for :owner_email). */
|
|
123
|
+
export function fixtureParameters(declaration) {
|
|
124
|
+
const names = new Set(), order = [];
|
|
125
|
+
const scan = (sql, where) => { order.length = 0; bind(sql, order, where); order.forEach(name => names.add(name)); };
|
|
126
|
+
for (const table of declaration.shape.tables) for (const [column, value] of Object.entries(table.values)) {
|
|
127
|
+
const where = { table: table.table, column };
|
|
128
|
+
if (typeof value === 'string') scan(value, where); else { scan(value.jsonArray.length, where); scan(value.jsonArray.item, where); }
|
|
129
|
+
}
|
|
130
|
+
for (const statement of declaration.shape.after ?? []) scan(statement, {});
|
|
131
|
+
return names;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** The column(s) SQLite names in a failure message ("NOT NULL constraint failed: items.label"), or undefined. */
|
|
135
|
+
export function failureColumn(message) {
|
|
136
|
+
const failed = /constraint failed: ([^:]+)/.exec(message)?.[1];
|
|
137
|
+
const named = failed ? [...failed.matchAll(/\w+\.(\w+)/g)].map(match => match[1]) : [];
|
|
138
|
+
return named.length ? named.join(', ') : /has no column named (\w+)/.exec(message)?.[1] ?? /no such column: (?:\w+\.)?(\w+)/.exec(message)?.[1];
|
|
139
|
+
}
|