@intentius/sql-yodeler 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -26
- package/dist/bin.js +15 -0
- package/dist/bin.js.map +1 -1
- package/dist/chant/apply.d.ts +9 -1
- package/dist/chant/apply.js +13 -1
- package/dist/chant/apply.js.map +1 -1
- package/dist/chant/build.js +7 -0
- package/dist/chant/build.js.map +1 -1
- package/dist/chant/drift.d.ts +69 -1
- package/dist/chant/drift.js +152 -5
- package/dist/chant/drift.js.map +1 -1
- package/dist/chant/runnable.d.ts +19 -0
- package/dist/chant/runnable.js +47 -0
- package/dist/chant/runnable.js.map +1 -0
- package/dist/ci/render.d.ts +289 -0
- package/dist/ci/render.js +688 -0
- package/dist/ci/render.js.map +1 -0
- package/dist/ci/replay.d.ts +17 -0
- package/dist/ci/replay.js +69 -0
- package/dist/ci/replay.js.map +1 -0
- package/dist/cli.d.ts +4 -0
- package/dist/cli.js +40 -0
- package/dist/cli.js.map +1 -1
- package/dist/command.d.ts +6 -0
- package/dist/command.js.map +1 -1
- package/dist/commands/apply.d.ts +113 -0
- package/dist/commands/apply.js +376 -49
- package/dist/commands/apply.js.map +1 -1
- package/dist/commands/approve.d.ts +5 -0
- package/dist/commands/approve.js +155 -0
- package/dist/commands/approve.js.map +1 -0
- package/dist/commands/checkpoint.d.ts +3 -0
- package/dist/commands/checkpoint.js +78 -0
- package/dist/commands/checkpoint.js.map +1 -0
- package/dist/commands/ci.d.ts +3 -0
- package/dist/commands/ci.js +118 -0
- package/dist/commands/ci.js.map +1 -0
- package/dist/commands/docs.d.ts +11 -0
- package/dist/commands/docs.js +108 -0
- package/dist/commands/docs.js.map +1 -0
- package/dist/commands/drift.d.ts +23 -0
- package/dist/commands/drift.js +263 -18
- package/dist/commands/drift.js.map +1 -1
- package/dist/commands/index.js +16 -0
- package/dist/commands/index.js.map +1 -1
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.js +15 -11
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/lint.js +25 -8
- package/dist/commands/lint.js.map +1 -1
- package/dist/commands/mcp.d.ts +3 -0
- package/dist/commands/mcp.js +50 -0
- package/dist/commands/mcp.js.map +1 -0
- package/dist/commands/new.d.ts +2 -0
- package/dist/commands/new.js +134 -10
- package/dist/commands/new.js.map +1 -1
- package/dist/commands/override.d.ts +3 -0
- package/dist/commands/override.js +150 -0
- package/dist/commands/override.js.map +1 -0
- package/dist/commands/plan.js +43 -27
- package/dist/commands/plan.js.map +1 -1
- package/dist/commands/repair.d.ts +11 -0
- package/dist/commands/repair.js +15 -7
- package/dist/commands/repair.js.map +1 -1
- package/dist/commands/report.d.ts +13 -0
- package/dist/commands/report.js +121 -0
- package/dist/commands/report.js.map +1 -0
- package/dist/commands/revert.d.ts +20 -0
- package/dist/commands/revert.js +246 -0
- package/dist/commands/revert.js.map +1 -0
- package/dist/commands/status.js +44 -17
- package/dist/commands/status.js.map +1 -1
- package/dist/config/base.d.ts +69 -0
- package/dist/config/base.js +121 -0
- package/dist/config/base.js.map +1 -0
- package/dist/config/define.d.ts +223 -3
- package/dist/config/define.js +3 -1
- package/dist/config/define.js.map +1 -1
- package/dist/config/index.d.ts +2 -0
- package/dist/config/index.js +2 -0
- package/dist/config/index.js.map +1 -1
- package/dist/config/load.d.ts +1 -1
- package/dist/config/load.js +150 -5
- package/dist/config/load.js.map +1 -1
- package/dist/config/settings.d.ts +18 -2
- package/dist/config/settings.js +40 -6
- package/dist/config/settings.js.map +1 -1
- package/dist/env-args.d.ts +6 -0
- package/dist/env-args.js +31 -1
- package/dist/env-args.js.map +1 -1
- package/dist/exit.d.ts +20 -0
- package/dist/exit.js +28 -0
- package/dist/exit.js.map +1 -0
- package/dist/history/client.d.ts +8 -0
- package/dist/history/client.js +24 -1
- package/dist/history/client.js.map +1 -1
- package/dist/history/lock.d.ts +5 -1
- package/dist/history/lock.js +8 -2
- package/dist/history/lock.js.map +1 -1
- package/dist/history/postgres/ddl.d.ts +18 -0
- package/dist/history/postgres/ddl.js +27 -2
- package/dist/history/postgres/ddl.js.map +1 -1
- package/dist/history/postgres/store.d.ts +2 -0
- package/dist/history/postgres/store.js +7 -1
- package/dist/history/postgres/store.js.map +1 -1
- package/dist/history/rows.d.ts +16 -3
- package/dist/history/rows.js +30 -1
- package/dist/history/rows.js.map +1 -1
- package/dist/history/store.d.ts +2 -0
- package/dist/history/store.js +2 -2
- package/dist/history/store.js.map +1 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/init/adopt.js +8 -3
- package/dist/init/adopt.js.map +1 -1
- package/dist/init/live.d.ts +11 -0
- package/dist/init/live.js +25 -2
- package/dist/init/live.js.map +1 -1
- package/dist/lint/check.d.ts +8 -1
- package/dist/lint/check.js +38 -4
- package/dist/lint/check.js.map +1 -1
- package/dist/lint/report.js +1 -1
- package/dist/lint/report.js.map +1 -1
- package/dist/lint/routines.d.ts +26 -0
- package/dist/lint/routines.js +65 -0
- package/dist/lint/routines.js.map +1 -0
- package/dist/lint/rules.d.ts +16 -4
- package/dist/lint/rules.js +9 -4
- package/dist/lint/rules.js.map +1 -1
- package/dist/mcp/server.d.ts +40 -0
- package/dist/mcp/server.js +106 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools.d.ts +72 -0
- package/dist/mcp/tools.js +178 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/migrations/checkpoint.d.ts +45 -0
- package/dist/migrations/checkpoint.js +110 -0
- package/dist/migrations/checkpoint.js.map +1 -0
- package/dist/migrations/checks.d.ts +28 -0
- package/dist/migrations/checks.js +97 -0
- package/dist/migrations/checks.js.map +1 -0
- package/dist/migrations/create.d.ts +3 -1
- package/dist/migrations/create.js +3 -0
- package/dist/migrations/create.js.map +1 -1
- package/dist/migrations/directory.js +4 -0
- package/dist/migrations/directory.js.map +1 -1
- package/dist/migrations/document.d.ts +33 -0
- package/dist/migrations/document.js +2 -0
- package/dist/migrations/document.js.map +1 -1
- package/dist/migrations/index.d.ts +3 -0
- package/dist/migrations/index.js +3 -0
- package/dist/migrations/index.js.map +1 -1
- package/dist/migrations/routines.d.ts +31 -0
- package/dist/migrations/routines.js +55 -0
- package/dist/migrations/routines.js.map +1 -0
- package/dist/notify/index.d.ts +124 -0
- package/dist/notify/index.js +210 -0
- package/dist/notify/index.js.map +1 -0
- package/dist/notify/verify.d.ts +53 -0
- package/dist/notify/verify.js +71 -0
- package/dist/notify/verify.js.map +1 -0
- package/dist/pr/forge.d.ts +4 -0
- package/dist/pr/forge.js +21 -7
- package/dist/pr/forge.js.map +1 -1
- package/dist/pr/issue.d.ts +68 -0
- package/dist/pr/issue.js +107 -0
- package/dist/pr/issue.js.map +1 -0
- package/dist/pr/model.d.ts +12 -1
- package/dist/pr/model.js +44 -6
- package/dist/pr/model.js.map +1 -1
- package/dist/pr/render.d.ts +3 -1
- package/dist/pr/render.js +63 -4
- package/dist/pr/render.js.map +1 -1
- package/dist/project.d.ts +19 -2
- package/dist/project.js +33 -1
- package/dist/project.js.map +1 -1
- package/dist/replay/index.d.ts +3 -0
- package/dist/replay/index.js +3 -0
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/replay.d.ts +9 -1
- package/dist/replay/replay.js +63 -32
- package/dist/replay/replay.js.map +1 -1
- package/dist/replay/report.js +1 -1
- package/dist/replay/report.js.map +1 -1
- package/dist/replay/shared.d.ts +28 -0
- package/dist/replay/shared.js +77 -0
- package/dist/replay/shared.js.map +1 -0
- package/dist/report/audit.d.ts +138 -0
- package/dist/report/audit.js +202 -0
- package/dist/report/audit.js.map +1 -0
- package/dist/report/jsonl.d.ts +21 -0
- package/dist/report/jsonl.js +52 -0
- package/dist/report/jsonl.js.map +1 -0
- package/dist/report/model.d.ts +68 -0
- package/dist/report/model.js +127 -0
- package/dist/report/model.js.map +1 -0
- package/dist/report/render.d.ts +18 -0
- package/dist/report/render.js +127 -0
- package/dist/report/render.js.map +1 -0
- package/dist/schema-docs/render.d.ts +49 -0
- package/dist/schema-docs/render.js +359 -0
- package/dist/schema-docs/render.js.map +1 -0
- package/dist/schemas.d.ts +37 -0
- package/dist/schemas.js +41 -0
- package/dist/schemas.js.map +1 -0
- package/dist/server.d.ts +40 -0
- package/dist/server.js +87 -0
- package/dist/server.js.map +1 -0
- package/dist/sources/ddl.d.ts +53 -0
- package/dist/sources/ddl.js +347 -0
- package/dist/sources/ddl.js.map +1 -0
- package/dist/sources/index.d.ts +69 -0
- package/dist/sources/index.js +170 -0
- package/dist/sources/index.js.map +1 -0
- package/dist/steps/backfill.d.ts +21 -17
- package/dist/steps/backfill.js +20 -20
- package/dist/steps/backfill.js.map +1 -1
- package/dist/steps/index.d.ts +2 -2
- package/dist/steps/index.js +3 -3
- package/dist/steps/index.js.map +1 -1
- package/dist/steps/postgres-migration.d.ts +35 -25
- package/dist/steps/postgres-migration.js +42 -29
- package/dist/steps/postgres-migration.js.map +1 -1
- package/dist/steps/rebuild.d.ts +15 -15
- package/dist/steps/rebuild.js +17 -24
- package/dist/steps/rebuild.js.map +1 -1
- package/dist/tenants/index.d.ts +40 -0
- package/dist/tenants/index.js +122 -0
- package/dist/tenants/index.js.map +1 -0
- package/dist/tenants/target.d.ts +23 -0
- package/dist/tenants/target.js +29 -0
- package/dist/tenants/target.js.map +1 -0
- package/dist/versioned/checks.d.ts +29 -0
- package/dist/versioned/checks.js +54 -0
- package/dist/versioned/checks.js.map +1 -0
- package/dist/versioned/clickhouse.js +8 -0
- package/dist/versioned/clickhouse.js.map +1 -1
- package/dist/versioned/driver.d.ts +10 -0
- package/dist/versioned/engine.d.ts +61 -1
- package/dist/versioned/engine.js +109 -6
- package/dist/versioned/engine.js.map +1 -1
- package/dist/versioned/explain.js +7 -2
- package/dist/versioned/explain.js.map +1 -1
- package/dist/versioned/index.d.ts +10 -0
- package/dist/versioned/index.js +25 -1
- package/dist/versioned/index.js.map +1 -1
- package/dist/versioned/op.d.ts +16 -1
- package/dist/versioned/op.js +43 -6
- package/dist/versioned/op.js.map +1 -1
- package/dist/versioned/policy.d.ts +98 -0
- package/dist/versioned/policy.js +212 -0
- package/dist/versioned/policy.js.map +1 -0
- package/dist/versioned/postgres/driver.d.ts +3 -1
- package/dist/versioned/postgres/driver.js +30 -3
- package/dist/versioned/postgres/driver.js.map +1 -1
- package/dist/versioned/postgres/waits.d.ts +46 -0
- package/dist/versioned/postgres/waits.js +78 -0
- package/dist/versioned/postgres/waits.js.map +1 -0
- package/dist/versioned/report.d.ts +3 -0
- package/dist/versioned/report.js +13 -1
- package/dist/versioned/report.js.map +1 -1
- package/dist/versioned/revert.d.ts +128 -0
- package/dist/versioned/revert.js +251 -0
- package/dist/versioned/revert.js.map +1 -0
- package/dist/versioned/stand-down.d.ts +30 -0
- package/dist/versioned/stand-down.js +56 -0
- package/dist/versioned/stand-down.js.map +1 -0
- package/dist/versioned/state.d.ts +25 -0
- package/dist/versioned/state.js +36 -3
- package/dist/versioned/state.js.map +1 -1
- package/dist/waves/index.d.ts +179 -0
- package/dist/waves/index.js +268 -0
- package/dist/waves/index.js.map +1 -0
- package/dist/waves/promotion.d.ts +56 -0
- package/dist/waves/promotion.js +108 -0
- package/dist/waves/promotion.js.map +1 -0
- package/dist/waves/record.d.ts +24 -0
- package/dist/waves/record.js +20 -0
- package/dist/waves/record.js.map +1 -0
- package/package.json +22 -8
- package/schemas/apply.schema.json +428 -0
- package/schemas/checkpoint.schema.json +19 -0
- package/schemas/common.schema.json +213 -0
- package/schemas/drift.schema.json +61 -0
- package/schemas/event.schema.json +36 -0
- package/schemas/init.schema.json +31 -0
- package/schemas/lint-rules.schema.json +54 -0
- package/schemas/lint.schema.json +294 -0
- package/schemas/mcp.schema.json +70 -0
- package/schemas/new.schema.json +71 -0
- package/schemas/plan.schema.json +489 -0
- package/schemas/rebase.schema.json +54 -0
- package/schemas/repair.schema.json +20 -0
- package/schemas/report.schema.json +114 -0
- package/schemas/revert.schema.json +69 -0
- package/schemas/status.schema.json +371 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* DDL sources (#67): schema that an ORM defines, read from the DDL it
|
|
3
|
+
* prints, beside the declarations in `src/`.
|
|
4
|
+
*
|
|
5
|
+
* ```ts
|
|
6
|
+
* // yodel.config.ts
|
|
7
|
+
* export default defineConfig({
|
|
8
|
+
* sources: {
|
|
9
|
+
* prisma: { command: "npx prisma migrate diff --from-empty --to-schema-datamodel prisma/schema.prisma --script", schema: "app" },
|
|
10
|
+
* legacy: { file: "db/legacy.sql" },
|
|
11
|
+
* },
|
|
12
|
+
* });
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Before each build, yodel runs each source's command (or reads its file)
|
|
16
|
+
* in the project directory and writes the DDL out as declarations in
|
|
17
|
+
* `<sourceDir>/<name>.generated.ts`, the file `chant import` would write.
|
|
18
|
+
* The build, and so plan, new, apply and drift, read it like any other
|
|
19
|
+
* declaration; the ApplyOp's own build finds it there too. Commit it: a
|
|
20
|
+
* build that does not run yodel (chant build, an Op run elsewhere) reads
|
|
21
|
+
* the committed file.
|
|
22
|
+
*
|
|
23
|
+
* Each object has one owner: an object a source creates that `src/` (or
|
|
24
|
+
* another source) also declares is an error, naming both.
|
|
25
|
+
*/
|
|
26
|
+
import { exec } from "node:child_process";
|
|
27
|
+
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
28
|
+
import { isAbsolute, join, relative } from "node:path";
|
|
29
|
+
import { YodelError } from "../project.js";
|
|
30
|
+
import { ddlToDeclarations } from "./ddl.js";
|
|
31
|
+
const SOURCE_NAME = /^[a-z][a-z0-9-]*$/;
|
|
32
|
+
/** The generated file a source writes, relative to the project directory. */
|
|
33
|
+
export const generatedPath = (project, name) => join(project.sourceDir, `${name}.generated.ts`);
|
|
34
|
+
/** Checks `sources` in yodel.config.ts. */
|
|
35
|
+
export function checkSources(value, where) {
|
|
36
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
37
|
+
throw new YodelError(`${where}: expected an object keyed by source name`);
|
|
38
|
+
for (const [name, source] of Object.entries(value)) {
|
|
39
|
+
const at = `${where}.${name}`;
|
|
40
|
+
if (!SOURCE_NAME.test(name))
|
|
41
|
+
throw new YodelError(`${at}: a source's name is lowercase letters, digits and -, starting with a letter`);
|
|
42
|
+
if (typeof source !== "object" || source === null || Array.isArray(source))
|
|
43
|
+
throw new YodelError(`${at}: expected { command } or { file }`);
|
|
44
|
+
for (const k of Object.keys(source))
|
|
45
|
+
if (!["command", "file", "schema"].includes(k))
|
|
46
|
+
throw new YodelError(`${at}.${k}: unknown setting (expected command, file, schema)`);
|
|
47
|
+
const s = source;
|
|
48
|
+
if ((s.command === undefined) === (s.file === undefined))
|
|
49
|
+
throw new YodelError(`${at}: give one of command and file`);
|
|
50
|
+
for (const k of ["command", "file", "schema"]) {
|
|
51
|
+
if (s[k] !== undefined && (typeof s[k] !== "string" || s[k] === ""))
|
|
52
|
+
throw new YodelError(`${at}.${k}: expected a non-empty string`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return value;
|
|
56
|
+
}
|
|
57
|
+
function runCommand(command, cwd, env) {
|
|
58
|
+
return new Promise((resolve, reject) => {
|
|
59
|
+
exec(command, { cwd, env: env ?? process.env, maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
|
|
60
|
+
if (error)
|
|
61
|
+
reject(new YodelError(`${command} failed (exit ${String(error.code ?? "?")})${stderr.trim() ? `:\n${stderr.trim()}` : ""}`));
|
|
62
|
+
else
|
|
63
|
+
resolve(stdout);
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
/** Reads one source's DDL. */
|
|
68
|
+
async function readDdl(project, name, source, env) {
|
|
69
|
+
if (source.command !== undefined) {
|
|
70
|
+
const origin = `sources.${name} (${source.command})`;
|
|
71
|
+
try {
|
|
72
|
+
return { ddl: await runCommand(source.command, project.dir, env), origin };
|
|
73
|
+
}
|
|
74
|
+
catch (e) {
|
|
75
|
+
throw new YodelError(`yodel.config.ts sources.${name}: ${e instanceof Error ? e.message : String(e)}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
const given = source.file ?? "";
|
|
79
|
+
const file = isAbsolute(given) ? given : join(project.dir, given);
|
|
80
|
+
if (!existsSync(file))
|
|
81
|
+
throw new YodelError(`yodel.config.ts sources.${name}: no file ${relative(project.dir, file)}`);
|
|
82
|
+
return { ddl: readFileSync(file, "utf8"), origin: `sources.${name} (${relative(project.dir, file)})` };
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Writes each source's declarations to its generated file, and returns what
|
|
86
|
+
* each declared. A file whose content is unchanged is not rewritten.
|
|
87
|
+
*/
|
|
88
|
+
export async function syncSources(project, options = {}) {
|
|
89
|
+
const sources = project.yodel?.config.sources ?? {};
|
|
90
|
+
const synced = [];
|
|
91
|
+
for (const [name, source] of Object.entries(sources)) {
|
|
92
|
+
const { ddl, origin } = await readDdl(project, name, source, options.env);
|
|
93
|
+
const how = source.command !== undefined ? `the output of: ${source.command}` : `the DDL in ${source.file ?? ""}`;
|
|
94
|
+
const declarations = await ddlToDeclarations(project.dialect, ddl, {
|
|
95
|
+
origin: `yodel.config.ts ${origin}`,
|
|
96
|
+
schema: source.schema,
|
|
97
|
+
header: `Generated by yodel from the DDL source "${name}" in yodel.config.ts,\n${how}\nDo not edit: yodel writes it again before each build.`,
|
|
98
|
+
});
|
|
99
|
+
const path = generatedPath(project, name);
|
|
100
|
+
const full = join(project.dir, path);
|
|
101
|
+
const before = existsSync(full) ? readFileSync(full, "utf8") : undefined;
|
|
102
|
+
if (before !== declarations.content)
|
|
103
|
+
writeFileSync(full, declarations.content);
|
|
104
|
+
synced.push({ name, path, objects: declarations.objects });
|
|
105
|
+
}
|
|
106
|
+
return synced;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* chant's build refuses two declarations exported under one name. When one
|
|
110
|
+
* of them is a source's, that is an object declared twice: say so, naming
|
|
111
|
+
* the source.
|
|
112
|
+
*/
|
|
113
|
+
export function duplicateExport(buildOutput, synced) {
|
|
114
|
+
const name = /Duplicate export name "([^"]+)"/.exec(buildOutput)?.[1];
|
|
115
|
+
if (!name)
|
|
116
|
+
return undefined;
|
|
117
|
+
const owners = synced.flatMap((s) => s.objects.filter((o) => o.export === name).map((o) => ({ source: s, object: o })));
|
|
118
|
+
const [first] = owners;
|
|
119
|
+
if (!first)
|
|
120
|
+
return undefined;
|
|
121
|
+
const elsewhere = owners.length > 1 ? `the source "${owners[1]?.source.name ?? ""}"` : "src/";
|
|
122
|
+
return new YodelError(`each object has one owner, and ${first.object.name} is declared twice: by ${elsewhere} (export ${name}) and by the source "${first.source.name}" (${first.source.path}). Leave it to one of them.`);
|
|
123
|
+
}
|
|
124
|
+
/** `schema.name` of a built object, read from its CREATE, or undefined when it cannot be read. */
|
|
125
|
+
async function builtName(dialect, type, ddl) {
|
|
126
|
+
try {
|
|
127
|
+
const first = ddl.split(/;\s*\n\s*COMMENT ON\b/i)[0] ?? ddl;
|
|
128
|
+
const [object] = (await ddlToDeclarations(dialect, first, { origin: "build" })).objects;
|
|
129
|
+
return object && (type === undefined || object.type === type) ? object.name : undefined;
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Each object declared once: an object a source creates that another export
|
|
137
|
+
* in the build (in `src/`, or another source) also creates is an error.
|
|
138
|
+
*/
|
|
139
|
+
export async function checkOwnership(output, synced) {
|
|
140
|
+
if (synced.length === 0)
|
|
141
|
+
return;
|
|
142
|
+
const owner = new Map();
|
|
143
|
+
for (const source of synced)
|
|
144
|
+
for (const object of source.objects)
|
|
145
|
+
owner.set(`${object.type} ${object.name}`, { source, object });
|
|
146
|
+
const problems = [];
|
|
147
|
+
for (const built of output.objects) {
|
|
148
|
+
if (synced.some((s) => s.objects.some((o) => o.export === built.export)))
|
|
149
|
+
continue;
|
|
150
|
+
const name = await builtName(output.dialect, built.type, built.ddl);
|
|
151
|
+
if (!name)
|
|
152
|
+
continue;
|
|
153
|
+
const hit = owner.get(`${built.type ?? ""} ${name}`);
|
|
154
|
+
if (hit)
|
|
155
|
+
problems.push(`${name} is declared by ${built.export} and by the source "${hit.source.name}" (${hit.object.export} in ${hit.source.path})`);
|
|
156
|
+
}
|
|
157
|
+
const seen = new Map();
|
|
158
|
+
for (const source of synced) {
|
|
159
|
+
for (const object of source.objects) {
|
|
160
|
+
const key = `${object.type} ${object.name}`;
|
|
161
|
+
const other = seen.get(key);
|
|
162
|
+
if (other && other !== source.name)
|
|
163
|
+
problems.push(`${object.name} is declared by the sources "${other}" and "${source.name}"`);
|
|
164
|
+
seen.set(key, source.name);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
if (problems.length > 0)
|
|
168
|
+
throw new YodelError(`each object has one owner, and ${problems.length === 1 ? "one is declared twice" : `${String(problems.length)} are declared twice`}:\n${problems.map((p) => ` - ${p}`).join("\n")}`);
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/sources/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAEvD,OAAO,EAAE,UAAU,EAAgB,MAAM,eAAe,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAkB,MAAM,UAAU,CAAC;AAuB7D,MAAM,WAAW,GAAG,mBAAmB,CAAC;AAExC,6EAA6E;AAC7E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAAgB,EAAE,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,IAAI,eAAe,CAAC,CAAC;AAEzH,2CAA2C;AAC3C,MAAM,UAAU,YAAY,CAAC,KAAc,EAAE,KAAa;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,2CAA2C,CAAC,CAAC;IACnJ,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QAC9E,MAAM,EAAE,GAAG,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,8EAA8E,CAAC,CAAC;QACvI,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,oCAAoC,CAAC,CAAC;QAC5I,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,oDAAoD,CAAC,CAAC;QAC1K,MAAM,CAAC,GAAG,MAAiC,CAAC;QAC5C,IAAI,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,gCAAgC,CAAC,CAAC;QACtH,KAAK,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC9C,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBAAE,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,+BAA+B,CAAC,CAAC;QACvI,CAAC;IACH,CAAC;IACD,OAAO,KAAkC,CAAC;AAC5C,CAAC;AAED,SAAS,UAAU,CAAC,OAAe,EAAE,GAAW,EAAE,GAAkC;IAClF,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;YACrG,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,UAAU,CAAC,GAAG,OAAO,iBAAiB,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;;gBACnI,OAAO,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8BAA8B;AAC9B,KAAK,UAAU,OAAO,CAAC,OAAgB,EAAE,IAAY,EAAE,MAAiB,EAAE,GAAkC;IAC1G,IAAI,MAAM,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,WAAW,IAAI,KAAK,MAAM,CAAC,OAAO,GAAG,CAAC;QACrD,IAAI,CAAC;YACH,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;QAC7E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,UAAU,CAAC,2BAA2B,IAAI,KAAK,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,2BAA2B,IAAI,aAAa,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IACvH,OAAO,EAAE,GAAG,EAAE,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,IAAI,KAAK,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;AACzG,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAAgB,EAAE,UAAuC,EAAE;IAC3F,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IACpD,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1E,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,cAAc,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;QAClH,MAAM,YAAY,GAAG,MAAM,iBAAiB,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,EAAE;YACjE,MAAM,EAAE,mBAAmB,MAAM,EAAE;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,2CAA2C,IAAI,0BAA0B,GAAG,yDAAyD;SAC9I,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACzE,IAAI,MAAM,KAAK,YAAY,CAAC,OAAO;YAAE,aAAa,CAAC,IAAI,EAAE,YAAY,CAAC,OAAO,CAAC,CAAC;QAC/E,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,WAAmB,EAAE,MAA+B;IAClF,MAAM,IAAI,GAAG,iCAAiC,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxH,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACvB,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9F,OAAO,IAAI,UAAU,CACnB,kCAAkC,KAAK,CAAC,MAAM,CAAC,IAAI,0BAA0B,SAAS,YAAY,IAAI,wBAAwB,KAAK,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,6BAA6B,CACpM,CAAC;AACJ,CAAC;AAED,kGAAkG;AAClG,KAAK,UAAU,SAAS,CAAC,OAA+B,EAAE,IAAwB,EAAE,GAAW;IAC7F,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACxF,OAAO,MAAM,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAmB,EAAE,MAA+B;IACvF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAChC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuD,CAAC;IAC7E,KAAK,MAAM,MAAM,IAAI,MAAM;QAAE,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACjI,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC;YAAE,SAAS;QACnF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,GAAG;YAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,mBAAmB,KAAK,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;IACvJ,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,KAAK,IAAI,KAAK,KAAK,MAAM,CAAC,IAAI;gBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,gCAAgC,KAAK,UAAU,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;YAC/H,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,qBAAqB,MAAM,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACvO,CAAC"}
|
package/dist/steps/backfill.d.ts
CHANGED
|
@@ -29,16 +29,16 @@
|
|
|
29
29
|
* to repeat (an idempotent `UPDATE`, or a delete of its rows before the
|
|
30
30
|
* insert), as the rebuild's backfill does per partition.
|
|
31
31
|
*
|
|
32
|
-
* The receipts are rows on the environment's server,
|
|
32
|
+
* The receipts are rows on the environment's server, kept by the sql
|
|
33
|
+
* lexicon's receipt store (`sqlReceiptStore`), addressed
|
|
33
34
|
* `yodel/<env>/<migration id>/<effect>`, so the same effect name in two
|
|
34
35
|
* migrations is two receipts:
|
|
35
36
|
*
|
|
36
|
-
* - ClickHouse:
|
|
37
|
-
*
|
|
38
|
-
* - Postgres: `<history schema>.__chant_receipts`,
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* transaction on the apply's connection, and its receipt is written last
|
|
37
|
+
* - ClickHouse: `chant_receipts.receipts` (the table `ClickHouseRebuildOp`
|
|
38
|
+
* keeps its partition receipts in).
|
|
39
|
+
* - Postgres: `<history schema>.__chant_receipts`, in the schema yodel's
|
|
40
|
+
* history is in, over the apply's connection. Each batch runs in one
|
|
41
|
+
* transaction on that connection, and its receipt is written last
|
|
42
42
|
* inside it: the batch's statements and its receipt commit together or
|
|
43
43
|
* not at all, so a run killed anywhere leaves no batch half done, and a
|
|
44
44
|
* resumed backfill runs no batch twice. A batch's statements must be ones
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
*/
|
|
56
56
|
import type { OpConfig } from "@intentius/chant/op/types";
|
|
57
57
|
import type { SqlDialect } from "@intentius/chant-lexicon-sql/dialects";
|
|
58
|
+
import type { ClickHouseEndpoint } from "@intentius/chant-lexicon-sql/clickhouse/http";
|
|
58
59
|
import type { ReceiptStore } from "@intentius/chant/op/receipt-store";
|
|
59
60
|
import type { StepRecord } from "@intentius/chant/op/local-executor";
|
|
60
61
|
import type { ClickHousePort } from "../history/index.js";
|
|
@@ -90,18 +91,21 @@ export interface BackfillSummary {
|
|
|
90
91
|
}
|
|
91
92
|
/** A receipt store whose every effect is under `prefix`, so receipts of different migrations never meet. */
|
|
92
93
|
export declare function prefixedStore(store: ReceiptStore, prefix: string): ReceiptStore;
|
|
93
|
-
/**
|
|
94
|
-
export
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
94
|
+
/** Where a backfill's receipts are kept: the environment's ClickHouse server, or the apply's Postgres connection and the schema to keep them in. */
|
|
95
|
+
export type ReceiptServer = {
|
|
96
|
+
clickhouse: ClickHouseEndpoint;
|
|
97
|
+
} | {
|
|
98
|
+
postgres: PgClient;
|
|
99
|
+
schema: string;
|
|
100
|
+
};
|
|
99
101
|
/**
|
|
100
|
-
* The receipt store on
|
|
101
|
-
*
|
|
102
|
-
*
|
|
102
|
+
* The sql lexicon's receipt store on the environment's server, under the
|
|
103
|
+
* stack `yodel`. On Postgres it writes over the apply's connection (so a
|
|
104
|
+
* receipt written inside a batch's transaction commits with it, and the
|
|
105
|
+
* store never closes it), and the table is made here, before any batch's
|
|
106
|
+
* transaction opens.
|
|
103
107
|
*/
|
|
104
|
-
export declare function
|
|
108
|
+
export declare function serverReceipts(server: ReceiptServer, environment: string): Promise<ReceiptStore>;
|
|
105
109
|
/** A backfill on Postgres: the connection its batches run on, and the environment's timeouts. */
|
|
106
110
|
export interface PgBackfillServer {
|
|
107
111
|
client: PgClient;
|
package/dist/steps/backfill.js
CHANGED
|
@@ -29,16 +29,16 @@
|
|
|
29
29
|
* to repeat (an idempotent `UPDATE`, or a delete of its rows before the
|
|
30
30
|
* insert), as the rebuild's backfill does per partition.
|
|
31
31
|
*
|
|
32
|
-
* The receipts are rows on the environment's server,
|
|
32
|
+
* The receipts are rows on the environment's server, kept by the sql
|
|
33
|
+
* lexicon's receipt store (`sqlReceiptStore`), addressed
|
|
33
34
|
* `yodel/<env>/<migration id>/<effect>`, so the same effect name in two
|
|
34
35
|
* migrations is two receipts:
|
|
35
36
|
*
|
|
36
|
-
* - ClickHouse:
|
|
37
|
-
*
|
|
38
|
-
* - Postgres: `<history schema>.__chant_receipts`,
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* transaction on the apply's connection, and its receipt is written last
|
|
37
|
+
* - ClickHouse: `chant_receipts.receipts` (the table `ClickHouseRebuildOp`
|
|
38
|
+
* keeps its partition receipts in).
|
|
39
|
+
* - Postgres: `<history schema>.__chant_receipts`, in the schema yodel's
|
|
40
|
+
* history is in, over the apply's connection. Each batch runs in one
|
|
41
|
+
* transaction on that connection, and its receipt is written last
|
|
42
42
|
* inside it: the batch's statements and its receipt commit together or
|
|
43
43
|
* not at all, so a run killed anywhere leaves no batch half done, and a
|
|
44
44
|
* resumed backfill runs no batch twice. A batch's statements must be ones
|
|
@@ -171,21 +171,21 @@ export function prefixedStore(store, prefix) {
|
|
|
171
171
|
write: (r, expectation) => store.write({ ...r, effect: `${prefix}${r.effect}` }, expectation),
|
|
172
172
|
};
|
|
173
173
|
}
|
|
174
|
-
/** The receipt store on a ClickHouse server: chant's, in `chant_receipts.receipts`. */
|
|
175
|
-
export async function serverReceipts(endpoint, environment) {
|
|
176
|
-
const { clickhouseReceiptStore } = await import("@intentius/chant-lexicon-sql/clickhouse/rebuild/receipts");
|
|
177
|
-
return clickhouseReceiptStore(endpoint, { stack: "yodel", env: environment });
|
|
178
|
-
}
|
|
179
174
|
/**
|
|
180
|
-
* The receipt store on
|
|
181
|
-
*
|
|
182
|
-
*
|
|
175
|
+
* The sql lexicon's receipt store on the environment's server, under the
|
|
176
|
+
* stack `yodel`. On Postgres it writes over the apply's connection (so a
|
|
177
|
+
* receipt written inside a batch's transaction commits with it, and the
|
|
178
|
+
* store never closes it), and the table is made here, before any batch's
|
|
179
|
+
* transaction opens.
|
|
183
180
|
*/
|
|
184
|
-
export async function
|
|
185
|
-
const {
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
181
|
+
export async function serverReceipts(server, environment) {
|
|
182
|
+
const { sqlReceiptStore } = await import("@intentius/chant-lexicon-sql/receipts");
|
|
183
|
+
const where = "clickhouse" in server
|
|
184
|
+
? { clickhouse: { endpoint: server.clickhouse } }
|
|
185
|
+
: { postgres: { query: (sql, params) => server.postgres.query(sql, params), end: () => Promise.resolve() }, schema: server.schema };
|
|
186
|
+
const store = sqlReceiptStore({ environment, stack: "yodel", config: {}, ...where });
|
|
187
|
+
if ("postgres" in server)
|
|
188
|
+
await store.ensure();
|
|
189
189
|
return store;
|
|
190
190
|
}
|
|
191
191
|
function clickhouseBatches(port) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/steps/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/steps/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAOzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAI3C,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,WAAW,EAAqB,MAAM,YAAY,CAAC;AAE3E,gGAAgG;AAChG,MAAM,CAAC,MAAM,YAAY,GAAG,UAAU,CAAC;AAEvC,2DAA2D;AAC3D,MAAM,UAAU,YAAY,CAAC,aAAqB,EAAE,IAAI,GAAG,aAAa;IACtE,OAAO;QACL,IAAI,EAAE,IAAI;QACV,MAAM,EAAE,UAAU;QAClB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,aAAa;QACnB,EAAE,EAAE,WAAW;QACf,UAAU,EAAE,KAAK,IAAI,EAAE;QACvB,OAAO,EAAE,EAAE,IAAI,EAAE;QACjB,WAAW,EAAE,sBAAsB,IAAI,gCAAgC;QACvE,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,oDAAoD,IAAI,0HAA0H;KAC3L,CAAC;AACJ,CAAC;AAED,uFAAuF;AACvF,MAAM,CAAC,MAAM,aAAa,GAAG,uBAAuB,CAAC;AAErD,oFAAoF;AACpF,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,UAAsB,YAAY;IAC/E,MAAM,IAAI,GACR,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC,sDAAsD,IAAI,2BAA2B;QACvF,CAAC,CAAC,4CAA4C,IAAI,oBAAoB,CAAC;IAC3E,OAAO;+BACsB,IAAI;;;sDAI/B,OAAO,KAAK,UAAU;QACpB,CAAC,CAAC;;iCAEyB;QAC3B,CAAC,CAAC;8DAEN;;;;;;;;;;;;;2BAayB,IAAI,yBAAyB,IAAI;;gBAE5C,YAAY,eAAe,IAAI;;;;oBAI3B,IAAI;sCACc,IAAI;;;CAGzC,CAAC;AACF,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAU,EAAgC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC;AAEnG,uEAAuE;AACvE,MAAM,UAAU,UAAU,CAAC,GAA4B;IACrD,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACxD,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;YAAE,OAAO,KAA4B,CAAC;IAC5H,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,qHAAqH;AACrH,MAAM,UAAU,gBAAgB,CAAC,MAAgB;IAC/C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACvF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAC7D,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;QACjC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,KAAK,WAAW,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,oGAAoG,CAAC,CAAC;IAC7O,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;IAC3H,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,yIAAyI;AACzI,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,IAAa;IAC5D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,0DAA0D,CAAC,CAAC;IAC/G,MAAM,IAAI,GAAG,IAAI,IAAI,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,GAA4B,CAAC;IACjC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,UAAU,OAAO,EAAE,CAAC,CAA4B,CAAC;IAClG,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,wBAAwB,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,qEAAqE,CAAC,CAAC;IAChH,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjF,OAAO,MAAM,CAAC;AAChB,CAAC;AAeD,4GAA4G;AAC5G,MAAM,UAAU,aAAa,CAAC,KAAmB,EAAE,MAAc;IAC/D,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;QACjE,KAAK,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,EAAE,WAAW,CAAC;KAC9F,CAAC;AACJ,CAAC;AAKD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAqB,EAAE,WAAmB;IAC7E,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,uCAAuC,CAAC,CAAC;IAClF,MAAM,KAAK,GACT,YAAY,IAAI,MAAM;QACpB,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE;QACjD,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAI,GAAW,EAAE,MAA2B,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAI,GAAG,EAAE,MAAM,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;IAC3K,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;IACrF,IAAI,UAAU,IAAI,MAAM;QAAE,MAAM,KAAK,CAAC,MAAM,EAAE,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAgBD,SAAS,iBAAiB,CAAC,IAAoB;IAC7C,OAAO;QACL,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM;QAC1F,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;QAC7B,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;KACjC,CAAC;AACJ,CAAC;AAED,mGAAmG;AACnG,SAAS,eAAe,CAAC,MAAwB;IAC/C,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IACpC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,oIAAoI;IACpI,IAAI,IAAwB,CAAC;IAC7B,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IAAI,CAAC,IAAI;YAAE,OAAO;QAClB,IAAI,GAAG,KAAK,CAAC;QACb,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC,CAAC;IACF,OAAO;QACL,KAAK,CAAC,IAAI;YACR,MAAM,QAAQ,EAAE,CAAC;YACjB,IAAI,GAAG,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ;YACrB,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,uDAAuD,CAAC,CAAC;YAClJ,IAAI,CAAC;gBACH,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC5B,IAAI,GAAG,IAAI,CAAC;oBACZ,MAAM,MAAM,CAAC,KAAK,CAAC,6BAA6B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;oBAC9G,MAAM,MAAM,CAAC,KAAK,CAAC,kCAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACrH,CAAC;gBACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;oBAAE,MAAM,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;gBAC5I,OAAO,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAC1C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,QAAQ,EAAE,CAAC;gBACjB,MAAM,CAAC,CAAC;YACV,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI;YACR,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,IAAI,GAAG,KAAK,CAAC;YACb,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,EAAE,QAAQ;KAClB,CAAC;AACJ,CAAC;AAiBD,gFAAgF;AAChF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,GAAuB;IACrE,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACjE,MAAM,OAAO,GAAoB,EAAE,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACjK,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAClH,IAAI,CAAC,OAAO;QAAE,MAAM,IAAI,UAAU,CAAC,gFAAgF,CAAC,CAAC;IACrH,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,mCAAmC,CAAC,CAAC;IAChF,MAAM,KAAK,GAAG,iBAAiB,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,GAAG,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IACpF,MAAM,UAAU,GAA+B;QAC7C,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC1B,uDAAuD;YACvD,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,IAA0D,CAAC,CAAC;YAC9F,IAAI,CAAC,CAAC,OAAO;gBAAE,OAAO,CAAC,OAAO,EAAE,CAAC;YACjC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC3B,IAAI,CAAiD,CAAC;YACtD,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAA2D,CAAC,CAAC;gBAC1F,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;gBACxB,MAAM,CAAC,CAAC;YACV,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,CAAC;QACX,CAAC;QACD,gBAAgB,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAA+D,CAAC;QACnH,CAAC,YAAY,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;YACrB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,GAAG,YAAY,gCAAgC,CAAC,CAAC;YACnH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAE,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAA4B,CAAC,CAAC,CAAC,SAAS,CAAC;YACrK,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7C,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;QACpD,CAAC;KACF,CAAC;IACF,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YAClC,GAAG,EAAE,GAAG,CAAC,UAAU;YACnB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,UAAU;YACV,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7C,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;QACH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,YAAY;YAAE,CAAC,CAAC,OAAO,GAAG,OAAO,CAAC;QACnD,MAAM,CAAC,CAAC;IACV,CAAC;YAAS,CAAC;QACT,8FAA8F;QAC9F,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC"}
|
package/dist/steps/index.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export interface StepContext {
|
|
|
32
32
|
}) | undefined;
|
|
33
33
|
/** Postgres: the environment's lock timeout, for a PostgresMigrationOp step whose options set none. */
|
|
34
34
|
lockTimeoutMs?: number | undefined;
|
|
35
|
-
/** The server, for
|
|
35
|
+
/** The server, for the backfill's receipt store; the port is enough for everything else. */
|
|
36
36
|
endpoint?: ClickHouseEndpoint | undefined;
|
|
37
37
|
migration: Migration;
|
|
38
38
|
index: number;
|
|
@@ -42,7 +42,7 @@ export interface StepContext {
|
|
|
42
42
|
run: "refused";
|
|
43
43
|
}>;
|
|
44
44
|
log: (line: string) => void;
|
|
45
|
-
/** A backfill's receipt store; default
|
|
45
|
+
/** A backfill's receipt store; default the sql lexicon's on the server. */
|
|
46
46
|
receipts?: ReceiptStore | undefined;
|
|
47
47
|
}
|
|
48
48
|
/** Runs one Op step to the end and returns the summary its history row records. Throws `StepOpFailed` when it stops part way. */
|
package/dist/steps/index.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* defaultStepRunner what the versioned apply calls for an Op step
|
|
10
10
|
*/
|
|
11
11
|
import { YodelError } from "../project.js";
|
|
12
|
-
import {
|
|
12
|
+
import { runBackfill, serverReceipts } from "./backfill.js";
|
|
13
13
|
import { rebuildChantConfig, runRebuild } from "./rebuild.js";
|
|
14
14
|
import { runPgMigration } from "./postgres-migration.js";
|
|
15
15
|
export * from "./kinds.js";
|
|
@@ -31,9 +31,9 @@ export const defaultStepRunner = async (ctx) => {
|
|
|
31
31
|
throw new YodelError("a backfill step runs its SQL on the environment's server, and this environment has none");
|
|
32
32
|
const receipts = ctx.receipts ??
|
|
33
33
|
(ctx.postgres
|
|
34
|
-
? await
|
|
34
|
+
? await serverReceipts({ postgres: ctx.postgres.client, schema: ctx.postgres.receiptsSchema }, ctx.environment)
|
|
35
35
|
: ctx.endpoint
|
|
36
|
-
? await serverReceipts(ctx.endpoint, ctx.environment)
|
|
36
|
+
? await serverReceipts({ clickhouse: ctx.endpoint }, ctx.environment)
|
|
37
37
|
: undefined);
|
|
38
38
|
if (!receipts)
|
|
39
39
|
throw new YodelError("a backfill step needs a receipt store, and this environment has no server endpoint for one");
|
package/dist/steps/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/steps/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/steps/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,WAAW,EAAE,cAAc,EAAyB,MAAM,eAAe,CAAC;AAEnF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAEzD,cAAc,YAAY,CAAC;AAC3B,cAAc,UAAU,CAAC;AACzB,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,yBAAyB,CAAC;AA0BxC,MAAM,CAAC,MAAM,iBAAiB,GAAe,KAAK,EAAE,GAAG,EAAE,EAAE;IACzD,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAClL,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa,EAAE,CAAC;QACnC,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC;QAChN,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,MAAM,IAAI,UAAU,CAAC,yFAAyF,CAAC,CAAC;IAChJ,MAAM,QAAQ,GACZ,GAAG,CAAC,QAAQ;QACZ,CAAC,GAAG,CAAC,QAAQ;YACX,CAAC,CAAC,MAAM,cAAc,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,cAAc,EAAE,EAAE,GAAG,CAAC,WAAW,CAAC;YAC/G,CAAC,CAAC,GAAG,CAAC,QAAQ;gBACZ,CAAC,CAAC,MAAM,cAAc,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,WAAW,CAAC;gBACrE,CAAC,CAAC,SAAS,CAAC,CAAC;IACnB,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,UAAU,CAAC,4FAA4F,CAAC,CAAC;IAClI,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;QACnD,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,WAAW,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE;QAC7B,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,GAAG;QACtB,IAAI,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;QAChD,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;QACvH,QAAQ;QACR,GAAG,EAAE,GAAG,CAAC,GAAG;KACb,CAAC,CAAC;IACH,OAAO,EAAE,OAAO,EAAE,CAAC;AACrB,CAAC,CAAC"}
|
|
@@ -15,40 +15,50 @@
|
|
|
15
15
|
*
|
|
16
16
|
* - The declaration is the migration's recorded schema (`migration.json`'s
|
|
17
17
|
* `schema`), written to a temporary build file, not a fresh `chant build`.
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* - It is built with `gates: "outer"`: the approval is the migration's (the
|
|
19
|
+
* step and its options are in the migration's checksum, so in the plan
|
|
20
|
+
* digest the migrations Op's gate approved), so the Op has neither of its
|
|
21
|
+
* gates (before the switch, before the contract) and no Contract phase of
|
|
22
|
+
* its own. The verification still runs before the switch, and a
|
|
22
23
|
* difference (a row whose new column is not the expression over the old,
|
|
23
24
|
* a NULL where NOT NULL is declared) fails the step with nothing switched.
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
25
|
+
* - After the Retain phase the step runs the lexicon's contract activity
|
|
26
|
+
* (`postgresMigrationContract`, the Op's own Contract step, with the same
|
|
27
|
+
* arguments), under the same approval. It drops the old column (and a
|
|
28
|
+
* rename's trigger) only once its retention date has passed. The
|
|
29
|
+
* retention is the step's `options.retain` when it sets one, else none
|
|
30
|
+
* (`0s`): the contract drops the old column in the same run, right after
|
|
31
|
+
* the switch, as the migration's recorded schema says it is gone. A step
|
|
32
|
+
* that sets `retain` keeps the old column that long; the step succeeds
|
|
33
|
+
* with it kept, its note says until when, and dropping it after the date
|
|
34
|
+
* is `chant run` of the Op's declaration (in `migration.json`), whose
|
|
35
|
+
* contract is convergent.
|
|
36
|
+
* - It is built with `onFailure: "keep"`, so a step that failed or was
|
|
37
|
+
* stopped keeps what the expand added and resumes on the next `yodel
|
|
38
|
+
* apply`. Every phase of the Op re-reads the server and does what is left
|
|
39
|
+
* of it: the expand finds the new column there and adds nothing, the dual
|
|
40
|
+
* write finds its trigger, the backfill skips every batch whose receipt
|
|
41
|
+
* is in `<schema>.__chant_receipts` (each receipt commits with its batch,
|
|
42
|
+
* so a batch is never filled twice), the carry over builds only the
|
|
43
|
+
* indexes and constraints not yet built (an INVALID one it left is built
|
|
44
|
+
* again), the verification runs again over every row, and a switch that
|
|
45
|
+
* already ran is not run again. To start over instead, drop the new
|
|
46
|
+
* column, its trigger and function (chant's working objects, named in
|
|
47
|
+
* their comments; a refusal says so).
|
|
42
48
|
*/
|
|
43
|
-
import type { OpConfig
|
|
49
|
+
import type { OpConfig } from "@intentius/chant/op/types";
|
|
44
50
|
import type { ActivityFn } from "@intentius/chant/op/activity-registry";
|
|
45
51
|
import type { StepRecord } from "@intentius/chant/op/local-executor";
|
|
46
52
|
import type { PgMigrationOptions } from "./kinds.js";
|
|
47
53
|
import { type RunOpOptions } from "./run.js";
|
|
48
54
|
/** The retention a step runs with when its options set none: the contract runs in the same apply. */
|
|
49
55
|
export declare const STEP_RETAIN = "0s";
|
|
50
|
-
/**
|
|
51
|
-
|
|
56
|
+
/**
|
|
57
|
+
* The Op's Contract step for a config built with `gates: "outer"`: the
|
|
58
|
+
* lexicon's contract activity with the Retain step's arguments. It drops the
|
|
59
|
+
* old column only once its retention date has passed.
|
|
60
|
+
*/
|
|
61
|
+
export declare function contractPhase(config: OpConfig): OpConfig["phases"][number];
|
|
52
62
|
/** The Op a step runs. `buildPath` is the recorded schema's file; `projectDir` holds chant.config.ts. */
|
|
53
63
|
export declare function pgMigrationOpConfig(options: PgMigrationOptions, ctx: {
|
|
54
64
|
environment: string;
|
|
@@ -15,30 +15,36 @@
|
|
|
15
15
|
*
|
|
16
16
|
* - The declaration is the migration's recorded schema (`migration.json`'s
|
|
17
17
|
* `schema`), written to a temporary build file, not a fresh `chant build`.
|
|
18
|
-
* -
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* - It is built with `gates: "outer"`: the approval is the migration's (the
|
|
19
|
+
* step and its options are in the migration's checksum, so in the plan
|
|
20
|
+
* digest the migrations Op's gate approved), so the Op has neither of its
|
|
21
|
+
* gates (before the switch, before the contract) and no Contract phase of
|
|
22
|
+
* its own. The verification still runs before the switch, and a
|
|
22
23
|
* difference (a row whose new column is not the expression over the old,
|
|
23
24
|
* a NULL where NOT NULL is declared) fails the step with nothing switched.
|
|
24
|
-
* -
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
25
|
+
* - After the Retain phase the step runs the lexicon's contract activity
|
|
26
|
+
* (`postgresMigrationContract`, the Op's own Contract step, with the same
|
|
27
|
+
* arguments), under the same approval. It drops the old column (and a
|
|
28
|
+
* rename's trigger) only once its retention date has passed. The
|
|
29
|
+
* retention is the step's `options.retain` when it sets one, else none
|
|
30
|
+
* (`0s`): the contract drops the old column in the same run, right after
|
|
31
|
+
* the switch, as the migration's recorded schema says it is gone. A step
|
|
32
|
+
* that sets `retain` keeps the old column that long; the step succeeds
|
|
33
|
+
* with it kept, its note says until when, and dropping it after the date
|
|
34
|
+
* is `chant run` of the Op's declaration (in `migration.json`), whose
|
|
35
|
+
* contract is convergent.
|
|
36
|
+
* - It is built with `onFailure: "keep"`, so a step that failed or was
|
|
37
|
+
* stopped keeps what the expand added and resumes on the next `yodel
|
|
38
|
+
* apply`. Every phase of the Op re-reads the server and does what is left
|
|
39
|
+
* of it: the expand finds the new column there and adds nothing, the dual
|
|
40
|
+
* write finds its trigger, the backfill skips every batch whose receipt
|
|
41
|
+
* is in `<schema>.__chant_receipts` (each receipt commits with its batch,
|
|
42
|
+
* so a batch is never filled twice), the carry over builds only the
|
|
43
|
+
* indexes and constraints not yet built (an INVALID one it left is built
|
|
44
|
+
* again), the verification runs again over every row, and a switch that
|
|
45
|
+
* already ran is not run again. To start over instead, drop the new
|
|
46
|
+
* column, its trigger and function (chant's working objects, named in
|
|
47
|
+
* their comments; a refusal says so).
|
|
42
48
|
*/
|
|
43
49
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
44
50
|
import { tmpdir } from "node:os";
|
|
@@ -46,9 +52,16 @@ import { join } from "node:path";
|
|
|
46
52
|
import { runOp, StepOpFailed } from "./run.js";
|
|
47
53
|
/** The retention a step runs with when its options set none: the contract runs in the same apply. */
|
|
48
54
|
export const STEP_RETAIN = "0s";
|
|
49
|
-
/**
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
/**
|
|
56
|
+
* The Op's Contract step for a config built with `gates: "outer"`: the
|
|
57
|
+
* lexicon's contract activity with the Retain step's arguments. It drops the
|
|
58
|
+
* old column only once its retention date has passed.
|
|
59
|
+
*/
|
|
60
|
+
export function contractPhase(config) {
|
|
61
|
+
const retain = config.phases.flatMap((p) => p.steps).find((s) => s.kind === "activity" && s.fn === "postgresMigrationRetain");
|
|
62
|
+
if (!retain)
|
|
63
|
+
throw new Error(`PostgresMigrationOp "${config.name}" has no Retain step`);
|
|
64
|
+
return { name: "Contract", steps: [{ kind: "activity", fn: "postgresMigrationContract", ...(retain.args ? { args: retain.args } : {}) }] };
|
|
52
65
|
}
|
|
53
66
|
/** The Op a step runs. `buildPath` is the recorded schema's file; `projectDir` holds chant.config.ts. */
|
|
54
67
|
export async function pgMigrationOpConfig(options, ctx) {
|
|
@@ -68,11 +81,11 @@ export async function pgMigrationOpConfig(options, ctx) {
|
|
|
68
81
|
build: false,
|
|
69
82
|
output: ctx.buildPath,
|
|
70
83
|
path: ctx.projectDir,
|
|
84
|
+
gates: "outer",
|
|
85
|
+
onFailure: "keep",
|
|
71
86
|
});
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
delete rest.onFailure;
|
|
75
|
-
return rest;
|
|
87
|
+
const config = op.props;
|
|
88
|
+
return { ...config, phases: [...config.phases, contractPhase(config)] };
|
|
76
89
|
}
|
|
77
90
|
/** The migration activities whose results the history's summary keeps. */
|
|
78
91
|
const SUMMARIZED = ["postgresMigrationPlan", "postgresMigrationBackfill", "postgresMigrationCarry", "postgresMigrationVerify", "postgresMigrationSwitch", "postgresMigrationContract"];
|