@intentius/chant-lexicon-sql 0.114.1 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clickhouse/apply/mutations.d.ts +7 -1
- package/dist/clickhouse/apply/mutations.d.ts.map +1 -1
- package/dist/clickhouse/container.d.ts +22 -7
- package/dist/clickhouse/container.d.ts.map +1 -1
- package/dist/clickhouse/plan/deep.d.ts +5 -0
- package/dist/clickhouse/plan/deep.d.ts.map +1 -1
- package/dist/clickhouse/plan/normalize.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/backfill.d.ts +11 -5
- package/dist/clickhouse/rebuild/backfill.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/op.d.ts +29 -0
- package/dist/clickhouse/rebuild/op.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/partitions.d.ts +8 -0
- package/dist/clickhouse/rebuild/partitions.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/receipts.d.ts +4 -6
- package/dist/clickhouse/rebuild/receipts.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/shards.d.ts +59 -0
- package/dist/clickhouse/rebuild/shards.d.ts.map +1 -0
- package/dist/clickhouse/rebuild/steps.d.ts +3 -0
- package/dist/clickhouse/rebuild/steps.d.ts.map +1 -1
- package/dist/clickhouse/rebuild/verify.d.ts +4 -0
- package/dist/clickhouse/rebuild/verify.d.ts.map +1 -1
- package/dist/codegen/registry.d.ts.map +1 -1
- package/dist/config.d.ts +14 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/core/receipts.d.ts +5 -3
- package/dist/core/receipts.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/integrity.json +5 -5
- package/dist/lint/rules/sqlpg001.d.ts.map +1 -1
- package/dist/lsp/postgres-template.d.ts.map +1 -1
- package/dist/manifest.json +1 -1
- package/dist/meta.json +15 -0
- package/dist/okf/index.md +3 -0
- package/dist/okf/types/PostgresFunction.md +9 -0
- package/dist/okf/types/PostgresProcedure.md +9 -0
- package/dist/okf/types/PostgresTrigger.md +9 -0
- package/dist/op/activities/clickhouse-rebuild.d.ts.map +1 -1
- package/dist/op/activities/index.d.ts +13 -1
- package/dist/op/activities/index.d.ts.map +1 -1
- package/dist/op/activities/sql-exec.d.ts +41 -0
- package/dist/op/activities/sql-exec.d.ts.map +1 -0
- package/dist/op/activity-contracts.d.ts +20 -0
- package/dist/op/activity-contracts.d.ts.map +1 -1
- package/dist/plugin.d.ts.map +1 -1
- package/dist/postgres/apply/apply.d.ts +6 -1
- package/dist/postgres/apply/apply.d.ts.map +1 -1
- package/dist/postgres/apply/statements.d.ts +9 -2
- package/dist/postgres/apply/statements.d.ts.map +1 -1
- package/dist/postgres/entities.d.ts +110 -3
- package/dist/postgres/entities.d.ts.map +1 -1
- package/dist/postgres/entity-types.d.ts +3 -0
- package/dist/postgres/entity-types.d.ts.map +1 -1
- package/dist/postgres/import/generator.d.ts.map +1 -1
- package/dist/postgres/import/live-export.d.ts.map +1 -1
- package/dist/postgres/live/catalog.d.ts +17 -5
- package/dist/postgres/live/catalog.d.ts.map +1 -1
- package/dist/postgres/live/describe-resources.d.ts +6 -1
- package/dist/postgres/live/describe-resources.d.ts.map +1 -1
- package/dist/postgres/migrate/receipts.d.ts +8 -5
- package/dist/postgres/migrate/receipts.d.ts.map +1 -1
- package/dist/postgres/parser.d.ts +82 -5
- package/dist/postgres/parser.d.ts.map +1 -1
- package/dist/postgres/plan/deep.d.ts.map +1 -1
- package/dist/postgres/plan/diff.d.ts +1 -1
- package/dist/postgres/plan/diff.d.ts.map +1 -1
- package/dist/postgres/plan/normalize.d.ts +21 -2
- package/dist/postgres/plan/normalize.d.ts.map +1 -1
- package/dist/postgres/plan/rules.d.ts +6 -0
- package/dist/postgres/plan/rules.d.ts.map +1 -1
- package/dist/postgres/plan/schema.d.ts +11 -1
- package/dist/postgres/plan/schema.d.ts.map +1 -1
- package/dist/postgres/plan/server-normalize.d.ts.map +1 -1
- package/dist/postgres.d.ts +1 -1
- package/dist/postgres.d.ts.map +1 -1
- package/dist/receipts.d.ts +102 -0
- package/dist/receipts.d.ts.map +1 -0
- package/dist/rules/postgres-templates.ts +2 -2
- package/dist/rules/sqlpg001.ts +3 -0
- package/dist/validate.d.ts.map +1 -1
- package/package.json +2 -2
- package/src/clickhouse/apply/mutations.ts +9 -3
- package/src/clickhouse/container.ts +39 -11
- package/src/clickhouse/plan/cluster-drift.e2e.test.ts +117 -0
- package/src/clickhouse/plan/deep.ts +102 -3
- package/src/clickhouse/plan/normalize.ts +10 -0
- package/src/clickhouse/plan/plan.test.ts +10 -0
- package/src/clickhouse/rebuild/backfill.ts +85 -42
- package/src/clickhouse/rebuild/op.ts +80 -35
- package/src/clickhouse/rebuild/partitions.ts +11 -0
- package/src/clickhouse/rebuild/rebuild-sharded.e2e.test.ts +161 -0
- package/src/clickhouse/rebuild/rebuild.e2e.test.ts +110 -4
- package/src/clickhouse/rebuild/rebuild.test.ts +56 -4
- package/src/clickhouse/rebuild/receipts.ts +4 -6
- package/src/clickhouse/rebuild/shards.ts +104 -0
- package/src/clickhouse/rebuild/steps.ts +4 -0
- package/src/clickhouse/rebuild/verify.ts +14 -5
- package/src/codegen/postgres-classifier-page.ts +1 -1
- package/src/codegen/registry.ts +3 -0
- package/src/config.ts +7 -0
- package/src/core/receipts.ts +5 -3
- package/src/generated/lexicon-sql.json +15 -0
- package/src/index.ts +1 -1
- package/src/lint/rules/postgres-templates.ts +2 -2
- package/src/lint/rules/sqlpg001.ts +3 -0
- package/src/lsp/postgres-template.ts +6 -0
- package/src/mcp/postgres.ts +1 -1
- package/src/op/activities/clickhouse-rebuild.ts +32 -7
- package/src/op/activities/index.ts +21 -4
- package/src/op/activities/sql-exec.ts +84 -0
- package/src/op/activity-contracts.ts +15 -0
- package/src/op/effect-receipts.e2e.test.ts +277 -0
- package/src/plugin.ts +3 -0
- package/src/postgres/apply/apply.ts +21 -5
- package/src/postgres/apply/statements.ts +53 -11
- package/src/postgres/entities.ts +332 -9
- package/src/postgres/entity-types.ts +3 -0
- package/src/postgres/import/generator.ts +5 -2
- package/src/postgres/import/live-export.ts +4 -0
- package/src/postgres/live/catalog.ts +81 -5
- package/src/postgres/live/describe-resources.ts +17 -7
- package/src/postgres/live/routines.e2e.test.ts +173 -0
- package/src/postgres/migrate/batches.e2e.test.ts +2 -1
- package/src/postgres/migrate/migrate.e2e.test.ts +2 -1
- package/src/postgres/migrate/migrate.test.ts +3 -2
- package/src/postgres/migrate/receipts.ts +18 -10
- package/src/postgres/parser.ts +314 -8
- package/src/postgres/plan/commands.ts +2 -2
- package/src/postgres/plan/deep.ts +57 -17
- package/src/postgres/plan/diff.ts +46 -6
- package/src/postgres/plan/normalize.ts +158 -4
- package/src/postgres/plan/rules.ts +36 -0
- package/src/postgres/plan/schema.ts +25 -6
- package/src/postgres/plan/server-normalize.ts +24 -0
- package/src/postgres/routines.test.ts +150 -0
- package/src/postgres.ts +11 -0
- package/src/receipts.test.ts +91 -0
- package/src/receipts.ts +259 -0
- package/src/skill-defs.ts +1 -1
- package/src/skills/chant-sql-postgres.md +18 -2
- package/src/skills/chant-sql-rebuild.md +5 -1
- package/src/validate.ts +3 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A project that uses only the sql lexicon runs an Op of `effect()` batches
|
|
3
|
+
* with `chant run`, and its receipts are kept on the environment's server
|
|
4
|
+
* (#3657), end to end on both dialects.
|
|
5
|
+
*
|
|
6
|
+
* For each dialect, a project in a temporary directory declares an Op of
|
|
7
|
+
* three batches. Each batch is an `effect()` whose steps run SQL with
|
|
8
|
+
* `sqlExec`: the batch records itself in a log table. The second batch fails
|
|
9
|
+
* while a stop table holds a row. The first `chant run --env admin` runs the
|
|
10
|
+
* first batch, fails on the second, and leaves one receipt. With the stop
|
|
11
|
+
* row deleted, the second run skips the first batch (the log holds it once),
|
|
12
|
+
* runs the other two, and leaves three receipts, in
|
|
13
|
+
* `chant_receipts.receipts` under `chant_e2e_3657/admin/`. A third run runs
|
|
14
|
+
* nothing.
|
|
15
|
+
*
|
|
16
|
+
* The test's own objects are in the database (ClickHouse) or schema
|
|
17
|
+
* (Postgres) `chant_e2e_3657`; its receipts are the rows of
|
|
18
|
+
* `chant_receipts.receipts` whose address starts `chant_e2e_3657/`, deleted
|
|
19
|
+
* afterwards.
|
|
20
|
+
*
|
|
21
|
+
* The servers: `CHANT_SQL_RECEIPTS_CLICKHOUSE_URL` and
|
|
22
|
+
* `CHANT_SQL_RECEIPTS_POSTGRES_URL` (a `postgres://user@host:port/db` URL, with
|
|
23
|
+
* `CHANT_SQL_RECEIPTS_POSTGRES_PASSWORD`) name running ones, such as the
|
|
24
|
+
* servers `chant emulator up --lexicon sql` starts. Without them, each dialect
|
|
25
|
+
* starts a throwaway server at the same pin, and skips cleanly when Docker is
|
|
26
|
+
* not available.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { execFile } from "node:child_process";
|
|
30
|
+
import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
|
31
|
+
import { tmpdir } from "node:os";
|
|
32
|
+
import { join } from "node:path";
|
|
33
|
+
import { promisify } from "node:util";
|
|
34
|
+
import { afterAll, beforeAll, describe, expect, test } from "vitest";
|
|
35
|
+
import { dockerAvailable, startScratchServer, type ScratchServer } from "../clickhouse/container";
|
|
36
|
+
import { clickhouseQuery, type ClickHouseEndpoint } from "../clickhouse/http";
|
|
37
|
+
import { clickhouseImage } from "../spec/pin";
|
|
38
|
+
import { startTestPostgres, type TestPostgres } from "../postgres/testing/server";
|
|
39
|
+
import { connectPostgres, type PostgresClient, type PostgresEndpoint } from "../postgres/live/client";
|
|
40
|
+
|
|
41
|
+
const exec = promisify(execFile);
|
|
42
|
+
|
|
43
|
+
const REPO = join(import.meta.dirname, "../../../..");
|
|
44
|
+
const BIN = join(REPO, "node_modules/.bin");
|
|
45
|
+
const NAME = "chant_e2e_3657";
|
|
46
|
+
const OP = "fill-batches";
|
|
47
|
+
const ENV = "admin";
|
|
48
|
+
const PREFIX = `${NAME}/${ENV}/`;
|
|
49
|
+
|
|
50
|
+
const givenClickHouse = process.env.CHANT_SQL_RECEIPTS_CLICKHOUSE_URL;
|
|
51
|
+
const givenPostgres = process.env.CHANT_SQL_RECEIPTS_POSTGRES_URL;
|
|
52
|
+
const docker = givenClickHouse && givenPostgres ? false : await dockerAvailable();
|
|
53
|
+
|
|
54
|
+
interface Run {
|
|
55
|
+
code: number;
|
|
56
|
+
output: string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const strip = (s: string) => s.replace(/\u001b\[[0-9;]*m/g, "");
|
|
60
|
+
|
|
61
|
+
/** A project directory chant can run in: the repository's `node_modules` linked in, and a git repository. */
|
|
62
|
+
async function project(files: Record<string, string>, env: Record<string, string>) {
|
|
63
|
+
const dir = mkdtempSync(join(tmpdir(), "chant-sql-receipts-"));
|
|
64
|
+
symlinkSync(join(REPO, "node_modules"), join(dir, "node_modules"), "dir");
|
|
65
|
+
for (const [path, text] of Object.entries({
|
|
66
|
+
"package.json": JSON.stringify({ name: "sql-receipts-e2e", private: true, type: "module" }),
|
|
67
|
+
".gitignore": "node_modules\ndist\n.chant\n",
|
|
68
|
+
...files,
|
|
69
|
+
})) {
|
|
70
|
+
mkdirSync(join(dir, path, ".."), { recursive: true });
|
|
71
|
+
writeFileSync(join(dir, path), text);
|
|
72
|
+
}
|
|
73
|
+
const git = (...args: string[]) => exec("git", ["-c", "user.email=e2e@chant.invalid", "-c", "user.name=e2e", ...args], { cwd: dir });
|
|
74
|
+
await git("init", "-q");
|
|
75
|
+
await git("add", "-A");
|
|
76
|
+
await git("commit", "-qm", "the op");
|
|
77
|
+
const chant = async (...args: string[]): Promise<Run> => {
|
|
78
|
+
try {
|
|
79
|
+
const r = await exec(join(BIN, "chant"), args, {
|
|
80
|
+
cwd: dir,
|
|
81
|
+
env: { ...process.env, ...env, PATH: `${BIN}:${process.env.PATH ?? ""}` },
|
|
82
|
+
maxBuffer: 16 * 1024 * 1024,
|
|
83
|
+
});
|
|
84
|
+
return { code: 0, output: strip(`${r.stdout}${r.stderr}`) };
|
|
85
|
+
} catch (err) {
|
|
86
|
+
const e = err as { code?: number; stdout?: string; stderr?: string };
|
|
87
|
+
return { code: typeof e.code === "number" ? e.code : 1, output: strip(`${e.stdout ?? ""}${e.stderr ?? ""}`) };
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
return { dir, chant };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const config = (sql: Record<string, unknown>) => `import type { ChantConfig } from "@intentius/chant";
|
|
94
|
+
import "@intentius/chant-lexicon-sql";
|
|
95
|
+
|
|
96
|
+
export default {
|
|
97
|
+
lexicons: ["sql"],
|
|
98
|
+
ownership: { stack: "${NAME}", env: "${ENV}" },
|
|
99
|
+
sql: ${JSON.stringify(sql, null, 2)},
|
|
100
|
+
} satisfies ChantConfig;
|
|
101
|
+
`;
|
|
102
|
+
|
|
103
|
+
/** The Op: three batches; each runs `stop` (which fails while the stop table holds a row) and then `record`. */
|
|
104
|
+
const opFile = (stop: string, record: (batch: string) => string) => `import { EffectReceipt } from "@intentius/chant";
|
|
105
|
+
import { Op, activity, effect, phase } from "@intentius/chant/op";
|
|
106
|
+
|
|
107
|
+
const batch = (b: string, steps: ReturnType<typeof activity>[]) =>
|
|
108
|
+
effect(EffectReceipt(\`fill-\${b}\`, { effect: \`fill/\${b}\`, flavor: "existence" }), steps);
|
|
109
|
+
const record = (b: string) => activity("sqlExec", { sql: ${JSON.stringify(record("__B__"))}.replace("__B__", b) }, "atMostOnce");
|
|
110
|
+
|
|
111
|
+
const op = Op({
|
|
112
|
+
name: "${OP}",
|
|
113
|
+
overview: "three batches, each with its receipt",
|
|
114
|
+
phases: [
|
|
115
|
+
phase("Fill", [
|
|
116
|
+
batch("b1", [record("b1")]),
|
|
117
|
+
batch("b2", [activity("sqlExec", { sql: ${JSON.stringify(stop)} }, "atMostOnce"), record("b2")]),
|
|
118
|
+
batch("b3", [record("b3")]),
|
|
119
|
+
]),
|
|
120
|
+
],
|
|
121
|
+
});
|
|
122
|
+
export default op;
|
|
123
|
+
`;
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The shared scenario. `log` reads the batches the log table holds, in
|
|
127
|
+
* order; `receipts` the receipt addresses under the test's prefix; `unstop`
|
|
128
|
+
* deletes the stop row.
|
|
129
|
+
*/
|
|
130
|
+
async function failThenResume(p: Awaited<ReturnType<typeof project>>, io: { log: () => Promise<string[]>; receipts: () => Promise<string[]>; unstop: () => Promise<void> }) {
|
|
131
|
+
const first = await p.chant("run", OP, "--env", ENV);
|
|
132
|
+
expect(first.code, first.output).not.toBe(0);
|
|
133
|
+
expect(await io.log()).toEqual(["b1"]);
|
|
134
|
+
expect(await io.receipts()).toEqual([`${PREFIX}fill/b1`]);
|
|
135
|
+
|
|
136
|
+
await io.unstop();
|
|
137
|
+
const second = await p.chant("run", OP, "--env", ENV);
|
|
138
|
+
expect(second.code, second.output).toBe(0);
|
|
139
|
+
// b1 ran once: the second run read its receipt and skipped it.
|
|
140
|
+
expect(await io.log()).toEqual(["b1", "b2", "b3"]);
|
|
141
|
+
expect(await io.receipts()).toEqual([`${PREFIX}fill/b1`, `${PREFIX}fill/b2`, `${PREFIX}fill/b3`]);
|
|
142
|
+
|
|
143
|
+
const third = await p.chant("run", OP, "--env", ENV);
|
|
144
|
+
expect(third.code, third.output).toBe(0);
|
|
145
|
+
expect(await io.log()).toEqual(["b1", "b2", "b3"]);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// ── ClickHouse ──────────────────────────────────────────────────────────────
|
|
149
|
+
|
|
150
|
+
describe.skipIf(!givenClickHouse && !docker)("effect() batches on ClickHouse keep their receipts in chant_receipts.receipts (#3657)", () => {
|
|
151
|
+
let server: ScratchServer | undefined;
|
|
152
|
+
let admin: ClickHouseEndpoint;
|
|
153
|
+
let p: Awaited<ReturnType<typeof project>>;
|
|
154
|
+
const q = <T = Record<string, unknown>>(sql: string) => clickhouseQuery<T>(admin, sql);
|
|
155
|
+
const forget = async () => {
|
|
156
|
+
const [t] = await q<{ n: string }>("SELECT count() AS n FROM system.tables WHERE database = 'chant_receipts' AND name = 'receipts'");
|
|
157
|
+
if (Number(t?.n ?? 0) > 0) await clickhouseQuery(admin, `ALTER TABLE chant_receipts.receipts DELETE WHERE startsWith(address, '${NAME}/')`, { settings: { mutations_sync: "1" } });
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
beforeAll(async () => {
|
|
161
|
+
if (givenClickHouse) admin = { url: givenClickHouse };
|
|
162
|
+
else {
|
|
163
|
+
server = await startScratchServer(clickhouseImage(), { namePrefix: "chant-sql-receipts" });
|
|
164
|
+
admin = server.endpoint;
|
|
165
|
+
}
|
|
166
|
+
await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`);
|
|
167
|
+
await forget();
|
|
168
|
+
await q(`CREATE DATABASE ${NAME}`);
|
|
169
|
+
await q(`CREATE TABLE ${NAME}.log (batch String, at DateTime64(3) DEFAULT now64(3)) ENGINE = MergeTree ORDER BY at`);
|
|
170
|
+
await q(`CREATE TABLE ${NAME}.stop (x UInt8) ENGINE = MergeTree ORDER BY x`);
|
|
171
|
+
await q(`INSERT INTO ${NAME}.stop VALUES (1)`);
|
|
172
|
+
p = await project(
|
|
173
|
+
{
|
|
174
|
+
"chant.config.ts": config({ dialect: "clickhouse", profiles: { [ENV]: { url: admin.url, databases: [NAME] } } }),
|
|
175
|
+
"ops/fill.op.ts": opFile(`SELECT throwIf(count() > 0, 'stopped') FROM ${NAME}.stop`, (b) => `INSERT INTO ${NAME}.log (batch) VALUES ('${b}')`),
|
|
176
|
+
},
|
|
177
|
+
{},
|
|
178
|
+
);
|
|
179
|
+
}, 600_000);
|
|
180
|
+
|
|
181
|
+
afterAll(async () => {
|
|
182
|
+
if (admin) {
|
|
183
|
+
await q(`DROP DATABASE IF EXISTS ${NAME} SYNC`).catch(() => undefined);
|
|
184
|
+
await forget().catch(() => undefined);
|
|
185
|
+
}
|
|
186
|
+
await server?.stop();
|
|
187
|
+
if (p) rmSync(p.dir, { recursive: true, force: true });
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
test("a failed run keeps the finished batch's receipt; the rerun skips it", async () => {
|
|
191
|
+
await failThenResume(p, {
|
|
192
|
+
log: async () => (await q<{ batch: string }>(`SELECT batch FROM ${NAME}.log ORDER BY at, batch`)).map((r) => r.batch),
|
|
193
|
+
receipts: async () =>
|
|
194
|
+
(await q<{ address: string }>(`SELECT DISTINCT address FROM chant_receipts.receipts WHERE startsWith(address, '${NAME}/') ORDER BY address`)).map((r) => r.address),
|
|
195
|
+
unstop: async () => {
|
|
196
|
+
await q(`TRUNCATE TABLE ${NAME}.stop`);
|
|
197
|
+
},
|
|
198
|
+
});
|
|
199
|
+
// The receipts database is chant's: its comment carries the receipts key.
|
|
200
|
+
const [db] = await q<{ comment: string }>("SELECT comment FROM system.databases WHERE name = 'chant_receipts'");
|
|
201
|
+
expect(db?.comment).toContain("receipts=effects");
|
|
202
|
+
}, 600_000);
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
// ── Postgres ────────────────────────────────────────────────────────────────
|
|
206
|
+
|
|
207
|
+
describe.skipIf(!givenPostgres && !docker)("effect() batches on Postgres keep their receipts in chant_receipts.receipts (#3657)", () => {
|
|
208
|
+
let server: TestPostgres | undefined;
|
|
209
|
+
let endpoint: PostgresEndpoint;
|
|
210
|
+
let admin: PostgresClient;
|
|
211
|
+
let p: Awaited<ReturnType<typeof project>>;
|
|
212
|
+
/** Whether chant_receipts was there before the test: the test drops it afterwards only when it made it. */
|
|
213
|
+
let hadReceiptsSchema = true;
|
|
214
|
+
const q = <T = Record<string, unknown>>(sql: string) => admin.query<T>(sql);
|
|
215
|
+
const forget = async () => {
|
|
216
|
+
const [t] = await q<{ present: boolean }>("SELECT pg_catalog.to_regclass('chant_receipts.receipts') IS NOT NULL AS present");
|
|
217
|
+
if (t?.present) await q(`DELETE FROM chant_receipts.receipts WHERE starts_with(address, '${NAME}/')`);
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
beforeAll(async () => {
|
|
221
|
+
if (givenPostgres) endpoint = { url: givenPostgres, password: process.env.CHANT_SQL_RECEIPTS_POSTGRES_PASSWORD };
|
|
222
|
+
else {
|
|
223
|
+
server = await startTestPostgres();
|
|
224
|
+
endpoint = server.endpoint();
|
|
225
|
+
}
|
|
226
|
+
admin = await connectPostgres(endpoint);
|
|
227
|
+
const [s] = await q<{ n: number }>("SELECT count(*)::int AS n FROM pg_catalog.pg_namespace WHERE nspname = 'chant_receipts'");
|
|
228
|
+
hadReceiptsSchema = (s?.n ?? 0) > 0;
|
|
229
|
+
await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`);
|
|
230
|
+
await forget();
|
|
231
|
+
await q(`CREATE SCHEMA ${NAME}`);
|
|
232
|
+
await q(`CREATE TABLE ${NAME}.log (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, batch text NOT NULL)`);
|
|
233
|
+
await q(`CREATE TABLE ${NAME}.stop (x int)`);
|
|
234
|
+
await q(`INSERT INTO ${NAME}.stop VALUES (1)`);
|
|
235
|
+
p = await project(
|
|
236
|
+
{
|
|
237
|
+
"chant.config.ts": config({
|
|
238
|
+
dialect: "postgres",
|
|
239
|
+
profiles: { [ENV]: { url: endpoint.url, password: { env: "ADMIN_PASSWORD" }, schemas: [NAME] } },
|
|
240
|
+
}),
|
|
241
|
+
"ops/fill.op.ts": opFile(
|
|
242
|
+
`DO $$ BEGIN IF EXISTS (SELECT 1 FROM ${NAME}.stop) THEN RAISE EXCEPTION 'stopped'; END IF; END $$`,
|
|
243
|
+
(b) => `INSERT INTO ${NAME}.log (batch) VALUES ('${b}')`,
|
|
244
|
+
),
|
|
245
|
+
},
|
|
246
|
+
{ ADMIN_PASSWORD: endpoint.password ?? "" },
|
|
247
|
+
);
|
|
248
|
+
}, 600_000);
|
|
249
|
+
|
|
250
|
+
afterAll(async () => {
|
|
251
|
+
if (admin) {
|
|
252
|
+
await q(`DROP SCHEMA IF EXISTS ${NAME} CASCADE`).catch(() => undefined);
|
|
253
|
+
await forget().catch(() => undefined);
|
|
254
|
+
if (!hadReceiptsSchema) await q("DROP SCHEMA IF EXISTS chant_receipts CASCADE").catch(() => undefined);
|
|
255
|
+
await admin.end();
|
|
256
|
+
}
|
|
257
|
+
await server?.stop();
|
|
258
|
+
if (p) rmSync(p.dir, { recursive: true, force: true });
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("a failed run keeps the finished batch's receipt; the rerun skips it", async () => {
|
|
262
|
+
await failThenResume(p, {
|
|
263
|
+
log: async () => (await q<{ batch: string }>(`SELECT batch FROM ${NAME}.log ORDER BY id`)).map((r) => r.batch),
|
|
264
|
+
receipts: async () =>
|
|
265
|
+
(await q<{ address: string }>(`SELECT address FROM chant_receipts.receipts WHERE starts_with(address, '${NAME}/') ORDER BY address`)).map((r) => r.address),
|
|
266
|
+
unstop: async () => {
|
|
267
|
+
await q(`DELETE FROM ${NAME}.stop`);
|
|
268
|
+
},
|
|
269
|
+
});
|
|
270
|
+
// The schema and the table are chant's: their comments carry the receipts key.
|
|
271
|
+
const [c] = await q<{ schema: string; tbl: string }>(
|
|
272
|
+
"SELECT pg_catalog.obj_description('chant_receipts'::regnamespace, 'pg_namespace') AS schema, pg_catalog.obj_description('chant_receipts.receipts'::regclass, 'pg_class') AS tbl",
|
|
273
|
+
);
|
|
274
|
+
expect(c?.schema).toContain("receipts=effects");
|
|
275
|
+
expect(c?.tbl).toContain("receipts=effects");
|
|
276
|
+
}, 600_000);
|
|
277
|
+
});
|
package/src/plugin.ts
CHANGED
|
@@ -103,6 +103,9 @@ export const sqlPlugin: LexiconPlugin = {
|
|
|
103
103
|
{ name: "type", isTag: true, description: "A Postgres CREATE TYPE ... AS ENUM, parsed into an enum entity" },
|
|
104
104
|
{ name: "domain", isTag: true, description: "A Postgres CREATE DOMAIN, parsed into a domain entity" },
|
|
105
105
|
{ name: "extension", isTag: true, description: "A Postgres CREATE EXTENSION, parsed into an extension entity" },
|
|
106
|
+
{ name: "func", isTag: true, description: "A Postgres CREATE FUNCTION, parsed into a function entity with its body kept verbatim" },
|
|
107
|
+
{ name: "procedure", isTag: true, description: "A Postgres CREATE PROCEDURE, parsed into a procedure entity with its body kept verbatim" },
|
|
108
|
+
{ name: "trigger", isTag: true, description: "A Postgres CREATE TRIGGER, parsed into a trigger entity on its table" },
|
|
106
109
|
{
|
|
107
110
|
name: "literal",
|
|
108
111
|
isTag: false,
|
|
@@ -138,9 +138,17 @@ export class PostgresApplyError extends SqlApplyError<PostgresApplyOutcome> {
|
|
|
138
138
|
}
|
|
139
139
|
}
|
|
140
140
|
|
|
141
|
-
/**
|
|
142
|
-
|
|
143
|
-
|
|
141
|
+
/**
|
|
142
|
+
* The name an apply result gives an object: `schema.name`, or a schema's or
|
|
143
|
+
* extension's own name; a routine's with its parameter types
|
|
144
|
+
* (`app.touch()`), a trigger's with its table (`touch ON app.users`).
|
|
145
|
+
*/
|
|
146
|
+
export const refName = (o: { kind: CanonicalKind; schema?: string; name: string; signature?: string }): string =>
|
|
147
|
+
o.kind === "trigger"
|
|
148
|
+
? `${o.name}${o.signature ?? ""}`
|
|
149
|
+
: o.kind === "schema" || o.kind === "extension" || o.schema === undefined
|
|
150
|
+
? o.name
|
|
151
|
+
: `${o.schema}.${o.name}${o.kind === "function" || o.kind === "procedure" ? (o.signature ?? "") : ""}`;
|
|
144
152
|
|
|
145
153
|
/**
|
|
146
154
|
* The objects a Postgres `chant build` output declares, in creation order,
|
|
@@ -434,7 +442,7 @@ export async function applyPostgres(
|
|
|
434
442
|
}
|
|
435
443
|
|
|
436
444
|
/** The address a live object is keyed by in the diff. */
|
|
437
|
-
function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?: string; name: string } {
|
|
445
|
+
function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?: string; name: string; signature?: string } {
|
|
438
446
|
const kind = (
|
|
439
447
|
{
|
|
440
448
|
"Postgres::Schema": "schema",
|
|
@@ -446,9 +454,17 @@ function liveCanonicalAddress(o: LivePgObject): { kind: CanonicalKind; schema?:
|
|
|
446
454
|
"Postgres::Enum": "enum",
|
|
447
455
|
"Postgres::Domain": "domain",
|
|
448
456
|
"Postgres::Extension": "extension",
|
|
457
|
+
"Postgres::Function": "function",
|
|
458
|
+
"Postgres::Procedure": "procedure",
|
|
459
|
+
"Postgres::Trigger": "trigger",
|
|
449
460
|
} as const
|
|
450
461
|
)[o.type];
|
|
451
|
-
return {
|
|
462
|
+
return {
|
|
463
|
+
kind,
|
|
464
|
+
...(o.schema !== undefined && kind !== "schema" && kind !== "extension" ? { schema: o.schema } : {}),
|
|
465
|
+
name: o.name,
|
|
466
|
+
...(o.signature !== undefined ? { signature: o.signature } : {}),
|
|
467
|
+
};
|
|
452
468
|
}
|
|
453
469
|
|
|
454
470
|
/** The comments the extensions' control files set, by extension name. */
|
|
@@ -80,9 +80,17 @@ export const pgString = (value: string): string => `'${value.replace(/'/g, "''")
|
|
|
80
80
|
/** `schema.name`, each part quoted where Postgres needs it. */
|
|
81
81
|
export const pgQualified = (schema: string | undefined, name: string): string => (schema ? `${quoteIdent(schema)}.${quoteIdent(name)}` : quoteIdent(name));
|
|
82
82
|
|
|
83
|
-
/**
|
|
84
|
-
|
|
85
|
-
|
|
83
|
+
/**
|
|
84
|
+
* An object's name in a statement. A schema and an extension have no schema;
|
|
85
|
+
* a routine is named with its parameter types (`app.f(integer,text)`), a
|
|
86
|
+
* trigger with its table (`touch ON app.users`).
|
|
87
|
+
*/
|
|
88
|
+
export const objectIdent = (o: { kind: CanonicalKind; schema?: string; name: string; signature?: string }): string =>
|
|
89
|
+
o.kind === "schema" || o.kind === "extension"
|
|
90
|
+
? quoteIdent(o.name)
|
|
91
|
+
: o.kind === "trigger"
|
|
92
|
+
? `${quoteIdent(o.name)}${o.signature ?? ""}`
|
|
93
|
+
: `${pgQualified(o.schema, o.name)}${o.signature ?? ""}`;
|
|
86
94
|
|
|
87
95
|
/** The words a statement names the object's kind with. */
|
|
88
96
|
export const KIND_WORDS: Readonly<Record<CanonicalKind, string>> = {
|
|
@@ -95,6 +103,9 @@ export const KIND_WORDS: Readonly<Record<CanonicalKind, string>> = {
|
|
|
95
103
|
enum: "TYPE",
|
|
96
104
|
domain: "DOMAIN",
|
|
97
105
|
extension: "EXTENSION",
|
|
106
|
+
function: "FUNCTION",
|
|
107
|
+
procedure: "PROCEDURE",
|
|
108
|
+
trigger: "TRIGGER",
|
|
98
109
|
};
|
|
99
110
|
|
|
100
111
|
/** The canonical kind of an entity type. */
|
|
@@ -168,7 +179,8 @@ export function createSteps(
|
|
|
168
179
|
splitStatements(obj.ddl).forEach((s, i) => {
|
|
169
180
|
if (i > 0 && isOwnComment(s, obj.canonical.kind)) return;
|
|
170
181
|
let sql = s.text;
|
|
171
|
-
|
|
182
|
+
const replaceable = obj.canonical.kind === "view" || obj.canonical.kind === "function" || obj.canonical.kind === "procedure" || (obj.canonical.kind === "trigger" && obj.canonical.fields.constraint !== true);
|
|
183
|
+
if (i === 0 && opts.orReplace && replaceable && !/^\s*CREATE\s+OR\s+REPLACE\b/i.test(sql)) sql = sql.replace(/\bCREATE\s+/i, "CREATE OR REPLACE ");
|
|
172
184
|
const concurrently = i === 0 && obj.canonical.kind === "index" && obj.props.concurrently === true;
|
|
173
185
|
out.push({ ...step(sql, cls, opts.rule, !concurrently), ...(concurrently ? { buildsIndex: objectIdent(obj.canonical) } : {}) });
|
|
174
186
|
});
|
|
@@ -177,10 +189,10 @@ export function createSteps(
|
|
|
177
189
|
}
|
|
178
190
|
|
|
179
191
|
/** The `DROP` for an object a prune removes. Never `CASCADE`: what something else still uses is kept. */
|
|
180
|
-
export function dropStatement(kind: CanonicalKind, schema: string | undefined, name: string): PgStep {
|
|
181
|
-
const what = objectIdent({ kind, ...(schema !== undefined ? { schema } : {}), name });
|
|
192
|
+
export function dropStatement(kind: CanonicalKind, schema: string | undefined, name: string, signature?: string): PgStep {
|
|
193
|
+
const what = objectIdent({ kind, ...(schema !== undefined ? { schema } : {}), name, ...(signature !== undefined ? { signature } : {}) });
|
|
182
194
|
if (kind === "index") return step(`DROP INDEX CONCURRENTLY ${what}`, "concurrently", "SQLPG242", false);
|
|
183
|
-
return step(`DROP ${KIND_WORDS[kind]} ${what}`, "drop", "SQLPG270");
|
|
195
|
+
return step(`DROP ${KIND_WORDS[kind]} ${what}`, "drop", kind === "trigger" ? "SQLPG285" : "SQLPG270");
|
|
184
196
|
}
|
|
185
197
|
|
|
186
198
|
// ── Columns ────────────────────────────────────────────────────────────
|
|
@@ -403,7 +415,7 @@ export const isRefused = (c: PgChange): boolean => c.class === "expand";
|
|
|
403
415
|
export const isDestructive = (c: PgChange): boolean => c.rule === "SQLPG204";
|
|
404
416
|
|
|
405
417
|
/** Changes that replace the object: drop it and create it again from its declaration. */
|
|
406
|
-
export const isRecreate = (c: PgChange): boolean => c.rule === "SQLPG252" || c.rule === "SQLPG243";
|
|
418
|
+
export const isRecreate = (c: PgChange): boolean => c.rule === "SQLPG252" || c.rule === "SQLPG243" || c.rule === "SQLPG281";
|
|
407
419
|
|
|
408
420
|
/**
|
|
409
421
|
* The statements for an object that exists on the server, change by change.
|
|
@@ -538,8 +550,15 @@ export function alterSteps(
|
|
|
538
550
|
steps.push(step(`ALTER INDEX ${objectIdent(live)} RENAME TO ${quoteIdent(d.name)}`, c.class, c.rule));
|
|
539
551
|
break;
|
|
540
552
|
case "SQLPG250":
|
|
553
|
+
case "SQLPG280":
|
|
541
554
|
steps.push(...createSteps(obj, opts.marker, { orReplace: true, cls: c.class, rule: c.rule }).slice(0, 1));
|
|
542
555
|
break;
|
|
556
|
+
case "SQLPG284":
|
|
557
|
+
if (d.fields.constraint === true || live.fields.constraint === true || d.signature !== live.signature) {
|
|
558
|
+
steps.push(step(`DROP TRIGGER ${objectIdent(live)}`, c.class, c.rule));
|
|
559
|
+
steps.push(...createSteps(obj, opts.marker, { cls: c.class, rule: c.rule }));
|
|
560
|
+
} else steps.push(...createSteps(obj, opts.marker, { orReplace: true, cls: c.class, rule: c.rule }).slice(0, 1));
|
|
561
|
+
break;
|
|
543
562
|
case "SQLPG260": {
|
|
544
563
|
const before = (live.fields.labels as string[] | undefined) ?? [];
|
|
545
564
|
const after = (d.fields.labels as string[] | undefined) ?? [];
|
|
@@ -631,6 +650,8 @@ export interface PgDropStatement {
|
|
|
631
650
|
kind: CanonicalKind;
|
|
632
651
|
schema?: string;
|
|
633
652
|
name: string;
|
|
653
|
+
/** A routine's parameter types, a trigger's table. */
|
|
654
|
+
signature?: string;
|
|
634
655
|
step: PgStep;
|
|
635
656
|
}
|
|
636
657
|
|
|
@@ -663,7 +684,21 @@ export interface PgStatementPlan {
|
|
|
663
684
|
}
|
|
664
685
|
|
|
665
686
|
/** Drop order: what reads from a table before the table, types after the tables using them, a schema last. */
|
|
666
|
-
const DROP_ORDER: Record<CanonicalKind, number> = {
|
|
687
|
+
const DROP_ORDER: Record<CanonicalKind, number> = {
|
|
688
|
+
trigger: 0,
|
|
689
|
+
materializedView: 1,
|
|
690
|
+
view: 2,
|
|
691
|
+
index: 3,
|
|
692
|
+
table: 4,
|
|
693
|
+
sequence: 5,
|
|
694
|
+
// A routine after what calls it (a view, a trigger, a default), before the types its parameters use.
|
|
695
|
+
function: 6,
|
|
696
|
+
procedure: 6,
|
|
697
|
+
domain: 7,
|
|
698
|
+
enum: 8,
|
|
699
|
+
extension: 9,
|
|
700
|
+
schema: 10,
|
|
701
|
+
};
|
|
667
702
|
|
|
668
703
|
/**
|
|
669
704
|
* The statements that take the current schema to the declared one, from the
|
|
@@ -735,11 +770,18 @@ export function planPgStatements(input: PgStatementPlanInput): PgStatementPlan {
|
|
|
735
770
|
|
|
736
771
|
const drops: PgDropStatement[] = [];
|
|
737
772
|
for (const c of input.changes) {
|
|
738
|
-
if (!((c.rule === "SQLPG270" || c.rule === "SQLPG242") && c.after === undefined)) continue;
|
|
773
|
+
if (!((c.rule === "SQLPG270" || c.rule === "SQLPG242" || c.rule === "SQLPG285") && c.after === undefined)) continue;
|
|
739
774
|
const o = currentByKey.get(c.object);
|
|
740
775
|
if (!o || o.foreign) continue;
|
|
741
776
|
const schema = o.kind === "schema" || o.kind === "extension" ? undefined : o.schema;
|
|
742
|
-
drops.push({
|
|
777
|
+
drops.push({
|
|
778
|
+
key: c.object,
|
|
779
|
+
kind: o.kind,
|
|
780
|
+
...(schema !== undefined ? { schema } : {}),
|
|
781
|
+
name: o.name,
|
|
782
|
+
...(o.signature !== undefined ? { signature: o.signature } : {}),
|
|
783
|
+
step: dropStatement(o.kind, schema, o.name, o.signature),
|
|
784
|
+
});
|
|
743
785
|
}
|
|
744
786
|
drops.sort((a, b) => DROP_ORDER[a.kind] - DROP_ORDER[b.kind]);
|
|
745
787
|
return { objects, drops };
|