@astralarium/repo-postgres 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +9 -2
- package/container/init/.containerignore +2 -0
- package/container/init/Containerfile +26 -0
- package/container/init/entrypoint.d/init-db.sh +20 -0
- package/container/snapshot/run-snapshot.sh +23 -0
- package/container/util/.containerignore +2 -0
- package/container/util/Containerfile +7 -0
- package/container/util/entrypoint.d/dump-db.sh +19 -0
- package/container/util/entrypoint.d/upgrade-db.sh +103 -0
- package/dist/bin/stage-schema.d.ts +2 -0
- package/dist/bin/stage-schema.js +20 -0
- package/dist/src/provision/connection.d.ts +11 -0
- package/dist/src/provision/connection.js +33 -0
- package/dist/src/provision/container.d.ts +87 -0
- package/dist/src/provision/container.js +244 -0
- package/dist/src/provision/data.d.ts +8 -0
- package/dist/src/provision/data.js +47 -0
- package/dist/src/provision/disposable.d.ts +49 -0
- package/dist/src/provision/disposable.js +188 -0
- package/dist/src/provision/image.d.ts +63 -0
- package/dist/src/provision/image.js +159 -0
- package/dist/src/provision/index.d.ts +17 -0
- package/dist/src/provision/index.js +205 -0
- package/dist/src/provision/options.d.ts +23 -0
- package/dist/src/provision/options.js +1 -0
- package/dist/src/provision/schema.d.ts +11 -0
- package/dist/src/provision/schema.js +134 -0
- package/dist/src/provision/test-database.d.ts +29 -0
- package/dist/src/provision/test-database.js +130 -0
- package/dist/src/schema/index.d.ts +3 -0
- package/dist/src/schema/index.js +3 -0
- package/dist/src/schema/module.d.ts +59 -0
- package/dist/src/schema/module.js +122 -0
- package/dist/src/schema/modules.d.ts +15 -0
- package/dist/src/schema/modules.js +48 -0
- package/dist/src/sql.d.ts +83 -0
- package/dist/src/sql.js +146 -0
- package/dist/testkit/index.d.ts +13 -0
- package/dist/testkit/index.js +31 -0
- package/dist/testkit/schema.d.ts +14 -0
- package/dist/testkit/schema.js +89 -0
- package/package.json +62 -3
- package/scripts/build-images.sh +52 -0
- package/scripts/dump-db.sh +32 -0
- package/sql/app/schema.sql +8 -0
- package/sql/bootstrap.sql +27 -0
- package/sql/database-grants.sql +3 -0
- package/sql/drop-schemas.sql +11 -0
- package/sql/extensions/cron/schema.sql +23 -0
- package/sql/extensions/partman/before-reset.sql +38 -0
- package/sql/extensions/partman/schema.sql +8 -0
- package/sql/extensions/trgm/schema.sql +1 -0
- package/sql/reset.sql +22 -0
- package/sql/roles.sql +10 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Postgres as a `DatabaseProvider`: the app's schema image, one persistent
|
|
3
|
+
* container per checkout for dev, a disposable one per test run.
|
|
4
|
+
*/
|
|
5
|
+
import { isAbsolute } from "node:path";
|
|
6
|
+
import { keepOrFindPort, RESOURCE_NAME, succeeds, } from "@astralarium/repo-dev";
|
|
7
|
+
import { ExpectedError, log } from "@astralarium/repo-telemetry";
|
|
8
|
+
import { z } from "zod";
|
|
9
|
+
import { isSchemaModule, loadOrder, stringList, } from "../schema/module.js";
|
|
10
|
+
import { adminConnectionString, APP_SERVICE_ROLE, appConnectionString, DEFAULT_PORT, probePostgres, } from "./connection.js";
|
|
11
|
+
import { claimable, detectRuntime, holders, inspectContainer, OWNER_LABEL, removeContainer, removeVolume, StaleSchemaError, startPostgresContainer, volumeName, } from "./container.js";
|
|
12
|
+
import { loadData } from "./data.js";
|
|
13
|
+
import { checkDumpExclude, migrationMounts, requireFile, runDisposableServer, runUpgradeDump, } from "./disposable.js";
|
|
14
|
+
import { buildImages, ensureImage, ensureUtilImage, schemaHash, writeTestImage, } from "./image.js";
|
|
15
|
+
import { moduleServerArgs, stageSchema } from "./schema.js";
|
|
16
|
+
import { startTestInstance, } from "./test-database.js";
|
|
17
|
+
export { APP_SERVICE_ROLE } from "./connection.js";
|
|
18
|
+
export { ensureUtilImage } from "./image.js";
|
|
19
|
+
/** Container names tried per base: `<base>`, `<base>-2`, … */
|
|
20
|
+
const NAME_ATTEMPTS = 100;
|
|
21
|
+
const postgresOptions = z.object({
|
|
22
|
+
modules: z
|
|
23
|
+
.array(z.custom(isSchemaModule, {
|
|
24
|
+
error: "modules must hold defineSchemaModule results",
|
|
25
|
+
}))
|
|
26
|
+
.min(1, { error: "modules lists no schema module" })
|
|
27
|
+
.readonly()
|
|
28
|
+
.check((ctx) => {
|
|
29
|
+
try {
|
|
30
|
+
loadOrder(ctx.value);
|
|
31
|
+
}
|
|
32
|
+
catch (cause) {
|
|
33
|
+
ctx.issues.push({
|
|
34
|
+
code: "custom",
|
|
35
|
+
input: ctx.value,
|
|
36
|
+
message: cause instanceof Error ? cause.message : String(cause),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
}),
|
|
40
|
+
name: z.string().regex(RESOURCE_NAME, {
|
|
41
|
+
error: "name must be a container resource name",
|
|
42
|
+
}),
|
|
43
|
+
serverArgs: stringList.optional(),
|
|
44
|
+
ephemeralArgs: stringList.optional(),
|
|
45
|
+
data: z
|
|
46
|
+
.array(z.string().refine(isAbsolute, { error: "data paths must be absolute" }))
|
|
47
|
+
.readonly()
|
|
48
|
+
.optional(),
|
|
49
|
+
});
|
|
50
|
+
class Postgres {
|
|
51
|
+
name;
|
|
52
|
+
#options;
|
|
53
|
+
#runtime;
|
|
54
|
+
constructor(options) {
|
|
55
|
+
this.#options = options;
|
|
56
|
+
this.name = options.name;
|
|
57
|
+
}
|
|
58
|
+
/** Detected once per process. */
|
|
59
|
+
get runtime() {
|
|
60
|
+
return (this.#runtime ??= detectRuntime());
|
|
61
|
+
}
|
|
62
|
+
async allocate({ base, owner, saved, }) {
|
|
63
|
+
const runtime = this.runtime;
|
|
64
|
+
const names = [
|
|
65
|
+
base,
|
|
66
|
+
...Array.from({ length: NAME_ATTEMPTS - 1 }, (_, i) => `${base}-${i + 2}`),
|
|
67
|
+
];
|
|
68
|
+
const name = [
|
|
69
|
+
...new Set([
|
|
70
|
+
...(saved.name && names.includes(saved.name) ? [saved.name] : []),
|
|
71
|
+
...names,
|
|
72
|
+
]),
|
|
73
|
+
].find((candidate) => claim(runtime, candidate, owner));
|
|
74
|
+
if (!name)
|
|
75
|
+
throw new ExpectedError(`no free container name from ${base} to ${names.at(-1)}`);
|
|
76
|
+
// A claimable container is this checkout's: a running one keeps its port.
|
|
77
|
+
const container = inspectContainer(runtime, name);
|
|
78
|
+
const port = container?.running && container.bindings[0]?.HostPort
|
|
79
|
+
? Number(container.bindings[0].HostPort)
|
|
80
|
+
: await keepOrFindPort(saved.port, DEFAULT_PORT);
|
|
81
|
+
return { name, port };
|
|
82
|
+
}
|
|
83
|
+
async ensure(resources, { allocation: { name, port }, image, owner, program, }) {
|
|
84
|
+
const options = this.#options;
|
|
85
|
+
const runtime = this.runtime;
|
|
86
|
+
const hash = schemaHash(options);
|
|
87
|
+
const resetHint = `${program} reset --db-name ${image} recreates it; this deletes local data.`;
|
|
88
|
+
try {
|
|
89
|
+
await startPostgresContainer(runtime, resources, {
|
|
90
|
+
name,
|
|
91
|
+
image: ensureImage(options, runtime, hash, image),
|
|
92
|
+
hash,
|
|
93
|
+
port,
|
|
94
|
+
volume: volumeName(name),
|
|
95
|
+
owner,
|
|
96
|
+
serverArgs: [
|
|
97
|
+
...moduleServerArgs(options.modules),
|
|
98
|
+
...(options.serverArgs ?? []),
|
|
99
|
+
],
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
catch (cause) {
|
|
103
|
+
if (cause instanceof StaleSchemaError)
|
|
104
|
+
throw new ExpectedError(`${cause.message} ${resetHint}`, { cause });
|
|
105
|
+
throw cause;
|
|
106
|
+
}
|
|
107
|
+
try {
|
|
108
|
+
await probePostgres(appConnectionString(port, "db_template"));
|
|
109
|
+
}
|
|
110
|
+
catch (cause) {
|
|
111
|
+
throw new ExpectedError(`${name}: ${APP_SERVICE_ROLE} cannot connect to db_template with the dev credentials. Inspect the database, or ${resetHint}`, { cause });
|
|
112
|
+
}
|
|
113
|
+
await loadData(adminConnectionString(port, "db_template"), options.data ?? []);
|
|
114
|
+
log.info("dev.db_ready", "{name}: ready on :{port}", { name, port });
|
|
115
|
+
}
|
|
116
|
+
reset(name, image) {
|
|
117
|
+
removeContainer(this.runtime, name);
|
|
118
|
+
removeVolume(this.runtime, volumeName(name));
|
|
119
|
+
this.build(image);
|
|
120
|
+
}
|
|
121
|
+
stop(name, owner) {
|
|
122
|
+
let runtime;
|
|
123
|
+
try {
|
|
124
|
+
runtime = this.runtime;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (claimable(runtime, name, owner))
|
|
130
|
+
succeeds(runtime, ["stop", name]);
|
|
131
|
+
}
|
|
132
|
+
build(image) {
|
|
133
|
+
buildImages(this.#options, this.runtime, schemaHash(this.#options), {
|
|
134
|
+
name: image,
|
|
135
|
+
refreshBase: true,
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
async runDisposable({ allocation, image, serverArgs, snapshot, signal, }) {
|
|
139
|
+
if (snapshot)
|
|
140
|
+
requireFile(snapshot.dump);
|
|
141
|
+
await runDisposableServer(this.runtime, {
|
|
142
|
+
name: `${allocation.name}-${snapshot ? "snapshot" : "temp"}`,
|
|
143
|
+
image: this.#image(image),
|
|
144
|
+
port: await keepOrFindPort(undefined, DEFAULT_PORT, new Set([allocation.port])),
|
|
145
|
+
serverArgs,
|
|
146
|
+
snapshot,
|
|
147
|
+
signal,
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
async upgradeDump({ allocation, image, migrations, signal, }) {
|
|
151
|
+
const mounts = migrationMounts(migrations);
|
|
152
|
+
// eslint-disable-next-line turbo/no-undeclared-env-vars -- no turbo task runs this CLI
|
|
153
|
+
checkDumpExclude(process.env.DB_DUMP_EXCLUDE);
|
|
154
|
+
await runUpgradeDump(this.runtime, {
|
|
155
|
+
name: `${allocation.name}-upgrade`,
|
|
156
|
+
image: this.#image(image),
|
|
157
|
+
util: ensureUtilImage(this.runtime, image),
|
|
158
|
+
mounts,
|
|
159
|
+
signal,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
/** This checkout's schema image under base name `image`, built when
|
|
163
|
+
* missing: its immutable ID. */
|
|
164
|
+
#image(image) {
|
|
165
|
+
return ensureImage(this.#options, this.runtime, schemaHash(this.#options), image);
|
|
166
|
+
}
|
|
167
|
+
connectionString({ port }) {
|
|
168
|
+
return appConnectionString(port, "db_template");
|
|
169
|
+
}
|
|
170
|
+
startTestInstance(resources, name) {
|
|
171
|
+
return startTestInstance(resources, name, this.#options);
|
|
172
|
+
}
|
|
173
|
+
writeTestImage(file) {
|
|
174
|
+
writeTestImage(this.#options, file);
|
|
175
|
+
}
|
|
176
|
+
stageSchema(dir) {
|
|
177
|
+
stageSchema(this.#options.modules, dir);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* `claimable`, naming each holder of `name` lacking `OWNER_LABEL` (a
|
|
182
|
+
* container or data volume) and the command freeing it: that holder keeps
|
|
183
|
+
* the name and its port from every checkout.
|
|
184
|
+
*/
|
|
185
|
+
function claim(runtime, name, owner) {
|
|
186
|
+
const held = holders(runtime, name);
|
|
187
|
+
if (held.every((holder) => holder.owner === owner))
|
|
188
|
+
return true;
|
|
189
|
+
for (const holder of held)
|
|
190
|
+
if (holder.owner === undefined)
|
|
191
|
+
log.warn("dev.db_name_held", "{kind, select, volume {Data volume} other {Container}} {name} carries no {label} label, so this checkout skips the name; free it with: {remove}", {
|
|
192
|
+
kind: holder.kind,
|
|
193
|
+
name: holder.name,
|
|
194
|
+
label: OWNER_LABEL,
|
|
195
|
+
remove: `${runtime} ${holder.kind === "volume" ? "volume rm" : "rm -f"} ${holder.name}`,
|
|
196
|
+
});
|
|
197
|
+
return false;
|
|
198
|
+
}
|
|
199
|
+
/** The app's Postgres provider; the declaration is validated (`ExpectedError`). */
|
|
200
|
+
export function postgresDatabase(options) {
|
|
201
|
+
const result = postgresOptions.safeParse(options);
|
|
202
|
+
if (!result.success)
|
|
203
|
+
throw new ExpectedError(`invalid Postgres options:\n${z.prettifyError(result.error)}`);
|
|
204
|
+
return new Postgres(options);
|
|
205
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { SchemaModule } from "../schema/module.ts";
|
|
2
|
+
/** What an app declares for its Postgres image and local instances. */
|
|
3
|
+
export interface PostgresOptions {
|
|
4
|
+
/**
|
|
5
|
+
* The app's schema modules; their `requires` load too. Files hash into
|
|
6
|
+
* the image tag; extensions install in the image; `preload` and
|
|
7
|
+
* `serverArgs` reach the dev instance.
|
|
8
|
+
*/
|
|
9
|
+
modules: readonly SchemaModule[];
|
|
10
|
+
/** Image and instance base name; `DB_NAME` and `--db-name` override it. */
|
|
11
|
+
name: string;
|
|
12
|
+
/** Postgres server arguments for the dev instance, after the modules'. */
|
|
13
|
+
serverArgs?: readonly string[];
|
|
14
|
+
/** Postgres server arguments for disposable (test) instances; stock Postgres by default. */
|
|
15
|
+
ephemeralArgs?: readonly string[];
|
|
16
|
+
/**
|
|
17
|
+
* Data SQL after the schema: absolute `.sql` files, or directories whose
|
|
18
|
+
* `*.sql` run in name order. Loads into `db_template` on every dev start
|
|
19
|
+
* and once per test run, before project databases clone it; idempotent.
|
|
20
|
+
* Its digest joins the test image manifest.
|
|
21
|
+
*/
|
|
22
|
+
data?: readonly string[];
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type SchemaModule } from "../schema/module.ts";
|
|
2
|
+
/** sha256 over the staged psql entries, then the module file, asset,
|
|
3
|
+
* `beforeReset` and `scrub` paths and contents, in load order. */
|
|
4
|
+
export declare function schemaDigest(modules: readonly SchemaModule[]): string;
|
|
5
|
+
/** Extensions every module installs, deduplicated in load order. */
|
|
6
|
+
export declare function moduleExtensions(modules: readonly SchemaModule[]): string[];
|
|
7
|
+
/** Persistent-instance server arguments: one `shared_preload_libraries`
|
|
8
|
+
* over every module's `preload`, then each module's `serverArgs`. */
|
|
9
|
+
export declare function moduleServerArgs(modules: readonly SchemaModule[]): string[];
|
|
10
|
+
/** Writes the staged directory under `dir`, creating it. */
|
|
11
|
+
export declare function stageSchema(modules: readonly SchemaModule[], dir: string): void;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The app's schema modules as psql loads them: a staged directory holding
|
|
3
|
+
* `schema.sql` (one `\i sql/<module>/<file>` per module file, in load
|
|
4
|
+
* order) beside `sql/` (each module's files and assets),
|
|
5
|
+
* `before-reset.sql` and `before-reset/` (each module's `beforeReset`),
|
|
6
|
+
* `scrub.sql` and `scrub/` (each module's `scrub`), and this package's
|
|
7
|
+
* `roles.sql`, `database-grants.sql`, `bootstrap.sql`, `reset.sql` and
|
|
8
|
+
* `drop-schemas.sql`. psql runs every entry with this directory as cwd.
|
|
9
|
+
* The image bakes it and runs `roles.sql`, `database-grants.sql`, then
|
|
10
|
+
* `schema.sql`, at first boot.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from "node:crypto";
|
|
13
|
+
import { copyFileSync, mkdirSync, readdirSync, readFileSync, writeFileSync, } from "node:fs";
|
|
14
|
+
import { basename, dirname, join } from "node:path";
|
|
15
|
+
import { packageRoot } from "@astralarium/repo-package-files";
|
|
16
|
+
import { ExpectedError } from "@astralarium/repo-telemetry";
|
|
17
|
+
import { loadOrder } from "../schema/module.js";
|
|
18
|
+
/** Entry psql runs from the staged directory. */
|
|
19
|
+
const SCHEMA_ENTRY = "schema.sql";
|
|
20
|
+
/** Directory of the staged module files, beside the entry. */
|
|
21
|
+
const SCHEMA_FILES = "sql";
|
|
22
|
+
/** Entry `reset.sql` runs ahead of its drop. */
|
|
23
|
+
const BEFORE_RESET_ENTRY = "before-reset.sql";
|
|
24
|
+
/** Directory of the staged `beforeReset` files. */
|
|
25
|
+
const BEFORE_RESET_FILES = "before-reset";
|
|
26
|
+
/** Entry a snapshot runs after its dump loads. */
|
|
27
|
+
const SCRUB_ENTRY = "scrub.sql";
|
|
28
|
+
/** Directory of the staged `scrub` files. */
|
|
29
|
+
const SCRUB_FILES = "scrub";
|
|
30
|
+
/** Each module's `files` or `assets` in load order, under `sql/<module>/`. */
|
|
31
|
+
function moduleFiles(order, kind) {
|
|
32
|
+
return order.flatMap((module) => module[kind].map((path) => ({
|
|
33
|
+
staged: `${SCHEMA_FILES}/${module.name}/${basename(path)}`,
|
|
34
|
+
path,
|
|
35
|
+
})));
|
|
36
|
+
}
|
|
37
|
+
/** Each module's `beforeReset` or `scrub` in load order, as
|
|
38
|
+
* `<dir>/<module>.sql`. */
|
|
39
|
+
function moduleScripts(order, kind, dir) {
|
|
40
|
+
return order.flatMap((module) => {
|
|
41
|
+
const path = module[kind];
|
|
42
|
+
return path ? [{ staged: `${dir}/${module.name}.sql`, path }] : [];
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
/** Every staged module file, asset, `beforeReset` and `scrub` file, in load
|
|
46
|
+
* order. */
|
|
47
|
+
function stagedFiles(order) {
|
|
48
|
+
return [
|
|
49
|
+
...moduleFiles(order, "files"),
|
|
50
|
+
...moduleFiles(order, "assets"),
|
|
51
|
+
...moduleScripts(order, "beforeReset", BEFORE_RESET_FILES),
|
|
52
|
+
...moduleScripts(order, "scrub", SCRUB_FILES),
|
|
53
|
+
];
|
|
54
|
+
}
|
|
55
|
+
/** psql entry including `files` in order. */
|
|
56
|
+
function entry(header, files) {
|
|
57
|
+
return [header, ...files.map((file) => `\\i ${file.staged}`), ""].join("\n");
|
|
58
|
+
}
|
|
59
|
+
/** Staged psql entries as `[file name, text]`. */
|
|
60
|
+
function entries(order) {
|
|
61
|
+
return [
|
|
62
|
+
[
|
|
63
|
+
SCHEMA_ENTRY,
|
|
64
|
+
entry("-- Schema load order, staged by @astralarium/repo-postgres from the app's schema modules.\n-- Run connected to the app database, after roles.sql and database-grants.sql.", moduleFiles(order, "files")),
|
|
65
|
+
],
|
|
66
|
+
[
|
|
67
|
+
BEFORE_RESET_ENTRY,
|
|
68
|
+
entry("-- Each schema module's beforeReset in load order, staged by @astralarium/repo-postgres.\n-- Run by reset.sql as app_admin, inside its transaction.", moduleScripts(order, "beforeReset", BEFORE_RESET_FILES)),
|
|
69
|
+
],
|
|
70
|
+
[
|
|
71
|
+
SCRUB_ENTRY,
|
|
72
|
+
entry("-- Each schema module's scrub in load order, staged by @astralarium/repo-postgres.\n-- Run by run-snapshot as superuser in db_template, inside one transaction.", moduleScripts(order, "scrub", SCRUB_FILES)),
|
|
73
|
+
],
|
|
74
|
+
];
|
|
75
|
+
}
|
|
76
|
+
/** The file's contents; a missing file names the path and its likely cause. */
|
|
77
|
+
function readSchemaFile({ path }) {
|
|
78
|
+
try {
|
|
79
|
+
return readFileSync(path);
|
|
80
|
+
}
|
|
81
|
+
catch (cause) {
|
|
82
|
+
throw new ExpectedError(`schema file ${path} is missing: check the module declaring it, or install the package providing it`, { cause });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** sha256 over the staged psql entries, then the module file, asset,
|
|
86
|
+
* `beforeReset` and `scrub` paths and contents, in load order. */
|
|
87
|
+
export function schemaDigest(modules) {
|
|
88
|
+
const order = loadOrder(modules);
|
|
89
|
+
const hash = createHash("sha256");
|
|
90
|
+
for (const [name, text] of entries(order)) {
|
|
91
|
+
hash.update(`${name}\0`);
|
|
92
|
+
hash.update(text);
|
|
93
|
+
hash.update("\0");
|
|
94
|
+
}
|
|
95
|
+
for (const file of stagedFiles(order)) {
|
|
96
|
+
hash.update(`${file.staged}\0`);
|
|
97
|
+
hash.update(readSchemaFile(file));
|
|
98
|
+
hash.update("\0");
|
|
99
|
+
}
|
|
100
|
+
return hash.digest("hex");
|
|
101
|
+
}
|
|
102
|
+
/** Extensions every module installs, deduplicated in load order. */
|
|
103
|
+
export function moduleExtensions(modules) {
|
|
104
|
+
return [
|
|
105
|
+
...new Set(loadOrder(modules).flatMap((module) => module.extensions)),
|
|
106
|
+
];
|
|
107
|
+
}
|
|
108
|
+
/** Persistent-instance server arguments: one `shared_preload_libraries`
|
|
109
|
+
* over every module's `preload`, then each module's `serverArgs`. */
|
|
110
|
+
export function moduleServerArgs(modules) {
|
|
111
|
+
const order = loadOrder(modules);
|
|
112
|
+
const preload = [...new Set(order.flatMap((module) => module.preload))];
|
|
113
|
+
return [
|
|
114
|
+
...(preload.length
|
|
115
|
+
? ["-c", `shared_preload_libraries=${preload.join(",")}`]
|
|
116
|
+
: []),
|
|
117
|
+
...order.flatMap((module) => module.serverArgs),
|
|
118
|
+
];
|
|
119
|
+
}
|
|
120
|
+
/** Writes the staged directory under `dir`, creating it. */
|
|
121
|
+
export function stageSchema(modules, dir) {
|
|
122
|
+
const order = loadOrder(modules);
|
|
123
|
+
for (const file of stagedFiles(order)) {
|
|
124
|
+
const target = join(dir, file.staged);
|
|
125
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
126
|
+
writeFileSync(target, readSchemaFile(file));
|
|
127
|
+
}
|
|
128
|
+
for (const [name, text] of entries(order))
|
|
129
|
+
writeFileSync(join(dir, name), text);
|
|
130
|
+
const scripts = join(packageRoot(import.meta.filename), "sql");
|
|
131
|
+
for (const name of readdirSync(scripts))
|
|
132
|
+
if (name.endsWith(".sql"))
|
|
133
|
+
copyFileSync(join(scripts, name), join(dir, name));
|
|
134
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type ResourceScope, type TestContainer, type TestDatabase } from "@astralarium/repo-dev";
|
|
2
|
+
import type { PostgresOptions } from "./options.ts";
|
|
3
|
+
/** Running seeded instance; fixtures may run tools in its container or network namespace. */
|
|
4
|
+
export declare class PostgresTestDatabase implements TestDatabase {
|
|
5
|
+
/** Published loopback port. */
|
|
6
|
+
readonly port: number;
|
|
7
|
+
/** Container running the instance; other containers join its network
|
|
8
|
+
* namespace. */
|
|
9
|
+
readonly container: TestContainer;
|
|
10
|
+
constructor(instance: {
|
|
11
|
+
port: number;
|
|
12
|
+
container: TestContainer;
|
|
13
|
+
});
|
|
14
|
+
/**
|
|
15
|
+
* `CREATE DATABASE … TEMPLATE db_template`; returns the `app_service`
|
|
16
|
+
* connection. A project whose name maps to an existing database is an
|
|
17
|
+
* `ExpectedError`.
|
|
18
|
+
*/
|
|
19
|
+
createDatabase(name: string): Promise<string>;
|
|
20
|
+
/** `postgres` connection to the project database. */
|
|
21
|
+
adminConnectionString(name: string): string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Sweeps stale test containers (`sweepTestInstances`), then starts
|
|
25
|
+
* `db-test-<name>-<uuid>`, labeled with this process's pid, on an assigned
|
|
26
|
+
* loopback port and loads the app's data into `db_template` (`testImage`).
|
|
27
|
+
* The run's resources remove the container and its data at teardown.
|
|
28
|
+
*/
|
|
29
|
+
export declare function startTestInstance(resources: ResourceScope, name: string, options: PostgresOptions): Promise<PostgresTestDatabase>;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/** Postgres instance per test run; a database per project. */
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { output, succeeds, } from "@astralarium/repo-dev";
|
|
5
|
+
import { ExpectedError } from "@astralarium/repo-telemetry";
|
|
6
|
+
import { isSqlstate, SQLSTATE, withSql } from "../sql.js";
|
|
7
|
+
import { adminConnectionString, appConnectionString, probePostgres, } from "./connection.js";
|
|
8
|
+
import { removeContainer, startPostgresContainer } from "./container.js";
|
|
9
|
+
import { dataDigest, loadData } from "./data.js";
|
|
10
|
+
import { prepareTestImage, readTestImage, schemaHash, } from "./image.js";
|
|
11
|
+
/** Name prefix of every test instance's container. */
|
|
12
|
+
const TEST_PREFIX = "db-test-";
|
|
13
|
+
/** Test container label carrying the pid of the process that owns it. */
|
|
14
|
+
const TEST_OWNER_LABEL = "db.test-pid";
|
|
15
|
+
/** Running seeded instance; fixtures may run tools in its container or network namespace. */
|
|
16
|
+
export class PostgresTestDatabase {
|
|
17
|
+
/** Published loopback port. */
|
|
18
|
+
port;
|
|
19
|
+
/** Container running the instance; other containers join its network
|
|
20
|
+
* namespace. */
|
|
21
|
+
container;
|
|
22
|
+
constructor(instance) {
|
|
23
|
+
this.port = instance.port;
|
|
24
|
+
this.container = instance.container;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* `CREATE DATABASE … TEMPLATE db_template`; returns the `app_service`
|
|
28
|
+
* connection. A project whose name maps to an existing database is an
|
|
29
|
+
* `ExpectedError`.
|
|
30
|
+
*/
|
|
31
|
+
async createDatabase(name) {
|
|
32
|
+
const database = testDatabaseName(name);
|
|
33
|
+
try {
|
|
34
|
+
await withSql(adminConnectionString(this.port, "postgres"), (sql) => sql `CREATE DATABASE ${sql(database)} TEMPLATE db_template`);
|
|
35
|
+
}
|
|
36
|
+
catch (cause) {
|
|
37
|
+
if (isSqlstate(cause, SQLSTATE.duplicateDatabase))
|
|
38
|
+
throw new ExpectedError(`Test project ${name} maps to database ${database}, which another project already uses; rename one`, { cause });
|
|
39
|
+
throw cause;
|
|
40
|
+
}
|
|
41
|
+
return appConnectionString(this.port, database);
|
|
42
|
+
}
|
|
43
|
+
/** `postgres` connection to the project database. */
|
|
44
|
+
adminConnectionString(name) {
|
|
45
|
+
return adminConnectionString(this.port, testDatabaseName(name));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The DB_TEST_IMAGE manifest when built for `options.name`, an
|
|
50
|
+
* `ExpectedError` when malformed or built from other schema or data
|
|
51
|
+
* sources; otherwise, the file unset or missing included, the schema
|
|
52
|
+
* image prepared locally.
|
|
53
|
+
*/
|
|
54
|
+
function testImage(options) {
|
|
55
|
+
const file = process.env.DB_TEST_IMAGE;
|
|
56
|
+
const supplied = file && existsSync(file) ? readTestImage(file) : undefined;
|
|
57
|
+
if (supplied?.name !== options.name)
|
|
58
|
+
return prepareTestImage(options);
|
|
59
|
+
if (supplied.hash !== schemaHash(options) ||
|
|
60
|
+
supplied.data !== dataDigest(options.data ?? []))
|
|
61
|
+
throw new ExpectedError(`${file} was prepared from other schema or data sources than the current ones; rerun the app's db:image task to regenerate it`);
|
|
62
|
+
return supplied;
|
|
63
|
+
}
|
|
64
|
+
/** Process `pid` has exited. */
|
|
65
|
+
function exited(pid) {
|
|
66
|
+
try {
|
|
67
|
+
process.kill(pid, 0);
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
catch (error) {
|
|
71
|
+
return error.code === "ESRCH";
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Removes test containers left by earlier runs whose `TEST_OWNER_LABEL`
|
|
76
|
+
* process has exited. Best effort; a container another run removes first
|
|
77
|
+
* is fine.
|
|
78
|
+
*/
|
|
79
|
+
function sweepTestInstances(runtime) {
|
|
80
|
+
const listed = output(runtime, [
|
|
81
|
+
"ps",
|
|
82
|
+
"-a",
|
|
83
|
+
"--filter",
|
|
84
|
+
`name=^${TEST_PREFIX}`,
|
|
85
|
+
"--format",
|
|
86
|
+
`{{.Names}}\t{{.Label "${TEST_OWNER_LABEL}"}}`,
|
|
87
|
+
]);
|
|
88
|
+
for (const line of listed?.split("\n") ?? []) {
|
|
89
|
+
const [name, label] = line.split("\t");
|
|
90
|
+
if (!name?.startsWith(TEST_PREFIX))
|
|
91
|
+
continue;
|
|
92
|
+
const pid = Number(label);
|
|
93
|
+
if (Number.isInteger(pid) && pid > 0 && exited(pid))
|
|
94
|
+
succeeds(runtime, ["rm", "-f", "-v", name]);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Sweeps stale test containers (`sweepTestInstances`), then starts
|
|
99
|
+
* `db-test-<name>-<uuid>`, labeled with this process's pid, on an assigned
|
|
100
|
+
* loopback port and loads the app's data into `db_template` (`testImage`).
|
|
101
|
+
* The run's resources remove the container and its data at teardown.
|
|
102
|
+
*/
|
|
103
|
+
export async function startTestInstance(resources, name, options) {
|
|
104
|
+
resources.signal.throwIfAborted();
|
|
105
|
+
const { runtime, hash, image } = testImage(options);
|
|
106
|
+
sweepTestInstances(runtime);
|
|
107
|
+
const container = `${TEST_PREFIX}${name}-${randomUUID()}`;
|
|
108
|
+
resources.own({ dispose: () => removeContainer(runtime, container) });
|
|
109
|
+
const { port } = await startPostgresContainer(runtime, resources, {
|
|
110
|
+
name: container,
|
|
111
|
+
hash,
|
|
112
|
+
image,
|
|
113
|
+
serverArgs: options.ephemeralArgs,
|
|
114
|
+
labels: { [TEST_OWNER_LABEL]: String(process.pid) },
|
|
115
|
+
});
|
|
116
|
+
const connectionString = adminConnectionString(port, "db_template");
|
|
117
|
+
await probePostgres(connectionString);
|
|
118
|
+
await loadData(connectionString, options.data ?? []);
|
|
119
|
+
return new PostgresTestDatabase({
|
|
120
|
+
port,
|
|
121
|
+
container: { runtime, name: container },
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
/** SQL identifier derived from a nonempty project name. */
|
|
125
|
+
function testDatabaseName(name) {
|
|
126
|
+
const database = name.replace(/[^a-z0-9_]/gi, "_").toLowerCase();
|
|
127
|
+
if (!database || database.length > 63)
|
|
128
|
+
throw new Error("Test project names must contain 1–63 characters");
|
|
129
|
+
return database;
|
|
130
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/** What a package declares for its slice of the app database. */
|
|
3
|
+
export interface SchemaModuleDeclaration {
|
|
4
|
+
/** Unique per install; names the module's staged directory. */
|
|
5
|
+
name: string;
|
|
6
|
+
/** Absolute directory `files`, `assets`, `beforeReset` and `scrub`
|
|
7
|
+
* resolve from: an app's SQL directory, or a package's
|
|
8
|
+
* `packageRoot(import.meta.filename)`. */
|
|
9
|
+
root: string;
|
|
10
|
+
/** SQL in load order, relative to `root`; basenames unique across `files`
|
|
11
|
+
* and `assets`. psql runs it connected to the app database, after the
|
|
12
|
+
* roles exist, with the staged directory as cwd: this module's files and
|
|
13
|
+
* assets sit at `sql/<name>/<basename>`. `reset.sql` runs every module's
|
|
14
|
+
* SQL inside one transaction. */
|
|
15
|
+
files: readonly string[];
|
|
16
|
+
/** Files the module's SQL reads via psql (e.g. `\copy` CSVs), relative to
|
|
17
|
+
* `root`, any extension; staged beside `files`. */
|
|
18
|
+
assets?: readonly string[];
|
|
19
|
+
/** Modules loading first; also those the module's queries read at runtime. */
|
|
20
|
+
requires?: readonly SchemaModule[];
|
|
21
|
+
/** Extensions the image installs: Debian `postgresql-<major>-<name>` packages. */
|
|
22
|
+
extensions?: readonly string[];
|
|
23
|
+
/** `shared_preload_libraries` entries for persistent instances. */
|
|
24
|
+
preload?: readonly string[];
|
|
25
|
+
/** Further Postgres server arguments for persistent instances. */
|
|
26
|
+
serverArgs?: readonly string[];
|
|
27
|
+
/** SQL relative to `root`, run as `app_admin` inside `reset.sql`'s
|
|
28
|
+
* transaction before it drops the schemas. */
|
|
29
|
+
beforeReset?: string;
|
|
30
|
+
/** SQL relative to `root`, run by `run-snapshot` as superuser in
|
|
31
|
+
* `db_template` after the dump loads, with the staged directory as cwd;
|
|
32
|
+
* every module's scrub runs inside one transaction. Clears
|
|
33
|
+
* environment-bound state (credentials, deploy state, build-keyed rows). */
|
|
34
|
+
scrub?: string;
|
|
35
|
+
}
|
|
36
|
+
/** A declared module: paths absolute, every list present. */
|
|
37
|
+
export interface SchemaModule {
|
|
38
|
+
readonly name: string;
|
|
39
|
+
readonly root: string;
|
|
40
|
+
readonly files: readonly string[];
|
|
41
|
+
readonly assets: readonly string[];
|
|
42
|
+
readonly requires: readonly SchemaModule[];
|
|
43
|
+
readonly extensions: readonly string[];
|
|
44
|
+
readonly preload: readonly string[];
|
|
45
|
+
readonly serverArgs: readonly string[];
|
|
46
|
+
readonly beforeReset?: string;
|
|
47
|
+
readonly scrub?: string;
|
|
48
|
+
}
|
|
49
|
+
/** Non-empty strings, e.g. extensions or server arguments. */
|
|
50
|
+
export declare const stringList: z.ZodReadonly<z.ZodArray<z.ZodString>>;
|
|
51
|
+
/** A value shaped as `defineSchemaModule` returns. */
|
|
52
|
+
export declare function isSchemaModule(value: unknown): value is SchemaModule;
|
|
53
|
+
/** A schema module; the declaration is validated (`ExpectedError`). */
|
|
54
|
+
export declare function defineSchemaModule(module: SchemaModuleDeclaration): SchemaModule;
|
|
55
|
+
/**
|
|
56
|
+
* `modules` and their `requires` closure in load order. One name on two
|
|
57
|
+
* module objects (two installed copies of a package) is an `ExpectedError`.
|
|
58
|
+
*/
|
|
59
|
+
export declare function loadOrder(modules: readonly SchemaModule[]): SchemaModule[];
|