@astralarium/repo-postgres 0.0.0-stage → 0.1.0

package/dist/src/provision/data.js

@@ -0,0 +1,47 @@
+/** The app's data SQL: loaded into `db_template` after the schema, its
+ *  digest recorded in the test image manifest. */
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+import { ExpectedError } from "@astralarium/repo-telemetry";
+import { withSql } from "../sql.js";
+/** `data`'s files in load order: each file as given, each directory's
+ *  `*.sql` in name order. */
+export function dataFiles(data) {
+    return data.flatMap((path) => {
+        let directory;
+        try {
+            directory = statSync(path).isDirectory();
+        }
+        catch (cause) {
+            throw new ExpectedError(`data path ${path} is missing`, { cause });
+        }
+        return directory
+            ? readdirSync(path)
+                .filter((file) => file.endsWith(".sql"))
+                .sort()
+                .map((file) => join(path, file))
+            : [path];
+    });
+}
+/** sha256 over the data files' names and contents, in load order; the
+ *  same across checkouts. */
+export function dataDigest(data) {
+    const hash = createHash("sha256");
+    for (const file of dataFiles(data)) {
+        hash.update(`${basename(file)}\0`);
+        hash.update(readFileSync(file));
+        hash.update("\0");
+    }
+    return hash.digest("hex");
+}
+/** Runs the data files in load order, each as one multi-statement script. */
+export async function loadData(connectionString, data) {
+    const files = dataFiles(data);
+    if (files.length === 0)
+        return;
+    await withSql(connectionString, async (sql) => {
+        for (const file of files)
+            await sql.file(file);
+    });
+}

package/dist/src/provision/disposable.d.ts

@@ -0,0 +1,49 @@
+/** Throws an `ExpectedError` unless `path` is an existing file. */
+export declare function requireFile(path: string): void;
+/** One foreground disposable server. */
+export interface DisposableServer {
+    /** Container name. */
+    name: string;
+    /** Immutable image ID. */
+    image: string;
+    /** Published loopback port. */
+    port: number;
+    /** Server arguments appended to the image command. */
+    serverArgs: readonly string[];
+    /** Full dump the initdb hook loads into `db_template`. */
+    snapshot?: {
+        dump: string;
+        scrub: boolean;
+    };
+    /** Stops the server: SIGINT, Postgres' fast shutdown. */
+    signal: AbortSignal;
+}
+/** `run --rm` in the foreground; logs the connection once Postgres
+ *  listens on the published port, resolves once the container is gone. */
+export declare function runDisposableServer(runtime: string, { name, image, port, serverArgs, snapshot, signal }: DisposableServer): Promise<void>;
+/** `--mount` arguments placing each migrate file at
+ *  `/app/migrate.d/<basename>`; each must be an existing `.sql` file with a
+ *  unique basename. */
+export declare function migrationMounts(files: readonly string[]): string[];
+/** Throws an `ExpectedError` for a `DB_DUMP_EXCLUDE` pattern lacking a
+ *  schema. */
+export declare function checkDumpExclude(patterns?: string): void;
+/** The upgrade dump's containers. */
+export interface UpgradeDump {
+    /** Name of the throwaway instance and its private network. */
+    name: string;
+    /** Immutable db image ID. */
+    image: string;
+    /** Util image tag. */
+    util: string;
+    /** `migrationMounts` of the migrate files. */
+    mounts: readonly string[];
+    /** Stops the dump; its containers go. */
+    signal: AbortSignal;
+}
+/**
+ * The util image loads the `PG*` source into a throwaway instance of
+ * `image` on a private network and prints that instance's full dump on
+ * stdout. Leftovers of an earlier run go first; everything goes on exit.
+ */
+export declare function runUpgradeDump(runtime: string, { name, image, util, mounts, signal }: UpgradeDump): Promise<void>;

package/dist/src/provision/disposable.js

@@ -0,0 +1,188 @@
+/** Disposable instances of the schema image: foreground servers and the
+ *  upgrade dump. */
+import { execFileSync, spawn, } from "node:child_process";
+import { once } from "node:events";
+import { chmodSync, copyFileSync, mkdtempSync, rmSync, statSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, extname, join } from "node:path";
+import { succeeds, waitFor } from "@astralarium/repo-dev";
+import { packageRoot } from "@astralarium/repo-package-files";
+import { ExpectedError, log } from "@astralarium/repo-telemetry";
+import { adminConnectionString } from "./connection.js";
+/** initdb hook loading `/app/data.sql`, relative to the package root. */
+const SNAPSHOT_HOOK = "container/snapshot/run-snapshot.sh";
+/** Container environment: trust auth, `postgres` as the default client. */
+const TRUST_ENV = [
+    "--env",
+    "POSTGRES_HOST_AUTH_METHOD=trust",
+    "--env",
+    "PGUSER=postgres",
+];
+/** Throws an `ExpectedError` unless `path` is an existing file. */
+export function requireFile(path) {
+    let file = false;
+    try {
+        file = statSync(path).isFile();
+    }
+    catch {
+        // Missing: reported below.
+    }
+    if (!file)
+        throw new ExpectedError(`${path} is not a file`);
+}
+/** Exit code of `child`, or its signal; rejects when it fails to spawn. */
+async function exit(child) {
+    const [code, signal] = (await once(child, "exit"));
+    return code === null ? String(signal) : String(code);
+}
+/** `run --rm` in the foreground; logs the connection once Postgres
+ *  listens on the published port, resolves once the container is gone. */
+export async function runDisposableServer(runtime, { name, image, port, serverArgs, snapshot, signal }) {
+    // The entrypoint sources non-executable hooks; packed packages drop mode bits.
+    const hooks = snapshot ? mkdtempSync(join(tmpdir(), "db-snapshot-")) : "";
+    try {
+        const snapshotArgs = [];
+        if (snapshot) {
+            const hook = join(hooks, basename(SNAPSHOT_HOOK));
+            copyFileSync(join(packageRoot(import.meta.filename), SNAPSHOT_HOOK), hook);
+            chmodSync(hook, 0o755);
+            snapshotArgs.push("--env", `SNAPSHOT_SCRUB=${snapshot.scrub ? 1 : 0}`, "--mount", `type=bind,source=${snapshot.dump},target=/app/data.sql,readonly`, "--mount", `type=bind,source=${hook},target=/docker-entrypoint-initdb.d/${basename(hook)},readonly`);
+        }
+        signal.throwIfAborted();
+        const server = spawn(runtime, [
+            "run",
+            "--rm",
+            "--name",
+            name,
+            ...TRUST_ENV,
+            "--publish",
+            `127.0.0.1:${port}:5432`,
+            ...snapshotArgs,
+            image,
+            ...serverArgs,
+        ], { stdio: ["ignore", "inherit", "inherit"] });
+        const exited = exit(server);
+        const gone = new AbortController();
+        exited.then(() => gone.abort(), () => gone.abort());
+        const stop = () => server.kill("SIGINT");
+        signal.addEventListener("abort", stop, { once: true });
+        try {
+            try {
+                // initdb, a snapshot load included, listens on the socket alone.
+                await waitFor(() => succeeds(runtime, [
+                    "exec",
+                    name,
+                    "pg_isready",
+                    "-h",
+                    "127.0.0.1",
+                    "-U",
+                    "postgres",
+                ]), `${name} never became ready`, Number.POSITIVE_INFINITY, AbortSignal.any([signal, gone.signal]));
+                log.info("dev.db_disposable_ready", "{name}: ready on 127.0.0.1:{port}, removed on exit: {url}", { name, port, url: adminConnectionString(port, "db_template") });
+            }
+            catch (error) {
+                if (!signal.aborted && !gone.signal.aborted)
+                    throw error;
+            }
+            const status = await exited;
+            if (!signal.aborted && status !== "0")
+                throw new ExpectedError(`${name} exited with ${status}`);
+        }
+        finally {
+            signal.removeEventListener("abort", stop);
+        }
+    }
+    finally {
+        if (hooks)
+            rmSync(hooks, { recursive: true, force: true });
+    }
+}
+/** `--mount` arguments placing each migrate file at
+ *  `/app/migrate.d/<basename>`; each must be an existing `.sql` file with a
+ *  unique basename. */
+export function migrationMounts(files) {
+    const targets = new Set();
+    return files.flatMap((file) => {
+        requireFile(file);
+        if (extname(file) !== ".sql")
+            throw new ExpectedError(`${file} lacks the .sql extension`);
+        const target = `/app/migrate.d/${basename(file)}`;
+        if (targets.has(target))
+            throw new ExpectedError(`two migrate files share the basename of ${file}`);
+        targets.add(target);
+        return ["--mount", `type=bind,source=${file},target=${target},readonly`];
+    });
+}
+/** Throws an `ExpectedError` for a `DB_DUMP_EXCLUDE` pattern lacking a
+ *  schema. */
+export function checkDumpExclude(patterns = "") {
+    for (const pattern of patterns.split(/\s+/))
+        if (pattern !== "" && !pattern.includes("."))
+            throw new ExpectedError(`DB_DUMP_EXCLUDE pattern ${pattern} lacks a schema`);
+}
+/**
+ * The util image loads the `PG*` source into a throwaway instance of
+ * `image` on a private network and prints that instance's full dump on
+ * stdout. Leftovers of an earlier run go first; everything goes on exit.
+ */
+export async function runUpgradeDump(runtime, { name, image, util, mounts, signal }) {
+    const utilName = `${name}-util`;
+    const cleanup = () => {
+        succeeds(runtime, ["rm", "-f", utilName]);
+        succeeds(runtime, ["rm", "-f", name]);
+        succeeds(runtime, ["network", "rm", name]);
+    };
+    cleanup();
+    try {
+        const quiet = {
+            stdio: ["ignore", "ignore", "inherit"],
+        };
+        execFileSync(runtime, ["network", "create", name], quiet);
+        execFileSync(runtime, [
+            "run",
+            "--rm",
+            "--detach",
+            "--name",
+            name,
+            ...TRUST_ENV,
+            "--network",
+            name,
+            "--expose",
+            "5432",
+            image,
+        ], quiet);
+        signal.throwIfAborted();
+        const dump = spawn(runtime, [
+            "run",
+            "--rm",
+            "--interactive",
+            "--name",
+            utilName,
+            "--env",
+            `CONTAINER_PGHOST=${name}`,
+            ...["PGHOST", "PGPORT", "PGUSER", "PGPASSWORD", "PGDATABASE"].flatMap((key) => ["--env", key]),
+            "--env",
+            "DB_DUMP_EXCLUDE",
+            "--network",
+            name,
+            ...mounts,
+            util,
+            "/entrypoint.d/upgrade-db.sh",
+        ], { stdio: "inherit" });
+        const stop = () => dump.kill("SIGINT");
+        signal.addEventListener("abort", stop, { once: true });
+        try {
+            const status = await exit(dump);
+            if (signal.aborted)
+                throw new ExpectedError(`${utilName} interrupted; the dump is partial`);
+            if (status !== "0")
+                throw new ExpectedError(`${utilName} exited with ${status}`);
+        }
+        finally {
+            signal.removeEventListener("abort", stop);
+        }
+    }
+    finally {
+        cleanup();
+    }
+}

package/dist/src/provision/image.d.ts

@@ -0,0 +1,63 @@
+import type { PostgresOptions } from "./options.ts";
+/**
+ * What the package contributes to the image: `name@version` once
+ * installed from a registry; while linked from a workspace (files change
+ * without a version bump), the Git-visible `container/init` and `sql`
+ * sources and the build script.
+ */
+export declare function packageFingerprint(dir?: string): string;
+/**
+ * sha256 of what the image bakes: the staged schema (`schemaDigest`), the
+ * package fingerprint and the modules' extensions.
+ */
+export declare function schemaHash(options: PostgresOptions): string;
+/** What `buildImages` builds and how. */
+interface BuildImagesOptions {
+    /** Images to build; defaults to `all`. */
+    target?: "db" | "util" | "all";
+    /** Image base name. */
+    name: string;
+    /** File receiving the built db image's ID. */
+    imageIdFile?: string;
+    /** Pulls the base image first. */
+    refreshBase?: boolean;
+}
+/**
+ * Builds database images with `build-images.sh` over a staged copy of the
+ * schema; base-image refresh is explicit.
+ */
+export declare function buildImages(options: PostgresOptions, runtime: string, hash: string, { target, name, imageIdFile, refreshBase, }: BuildImagesOptions): void;
+/**
+ * Reuses the schema-tagged local image or builds it; returns its immutable
+ * ID. A reused image is retagged `<name>:latest`, as a build tags it.
+ */
+export declare function ensureImage(options: PostgresOptions, runtime: string, hash: string, name: string): string;
+/** Builds `<name>-util:latest` from the package's util sources
+ *  (layer-cached) and returns the tag. */
+export declare function ensureUtilImage(runtime: string, name: string): string;
+/** Database image and host runtime captured by the image prerequisite. */
+export interface TestImage {
+    /** `PostgresOptions.name` the image was built for. */
+    name: string;
+    /** Container runtime the image was built with. */
+    runtime: string;
+    /** Immutable image ID (`sha256:…`). */
+    image: string;
+    /** `schemaHash` the image was built from. */
+    hash: string;
+    /** `dataDigest` of `PostgresOptions.data`, which loads after the image's
+     *  schema. */
+    data: string;
+    /** Node version and platform captured by the prerequisite. */
+    host: string;
+}
+/** Selects the schema image, tagged `name`, and records its immutable ID,
+ *  runtime and data digest. */
+export declare function prepareTestImage(options: PostgresOptions, name?: string): TestImage;
+/** Publishes the manifest of the image tagged `envDbName` at `file`,
+ *  replacing it whole. */
+export declare function writeTestImage(options: PostgresOptions, file: URL): void;
+/** Reads a prepared database-image manifest; a malformed one is an
+ *  `ExpectedError`. */
+export declare function readTestImage(path: string): TestImage;
+export {};

package/dist/src/provision/image.js

@@ -0,0 +1,159 @@
+/** The app's database image: schema hash, build, selection for test runs. */
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { mkdirSync, mkdtempSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { envDbName, output, sourceHash } from "@astralarium/repo-dev";
+import { packageRoot, readPackageManifest, } from "@astralarium/repo-package-files";
+import { ExpectedError } from "@astralarium/repo-telemetry";
+import { z } from "zod";
+import { detectRuntime, SCHEMA_LABEL } from "./container.js";
+import { dataDigest } from "./data.js";
+import { moduleExtensions, schemaDigest, stageSchema } from "./schema.js";
+/** `build-images.sh <runtime> <schema-hash> [db|util|all] [db-iidfile]`. */
+const BUILD_IMAGES = "scripts/build-images.sh";
+/** SQL the image bakes beside the schema: roles, database grants,
+ *  bootstrap, reset, drop. */
+const PACKAGE_SQL = "sql";
+/**
+ * What the package contributes to the image: `name@version` once
+ * installed from a registry; while linked from a workspace (files change
+ * without a version bump), the Git-visible `container/init` and `sql`
+ * sources and the build script.
+ */
+export function packageFingerprint(dir = packageRoot(import.meta.filename)) {
+    const real = realpathSync(dir);
+    if (real.split(sep).includes("node_modules")) {
+        const { name, version } = readPackageManifest(join(real, "package.json"));
+        return `${name}@${version}`;
+    }
+    return sourceHash(real, ["container/init", PACKAGE_SQL, BUILD_IMAGES]);
+}
+/**
+ * sha256 of what the image bakes: the staged schema (`schemaDigest`), the
+ * package fingerprint and the modules' extensions.
+ */
+export function schemaHash(options) {
+    return createHash("sha256")
+        .update(schemaDigest(options.modules))
+        .update(packageFingerprint())
+        .update(extensionList(options))
+        .digest("hex");
+}
+/** `DB_EXTENSIONS` for the image build. */
+function extensionList(options) {
+    return moduleExtensions(options.modules).join(" ");
+}
+/** Runs `build-images.sh` (through bash: packed scripts lose their mode
+ *  bits) with the image base name and `env` added. */
+function runBuildImages(args, name, env = {}) {
+    execFileSync("bash", [join(packageRoot(import.meta.filename), BUILD_IMAGES), ...args], {
+        stdio: "inherit",
+        env: { ...process.env, DB_NAME: name, ...env },
+    });
+}
+/**
+ * Builds database images with `build-images.sh` over a staged copy of the
+ * schema; base-image refresh is explicit.
+ */
+export function buildImages(options, runtime, hash, { target = "all", name, imageIdFile, refreshBase = false, }) {
+    const staged = mkdtempSync(join(tmpdir(), "db-schema-"));
+    try {
+        stageSchema(options.modules, staged);
+        runBuildImages([runtime, hash, target, ...(imageIdFile ? [imageIdFile] : [])], name, {
+            DB_SCHEMA_DIR: staged,
+            DB_EXTENSIONS: extensionList(options),
+            DB_REFRESH_BASE: refreshBase ? "true" : "",
+        });
+    }
+    finally {
+        rmSync(staged, { recursive: true, force: true });
+    }
+}
+/**
+ * Reuses the schema-tagged local image or builds it; returns its immutable
+ * ID. A reused image is retagged `<name>:latest`, as a build tags it.
+ */
+export function ensureImage(options, runtime, hash, name) {
+    const cached = output(runtime, [
+        "image",
+        "inspect",
+        "-f",
+        `{{.Id}}\t{{index .Config.Labels "${SCHEMA_LABEL}"}}`,
+        `${name}:${hash}`,
+    ]);
+    if (cached) {
+        const [id, schema] = cached.split("\t");
+        if (id && schema === hash) {
+            const image = id.startsWith("sha256:") ? id : `sha256:${id}`;
+            execFileSync(runtime, ["tag", image, `${name}:latest`]);
+            return image;
+        }
+    }
+    const dir = mkdtempSync(join(tmpdir(), "db-image-"));
+    try {
+        const imageIdFile = join(dir, "id");
+        buildImages(options, runtime, hash, { target: "db", name, imageIdFile });
+        return readFileSync(imageIdFile, "utf8").trim();
+    }
+    finally {
+        rmSync(dir, { recursive: true, force: true });
+    }
+}
+/** Builds `<name>-util:latest` from the package's util sources
+ *  (layer-cached) and returns the tag. */
+export function ensureUtilImage(runtime, name) {
+    runBuildImages([runtime, "", "util"], name);
+    return `${name}-util:latest`;
+}
+/** Selects the schema image, tagged `name`, and records its immutable ID,
+ *  runtime and data digest. */
+export function prepareTestImage(options, name = options.name) {
+    const runtime = detectRuntime();
+    const hash = schemaHash(options);
+    return {
+        name: options.name,
+        runtime,
+        image: ensureImage(options, runtime, hash, name),
+        hash,
+        data: dataDigest(options.data ?? []),
+        host: `${process.version}/${process.platform}/${process.arch}`,
+    };
+}
+/** Publishes the manifest of the image tagged `envDbName` at `file`,
+ *  replacing it whole. */
+export function writeTestImage(options, file) {
+    const image = prepareTestImage(options, envDbName(options));
+    const path = fileURLToPath(file);
+    mkdirSync(dirname(path), { recursive: true });
+    const temp = `${path}.${process.pid}.tmp`;
+    writeFileSync(temp, JSON.stringify(image) + "\n");
+    renameSync(temp, path);
+}
+const SHA256 = /^[a-f0-9]{64}$/;
+const testImageManifest = z.object({
+    name: z.string(),
+    runtime: z.string(),
+    image: z.string().regex(/^sha256:[a-f0-9]{64}$/),
+    hash: z.string().regex(SHA256),
+    data: z.string().regex(SHA256),
+    host: z.string(),
+});
+/** Reads a prepared database-image manifest; a malformed one is an
+ *  `ExpectedError`. */
+export function readTestImage(path) {
+    const text = readFileSync(path, "utf8");
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        value = undefined;
+    }
+    const result = testImageManifest.safeParse(value);
+    if (!result.success)
+        throw new ExpectedError(`${path}: invalid prepared test image; run the app's db:image task to regenerate it`);
+    return result.data;
+}

package/dist/src/provision/index.d.ts

@@ -0,0 +1,17 @@
+import { type DatabaseProvider } from "@astralarium/repo-dev";
+import type { PostgresOptions } from "./options.ts";
+import { type PostgresTestDatabase } from "./test-database.ts";
+export { APP_SERVICE_ROLE } from "./connection.ts";
+export { ensureUtilImage } from "./image.ts";
+export type { PostgresOptions } from "./options.ts";
+export type { PostgresTestDatabase } from "./test-database.ts";
+/** The app's Postgres, as `DevConfig.database` and `TestApp.database` take it. */
+export interface PostgresProvider extends DatabaseProvider<PostgresTestDatabase> {
+    /**
+     * Writes the staged schema directory under `dir`, as the image bakes it
+     * (`stageSchema`).
+     */
+    stageSchema(dir: string): void;
+}
+/** The app's Postgres provider; the declaration is validated (`ExpectedError`). */
+export declare function postgresDatabase(options: PostgresOptions): PostgresProvider;