@astralarium/repo-postgres 0.0.0-stage → 0.2.0

package/dist/src/provision/container.js

@@ -0,0 +1,244 @@
+/** Postgres containers: runtime detection, inspection, startup, readiness. */
+import { execFileSync, spawnSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import { output, READY_TIMEOUT_MS, succeeds, waitFor, } from "@astralarium/repo-dev";
+import { ExpectedError } from "@astralarium/repo-telemetry";
+/** Image label carrying the `schemaHash()` it was built from. */
+export const SCHEMA_LABEL = "db.schema";
+/** Container and volume label naming the checkout that owns it. */
+export const OWNER_LABEL = "dev.owner";
+/** Container label carrying the sha256 of its Postgres server arguments. */
+const ARGS_LABEL = "db.args";
+/** File the init script writes the image's schema hash to, inside the data volume. */
+const SCHEMA_STAMP = "/var/lib/postgresql/db-schema";
+/** `CONTAINER_RUNTIME`, else the first of podman, docker that answers `info`. */
+export function detectRuntime() {
+    const override = process.env.CONTAINER_RUNTIME;
+    if (override)
+        return override;
+    for (const bin of ["podman", "docker"]) {
+        if (succeeds(bin, ["info"]))
+            return bin;
+    }
+    throw new ExpectedError("neither podman nor docker is running");
+}
+/** Schema hash the container's data was initialised from; null while
+ *  init has yet to complete. */
+function schemaStamp(runtime, container) {
+    return output(runtime, ["exec", container, "cat", SCHEMA_STAMP]);
+}
+/** The container's data was initialised from a schema other than its image's. */
+export class StaleSchemaError extends Error {
+    constructor(runtime, container, stamp) {
+        super(stamp === null
+            ? `${container} has no schema stamp: its first boot is incomplete (see ${runtime} logs ${container}).`
+            : `${container} data was initialised from another schema.`);
+    }
+}
+/** Host port published for Postgres; the container must be running. */
+function publishedPort(runtime, container) {
+    const port = Number(execFileSync(runtime, [
+        "container",
+        "inspect",
+        "-f",
+        '{{(index (index .NetworkSettings.Ports "5432/tcp") 0).HostPort}}',
+        container,
+    ], { encoding: "utf8" }).trim());
+    if (!Number.isInteger(port) || port < 1 || port > 65535)
+        throw new Error(`${container} has no published Postgres port: ${port}`);
+    return port;
+}
+/** One `container inspect`; null for an absent `name`. */
+export function inspectContainer(runtime, name) {
+    const state = output(runtime, [
+        "container",
+        "inspect",
+        "-f",
+        "{{json .}}",
+        name,
+    ]);
+    if (state === null)
+        return null;
+    const inspected = JSON.parse(state);
+    const image = inspected.Image;
+    return {
+        running: inspected.State.Running,
+        image: image.startsWith("sha256:") ? image : `sha256:${image}`,
+        schema: inspected.Config.Labels[SCHEMA_LABEL],
+        owner: inspected.Config.Labels[OWNER_LABEL],
+        args: inspected.Config.Labels[ARGS_LABEL],
+        bindings: inspected.HostConfig.PortBindings["5432/tcp"] ?? [],
+        dataMount: inspected.Mounts.find((mount) => mount.Destination === "/var/lib/postgresql"),
+    };
+}
+/** Persistent data volume of dev instance `name`. */
+export function volumeName(name) {
+    return `${name}-data`;
+}
+/** A named volume's `OWNER_LABEL`; null for an absent `name`. */
+function inspectVolume(runtime, name) {
+    const labels = output(runtime, [
+        "volume",
+        "inspect",
+        "-f",
+        "{{json .Labels}}",
+        name,
+    ]);
+    if (labels === null)
+        return null;
+    const parsed = JSON.parse(labels || "null");
+    return { owner: parsed?.[OWNER_LABEL] };
+}
+/** The existing container and data volume of dev instance `name`. */
+export function holders(runtime, name) {
+    const found = [];
+    const container = inspectContainer(runtime, name);
+    if (container)
+        found.push({ kind: "container", name, owner: container.owner });
+    const volume = inspectVolume(runtime, volumeName(name));
+    if (volume)
+        found.push({ kind: "volume", name: volumeName(name), owner: volume.owner });
+    return found;
+}
+/** Container `name` and its data volume are each absent or labeled as
+ *  owned by `owner`, so a volume surviving its container stays with the
+ *  checkout that made it. */
+export function claimable(runtime, name, owner) {
+    return holders(runtime, name).every((holder) => holder.owner === owner);
+}
+/** Creates `volume` labeled for `owner` when absent; one labeled
+ *  otherwise throws, keeping each checkout on its own data. */
+function ensureVolume(runtime, volume, owner) {
+    const current = inspectVolume(runtime, volume);
+    if (current === null) {
+        execFileSync(runtime, [
+            "volume",
+            "create",
+            ...(owner ? ["--label", `${OWNER_LABEL}=${owner}`] : []),
+            volume,
+        ], { stdio: ["ignore", "ignore", "inherit"] });
+        return;
+    }
+    if (current.owner !== owner)
+        throw new ExpectedError(`data volume ${volume} is ${current.owner === undefined ? "unlabeled" : `labeled for ${current.owner}`}; rerun to allocate another instance, or free the name with: ${runtime} volume rm ${volume}`);
+}
+/** `rm -f`; a missing container is fine, any other failure throws. */
+export function removeContainer(runtime, name) {
+    if (inspectContainer(runtime, name) === null)
+        return;
+    execFileSync(runtime, ["rm", "-f", "-v", name], {
+        stdio: ["ignore", "ignore", "inherit"],
+    });
+}
+/** `volume rm -f`; a missing volume is fine, any other failure throws. */
+export function removeVolume(runtime, name) {
+    if (inspectVolume(runtime, name) === null)
+        return;
+    execFileSync(runtime, ["volume", "rm", "-f", name], {
+        stdio: ["ignore", "ignore", "inherit"],
+    });
+}
+/**
+ * Reuses a container of exactly `image`, `owner`, server arguments and
+ * settings, else recreates it (named volumes survive). Resolves once
+ * Postgres accepts connections and the data carries `hash`, with the
+ * published host port. `resources` stops a persistent container this call
+ * started; one already running is left alone. The caller removes an
+ * ephemeral one (`volume` unset).
+ */
+export async function startPostgresContainer(runtime, resources, { name, image, hash, port, volume, owner, serverArgs = [], labels = {}, }) {
+    const args = [
+        ...serverArgs,
+        ...(volume
+            ? []
+            : [
+                "-c",
+                "fsync=off",
+                "-c",
+                "synchronous_commit=off",
+                "-c",
+                "full_page_writes=off",
+            ]),
+    ];
+    const argsHash = createHash("sha256")
+        .update(JSON.stringify(args))
+        .digest("hex");
+    const current = inspectContainer(runtime, name);
+    const reuse = current !== null &&
+        current.schema === hash &&
+        current.image === image &&
+        current.owner === owner &&
+        current.args === argsHash &&
+        current.bindings.length === 1 &&
+        current.bindings[0]?.HostIp === "127.0.0.1" &&
+        (port === undefined || current.bindings[0].HostPort === String(port)) &&
+        (volume
+            ? current.dataMount?.Type === "volume" &&
+                current.dataMount.Name === volume
+            : current.dataMount === undefined);
+    if (!reuse && volume)
+        ensureVolume(runtime, volume, owner);
+    if (current && !reuse)
+        removeContainer(runtime, name);
+    if (!reuse) {
+        execFileSync(runtime, [
+            "create",
+            "--name",
+            name,
+            "--env",
+            "POSTGRES_HOST_AUTH_METHOD=trust",
+            "--env",
+            "PGUSER=postgres",
+            ...(owner ? ["--label", `${OWNER_LABEL}=${owner}`] : []),
+            "--label",
+            `${ARGS_LABEL}=${argsHash}`,
+            ...Object.entries(labels).flatMap(([key, value]) => [
+                "--label",
+                `${key}=${value}`,
+            ]),
+            "--publish",
+            `127.0.0.1:${port ?? ""}:5432`,
+            ...(volume ? ["--volume", `${volume}:/var/lib/postgresql`] : []),
+            image,
+            ...args,
+        ], { stdio: ["ignore", "ignore", "inherit"] });
+    }
+    if (!(reuse && current.running)) {
+        if (volume)
+            resources.own({
+                dispose: () => {
+                    execFileSync(runtime, ["stop", name], { stdio: "ignore" });
+                },
+            });
+        execFileSync(runtime, ["start", name], { stdio: "ignore" });
+    }
+    try {
+        await waitForPostgres(runtime, name, resources.signal);
+    }
+    catch (error) {
+        spawnSync(runtime, ["logs", "--tail", "50", name], { stdio: "inherit" });
+        throw error;
+    }
+    const stamp = schemaStamp(runtime, name);
+    if (stamp !== hash)
+        throw new StaleSchemaError(runtime, name, stamp);
+    return { port: publishedPort(runtime, name) };
+}
+/** Resolves once `pg_isready` succeeds in `container`: init SQL has been applied. */
+function waitForPostgres(runtime, container, signal) {
+    return waitFor(() => {
+        if (succeeds(runtime, [
+            "exec",
+            container,
+            "pg_isready",
+            "-h",
+            "127.0.0.1",
+            "-U",
+            "postgres",
+        ]))
+            return true;
+        if (!inspectContainer(runtime, container)?.running)
+            throw new ExpectedError(`${container} exited during startup. Inspect: ${runtime} logs ${container}`);
+        return false;
+    }, `${container} not ready after ${READY_TIMEOUT_MS}ms. Inspect: ${runtime} logs ${container}`, READY_TIMEOUT_MS, signal);
+}

package/dist/src/provision/data.d.ts

@@ -0,0 +1,8 @@
+/** `data`'s files in load order: each file as given, each directory's
+ *  `*.sql` in name order. */
+export declare function dataFiles(data: readonly string[]): string[];
+/** sha256 over the data files' names and contents, in load order; the
+ *  same across checkouts. */
+export declare function dataDigest(data: readonly string[]): string;
+/** Runs the data files in load order, each as one multi-statement script. */
+export declare function loadData(connectionString: string, data: readonly string[]): Promise<void>;

package/dist/src/provision/data.js

@@ -0,0 +1,47 @@
+/** The app's data SQL: loaded into `db_template` after the schema, its
+ *  digest recorded in the test image manifest. */
+import { createHash } from "node:crypto";
+import { readdirSync, readFileSync, statSync } from "node:fs";
+import { basename, join } from "node:path";
+import { ExpectedError } from "@astralarium/repo-telemetry";
+import { withSql } from "../sql.js";
+/** `data`'s files in load order: each file as given, each directory's
+ *  `*.sql` in name order. */
+export function dataFiles(data) {
+    return data.flatMap((path) => {
+        let directory;
+        try {
+            directory = statSync(path).isDirectory();
+        }
+        catch (cause) {
+            throw new ExpectedError(`data path ${path} is missing`, { cause });
+        }
+        return directory
+            ? readdirSync(path)
+                .filter((file) => file.endsWith(".sql"))
+                .sort()
+                .map((file) => join(path, file))
+            : [path];
+    });
+}
+/** sha256 over the data files' names and contents, in load order; the
+ *  same across checkouts. */
+export function dataDigest(data) {
+    const hash = createHash("sha256");
+    for (const file of dataFiles(data)) {
+        hash.update(`${basename(file)}\0`);
+        hash.update(readFileSync(file));
+        hash.update("\0");
+    }
+    return hash.digest("hex");
+}
+/** Runs the data files in load order, each as one multi-statement script. */
+export async function loadData(connectionString, data) {
+    const files = dataFiles(data);
+    if (files.length === 0)
+        return;
+    await withSql(connectionString, async (sql) => {
+        for (const file of files)
+            await sql.file(file);
+    });
+}

package/dist/src/provision/disposable.d.ts

@@ -0,0 +1,49 @@
+/** Throws an `ExpectedError` unless `path` is an existing file. */
+export declare function requireFile(path: string): void;
+/** One foreground disposable server. */
+export interface DisposableServer {
+    /** Container name. */
+    name: string;
+    /** Immutable image ID. */
+    image: string;
+    /** Published loopback port. */
+    port: number;
+    /** Server arguments appended to the image command. */
+    serverArgs: readonly string[];
+    /** Full dump the initdb hook loads into `db_template`. */
+    snapshot?: {
+        dump: string;
+        scrub: boolean;
+    };
+    /** Stops the server: SIGINT, Postgres' fast shutdown. */
+    signal: AbortSignal;
+}
+/** `run --rm` in the foreground; logs the connection once Postgres
+ *  listens on the published port, resolves once the container is gone. */
+export declare function runDisposableServer(runtime: string, { name, image, port, serverArgs, snapshot, signal }: DisposableServer): Promise<void>;
+/** `--mount` arguments placing each migrate file at
+ *  `/app/migrate.d/<basename>`; each must be an existing `.sql` file with a
+ *  unique basename. */
+export declare function migrationMounts(files: readonly string[]): string[];
+/** Throws an `ExpectedError` for a `DB_DUMP_EXCLUDE` pattern lacking a
+ *  schema. */
+export declare function checkDumpExclude(patterns?: string): void;
+/** The upgrade dump's containers. */
+export interface UpgradeDump {
+    /** Name of the throwaway instance and its private network. */
+    name: string;
+    /** Immutable db image ID. */
+    image: string;
+    /** Util image tag. */
+    util: string;
+    /** `migrationMounts` of the migrate files. */
+    mounts: readonly string[];
+    /** Stops the dump; its containers go. */
+    signal: AbortSignal;
+}
+/**
+ * The util image loads the `PG*` source into a throwaway instance of
+ * `image` on a private network and prints that instance's full dump on
+ * stdout. Leftovers of an earlier run go first; everything goes on exit.
+ */
+export declare function runUpgradeDump(runtime: string, { name, image, util, mounts, signal }: UpgradeDump): Promise<void>;

package/dist/src/provision/disposable.js

@@ -0,0 +1,188 @@
+/** Disposable instances of the schema image: foreground servers and the
+ *  upgrade dump. */
+import { execFileSync, spawn, } from "node:child_process";
+import { once } from "node:events";
+import { chmodSync, copyFileSync, mkdtempSync, rmSync, statSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, extname, join } from "node:path";
+import { succeeds, waitFor } from "@astralarium/repo-dev";
+import { packageRoot } from "@astralarium/repo-package-files";
+import { ExpectedError, log } from "@astralarium/repo-telemetry";
+import { adminConnectionString } from "./connection.js";
+/** initdb hook loading `/app/data.sql`, relative to the package root. */
+const SNAPSHOT_HOOK = "container/snapshot/run-snapshot.sh";
+/** Container environment: trust auth, `postgres` as the default client. */
+const TRUST_ENV = [
+    "--env",
+    "POSTGRES_HOST_AUTH_METHOD=trust",
+    "--env",
+    "PGUSER=postgres",
+];
+/** Throws an `ExpectedError` unless `path` is an existing file. */
+export function requireFile(path) {
+    let file = false;
+    try {
+        file = statSync(path).isFile();
+    }
+    catch {
+        // Missing: reported below.
+    }
+    if (!file)
+        throw new ExpectedError(`${path} is not a file`);
+}
+/** Exit code of `child`, or its signal; rejects when it fails to spawn. */
+async function exit(child) {
+    const [code, signal] = (await once(child, "exit"));
+    return code === null ? String(signal) : String(code);
+}
+/** `run --rm` in the foreground; logs the connection once Postgres
+ *  listens on the published port, resolves once the container is gone. */
+export async function runDisposableServer(runtime, { name, image, port, serverArgs, snapshot, signal }) {
+    // The entrypoint sources non-executable hooks; packed packages drop mode bits.
+    const hooks = snapshot ? mkdtempSync(join(tmpdir(), "db-snapshot-")) : "";
+    try {
+        const snapshotArgs = [];
+        if (snapshot) {
+            const hook = join(hooks, basename(SNAPSHOT_HOOK));
+            copyFileSync(join(packageRoot(import.meta.filename), SNAPSHOT_HOOK), hook);
+            chmodSync(hook, 0o755);
+            snapshotArgs.push("--env", `SNAPSHOT_SCRUB=${snapshot.scrub ? 1 : 0}`, "--mount", `type=bind,source=${snapshot.dump},target=/app/data.sql,readonly`, "--mount", `type=bind,source=${hook},target=/docker-entrypoint-initdb.d/${basename(hook)},readonly`);
+        }
+        signal.throwIfAborted();
+        const server = spawn(runtime, [
+            "run",
+            "--rm",
+            "--name",
+            name,
+            ...TRUST_ENV,
+            "--publish",
+            `127.0.0.1:${port}:5432`,
+            ...snapshotArgs,
+            image,
+            ...serverArgs,
+        ], { stdio: ["ignore", "inherit", "inherit"] });
+        const exited = exit(server);
+        const gone = new AbortController();
+        exited.then(() => gone.abort(), () => gone.abort());
+        const stop = () => server.kill("SIGINT");
+        signal.addEventListener("abort", stop, { once: true });
+        try {
+            try {
+                // initdb, a snapshot load included, listens on the socket alone.
+                await waitFor(() => succeeds(runtime, [
+                    "exec",
+                    name,
+                    "pg_isready",
+                    "-h",
+                    "127.0.0.1",
+                    "-U",
+                    "postgres",
+                ]), `${name} never became ready`, Number.POSITIVE_INFINITY, AbortSignal.any([signal, gone.signal]));
+                log.info("dev.db_disposable_ready", "{name}: ready on 127.0.0.1:{port}, removed on exit: {url}", { name, port, url: adminConnectionString(port, "db_template") });
+            }
+            catch (error) {
+                if (!signal.aborted && !gone.signal.aborted)
+                    throw error;
+            }
+            const status = await exited;
+            if (!signal.aborted && status !== "0")
+                throw new ExpectedError(`${name} exited with ${status}`);
+        }
+        finally {
+            signal.removeEventListener("abort", stop);
+        }
+    }
+    finally {
+        if (hooks)
+            rmSync(hooks, { recursive: true, force: true });
+    }
+}
+/** `--mount` arguments placing each migrate file at
+ *  `/app/migrate.d/<basename>`; each must be an existing `.sql` file with a
+ *  unique basename. */
+export function migrationMounts(files) {
+    const targets = new Set();
+    return files.flatMap((file) => {
+        requireFile(file);
+        if (extname(file) !== ".sql")
+            throw new ExpectedError(`${file} lacks the .sql extension`);
+        const target = `/app/migrate.d/${basename(file)}`;
+        if (targets.has(target))
+            throw new ExpectedError(`two migrate files share the basename of ${file}`);
+        targets.add(target);
+        return ["--mount", `type=bind,source=${file},target=${target},readonly`];
+    });
+}
+/** Throws an `ExpectedError` for a `DB_DUMP_EXCLUDE` pattern lacking a
+ *  schema. */
+export function checkDumpExclude(patterns = "") {
+    for (const pattern of patterns.split(/\s+/))
+        if (pattern !== "" && !pattern.includes("."))
+            throw new ExpectedError(`DB_DUMP_EXCLUDE pattern ${pattern} lacks a schema`);
+}
+/**
+ * The util image loads the `PG*` source into a throwaway instance of
+ * `image` on a private network and prints that instance's full dump on
+ * stdout. Leftovers of an earlier run go first; everything goes on exit.
+ */
+export async function runUpgradeDump(runtime, { name, image, util, mounts, signal }) {
+    const utilName = `${name}-util`;
+    const cleanup = () => {
+        succeeds(runtime, ["rm", "-f", utilName]);
+        succeeds(runtime, ["rm", "-f", name]);
+        succeeds(runtime, ["network", "rm", name]);
+    };
+    cleanup();
+    try {
+        const quiet = {
+            stdio: ["ignore", "ignore", "inherit"],
+        };
+        execFileSync(runtime, ["network", "create", name], quiet);
+        execFileSync(runtime, [
+            "run",
+            "--rm",
+            "--detach",
+            "--name",
+            name,
+            ...TRUST_ENV,
+            "--network",
+            name,
+            "--expose",
+            "5432",
+            image,
+        ], quiet);
+        signal.throwIfAborted();
+        const dump = spawn(runtime, [
+            "run",
+            "--rm",
+            "--interactive",
+            "--name",
+            utilName,
+            "--env",
+            `CONTAINER_PGHOST=${name}`,
+            ...["PGHOST", "PGPORT", "PGUSER", "PGPASSWORD", "PGDATABASE"].flatMap((key) => ["--env", key]),
+            "--env",
+            "DB_DUMP_EXCLUDE",
+            "--network",
+            name,
+            ...mounts,
+            util,
+            "/entrypoint.d/upgrade-db.sh",
+        ], { stdio: "inherit" });
+        const stop = () => dump.kill("SIGINT");
+        signal.addEventListener("abort", stop, { once: true });
+        try {
+            const status = await exit(dump);
+            if (signal.aborted)
+                throw new ExpectedError(`${utilName} interrupted; the dump is partial`);
+            if (status !== "0")
+                throw new ExpectedError(`${utilName} exited with ${status}`);
+        }
+        finally {
+            signal.removeEventListener("abort", stop);
+        }
+    }
+    finally {
+        cleanup();
+    }
+}

package/dist/src/provision/image.d.ts

@@ -0,0 +1,63 @@
+import type { PostgresOptions } from "./options.ts";
+/**
+ * What the package contributes to the image: `name@version` once
+ * installed from a registry; while linked from a workspace (files change
+ * without a version bump), the Git-visible `container/init` and `sql`
+ * sources and the build script.
+ */
+export declare function packageFingerprint(dir?: string): string;
+/**
+ * sha256 of what the image bakes: the staged schema (`schemaDigest`), the
+ * package fingerprint and the modules' extensions.
+ */
+export declare function schemaHash(options: PostgresOptions): string;
+/** What `buildImages` builds and how. */
+interface BuildImagesOptions {
+    /** Images to build; defaults to `all`. */
+    target?: "db" | "util" | "all";
+    /** Image base name. */
+    name: string;
+    /** File receiving the built db image's ID. */
+    imageIdFile?: string;
+    /** Pulls the base image first. */
+    refreshBase?: boolean;
+}
+/**
+ * Builds database images with `build-images.sh` over a staged copy of the
+ * schema; base-image refresh is explicit.
+ */
+export declare function buildImages(options: PostgresOptions, runtime: string, hash: string, { target, name, imageIdFile, refreshBase, }: BuildImagesOptions): void;
+/**
+ * Reuses the schema-tagged local image or builds it; returns its immutable
+ * ID. A reused image is retagged `<name>:latest`, as a build tags it.
+ */
+export declare function ensureImage(options: PostgresOptions, runtime: string, hash: string, name: string): string;
+/** Builds `<name>-util:latest` from the package's util sources
+ *  (layer-cached) and returns the tag. */
+export declare function ensureUtilImage(runtime: string, name: string): string;
+/** Database image and host runtime captured by the image prerequisite. */
+export interface TestImage {
+    /** `PostgresOptions.name` the image was built for. */
+    name: string;
+    /** Container runtime the image was built with. */
+    runtime: string;
+    /** Immutable image ID (`sha256:…`). */
+    image: string;
+    /** `schemaHash` the image was built from. */
+    hash: string;
+    /** `dataDigest` of `PostgresOptions.data`, which loads after the image's
+     *  schema. */
+    data: string;
+    /** Node version and platform captured by the prerequisite. */
+    host: string;
+}
+/** Selects the schema image, tagged `name`, and records its immutable ID,
+ *  runtime and data digest. */
+export declare function prepareTestImage(options: PostgresOptions, name?: string): TestImage;
+/** Publishes the manifest of the image tagged `envDbName` at `file`,
+ *  replacing it whole. */
+export declare function writeTestImage(options: PostgresOptions, file: URL): void;
+/** Reads a prepared database-image manifest; a malformed one is an
+ *  `ExpectedError`. */
+export declare function readTestImage(path: string): TestImage;
+export {};