@intentius/sql-yodeler 0.2.0 → 0.4.0

package/dist/ci/render.d.ts

@@ -0,0 +1,333 @@
+/**
+ * `yodel ci` (#65): a project's CI for GitHub Actions, GitLab CI and Forgejo
+ * Actions, rendered by the installed package, so a fix to the pipelines
+ * reaches a project when it upgrades yodel and renders again. Until #65 the
+ * templates carried this as scripts/ci.ts.
+ *
+ *   yodel ci            write the files
+ *   yodel ci --check    exit 1 when a file differs from what this renders
+ *   yodel ci --replay   the lint job's replay check on this machine (./replay.ts)
+ *
+ * yodel.config.ts `ci` picks the forges (default all three) and names the
+ * module that declares the project's own jobs, which are merged after
+ * yodel's into the pull request pipeline and kept on every render
+ * ({@link ProjectJobs}).
+ *
+ * Environments are the keys of `sql.profiles` in chant.config.ts, in order,
+ * but `replay`, which is the lint job's throwaway server.
+ * Each one needs ops/migrate-<env>.op.ts and ops/watch-<env>.op.ts.
+ * The apply pipeline's waves are yodel.config.ts `waves`, in order; without
+ * it, one wave per environment in profile order, each with gate `always`.
+ *
+ * Three pipelines per forge, each declared with chant: yodel-pr with the
+ * github lexicon's Workflow, Job and Step (Forgejo renders the same
+ * declaration through the forgejo lexicon's dialect) and the gitlab
+ * lexicon's Workflow and Job; yodel-apply with chant's
+ * generateOpWavesPipeline, from the waves spec this writes
+ * (yodel-waves.json); the watch with generateOpsPipeline.
+ *
+ * - yodel-pr: on a pull (merge) request, `yodel ci --check` (these files are
+ *   what yodel ci renders), `yodel lint`, and the replay check into a
+ *   throwaway server the job starts (the `replay` profile), then for each
+ *   environment `yodel plan <env> --comment`, which posts the pending
+ *   migrations and the digest to approve. One job per environment, holding
+ *   that environment's reader credentials and nothing else.
+ * - yodel-apply: on a push to main, one job per wave, each needing the one
+ *   before, running `chant run wave`: it plans the environment (`yodel apply
+ *   <env> --wave-plan`), reads the wave's gate policy from yodel-waves.json
+ *   as the commit before the push has it, waits for an approval when that
+ *   policy asks for one (exit 3, and the waves after it do not run), and
+ *   applies (`yodel apply <env> --wave-apply`). The only jobs that hold a
+ *   writer's credentials, one environment's each, with
+ *   YODEL_CREDENTIALS=writer. Two merges in quick succession do not fail the
+ *   second apply: it waits for the first's lock (YODEL_LOCK_WAIT), and an
+ *   apply whose commit is no longer main's tip stands down
+ *   (YODEL_STAND_DOWN), since the newer commit's waves apply the same
+ *   environments.
+ * - watch-<env>: the environment's WatchOp on its own schedule, rendered by
+ *   chant's generateOpsPipeline, with the reader's credentials. After the
+ *   Op, `yodel drift <env> --issue` compares the server with the schema the
+ *   newest applied migration records (so a migration waiting at its gate is
+ *   not drift), keeps one tracking issue for the environment (opened on
+ *   drift, updated while it changes, closed when it is gone), and fails the
+ *   job on drift. The issue is written with the job's token on GitHub
+ *   (issues: write) and Forgejo, and with GITLAB_TOKEN on GitLab.
+ *
+ * The scheduled watch never pushes. Its first phase (chant lifecycle
+ * snapshot) commits a snapshot to chant/lifecycle and tries to push it; the
+ * drift it reports comes from `chant lifecycle diff --live`, declared against
+ * live, which needs no snapshot on the remote. So the job sets a push URL
+ * that cannot be pushed to: the push fails, chant carries on, and the job
+ * works the same with a read-only token (GitHub `contents: read`, GitLab's
+ * CI_JOB_TOKEN) as with Forgejo's token, which can write. A push that could
+ * land would race the approvals people push to chant/lifecycle.
+ *
+ * Every job installs @intentius/sql-yodeler from npm with `npm ci`, which
+ * needs no token, in IMAGE. With yodel.config.ts `ci.image` (#66), the CI
+ * image ghcr.io/intentius/sql-yodeler pinned by its digest, every job runs
+ * in that image instead, which carries node, yodel, chant and the lexicons
+ * on PATH, and no job runs `npm ci` ({@link Toolchain}).
+ */
+import { type ApprovalMode } from "../config/define.js";
+import { type CloudToken, type Role } from "../credentials.js";
+import type { Declarable } from "@intentius/chant/declarable";
+export type Forge = "github" | "gitlab" | "forgejo";
+export declare const FORGES: readonly Forge[];
+export interface RenderedFile {
+    /** Relative to the project. */
+    path: string;
+    content: string;
+}
+/** The image every job runs in, where the forge runs jobs in a container. It needs git, so not -slim. */
+export declare const IMAGE = "node:22-bookworm";
+/** The CI image (#66): yodel, chant and the sql lexicon, published for each release. */
+export declare const YODEL_IMAGE = "ghcr.io/intentius/sql-yodeler";
+/**
+ * What the jobs run in. By default IMAGE, and each job installs the
+ * project's packages with `npm ci` and puts node_modules/.bin on PATH. With
+ * `ci.image` the yodel image (pinned by digest): its yodel, chant and tsx
+ * are on PATH, and a checkout resolves chant and the lexicons from the
+ * image's /node_modules, so nothing is installed.
+ */
+export interface Toolchain {
+    image: string;
+    /** The yodel image: no `npm ci`, and every job in the container. */
+    yodelImage: boolean;
+}
+export declare const toolchainOf: (envs: Pick<Environments, "image">) => Toolchain;
+export declare const MAIN = "main";
+/** DEV for dev: the prefix of an environment's variables. */
+export declare const prefixOf: (env: string) => string;
+export type Dialect = "clickhouse" | "postgres";
+/** The environments to render for: chant.config.ts's sql.dialect and its profiles, in order. */
+export interface Environments {
+    dialect: Dialect;
+    names: readonly string[];
+    /** The forges to render for: yodel.config.ts `ci.forges`, default every one. */
+    forges?: readonly Forge[];
+    /** The project's own jobs (yodel.config.ts `ci.jobs`), merged after yodel's. */
+    jobs?: ProjectJobs;
+    /** yodel.config.ts `ci.image`: the yodel image every job runs in, by digest. */
+    image?: string;
+    /** yodel.config.ts `ci.resume`: the scheduled job that resumes a wave whose approval arrived. */
+    resume?: {
+        schedule: string;
+    };
+    /** The apply pipeline's waves, in order. */
+    waves: readonly Wave[];
+    /** chant.config.ts declares the `replay` profile, so the lint job runs the replay check. */
+    replay: boolean;
+    /** The project has tests/*.test.ts: with the replay profile, the lint job runs yodel test on its server too (#54). */
+    tests?: boolean;
+    /**
+     * The environments whose profile is made with credentials() (#50): the
+     * variables each role reads, and the cloud token source that mints its
+     * password, if any. An environment not here reads the default variables.
+     */
+    credentials?: Record<string, EnvCredentials>;
+}
+/** What a job holding one role of an environment needs: the variables it reads, and the cloud identity its token source mints from. */
+export interface RoleNeeds {
+    variables: string[];
+    cloud?: CloudToken;
+}
+export type EnvCredentials = Record<Role, RoleNeeds>;
+/** DEV_CLICKHOUSE for dev on ClickHouse: what an environment's variables start with. */
+export declare const variablePrefix: (env: string, dialect: Dialect) => string;
+/** The variables a job reading `env` holds: the URL and the reader (the variables credentials() names; none for a token-minted password). */
+export declare const readerVariables: (env: string, dialect: Dialect, creds?: EnvCredentials) => string[];
+/** The variables the apply job for `env` holds: the URL and the writer. */
+export declare const writerVariables: (env: string, dialect: Dialect, creds?: EnvCredentials) => string[];
+/** The GitLab ID token per cloud: its variable and its audience. */
+export declare const ID_TOKENS: Record<CloudToken, {
+    variable: string;
+    aud: string;
+}>;
+/** The write probe a pull request job runs before its plan: it holds no writer for `envs`. */
+export declare const writeProbe: (envs: readonly string[]) => string;
+/**
+ * The approvals: `chant approve` records them on the chant/lifecycle branch,
+ * and approvers push it. A CI checkout fetches one ref, so fetch this one too.
+ * The remote-tracking ref is what chant's push lease reads.
+ */
+export declare const FETCH_APPROVALS = "git fetch --no-tags origin '+refs/heads/chant/lifecycle:refs/remotes/origin/chant/lifecycle' && git branch -f chant/lifecycle origin/chant/lifecycle || echo 'no chant/lifecycle branch yet, so nothing is approved'";
+/** The apply jobs' lock settings: wait for another apply's lock, and stand down for a newer commit on main. */
+export declare const APPLY_LOCK: Record<string, string>;
+/** The watch reads; it never pushes. See the comment at the top. */
+export declare const NO_PUSH = "git remote set-url --push origin no-push://the-drift-watch-only-reads";
+/** After the watch's `chant run`: drift against the history, the environment's tracking issue, and exit 2 on drift. */
+export declare const watchDrift: (env: string) => string;
+/**
+ * The replay check: `yodel lint --replay replay --reset` replays every
+ * migration, in order, into a fresh database on the `replay` profile's
+ * server and fails when one does not give the schema it recorded. That
+ * server is a throwaway container the lint job starts as a service, the
+ * image chant's emulator runs, which the job (in IMAGE, on every forge)
+ * reaches by the service's name; never a server whose data matters, since
+ * --reset drops what the history names there first.
+ */
+export declare const REPLAY_PROFILE = "replay";
+interface ReplayService {
+    /**
+     * The service's name, and its host where the job runs in a container.
+     * Prefixed, so it cannot meet another container's alias on a runner's
+     * shared network: a host that answers for two servers would split the replay.
+     */
+    name: string;
+    image: string;
+    port: number;
+    /** The service container's environment. */
+    env: Record<string, string>;
+    health: string;
+    /** The server's URL at `host`, on `port` (default: the service's own). */
+    url: (host: string, port?: number) => string;
+    /** What the lint job sets besides the URL: the throwaway server's credentials. */
+    jobEnv: Record<string, string>;
+}
+export declare const REPLAY_SERVICE: Record<Dialect, ReplayService>;
+/** Waits up to 90 s for the replay server's port; a shell-free one-liner, so every forge runs it the same. */
+export declare const waitFor: (host: string, port: number) => string;
+export declare const LINT = "yodel lint";
+export declare const REPLAY = "yodel lint --replay replay --reset";
+/** yodel test (#54), after the replay check on the same throwaway server, its outcome as JUnit XML. */
+export declare const TEST = "yodel test --env replay --reset --junit yodel-test.xml";
+/** Fails a pull request whose pipelines differ from what yodel ci renders. */
+export declare const CI_CHECK = "yodel ci --check";
+/**
+ * Webhook events (yodel.config.ts `notify`): the variables the apply, plan
+ * and watch jobs pass from the forge's secrets. yodel sends nothing while
+ * they are unset (empty), so an unconfigured repository needs neither.
+ * GitLab jobs have the project's variables already.
+ */
+export declare const WEBHOOK_VARIABLES: string[];
+/** Where the plan and apply jobs write `yodel report`, kept as the job's artifact. */
+export declare const REPORT_DIR = "yodel-report";
+/**
+ * `yodel report <env>` as JSON and as the HTML run view, into REPORT_DIR.
+ * It runs whatever the job's outcome, and a gap it reports (exit 4) does not
+ * fail the job: the report is for reading. `yodel` is the project's, named
+ * by path, since GitLab's after_script does not keep before_script's PATH;
+ * in the yodel image, the image's, which is on PATH.
+ */
+export declare const reportCommand: (env: string, tc?: Toolchain) => string;
+/** The artifact a job keeps its report in. */
+export declare const reportArtifact: (job: string) => string;
+type Actions = "github" | "forgejo";
+export declare function actionsReview(forge: Actions, envs_: Environments): RenderedFile;
+export declare const GITLAB_WATCH_FILE = ".gitlab/yodel-watch.gitlab-ci.yml";
+export declare const GITLAB_APPLY_FILE = ".gitlab/yodel-apply.gitlab-ci.yml";
+export declare function gitlabPipeline(envs_: Environments): RenderedFile;
+/** When a wave waits for an approval; yodel.config.ts `waves[].gate`. */
+export type WaveGate = "always" | "on-destructive" | "never";
+export interface Wave {
+    env: string;
+    gate: WaveGate;
+    /**
+     * The environment whose history must record a migration applied before
+     * this one applies it (yodel.config.ts `waves[].requires`, default the
+     * wave before): this wave's jobs, and its plan job, read it with its reader.
+     */
+    requires?: string;
+    /**
+     * A tenant set's wave (#56): a run per tenant, `<env>/<tenant>`, or per
+     * slot of a set read at run time, `<env>/@<i>of<n>`, in place of the
+     * environment's one, all behind the wave's one gate.
+     */
+    targets?: readonly string[];
+    /** Split the wave into a deciding job and this many share jobs (yodel.config.ts `waves[].shares`). */
+    shares?: number;
+    /** Which approvals of the wave's digest count (yodel.config.ts `waves[].approval`, #48); absent, `ledger`. */
+    approval?: ApprovalMode;
+}
+/** The waves spec: committed, and read by every wave job (its gate policy at the commit before the push). */
+export declare const WAVES_SPEC = "yodel-waves.json";
+export declare const APPLY_PIPELINE = "yodel-apply";
+interface WavesSpec {
+    name: string;
+    op: string;
+    plan: string[];
+    apply: string[];
+    branches: string[];
+    waves: Array<{
+        name: string;
+        runs: Array<{
+            target: string;
+            op: string;
+        }>;
+        gate: WaveGate;
+        shares?: number;
+        approval?: ApprovalMode;
+        environment?: {
+            name: string;
+        };
+        variables?: Record<string, string>;
+    }>;
+    /** The scheduled resume job (yodel.config.ts `ci.resume`); only the pipelines are rendered from it, the spec file leaves it out. */
+    resume?: {
+        schedule: string;
+    };
+}
+/**
+ * The waves spec. What the file holds is the same for every forge; each
+ * forge's jobs get their own variables on top (`variables`).
+ */
+export declare function wavesSpec(waves: readonly Wave[], variables?: (env: string) => Record<string, string>, resume?: {
+    schedule: string;
+}): WavesSpec;
+/** The waves spec file and the apply pipeline on each forge. */
+export declare function applyPipelines(dir: string, envs: Environments): Promise<RenderedFile[]>;
+/** yodel.config.ts `waves`, checked against the environments; one wave per environment, gate `always`, without it. */
+export declare function readWaves(dir: string, envs: readonly string[]): Promise<Wave[]>;
+/**
+ * Each wave of an environment with a tenant set (#56), with its runs: a
+ * run per tenant of a list or a file, or for a set given by a query,
+ * `shares` slots (one without) that read the tenants when the wave runs.
+ * The query is not run here, so the pipeline stays the same as tenants come
+ * and go; a list or a file is in the repository, and changing it is
+ * rendering again.
+ */
+export declare function withTenants(dir: string, waves: readonly Wave[]): Promise<Wave[]>;
+/**
+ * The watch's spec for one environment: the Op, and on GitHub and Forgejo the
+ * reader's secrets. A GitLab job already has the project's variables (a
+ * writer's only under its environment, which the watch does not declare), and
+ * `NAME: $NAME` there would refer to itself.
+ */
+export declare function watchSpec(env: string, dialect: Dialect, forge: Forge, creds?: EnvCredentials): {
+    name: string;
+    variables?: Record<string, string>;
+    permissions?: Record<string, string>;
+};
+export declare function watchPipelines(dir: string, envs_: Environments): Promise<RenderedFile[]>;
+/** The dialect and the environments chant.config.ts declares, in order. */
+export declare function environments(dir: string): Promise<Environments>;
+/** The roles of each environment whose profile is made with credentials(). */
+export declare function credentialsIn(profiles: Record<string, unknown>, envs: readonly string[]): Record<string, EnvCredentials>;
+/**
+ * The project's own jobs (#65): the module yodel.config.ts `ci.jobs` names,
+ * relative to the project, declaring jobs with chant's lexicons. `actions`
+ * (the github lexicon's Job) goes into yodel-pr on GitHub and Forgejo, after
+ * yodel's jobs; `gitlab` (the gitlab lexicon's Job) into .gitlab-ci.yml,
+ * after yodel's, its stages after `review`. They are declared here, not
+ * edited into the rendered files, so every `yodel ci` keeps them:
+ *
+ * ```ts
+ * // ci/jobs.ts
+ * import { Job, Step } from "@intentius/chant-lexicon-github";
+ * import { Job as GitLabJob, Image } from "@intentius/chant-lexicon-gitlab";
+ * export const actions = { test: new Job({ "runs-on": "ubuntu-latest", steps: [new Step({ uses: "actions/checkout@v7" }), new Step({ run: "npm test" })] }) };
+ * export const gitlab = { test: new GitLabJob({ stage: "test", image: new Image({ name: "node:22" }), script: ["npm test"] }) };
+ * ```
+ */
+export interface ProjectJobs {
+    actions?: Record<string, Declarable>;
+    gitlab?: Record<string, Declarable>;
+}
+/** Loads the module `ci.jobs` names. */
+export declare function loadProjectJobs(dir: string, file: string): Promise<ProjectJobs>;
+/** The pipelines that need no Op discovery: pull requests, and GitLab's main file. */
+export declare function review(envs: Environments): RenderedFile[];
+/** The forge a rendered file is for; undefined for the waves spec, which every forge's apply reads. */
+export declare function forgeOf(path: string): Forge | undefined;
+export declare function render(dir: string): Promise<RenderedFile[]>;
+export {};