@dbx-tools/projen 0.9.18 → 0.9.21

package/tasks/release-assets.ts

@@ -14,7 +14,6 @@ import {
 } from "node:fs";
 import { platform } from "node:os";
 import { basename, delimiter, dirname, join, relative, resolve } from "node:path";
-import * as exec from "@dbx-tools/core/exec";
 import * as projectUtils from "@dbx-tools/core/project-utils";
 import { json, log } from "@dbx-tools/shared-core";
 import { Command } from "commander";
@@ -23,6 +22,11 @@ import type {
   RustReleaseConfiguration,
   RustReleaseTargetPlan,
 } from "../src/_rust-release-workflow.ts";
+import {
+  captureTaskCommand,
+  runLoggedTaskCommand,
+  taskCommandSucceeds,
+} from "../src/_task-command.ts";
 
 const logger = log.logger("projen:release-assets");
 const emoji = /[\p{Extended_Pictographic}\uFE0F\u200D]/gu;
@@ -39,55 +43,8 @@ interface ReleaseAsset {
   readonly size: number;
 }
 
-function run(
-  root: string,
-  command: string,
-  args: readonly string[],
-  env: NodeJS.ProcessEnv = process.env,
-): void {
-  const result = exec.spawnSync(command, [...args], {
-    cwd: root,
-    env,
-    stdin: "ignore",
-    stdout: "capture",
-    stderr: "capture",
-    check: false,
-  });
-  for (const line of `${result.stdout}\n${result.stderr}`.split(/\r?\n/)) {
-    if (line) logger.info(line.replace(emoji, ""));
-  }
-  if (result.exitCode !== 0) {
-    throw new Error(`${command} exited with ${result.exitCode}`);
-  }
-}
-
-function capture(
-  root: string,
-  command: string,
-  args: readonly string[],
-  env: NodeJS.ProcessEnv,
-): string {
-  return exec.spawnSync(command, [...args], {
-    cwd: root,
-    env,
-    stdin: "ignore",
-    stdout: "capture",
-    stderr: "inherit",
-    check: true,
-  }).stdout;
-}
-
-function commandSucceeds(root: string, command: string, args: readonly string[]): boolean {
-  return (
-    exec.spawnSync(command, [...args], {
-      cwd: root,
-      env: { ...process.env, PATH: releasePath() },
-      stdin: "ignore",
-      stdout: "ignore",
-      stderr: "ignore",
-      check: false,
-    }).exitCode === 0
-  );
+function logReleaseOutput(line: string): void {
+  logger.info(line.replace(emoji, ""));
 }
 
 function releasePath(): string {
@@ -110,7 +67,12 @@ export function missingLocalReleaseTools(root: string): string[] {
     ["zip", "zip", ["-v"]],
   ];
   return checks
-    .filter(([, command, args]) => !commandSucceeds(root, command, args))
+    .filter(
+      ([, command, args]) =>
+        !taskCommandSucceeds(root, command, args, {
+          env: { ...process.env, PATH: releasePath() },
+        }),
+    )
     .map(([name]) => name);
 }
 
@@ -205,7 +167,11 @@ function targetBuildEnvironment(
 
 function cargoTargetRoot(root: string, env: NodeJS.ProcessEnv): string {
   const metadata = json.parseRecord(
-    capture(root, "cargo", ["metadata", "--format-version", "1", "--no-deps"], env),
+    captureTaskCommand(root, "cargo", ["metadata", "--format-version", "1", "--no-deps"], {
+      env,
+      check: true,
+      stderr: "inherit",
+    }),
   );
   if (typeof metadata?.target_directory !== "string") {
     throw new Error("Cargo metadata returned no target directory");
@@ -214,7 +180,13 @@ function cargoTargetRoot(root: string, env: NodeJS.ProcessEnv): string {
 }
 
 function rustHostTarget(root: string, env: NodeJS.ProcessEnv): string {
-  const host = /^host:\s+(.+)$/m.exec(capture(root, "rustc", ["-vV"], env))?.[1];
+  const host = /^host:\s+(.+)$/m.exec(
+    captureTaskCommand(root, "rustc", ["-vV"], {
+      env,
+      check: true,
+      stderr: "inherit",
+    }),
+  )?.[1];
   if (!host) throw new Error("rustc did not report a host target");
   return host;
 }
@@ -226,7 +198,7 @@ function buildBindingGenerators(
   env: NodeJS.ProcessEnv,
 ): void {
   if (configuration.bindings.length === 0) return;
-  run(
+  runLoggedTaskCommand(
     root,
     "cargo",
     [
@@ -241,7 +213,7 @@ function buildBindingGenerators(
       "--target",
       hostTarget,
     ],
-    env,
+    { env, onLine: logReleaseOutput },
   );
 }
 
@@ -255,7 +227,7 @@ function generatePythonBindings(
   for (const binding of configuration.bindings) {
     if (!binding.python) continue;
     const output = join(root, "dist/local-bindings", binding.crate);
-    run(
+    runLoggedTaskCommand(
       root,
       "node",
       [
@@ -270,7 +242,7 @@ function generatePythonBindings(
         "--output",
         output,
       ],
-      env,
+      { env, onLine: logReleaseOutput },
     );
     generated.set(binding.crate, join(output, `${binding.crate.replaceAll("-", "_")}.py`));
   }
@@ -286,7 +258,7 @@ function packageBindings(
   env: NodeJS.ProcessEnv,
 ): void {
   for (const binding of configuration.bindings) {
-    run(
+    runLoggedTaskCommand(
       root,
       "node",
       [
@@ -325,7 +297,7 @@ function packageBindings(
           ? ["--python-bindings", pythonBindings.get(binding.crate)!]
           : []),
       ],
-      env,
+      { env, onLine: logReleaseOutput },
     );
   }
 }
@@ -352,9 +324,15 @@ function packageBinaries(
     const output = join(root, "dist/release", binary.crate, target.node, "binary");
     const asset = join(output, releaseBinaryAssetName(binary.binary, target.node, target.os));
     if (target.os === "win32") {
-      run(root, "zip", ["-j", asset, source], env);
+      runLoggedTaskCommand(root, "zip", ["-j", asset, source], {
+        env,
+        onLine: logReleaseOutput,
+      });
     } else {
-      run(root, "tar", ["-C", dirname(source), "-czf", asset, basename(source)], env);
+      runLoggedTaskCommand(root, "tar", ["-C", dirname(source), "-czf", asset, basename(source)], {
+        env,
+        onLine: logReleaseOutput,
+      });
     }
   }
 }
@@ -418,8 +396,15 @@ function uploadAssets(
   assets: readonly ReleaseAsset[],
   manifest: string,
 ): void {
-  run(root, "gh", ["release", "upload", tag, ...assets.map((asset) => asset.path), "--clobber"]);
-  run(root, "gh", ["release", "upload", tag, manifest, "--clobber"]);
+  runLoggedTaskCommand(
+    root,
+    "gh",
+    ["release", "upload", tag, ...assets.map((asset) => asset.path), "--clobber"],
+    { onLine: logReleaseOutput },
+  );
+  runLoggedTaskCommand(root, "gh", ["release", "upload", tag, manifest, "--clobber"], {
+    onLine: logReleaseOutput,
+  });
 }
 
 /** Build all configured targets sequentially and optionally upload them. */
@@ -443,11 +428,17 @@ export function buildReleaseAssets(options: {
   buildBindingGenerators(root, configuration, hostTarget, env);
   const pythonBindings = generatePythonBindings(root, configuration, hostTarget, env);
   for (const target of configuration.targets) {
-    run(root, "rustup", ["target", "add", target.cargo], env);
+    runLoggedTaskCommand(root, "rustup", ["target", "add", target.cargo], {
+      env,
+      onLine: logReleaseOutput,
+    });
     const invocation = localTargetCommand(configuration, target);
     const buildEnv = targetBuildEnvironment(root, target, env);
     logger.info("building Rust release target", { target: target.cargo });
-    run(root, invocation.command, invocation.args, buildEnv);
+    runLoggedTaskCommand(root, invocation.command, invocation.args, {
+      env: buildEnv,
+      onLine: logReleaseOutput,
+    });
     packageBindings(root, configuration, target, options.version, pythonBindings, env);
     packageBinaries(root, configuration, target, targetRoot, env);
   }

package/tasks/release-candidate.ts

@@ -0,0 +1,416 @@
+#!/usr/bin/env -S bun
+/** Build and optionally upload the complete locally approved release candidate. */
+
+import { existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import * as projectUtils from "@dbx-tools/core/project-utils";
+import { json, log } from "@dbx-tools/shared-core";
+import { Command } from "commander";
+import { readNpmArchiveIdentity } from "./publish-npm.ts";
+import { buildPythonProjects } from "./publish-python.ts";
+import { buildReleaseAssets } from "./release-assets.ts";
+import {
+  type ReleaseArtifactInput,
+  type ReleaseArtifactRole,
+  verifyReleaseManifest,
+  writeReleaseManifest,
+} from "./release-manifest.ts";
+import type { RustReleaseConfiguration } from "../src/_rust-release-workflow.ts";
+import { captureTaskCommand, runTaskCommand, taskCommandSucceeds } from "../src/_task-command.ts";
+
+const logger = log.logger("projen:release-candidate");
+
+function files(directory: string): string[] {
+  if (!existsSync(directory)) return [];
+  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    const path = join(directory, entry.name);
+    return entry.isDirectory() ? files(path) : [path];
+  });
+}
+
+function pythonPackageName(path: string, version: string): string {
+  const name = basename(path);
+  const marker = `-${version}`;
+  const index = name.indexOf(marker);
+  if (index <= 0) throw new Error(`Cannot resolve Python package name from ${name}`);
+  return name.slice(0, index).replace(/[_.]+/g, "-").toLowerCase();
+}
+
+function role(path: string): ReleaseArtifactRole {
+  const normalized = path.split("\\").join("/");
+  if (normalized.includes("/npm-facade/")) return "facade";
+  if (normalized.includes("/npm-workspace/") || normalized.includes("/python-workspace/")) {
+    return "workspace";
+  }
+  return "native";
+}
+
+export function releaseArtifacts(root: string, version: string): ReleaseArtifactInput[] {
+  const releaseRoot = join(root, "dist/release");
+  return files(releaseRoot)
+    .filter((path) => !path.includes(`${join(releaseRoot, "upload")}/`))
+    .flatMap((path): ReleaseArtifactInput[] => {
+      const normalized = path.split("\\").join("/");
+      if (
+        ![
+          "/binary/",
+          "/npm/",
+          "/npm-facade/",
+          "/npm-workspace/",
+          "/python/",
+          "/python-workspace/",
+        ].some((directory) => normalized.includes(directory))
+      ) {
+        return [];
+      }
+      if (path.endsWith(".tgz")) {
+        const identity = readNpmArchiveIdentity(path);
+        if (identity.version !== version) {
+          throw new Error(`${identity.name} carries ${identity.version}, expected ${version}`);
+        }
+        return [
+          {
+            kind: "npm",
+            packageName: identity.name,
+            packageVersion: identity.version,
+            path,
+            role: role(path),
+          },
+        ];
+      }
+      if (
+        path.endsWith(".whl") ||
+        (path.endsWith(".tar.gz") &&
+          (normalized.includes("/python/") || normalized.includes("/python-workspace/")))
+      ) {
+        return [
+          {
+            kind: "pypi",
+            packageName: pythonPackageName(path, version),
+            packageVersion: version,
+            path,
+            role: role(path),
+          },
+        ];
+      }
+      if (path.endsWith(".zip") || path.endsWith(".tar.gz")) {
+        return [{ kind: "binary", path }];
+      }
+      return [];
+    });
+}
+
+function pruneDraftAssets(
+  root: string,
+  tag: string,
+  desired: ReadonlySet<string>,
+  env: NodeJS.ProcessEnv,
+): void {
+  const current = captureTaskCommand(
+    root,
+    "gh",
+    ["release", "view", tag, "--json", "assets", "--jq", ".assets[].name"],
+    { env, check: true, stderr: "inherit" },
+  )
+    .split(/\r?\n/)
+    .filter(Boolean);
+  for (const name of current) {
+    if (!desired.has(name)) {
+      runTaskCommand(root, "gh", ["release", "delete-asset", tag, name, "--yes"], { env });
+    }
+  }
+}
+
+function rustReleaseConfiguration(root: string): RustReleaseConfiguration | undefined {
+  const path = join(root, ".projen/rust-release.json");
+  return existsSync(path)
+    ? (json.parse(readFileSync(path, "utf8")) as RustReleaseConfiguration)
+    : undefined;
+}
+
+function buildFacades(root: string, version: string): void {
+  const configuration = rustReleaseConfiguration(root);
+  if (!configuration) return;
+  for (const binding of configuration.bindings) {
+    if (!binding.node || !binding.nodePackage) continue;
+    runTaskCommand(root, "node", [
+      ".projen/uniffi-release.mjs",
+      "facade",
+      "--node",
+      binding.node,
+      "--node-package",
+      binding.nodePackage,
+      "--node-triple",
+      "linux-x64-gnu",
+      "--version",
+      version,
+      "--output",
+      `dist/release/${binding.crate}/facade`,
+    ]);
+  }
+}
+
+function ensureDraftRelease(options: {
+  readonly root: string;
+  readonly sha: string;
+  readonly tag: string;
+  readonly notesFile?: string;
+  readonly env: NodeJS.ProcessEnv;
+}): void {
+  const existingTag = captureTaskCommand(
+    options.root,
+    "git",
+    ["rev-parse", "--verify", `${options.tag}^{commit}`],
+    { env: options.env },
+  );
+  if (existingTag) {
+    if (existingTag !== options.sha) {
+      throw new Error(`Release tag ${options.tag} does not point to ${options.sha}`);
+    }
+    if (
+      captureTaskCommand(options.root, "git", ["cat-file", "-t", options.tag], {
+        env: options.env,
+        check: true,
+        stderr: "inherit",
+      }) !== "tag"
+    ) {
+      throw new Error(`Release tag ${options.tag} must be annotated`);
+    }
+  } else {
+    runTaskCommand(
+      options.root,
+      "git",
+      ["tag", "-a", options.tag, options.sha, "-m", options.tag],
+      { env: options.env },
+    );
+  }
+  const remoteTag = taskCommandSucceeds(
+    options.root,
+    "git",
+    ["ls-remote", "--exit-code", "--tags", "origin", `refs/tags/${options.tag}`],
+    { env: options.env },
+  );
+  if (remoteTag) {
+    runTaskCommand(
+      options.root,
+      "git",
+      ["fetch", "--force", "origin", `+refs/tags/${options.tag}:refs/tags/${options.tag}`],
+      { env: options.env },
+    );
+    if (
+      captureTaskCommand(options.root, "git", ["rev-parse", `${options.tag}^{commit}`], {
+        env: options.env,
+        check: true,
+        stderr: "inherit",
+      }) !== options.sha
+    ) {
+      throw new Error(`Remote release tag ${options.tag} does not point to ${options.sha}`);
+    }
+    if (
+      captureTaskCommand(options.root, "git", ["cat-file", "-t", options.tag], {
+        env: options.env,
+        check: true,
+        stderr: "inherit",
+      }) !== "tag"
+    ) {
+      throw new Error(`Remote release tag ${options.tag} must be annotated`);
+    }
+  } else {
+    runTaskCommand(options.root, "git", ["push", "origin", `refs/tags/${options.tag}`], {
+      env: options.env,
+    });
+  }
+
+  const release = captureTaskCommand(
+    options.root,
+    "gh",
+    ["release", "view", options.tag, "--json", "isDraft"],
+    { env: options.env },
+  );
+  if (release) {
+    const state = JSON.parse(release) as { isDraft?: boolean };
+    if (state.isDraft !== true) {
+      throw new Error(`GitHub Release ${options.tag} is already published`);
+    }
+    if (options.notesFile) {
+      runTaskCommand(
+        options.root,
+        "gh",
+        ["release", "edit", options.tag, "--title", options.tag, "--notes-file", options.notesFile],
+        { env: options.env },
+      );
+    }
+    return;
+  }
+  const notes =
+    options.notesFile && existsSync(options.notesFile) ? ["--notes-file", options.notesFile] : [];
+  runTaskCommand(
+    options.root,
+    "gh",
+    ["release", "create", options.tag, "--draft", "--verify-tag", "--title", options.tag, ...notes],
+    { env: options.env },
+  );
+}
+
+/** Build the exact release candidate and optionally attach it to a draft release. */
+export function buildReleaseCandidate(options: {
+  readonly root: string;
+  readonly pythonRoot: string;
+  readonly sha: string;
+  readonly tag: string;
+  readonly version: string;
+  readonly notesFile?: string;
+  readonly upload?: boolean;
+  readonly env?: NodeJS.ProcessEnv;
+}): void {
+  const root = resolve(options.root);
+  const env = options.env ?? process.env;
+  if (
+    captureTaskCommand(root, "git", ["rev-parse", "HEAD"], {
+      env,
+      check: true,
+      stderr: "inherit",
+    }) !== options.sha
+  ) {
+    throw new Error(`Release candidate must be built from ${options.sha}`);
+  }
+  if (readFileSync(join(root, "VERSION"), "utf8").trim() !== options.version) {
+    throw new Error(`VERSION does not match ${options.version}`);
+  }
+  const status = captureTaskCommand(
+    root,
+    "git",
+    ["status", "--porcelain=v1", "--untracked-files=all"],
+    { env, check: true, stderr: "inherit" },
+  );
+  if (status) throw new Error("Release candidate source contains tracked changes");
+
+  rmSync(join(root, "dist/release"), { recursive: true, force: true });
+  if ((rustReleaseConfiguration(root)?.targets.length ?? 0) > 0) {
+    buildReleaseAssets({ root, version: options.version });
+    buildFacades(root, options.version);
+  }
+  const publishScript = fileURLToPath(new URL("./publish.ts", import.meta.url));
+  runTaskCommand(root, process.execPath, [
+    publishScript,
+    options.version,
+    "--output",
+    "dist/release/npm-workspace",
+  ]);
+  const pythonRoot = resolve(root, options.pythonRoot);
+  if (existsSync(pythonRoot)) {
+    buildPythonProjects({
+      allowEmpty: true,
+      excludeUniFFI: true,
+      output: join(root, "dist/release/python-workspace"),
+      root: pythonRoot,
+      version: options.version,
+    });
+  }
+  const uploadDirectory = join(root, "dist/release/upload");
+  const manifest = writeReleaseManifest({
+    artifacts: releaseArtifacts(root, options.version),
+    directory: uploadDirectory,
+    gitSha: options.sha,
+    tag: options.tag,
+    version: options.version,
+  });
+  logger.success("built release candidate", {
+    artifacts: manifest.artifacts.length,
+    sha: options.sha,
+    tag: options.tag,
+  });
+  if (!options.upload) return;
+  uploadReleaseCandidate(options);
+}
+
+/** Attach an already-built and verified candidate to its draft GitHub Release. */
+export function uploadReleaseCandidate(options: {
+  readonly root: string;
+  readonly sha: string;
+  readonly tag: string;
+  readonly version: string;
+  readonly notesFile?: string;
+  readonly env?: NodeJS.ProcessEnv;
+}): void {
+  const root = resolve(options.root);
+  const env = options.env ?? process.env;
+  const uploadDirectory = join(root, "dist/release/upload");
+  verifyReleaseManifest({
+    directory: uploadDirectory,
+    gitSha: options.sha,
+    tag: options.tag,
+    version: options.version,
+  });
+  const notesFile = join(root, "dist/release/release-notes.md");
+  const summary =
+    options.notesFile && existsSync(resolve(root, options.notesFile))
+      ? readFileSync(resolve(root, options.notesFile), "utf8").trim()
+      : "";
+  writeFileSync(
+    notesFile,
+    [
+      `Release commit: \`${options.sha}\``,
+      "",
+      "Candidate manifest: `release-manifest.json`",
+      "",
+      "Candidate checksums: `SHA256SUMS`",
+      ...(summary ? ["", summary] : []),
+      "",
+    ].join("\n"),
+  );
+  ensureDraftRelease({
+    root,
+    sha: options.sha,
+    tag: options.tag,
+    notesFile,
+    env,
+  });
+  const uploadFiles = files(uploadDirectory).sort((left, right) => {
+    const priority = (path: string): number =>
+      basename(path) === "release-manifest.json" ? 2 : basename(path) === "SHA256SUMS" ? 1 : 0;
+    return priority(left) - priority(right) || left.localeCompare(right);
+  });
+  pruneDraftAssets(root, options.tag, new Set(uploadFiles.map((path) => basename(path))), env);
+  runTaskCommand(root, "gh", ["release", "upload", options.tag, ...uploadFiles, "--clobber"], {
+    env,
+  });
+  logger.success(`uploaded draft release candidate ${options.tag}`);
+}
+
+if (import.meta.main) {
+  new Command()
+    .requiredOption("--version <version>", "release version")
+    .requiredOption("--tag <tag>", "annotated release tag")
+    .requiredOption("--sha <sha>", "exact release commit")
+    .option("--root <path>", "repository root")
+    .option("--python-root <path>", "Python package root", "packages/py")
+    .option("--notes-file <path>", "draft release notes file")
+    .option("--upload", "create or update the draft GitHub Release after building")
+    .option("--upload-existing", "upload the existing verified candidate without rebuilding")
+    .action(
+      (options: {
+        version: string;
+        tag: string;
+        sha: string;
+        root?: string;
+        pythonRoot: string;
+        notesFile?: string;
+        upload?: boolean;
+        uploadExisting?: boolean;
+      }) => {
+        const candidateOptions = {
+          root: options.root ?? projectUtils.root() ?? process.cwd(),
+          pythonRoot: options.pythonRoot,
+          sha: options.sha,
+          tag: options.tag,
+          version: options.version,
+          ...(options.notesFile ? { notesFile: options.notesFile } : {}),
+        };
+        if (options.uploadExisting) uploadReleaseCandidate(candidateOptions);
+        else buildReleaseCandidate({ ...candidateOptions, upload: options.upload });
+      },
+    )
+    .parse();
+}