@dbx-tools/projen 0.9.18 → 0.9.21

package/tasks/publish-npm.ts

@@ -2,6 +2,7 @@
 /** Idempotent npm archive publication for release recovery. */
 import { createHash } from "node:crypto";
 import {
+  chmodSync,
   lstatSync,
   mkdtempSync,
   readFileSync,
@@ -15,6 +16,7 @@ import { join, relative, resolve } from "node:path";
 import * as exec from "@dbx-tools/core/exec";
 import { log } from "@dbx-tools/shared-core";
 import { Command } from "commander";
+import { runTaskCommand } from "../src/_task-command.ts";
 
 const DEFAULT_REGISTRY = "https://registry.npmjs.org";
 const logger = log.logger("projen:publish-npm");
@@ -28,6 +30,20 @@ export interface NpmReleaseIdentity {
   readonly version: string;
 }
 
+interface NpmArchiveManifest {
+  readonly [key: string]: unknown;
+  readonly dependencies?: Readonly<Record<string, string>>;
+  readonly optionalDependencies?: Readonly<Record<string, string>>;
+  readonly name?: string;
+  readonly publishConfig?: { readonly access?: unknown };
+  readonly repository?: unknown;
+  readonly version?: string;
+}
+
+export type NpmArchiveManifestTransform = (
+  manifest: NpmArchiveManifest,
+) => Readonly<Record<string, unknown>>;
+
 function normalizedRepository(value: unknown): string | undefined {
   const url =
     typeof value === "string"
@@ -120,7 +136,7 @@ export async function publishedNpmRelease(
   };
 }
 
-export function readNpmArchiveIdentity(path: string): NpmReleaseIdentity {
+function readNpmArchiveManifest(path: string): NpmArchiveManifest {
   const result = exec.spawnSync("tar", ["-xOf", path, "package/package.json"], {
     cwd: process.cwd(),
     stdout: "capture",
@@ -131,12 +147,11 @@ export function readNpmArchiveIdentity(path: string): NpmReleaseIdentity {
   if (result.exitCode !== 0 || !result.stdout) {
     throw new Error(`Cannot read npm package manifest from ${path}: ${result.stderr}`);
   }
-  const manifest = JSON.parse(result.stdout) as {
-    name?: string;
-    publishConfig?: { access?: unknown };
-    repository?: unknown;
-    version?: string;
-  };
+  return JSON.parse(result.stdout) as NpmArchiveManifest;
+}
+
+export function readNpmArchiveIdentity(path: string): NpmReleaseIdentity {
+  const manifest = readNpmArchiveManifest(path);
   if (!manifest.name || !manifest.version) {
     throw new Error(`npm archive has no package name or version: ${path}`);
   }
@@ -154,6 +169,48 @@ export function readNpmArchiveIdentity(path: string): NpmReleaseIdentity {
   };
 }
 
+function orderNpmArchives(archives: readonly string[]): string[] {
+  const byName = new Map(
+    archives.map((archive) => {
+      const manifest = readNpmArchiveManifest(archive);
+      if (!manifest.name) throw new Error(`npm archive has no package name: ${archive}`);
+      return [manifest.name, { archive, manifest }] as const;
+    }),
+  );
+  if (byName.size !== archives.length) throw new Error("Release contains duplicate npm packages");
+  const remaining = new Map(
+    [...byName].map(([name, value]) => [
+      name,
+      new Set(
+        [
+          ...Object.keys(value.manifest.dependencies ?? {}),
+          ...Object.keys(value.manifest.optionalDependencies ?? {}),
+        ].filter((dependency) => byName.has(dependency)),
+      ),
+    ]),
+  );
+  const ordered: string[] = [];
+  while (remaining.size > 0) {
+    const ready = [...remaining]
+      .filter(([, dependencies]) => dependencies.size === 0)
+      .map(([name]) => name)
+      .sort();
+    if (ready.length === 0) {
+      throw new Error(
+        `Cyclic npm release dependencies: ${[...remaining.keys()].sort().join(", ")}`,
+      );
+    }
+    for (const name of ready) {
+      ordered.push(byName.get(name)!.archive);
+      remaining.delete(name);
+    }
+    for (const dependencies of remaining.values()) {
+      for (const name of ready) dependencies.delete(name);
+    }
+  }
+  return ordered;
+}
+
 /** Hash paths, executable bits, symlink targets, and bytes while ignoring tar metadata. */
 export function npmArchiveContentDigest(path: string): string {
   const temp = mkdtempSync(join(tmpdir(), "projen-npm-content-"));
@@ -191,25 +248,60 @@ export function packNpmPackage(
   directory: string,
   destination: string,
   path = process.env.PATH ?? "",
+  transformManifest?: NpmArchiveManifestTransform,
 ): string {
   const executable = process.versions.bun ? process.execPath : "bun";
-  exec.spawnSync(
-    executable,
-    ["pm", "pack", "--destination", destination, "--ignore-scripts", "--quiet"],
-    {
-      cwd: directory,
-      env: { ...process.env, PATH: path },
-      stdout: "inherit",
-      stderr: "inherit",
-      stdin: "ignore",
-      check: true,
-    },
-  );
+  const manifest = join(directory, "package.json");
+  const manifestMode = lstatSync(manifest).mode & 0o777;
+  const restoreManifestMode = (manifestMode & 0o200) === 0;
+  if (restoreManifestMode) chmodSync(manifest, manifestMode | 0o200);
+  try {
+    exec.spawnSync(
+      executable,
+      ["pm", "pack", "--destination", destination, "--ignore-scripts", "--quiet"],
+      {
+        cwd: directory,
+        env: { ...process.env, PATH: path },
+        stdout: "inherit",
+        stderr: "inherit",
+        stdin: "ignore",
+        check: true,
+      },
+    );
+  } finally {
+    if (restoreManifestMode) chmodSync(manifest, manifestMode);
+  }
   const archives = readdirSync(destination).filter((file) => file.endsWith(".tgz"));
   if (archives.length !== 1) {
     throw new Error(`Expected one packed npm archive, found ${archives.length}`);
   }
-  return join(destination, archives[0]);
+  const archive = join(destination, archives[0]);
+  if (!transformManifest) return archive;
+
+  const temp = mkdtempSync(join(tmpdir(), "projen-npm-project-"));
+  try {
+    exec.spawnSync("tar", ["-xzf", archive, "-C", temp], {
+      cwd: process.cwd(),
+      stdout: "ignore",
+      stderr: "inherit",
+      stdin: "ignore",
+      check: true,
+    });
+    const manifestPath = join(temp, "package", "package.json");
+    const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as NpmArchiveManifest;
+    writeFileSync(manifestPath, `${JSON.stringify(transformManifest(manifest), null, 2)}\n`);
+    rmSync(archive, { force: true });
+    exec.spawnSync("tar", ["-czf", archive, "-C", temp, "package"], {
+      cwd: process.cwd(),
+      stdout: "ignore",
+      stderr: "inherit",
+      stdin: "ignore",
+      check: true,
+    });
+    return archive;
+  } finally {
+    rmSync(temp, { recursive: true, force: true });
+  }
 }
 
 export async function publishNpmArchives(options: {
@@ -219,10 +311,12 @@ export async function publishNpmArchives(options: {
   readonly version?: string;
 }): Promise<void> {
   const directory = resolve(options.directory);
-  const archives = readdirSync(directory)
-    .filter((file) => file.endsWith(".tgz"))
-    .sort()
-    .map((file) => join(directory, file));
+  const archives = orderNpmArchives(
+    readdirSync(directory)
+      .filter((file) => file.endsWith(".tgz"))
+      .sort()
+      .map((file) => join(directory, file)),
+  );
   if (archives.length === 0) throw new Error(`No npm archives found in ${directory}`);
 
   for (const archive of archives) {
@@ -239,23 +333,13 @@ export async function publishNpmArchives(options: {
         continue;
       }
     }
-    exec.spawnSync(
-      "npm",
-      [
-        "publish",
-        archive,
-        ...(local.access ? ["--access", local.access] : []),
-        ...(options.registry ? ["--registry", options.registry] : []),
-        ...(options.dryRun ? ["--dry-run"] : []),
-      ],
-      {
-        cwd: process.cwd(),
-        stdout: "inherit",
-        stderr: "inherit",
-        stdin: "ignore",
-        check: true,
-      },
-    );
+    runTaskCommand(process.cwd(), "npm", [
+      "publish",
+      archive,
+      ...(local.access ? ["--access", local.access] : []),
+      ...(options.registry ? ["--registry", options.registry] : []),
+      ...(options.dryRun ? ["--dry-run"] : []),
+    ]);
   }
 }
 

package/tasks/publish-python.ts

@@ -1,25 +1,24 @@
 #!/usr/bin/env -S bun
 import {
   chmodSync,
+  cpSync,
   existsSync,
+  mkdirSync,
   mkdtempSync,
   readFileSync,
   readdirSync,
   rmSync,
-  statSync,
   writeFileSync,
 } from "node:fs";
 import { tmpdir } from "node:os";
-import { basename, join, resolve } from "node:path";
-import * as exec from "@dbx-tools/core/exec";
+import { basename, dirname, join, resolve } from "node:path";
 import { Command } from "commander";
 import { parse, stringify } from "smol-toml";
-import { pythonProjectInfo, stampPythonProject } from "./uniffi-python.js";
-import type { ReleasePlan } from "../src/release-plan.ts";
+import { preparePythonProjectForPublication, pythonProjectInfo } from "./uniffi-python.js";
+import { runTaskCommand } from "../src/_task-command.ts";
 
 interface PythonProjectFile {
   readonly directory: string;
-  readonly mode: number;
   readonly name: string;
   readonly version: string;
   readonly path: string;
@@ -28,21 +27,15 @@ interface PythonProjectFile {
   readonly uniffi: boolean;
 }
 
-export interface StampPythonProjectsOptions {
-  readonly rewriteDependencies?: boolean;
-  readonly versions?: ReadonlyMap<string, string>;
+/** Python wheel and source archives accepted by Twine and package indexes. */
+export function pythonDistributionPaths(directory: string): string[] {
+  return readdirSync(directory)
+    .filter((file) => file.endsWith(".whl") || file.endsWith(".tar.gz"))
+    .sort()
+    .map((file) => join(directory, file));
 }
 
-export interface RestorePythonProjects {
-  (): void;
-  readonly paths: readonly string[];
-}
-
-export function stampPythonProjects(
-  root: string,
-  version: string,
-  options: StampPythonProjectsOptions = {},
-): RestorePythonProjects {
+function pythonProjects(root: string): PythonProjectFile[] {
   const packageFiles = readdirSync(root, { withFileTypes: true })
     .filter((entry) => entry.isDirectory())
     .map((entry) => resolve(root, entry.name, "pyproject.toml"))
@@ -53,7 +46,6 @@ export function stampPythonProjects(
     const info = pythonProjectInfo(source, { parse, stringify });
     return {
       directory: basename(resolve(path, "..")),
-      mode: statSync(path).mode,
       name: info.name,
       version: info.version,
       path,
@@ -62,44 +54,7 @@ export function stampPythonProjects(
       uniffi: info.uniffi,
     };
   });
-  const projects = allProjects.filter(
-    (project) => !project.private && (!options.versions || options.versions.has(project.directory)),
-  );
-  if (projects.length === 0) throw new Error(`No Python packages found under ${root}`);
-
-  try {
-    for (const project of projects) {
-      const stamped = stampPythonProject(project.source, {
-        packages: allProjects,
-        rewriteDependencies: options.rewriteDependencies,
-        usePackageVersions: options.versions !== undefined,
-        toml: { parse, stringify },
-        version: options.versions?.get(project.directory) ?? version,
-      });
-      chmodSync(project.path, project.mode | 0o200);
-      writeFileSync(project.path, stamped);
-      chmodSync(project.path, project.mode);
-    }
-  } catch (error) {
-    for (const project of projects) {
-      chmodSync(project.path, project.mode | 0o200);
-      writeFileSync(project.path, project.source);
-      chmodSync(project.path, project.mode);
-    }
-    throw error;
-  }
-
-  const restore = () => {
-    for (const project of projects) {
-      chmodSync(project.path, project.mode | 0o200);
-      writeFileSync(project.path, project.source);
-      chmodSync(project.path, project.mode);
-    }
-  };
-  Object.defineProperty(restore, "paths", {
-    value: projects.map((project) => project.path),
-  });
-  return restore as RestorePythonProjects;
+  return allProjects;
 }
 
 export function publishPythonProjects(options: {
@@ -108,38 +63,12 @@ export function publishPythonProjects(options: {
   readonly publishUrl: string;
   readonly root: string;
   readonly version: string;
-  readonly plan?: ReleasePlan;
 }): void {
-  const root = resolve(options.root);
   const output = mkdtempSync(join(tmpdir(), "projen-python-publish-"));
-  const planned = new Map(
-    options.plan?.pythonPackages.map((pkg) => [pkg.path.replace(/^.*\//, ""), pkg.version]) ?? [],
-  );
-  const stamp = stampPythonProjects(root, options.version, {
-    ...(options.plan ? { versions: planned } : {}),
-  });
   try {
-    const packages = options.plan?.pythonPackages ?? [];
-    if (packages.length > 0) {
-      for (const pkg of packages) {
-        exec.spawnSync("uv", ["build", "--package", pkg.identity, "--out-dir", output], {
-          cwd: process.cwd(),
-          stdout: "inherit",
-          stderr: "inherit",
-          stdin: "ignore",
-          check: true,
-        });
-      }
-    } else {
-      exec.spawnSync("uv", ["build", "--all-packages", "--out-dir", output], {
-        cwd: process.cwd(),
-        stdout: "inherit",
-        stderr: "inherit",
-        stdin: "ignore",
-        check: true,
-      });
-    }
-    exec.spawnSync(
+    buildPythonProjects({ ...options, output });
+    runTaskCommand(
+      process.cwd(),
       "uvx",
       [
         "--from",
@@ -152,47 +81,91 @@ export function publishPythonProjects(options: {
         ...(options.dryRun ? ["--dry-run"] : []),
         output,
       ],
-      {
-        cwd: process.cwd(),
-        env: { ...process.env, UV_DEFAULT_INDEX: options.indexUrl },
-        stdout: "inherit",
-        stderr: "inherit",
-        stdin: "ignore",
-        check: true,
-      },
+      { env: { ...process.env, UV_DEFAULT_INDEX: options.indexUrl } },
     );
   } finally {
-    stamp();
     rmSync(output, { recursive: true, force: true });
   }
 }
 
+/** Build the exact Python release distributions without publishing them. */
+export function buildPythonProjects(options: {
+  readonly allowEmpty?: boolean;
+  readonly excludeUniFFI?: boolean;
+  readonly output: string;
+  readonly root: string;
+  readonly version: string;
+}): void {
+  const root = resolve(options.root);
+  const output = resolve(options.output);
+  rmSync(output, { recursive: true, force: true });
+  const allProjects = pythonProjects(root);
+  const projects = allProjects.filter(
+    (project) => !project.private && (!options.excludeUniFFI || !project.uniffi),
+  );
+  if (projects.length === 0 && options.allowEmpty) {
+    mkdirSync(output, { recursive: true });
+    return;
+  }
+  if (projects.length === 0) throw new Error(`No Python packages found under ${root}`);
+
+  const temporaryRoot = mkdtempSync(join(tmpdir(), "projen-python-release-"));
+  try {
+    mkdirSync(output, { recursive: true });
+    for (const project of projects) {
+      const packageRoot = join(temporaryRoot, project.directory);
+      cpSync(dirname(project.path), packageRoot, { recursive: true });
+      const manifestPath = join(packageRoot, "pyproject.toml");
+      chmodSync(manifestPath, 0o644);
+      writeFileSync(
+        manifestPath,
+        preparePythonProjectForPublication(project.source, {
+          packages: allProjects,
+          toml: { parse, stringify },
+          version: options.version,
+        }),
+      );
+      runTaskCommand(root, "uv", ["build", "--out-dir", output, packageRoot]);
+    }
+    const distributions = pythonDistributionPaths(output);
+    if (distributions.length === 0) {
+      throw new Error(`No Python distributions found in ${output}`);
+    }
+    runTaskCommand(process.cwd(), "uvx", ["twine", "check", ...distributions]);
+  } finally {
+    rmSync(temporaryRoot, { recursive: true, force: true });
+  }
+}
+
 if (import.meta.main) {
   const program = new Command();
   program
-    .argument("[version]", "Python package version")
+    .argument("<version>", "Python package version")
     .requiredOption("--index-url <url>", "devpi Simple API URL")
     .requiredOption("--publish-url <url>", "devpi writable index URL")
     .option("--root <path>", "Python workspace package root", "packages/py")
-    .option("--plan <path>", "affected release plan")
+    .option("--output <path>", "Build distributions into a directory without publishing")
     .option("--dry-run", "build and inspect distributions without uploading")
     .action(
       (
-        version: string | undefined,
+        version: string,
         options: {
           dryRun?: boolean;
           indexUrl: string;
           publishUrl: string;
           root: string;
-          plan?: string;
+          output?: string;
         },
       ) => {
-        const plan = options.plan
-          ? (JSON.parse(readFileSync(options.plan, "utf8")) as ReleasePlan)
-          : undefined;
-        const fallbackVersion = version ?? plan?.pythonPackages[0]?.version;
-        if (!fallbackVersion) throw new Error("Python publication requires a version or plan");
-        publishPythonProjects({ ...options, version: fallbackVersion, plan });
+        if (options.output) {
+          buildPythonProjects({
+            output: options.output,
+            root: options.root,
+            version,
+          });
+        } else {
+          publishPythonProjects({ ...options, version });
+        }
       },
     );
   await program.parseAsync();