@dbx-tools/projen 0.6.228 → 0.6.230

package/src/release.ts

@@ -17,6 +17,7 @@ const NPM_REGISTRY_URL = "https://registry.npmjs.org";
 const nodeReleaseProjects = new WeakSet<DBXToolsJavaScriptProject>();
 const releaseTagPrefixes = new WeakMap<DBXToolsJavaScriptProject, string>();
 const releaseWorkflows = new WeakMap<DBXToolsJavaScriptProject, GithubWorkflow>();
+const independentPublicationJobs = new WeakMap<GithubWorkflow, Set<string>>();
 
 /** Independently recoverable portions of the release workflow. */
 export type ReleaseStage = "all" | "node" | "python" | "docs";
@@ -74,6 +75,13 @@ export function tryReleaseWorkflow(project: DBXToolsJavaScriptProject): GithubWo
   return releaseWorkflows.get(project);
 }
 
+/** Register one publication job for the independent release completion barrier. */
+export function registerIndependentPublicationJob(workflow: GithubWorkflow, jobId: string): void {
+  const jobs = independentPublicationJobs.get(workflow) ?? new Set<string>();
+  jobs.add(jobId);
+  independentPublicationJobs.set(workflow, jobs);
+}
+
 /** Whether the unified workflow publishes the normal npm workspace. */
 export function hasNodeRelease(project: DBXToolsJavaScriptProject): boolean {
   return nodeReleaseProjects.has(project);
@@ -336,10 +344,387 @@ function addDocsJobs(
   });
 }
 
+function independentReleasePleaseJob(project: DBXToolsJavaScriptProject, branch: string): Job {
+  return {
+    if: "${{ github.event_name == 'push' }}",
+    runsOn: ["ubuntu-latest"],
+    permissions: {
+      contents: JobPermission.WRITE,
+      pullRequests: JobPermission.WRITE,
+    },
+    timeoutMinutes: 30,
+    env: { BUN_VERSION },
+    outputs: {
+      releases_created: { stepId: "release", outputName: "releases_created" },
+      prs_created: { stepId: "release", outputName: "prs_created" },
+      releases: { stepId: "release", outputName: "releases" },
+      prs: { stepId: "release", outputName: "prs" },
+      docs_changed: { stepId: "changes", outputName: "docs_changed" },
+      recovery_requested: { stepId: "recovery", outputName: "requested" },
+      recovery_component: { stepId: "recovery", outputName: "component" },
+      recovery_version: { stepId: "recovery", outputName: "version" },
+    },
+    steps: [
+      {
+        name: "Checkout source",
+        uses: "actions/checkout@v6",
+        with: { "fetch-depth": 2 },
+      },
+      ...bunCacheRestoreSteps(project, {
+        ignorePaths: project.workflowCacheIgnorePaths,
+      }),
+      { name: "Install dependencies", run: "bun install" },
+      {
+        name: "Refresh Release Please state",
+        id: "release",
+        env: { GITHUB_TOKEN: "${{ github.token }}" },
+        run: [
+          'OWNER="${GITHUB_REPOSITORY%%/*}"',
+          'REPO="${GITHUB_REPOSITORY#*/}"',
+          `bun node_modules/@dbx-tools/projen/tasks/release-please.ts --owner "$OWNER" --repo "$REPO" --target-branch ${JSON.stringify(branch)}`,
+        ].join("\n"),
+      },
+      {
+        name: "Detect documentation changes",
+        id: "changes",
+        shell: "bash",
+        run: [
+          "if git diff --quiet HEAD^ HEAD -- README.md AGENTS.md ':(glob)**/README.md' docs; then",
+          '  echo "docs_changed=false" >> "$GITHUB_OUTPUT"',
+          "else",
+          '  echo "docs_changed=true" >> "$GITHUB_OUTPUT"',
+          "fi",
+        ].join("\n"),
+      },
+      {
+        name: "Detect committed recovery request",
+        id: "recovery",
+        shell: "bash",
+        run: [
+          "if [ -f .release-recovery.json ]; then",
+          '  echo "requested=true" >> "$GITHUB_OUTPUT"',
+          '  echo "component=$(jq -r .component .release-recovery.json)" >> "$GITHUB_OUTPUT"',
+          '  echo "version=$(jq -r .version .release-recovery.json)" >> "$GITHUB_OUTPUT"',
+          "else",
+          '  echo "requested=false" >> "$GITHUB_OUTPUT"',
+          "fi",
+        ].join("\n"),
+      },
+      {
+        name: "Reconcile generated release PR files",
+        if: "${{ steps.release.outputs.prs_created == 'true' }}",
+        env: { RELEASE_PRS: "${{ steps.release.outputs.prs }}" },
+        shell: "bash",
+        run: [
+          'BRANCH="$(jq -r \'.[0].headBranchName\' <<<"$RELEASE_PRS")"',
+          'test -n "$BRANCH" && test "$BRANCH" != "null"',
+          'git fetch origin "$BRANCH"',
+          'git switch --force-create "$BRANCH" "origin/$BRANCH"',
+          "bun install",
+          "bunx projen",
+          "bun node_modules/@dbx-tools/projen/tasks/version-check.ts",
+          `bun node_modules/@dbx-tools/projen/tasks/release-plan.ts --base-ref origin/${branch}`,
+          `bun node_modules/@dbx-tools/projen/tasks/release-summary-units.ts --from-ref origin/${branch}`,
+          'if ! git diff --quiet || test -n "$(git ls-files --others --exclude-standard)"; then',
+          '  git config user.name "github-actions[bot]"',
+          '  git config user.email "41898282+github-actions[bot]@users.noreply.github.com"',
+          "  git add -A",
+          '  git commit -m "chore: reconcile release metadata"',
+          '  git push origin "HEAD:$BRANCH"',
+          "fi",
+        ].join("\n"),
+      },
+    ],
+  };
+}
+
+function independentReleasePlanJob(project: DBXToolsJavaScriptProject): Job {
+  return {
+    if: "${{ always() && (github.event_name == 'workflow_dispatch' || needs.release-please.outputs.releases_created == 'true' || needs.release-please.outputs.docs_changed == 'true' || needs.release-please.outputs.recovery_requested == 'true') }}",
+    needs: ["release-please"],
+    runsOn: ["ubuntu-latest"],
+    permissions: { contents: JobPermission.READ },
+    timeoutMinutes: 15,
+    env: { BUN_VERSION },
+    outputs: {
+      release_sha: { stepId: "plan", outputName: "release_sha" },
+      rust: { stepId: "plan", outputName: "rust" },
+      python: { stepId: "plan", outputName: "python" },
+      node: { stepId: "plan", outputName: "node" },
+      github: { stepId: "plan", outputName: "github" },
+      docs: { stepId: "plan", outputName: "docs" },
+      units: { stepId: "plan", outputName: "units" },
+      python_packages: { stepId: "plan", outputName: "python_packages" },
+      rust_packages: { stepId: "plan", outputName: "rust_packages" },
+      artifacts: { stepId: "plan", outputName: "artifacts" },
+      rust_targets: { stepId: "plan", outputName: "rust_targets" },
+    },
+    steps: [
+      {
+        name: "Checkout release source",
+        uses: "actions/checkout@v6",
+        with: { "fetch-depth": 2 },
+      },
+      ...bunCacheRestoreSteps(project, {
+        ignorePaths: project.workflowCacheIgnorePaths,
+      }),
+      { name: "Install dependencies", run: "bun install" },
+      {
+        name: "Build affected release plan",
+        id: "plan",
+        shell: "bash",
+        env: {
+          COMPONENT:
+            "${{ inputs.component || needs.release-please.outputs.recovery_component || '' }}",
+          VERSION: "${{ inputs.version || needs.release-please.outputs.recovery_version || '' }}",
+          DOCS_CHANGED: "${{ needs.release-please.outputs.docs_changed || 'false' }}",
+        },
+        run: [
+          "ARGS=()",
+          'if [ -n "$COMPONENT" ]; then ARGS+=(--component "$COMPONENT" --version "$VERSION"); fi',
+          'if [ "$DOCS_CHANGED" = "true" ]; then ARGS+=(--docs); fi',
+          'bun node_modules/@dbx-tools/projen/tasks/release-plan.ts "${ARGS[@]}"',
+        ].join("\n"),
+      },
+      {
+        name: "Upload release plan",
+        uses: "actions/upload-artifact@v7",
+        with: {
+          name: "release-plan",
+          path: "dist/release-plan.json",
+          "if-no-files-found": "error",
+        },
+      },
+    ],
+  };
+}
+
+/** Checkout and install the immutable source selected by an independent release plan. */
+export function independentReleaseSetupSteps(
+  project: DBXToolsJavaScriptProject,
+): readonly JobStep[] {
+  return [
+    {
+      name: "Checkout release source",
+      uses: "actions/checkout@v6",
+      with: { ref: "${{ needs.release-plan.outputs.release_sha }}", "fetch-depth": 1 },
+    },
+    ...bunCacheRestoreSteps(project, {
+      ignorePaths: project.workflowCacheIgnorePaths,
+    }),
+    { name: "Install dependencies", run: "bun install" },
+    {
+      name: "Download release plan",
+      uses: "actions/download-artifact@v8",
+      with: { name: "release-plan", path: "dist" },
+    },
+  ];
+}
+
+function independentNodePublishJob(project: DBXToolsJavaScriptProject): Job {
+  return {
+    if: "${{ needs.release-plan.outputs.node == 'true' && (github.event_name == 'push' || inputs.stage == 'all' || inputs.stage == 'node') }}",
+    needs: ["release-plan"],
+    runsOn: ["ubuntu-latest"],
+    permissions: { contents: JobPermission.READ, idToken: JobPermission.WRITE },
+    timeoutMinutes: 60,
+    env: { BUN_VERSION, CI: "true" },
+    steps: [
+      ...independentReleaseSetupSteps(project),
+      {
+        name: "Publish affected npm packages",
+        env: npmPublishEnvironment(),
+        run: "bun node_modules/@dbx-tools/projen/tasks/publish.ts --plan dist/release-plan.json",
+      },
+    ],
+  };
+}
+
+function independentReleaseNotesJob(project: DBXToolsJavaScriptProject): Job {
+  return {
+    if: "${{ needs.release-plan.outputs.units != '[]' }}",
+    needs: ["release-plan"],
+    runsOn: ["ubuntu-latest"],
+    permissions: { contents: JobPermission.WRITE },
+    env: { BUN_VERSION },
+    steps: [
+      ...independentReleaseSetupSteps(project),
+      {
+        name: "Publish component release notes",
+        env: { GH_TOKEN: "${{ github.token }}" },
+        shell: "bash",
+        run: [
+          "jq -c '.units[]' dist/release-plan.json | while read -r UNIT; do",
+          '  COMPONENT="$(jq -r .component <<<"$UNIT")"',
+          '  VERSION="$(jq -r .newVersion <<<"$UNIT")"',
+          '  gh release edit "$COMPONENT-v$VERSION" --notes-file "docs/releases/$COMPONENT-v$VERSION.md"',
+          "done",
+        ].join("\n"),
+      },
+    ],
+  };
+}
+
+function addIndependentDocsJobs(
+  workflow: GithubWorkflow,
+  project: DBXToolsJavaScriptProject,
+  options: ReleaseDocsOptions,
+): void {
+  workflow.addJob("build-docs", {
+    if: "${{ needs.release-plan.outputs.docs == 'true' && needs.publication-complete.result == 'success' && (github.event_name == 'push' || inputs.stage == 'all' || inputs.stage == 'docs') }}",
+    needs: ["release-plan", "publication-complete"],
+    runsOn: ["ubuntu-latest"],
+    permissions: {
+      contents: JobPermission.READ,
+      pages: JobPermission.WRITE,
+      idToken: JobPermission.WRITE,
+    },
+    timeoutMinutes: 30,
+    env: {
+      BUN_VERSION,
+      GITHUB_PAGES: "true",
+      SITE_URL: options.siteUrl,
+      BASE_PATH: options.base ?? "${{ github.event.repository.name }}",
+    },
+    steps: [
+      ...independentReleaseSetupSteps(project),
+      ...options.prepareSteps,
+      bunCacheSaveStep(),
+      ...options.buildSteps,
+      {
+        name: "Upload Pages artifact",
+        uses: "actions/upload-pages-artifact@v4",
+        with: { path: options.artifactPath },
+      },
+    ],
+  });
+  workflow.addJob("deploy-docs", {
+    needs: ["build-docs"],
+    environment: {
+      name: "github-pages",
+      url: "${{ steps.deployment.outputs.page_url }}",
+    },
+    runsOn: ["ubuntu-latest"],
+    permissions: { pages: JobPermission.WRITE, idToken: JobPermission.WRITE },
+    timeoutMinutes: 15,
+    steps: [
+      {
+        name: "Deploy to GitHub Pages",
+        id: "deployment",
+        uses: "actions/deploy-pages@v4",
+      },
+    ],
+  });
+}
+
+class IndependentReleaseFinalizer extends Component {
+  private finalized = false;
+
+  constructor(
+    project: DBXToolsJavaScriptProject,
+    private readonly workflow: GithubWorkflow,
+    private readonly docs: ReleaseDocsOptions | undefined,
+  ) {
+    super(project);
+  }
+
+  public override preSynthesize(): void {
+    if (this.finalized) return;
+    this.finalized = true;
+    const jobs = [...(independentPublicationJobs.get(this.workflow) ?? [])].sort();
+    const successful = jobs
+      .map((job) => `needs.${job}.result != 'failure' && needs.${job}.result != 'cancelled'`)
+      .join(" && ");
+    this.workflow.addJob("publication-complete", {
+      if: `\${{ always()${successful ? ` && ${successful}` : ""} }}`,
+      needs: jobs,
+      runsOn: ["ubuntu-latest"],
+      permissions: { contents: JobPermission.READ },
+      steps: [{ name: "Confirm publication stages", run: "true" }],
+    });
+    if (this.docs) {
+      addIndependentDocsJobs(this.workflow, this.project as DBXToolsJavaScriptProject, this.docs);
+    }
+  }
+}
+
+function configureIndependentRelease(
+  project: DBXToolsJavaScriptProject,
+  options: DBXToolsReleaseOptions,
+): void {
+  const branch = projectReleaseBranch(project);
+  if (options.nodeRelease !== false) nodeReleaseProjects.add(project);
+  applyTasks(project, {
+    "version:check": {
+      exec: taskScript(project, "version-check.ts"),
+      description: "Verify every package against its release unit",
+    },
+    "release:plan": {
+      exec: taskScript(project, "release-plan.ts"),
+      receiveArgs: true,
+      description: "Build the affected release plan",
+    },
+    release: {
+      exec: taskScript(project, "release-please.ts", `--target-branch ${JSON.stringify(branch)}`),
+      receiveArgs: true,
+      description: "Refresh independent release PRs, tags, and GitHub Releases",
+    },
+  });
+  if (!project.github) return;
+  const workflow = new GithubWorkflow(project.github, "release", {
+    fileName: "release.yml",
+    limitConcurrency: true,
+    concurrencyOptions: { group: "release", cancelInProgress: false },
+  });
+  workflow.runName =
+    "release units " + "${{ github.event_name == 'push' && github.sha || inputs.component }}";
+  workflow.on({
+    push: { branches: [branch] },
+    workflowDispatch: {
+      inputs: {
+        component: {
+          description: "Component to recover",
+          type: "string",
+          default: "",
+          required: false,
+        },
+        version: {
+          description: "Component version to recover",
+          type: "string",
+          default: "",
+          required: false,
+        },
+        stage: {
+          description: "Publication target to recover",
+          type: "choice",
+          options: ["all", "node", "python", "rust", "github", "docs"],
+          default: "all",
+          required: true,
+        },
+      },
+    },
+  });
+  workflow.file?.addOverride("permissions.contents", "read");
+  workflow.addJob("release-please", independentReleasePleaseJob(project, branch));
+  workflow.addJob("release-plan", independentReleasePlanJob(project));
+  workflow.addJob("publish-release-notes", independentReleaseNotesJob(project));
+  registerIndependentPublicationJob(workflow, "publish-release-notes");
+  if (options.nodeRelease !== false) {
+    workflow.addJob("publish-node", independentNodePublishJob(project));
+    registerIndependentPublicationJob(workflow, "publish-node");
+  }
+  new IndependentReleaseFinalizer(project, workflow, options.docs);
+  releaseWorkflows.set(project, workflow);
+}
+
 /** Owns the single release workflow and local release preparation tasks. */
 export class DBXToolsRelease extends Component {
   constructor(project: DBXToolsJavaScriptProject, options: DBXToolsReleaseOptions = {}) {
     super(project);
+    if (project.releaseCatalog.mode === "independent") {
+      configureIndependentRelease(project, options);
+      return;
+    }
     const tagPrefix = options.tagPrefix ?? "v";
     const releaseBranch = projectReleaseBranch(project);
     const summary = options.summary ?? true;

package/tasks/publish-npm.ts

@@ -209,7 +209,7 @@ export async function publishNpmArchives(options: {
   readonly directory: string;
   readonly dryRun?: boolean;
   readonly registry?: string;
-  readonly version: string;
+  readonly version?: string;
 }): Promise<void> {
   const directory = resolve(options.directory);
   const archives = readdirSync(directory)
@@ -220,7 +220,7 @@ export async function publishNpmArchives(options: {
 
   for (const archive of archives) {
     const local = readNpmArchiveIdentity(archive);
-    if (local.version !== options.version) {
+    if (options.version && local.version !== options.version) {
       throw new Error(
         `npm archive ${archive} carries ${local.version}, expected ${options.version}`,
       );
@@ -257,11 +257,11 @@ if (import.meta.main) {
   const program = new Command();
   program
     .requiredOption("--directory <path>", "Directory containing npm archives")
-    .requiredOption("--version <version>", "Exact npm release version")
+    .option("--version <version>", "Expected npm release version")
     .option("--registry <url>", "npm registry URL")
     .option("--dry-run", "Validate archives without publishing")
     .action(
-      (options: { directory: string; dryRun?: boolean; registry?: string; version: string }) =>
+      (options: { directory: string; dryRun?: boolean; registry?: string; version?: string }) =>
         publishNpmArchives(options),
     );
   await program.parseAsync();

package/tasks/publish-python.ts

@@ -15,18 +15,22 @@ import { exec } from "@dbx-tools/core";
 import { Command } from "commander";
 import { parse, stringify } from "smol-toml";
 import { pythonProjectInfo, stampPythonProject } from "./uniffi-python.js";
+import type { ReleasePlan } from "../src/release-plan.ts";
 
 interface PythonProjectFile {
   readonly directory: string;
   readonly mode: number;
   readonly name: string;
+  readonly version: string;
   readonly path: string;
   readonly private: boolean;
   readonly source: string;
+  readonly uniffi: boolean;
 }
 
 export interface StampPythonProjectsOptions {
   readonly rewriteDependencies?: boolean;
+  readonly versions?: ReadonlyMap<string, string>;
 }
 
 export interface RestorePythonProjects {
@@ -51,12 +55,16 @@ export function stampPythonProjects(
       directory: basename(resolve(path, "..")),
       mode: statSync(path).mode,
       name: info.name,
+      version: info.version,
       path,
       private: info.private,
       source,
+      uniffi: info.uniffi,
     };
   });
-  const projects = allProjects.filter((project) => !project.private);
+  const projects = allProjects.filter(
+    (project) => !project.private && (!options.versions || options.versions.has(project.directory)),
+  );
   if (projects.length === 0) throw new Error(`No Python packages found under ${root}`);
 
   try {
@@ -64,8 +72,9 @@ export function stampPythonProjects(
       const stamped = stampPythonProject(project.source, {
         packages: allProjects,
         rewriteDependencies: options.rewriteDependencies,
+        usePackageVersions: options.versions !== undefined,
         toml: { parse, stringify },
-        version,
+        version: options.versions?.get(project.directory) ?? version,
       });
       chmodSync(project.path, project.mode | 0o200);
       writeFileSync(project.path, stamped);
@@ -99,18 +108,37 @@ export function publishPythonProjects(options: {
   readonly publishUrl: string;
   readonly root: string;
   readonly version: string;
+  readonly plan?: ReleasePlan;
 }): void {
   const root = resolve(options.root);
   const output = mkdtempSync(join(tmpdir(), "dbx-tools-python-publish-"));
-  const stamp = stampPythonProjects(root, options.version);
+  const planned = new Map(
+    options.plan?.pythonPackages.map((pkg) => [pkg.path.replace(/^.*\//, ""), pkg.version]) ?? [],
+  );
+  const stamp = stampPythonProjects(root, options.version, {
+    ...(options.plan ? { versions: planned } : {}),
+  });
   try {
-    exec.spawnSync("uv", ["build", "--all-packages", "--out-dir", output], {
-      cwd: process.cwd(),
-      stdout: "inherit",
-      stderr: "inherit",
-      stdin: "ignore",
-      check: true,
-    });
+    const packages = options.plan?.pythonPackages ?? [];
+    if (packages.length > 0) {
+      for (const pkg of packages) {
+        exec.spawnSync("uv", ["build", "--package", pkg.identity, "--out-dir", output], {
+          cwd: process.cwd(),
+          stdout: "inherit",
+          stderr: "inherit",
+          stdin: "ignore",
+          check: true,
+        });
+      }
+    } else {
+      exec.spawnSync("uv", ["build", "--all-packages", "--out-dir", output], {
+        cwd: process.cwd(),
+        stdout: "inherit",
+        stderr: "inherit",
+        stdin: "ignore",
+        check: true,
+      });
+    }
     exec.spawnSync(
       "uvx",
       [
@@ -142,16 +170,30 @@ export function publishPythonProjects(options: {
 if (import.meta.main) {
   const program = new Command();
   program
-    .argument("<version>", "Python package version")
+    .argument("[version]", "Python package version")
     .requiredOption("--index-url <url>", "devpi Simple API URL")
     .requiredOption("--publish-url <url>", "devpi writable index URL")
     .option("--root <path>", "Python workspace package root", "packages/py")
+    .option("--plan <path>", "affected release plan")
     .option("--dry-run", "build and inspect distributions without uploading")
     .action(
       (
-        version: string,
-        options: { dryRun?: boolean; indexUrl: string; publishUrl: string; root: string },
-      ) => publishPythonProjects({ ...options, version }),
+        version: string | undefined,
+        options: {
+          dryRun?: boolean;
+          indexUrl: string;
+          publishUrl: string;
+          root: string;
+          plan?: string;
+        },
+      ) => {
+        const plan = options.plan
+          ? (JSON.parse(readFileSync(options.plan, "utf8")) as ReleasePlan)
+          : undefined;
+        const fallbackVersion = version ?? plan?.pythonPackages[0]?.version;
+        if (!fallbackVersion) throw new Error("Python publication requires a version or plan");
+        publishPythonProjects({ ...options, version: fallbackVersion, plan });
+      },
     );
   await program.parseAsync();
 }