@dbx-tools/projen 0.9.21 → 0.9.29

package/tasks/local-publish.ts

@@ -1,27 +1,20 @@
-/** Publish one verified release candidate to configured loopback registries. */
-import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
-import { homedir, tmpdir } from "node:os";
-import { join, resolve } from "node:path";
+/** Build and publish the current workspace to configured loopback registries. */
+import { resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import * as projectUtils from "@dbx-tools/core/project-utils";
 import { log, net } from "@dbx-tools/shared-core";
 import { Command } from "commander";
 import { activePythonIndexes, resolveLocalPypi } from "./python-registry.ts";
-import { verifyReleaseManifest } from "./release-manifest.ts";
 import { runTaskCommandAsync } from "../src/_task-command.ts";
-import { readDbxToolsConfig } from "../src/packages.ts";
 
 const logger = log.logger("projen:local-publish");
 
-/** Options for publishing an immutable release candidate to local registries. */
+/** Options for publishing the current workspace to local registries. */
 export interface LocalPublishOptions {
-  readonly candidateDirectory: string;
-  readonly gitSha: string;
-  readonly localCargo: boolean;
   readonly localPypi: string;
   readonly localRegistry: string;
+  readonly pythonRoot: string;
   readonly root: string;
-  readonly tag: string;
   readonly version: string;
 }
 
@@ -35,43 +28,9 @@ export function resolveLocalRegistry(value: string): string | undefined {
   return trimmed;
 }
 
-export function localCargoRegistry(): string | undefined {
-  if (process.env.LOCAL_CARGO_REGISTRY) return process.env.LOCAL_CARGO_REGISTRY;
-  const config = join(homedir(), ".cargo", "config.toml");
-  if (!existsSync(config)) return undefined;
-  const source = readFileSync(config, "utf8");
-  const sections = [
-    ...source.matchAll(/^\[registries\.([^\]]+)\]\s*\n([\s\S]*?)(?=^\[|(?![\s\S]))/gm),
-  ];
-  for (const section of sections) {
-    const index = section[2]?.match(/^\s*index\s*=\s*["']([^"']+)["']/m)?.[1];
-    if (index && net.isLoopbackHost(index.replace(/^sparse\+/, ""))) return section[1];
-  }
-  return undefined;
-}
-
-function hasRustCrates(root: string): boolean {
-  const rust = readDbxToolsConfig(root)?.rust;
-  return Boolean(
-    rust &&
-    typeof rust === "object" &&
-    !Array.isArray(rust) &&
-    Array.isArray((rust as { crates?: unknown }).crates) &&
-    (rust as { crates: unknown[] }).crates.length > 0,
-  );
-}
-
-/** Publish the exact candidate archives before they are attached to the draft release. */
+/** Build and publish the current workspace directly to local registries. */
 export async function publishLocalRelease(options: LocalPublishOptions): Promise<void> {
   const root = resolve(options.root);
-  const candidateDirectory = resolve(options.candidateDirectory);
-  verifyReleaseManifest({
-    directory: candidateDirectory,
-    gitSha: options.gitSha,
-    tag: options.tag,
-    version: options.version,
-  });
-
   const localRegistry = resolveLocalRegistry(options.localRegistry);
   const activeIndexes = activePythonIndexes();
   const localPypi = resolveLocalPypi(options.localPypi, activeIndexes);
@@ -88,14 +47,11 @@ export async function publishLocalRelease(options: LocalPublishOptions): Promise
   }
 
   if (localRegistry) {
-    const publishNpmScript = fileURLToPath(new URL("./publish-npm.ts", import.meta.url));
-    logger.info(`publishing approved npm archives to ${localRegistry}`);
+    const publishNpmScript = fileURLToPath(new URL("./publish.ts", import.meta.url));
+    logger.info(`building and publishing npm workspace to ${localRegistry}`);
     publishes.push(
       runTaskCommandAsync(root, process.execPath, [
         publishNpmScript,
-        "--directory",
-        candidateDirectory,
-        "--version",
         options.version,
         "--registry",
         localRegistry,
@@ -104,86 +60,46 @@ export async function publishLocalRelease(options: LocalPublishOptions): Promise
   }
 
   if (localPypi) {
-    const pythonDirectory = mkdtempSync(join(tmpdir(), "dbx-tools-local-pypi-"));
-    verifyReleaseManifest({
-      directory: candidateDirectory,
-      gitSha: options.gitSha,
-      kind: "pypi",
-      output: pythonDirectory,
-      tag: options.tag,
-      version: options.version,
-    });
-    logger.info(`publishing approved Python distributions to ${localPypi.publishUrl}`);
+    const publishPythonScript = fileURLToPath(new URL("./publish-python.ts", import.meta.url));
+    logger.info(`building and publishing Python workspace to ${localPypi.publishUrl}`);
     publishes.push(
-      runTaskCommandAsync(
-        root,
-        "uvx",
-        [
-          "--from",
-          "devpi-client",
-          "devpi",
-          "upload",
-          "--index",
-          localPypi.publishUrl,
-          "--from-dir",
-          pythonDirectory,
-        ],
-        { env: { ...process.env, UV_DEFAULT_INDEX: localPypi.indexUrl } },
-      ).finally(() => rmSync(pythonDirectory, { recursive: true, force: true })),
+      runTaskCommandAsync(root, process.execPath, [
+        publishPythonScript,
+        options.version,
+        "--root",
+        options.pythonRoot,
+        "--index-url",
+        localPypi.indexUrl,
+        "--publish-url",
+        localPypi.publishUrl,
+      ]),
     );
   }
 
   await Promise.all(publishes);
-
-  if (options.localCargo && hasRustCrates(root)) {
-    const registry = localCargoRegistry();
-    if (registry) {
-      const publishCargoScript = fileURLToPath(
-        new URL("./publish-uniffi-local.ts", import.meta.url),
-      );
-      await runTaskCommandAsync(root, process.execPath, [
-        publishCargoScript,
-        "--version",
-        options.version,
-        "--cargo-registry",
-        registry,
-      ]);
-      logger.success(`published Rust ${options.version} to ${registry}`);
-    }
-  }
-
-  logger.success(`published approved local candidate ${options.tag}`);
+  logger.success(`published local workspace ${options.version}`);
 }
 
 if (import.meta.main) {
   await new Command()
-    .requiredOption("--candidate-directory <path>", "verified release candidate directory")
-    .requiredOption("--sha <sha>", "exact release commit")
-    .requiredOption("--tag <tag>", "annotated release tag")
     .requiredOption("--version <version>", "release version")
     .option("--root <path>", "repository root")
+    .option("--python-root <path>", "Python package root", "packages/py")
     .option("--local-registry <url>", "local npm registry", "auto")
     .option("--local-pypi <url>", "local devpi index", "auto")
-    .option("--local-cargo", "publish Cargo crates to a configured local registry")
     .action(
       (options: {
-        candidateDirectory: string;
-        sha: string;
-        tag: string;
         version: string;
         root?: string;
+        pythonRoot: string;
         localRegistry: string;
         localPypi: string;
-        localCargo?: boolean;
       }) =>
         publishLocalRelease({
-          candidateDirectory: options.candidateDirectory,
-          gitSha: options.sha,
-          localCargo: Boolean(options.localCargo),
           localPypi: options.localPypi,
           localRegistry: options.localRegistry,
+          pythonRoot: options.pythonRoot,
           root: options.root ?? projectUtils.root() ?? process.cwd(),
-          tag: options.tag,
           version: options.version,
         }),
     )

package/tasks/publish-npm.ts

@@ -44,6 +44,15 @@ export type NpmArchiveManifestTransform = (
   manifest: NpmArchiveManifest,
 ) => Readonly<Record<string, unknown>>;
 
+function archiveFileEntries(directory: string, prefix = ""): string[] {
+  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+    return entry.isDirectory()
+      ? archiveFileEntries(join(directory, entry.name), relativePath)
+      : [relativePath];
+  });
+}
+
 function normalizedRepository(value: unknown): string | undefined {
   const url =
     typeof value === "string"
@@ -291,7 +300,7 @@ export function packNpmPackage(
     const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as NpmArchiveManifest;
     writeFileSync(manifestPath, `${JSON.stringify(transformManifest(manifest), null, 2)}\n`);
     rmSync(archive, { force: true });
-    exec.spawnSync("tar", ["-czf", archive, "-C", temp, "package"], {
+    exec.spawnSync("tar", ["-czf", archive, "-C", temp, ...archiveFileEntries(temp)], {
       cwd: process.cwd(),
       stdout: "ignore",
       stderr: "inherit",

package/tasks/publish-python.ts

@@ -14,7 +14,7 @@ import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
 import { Command } from "commander";
 import { parse, stringify } from "smol-toml";
-import { preparePythonProjectForPublication, pythonProjectInfo } from "./uniffi-python.js";
+import { preparePythonProjectForPublication, pythonProjectInfo } from "./python-release.ts";
 import { runTaskCommand } from "../src/_task-command.ts";
 
 interface PythonProjectFile {
@@ -24,7 +24,6 @@ interface PythonProjectFile {
   readonly path: string;
   readonly private: boolean;
   readonly source: string;
-  readonly uniffi: boolean;
 }
 
 /** Python wheel and source archives accepted by Twine and package indexes. */
@@ -43,7 +42,7 @@ function pythonProjects(root: string): PythonProjectFile[] {
     .sort();
   const allProjects: PythonProjectFile[] = packageFiles.map((path) => {
     const source = readFileSync(path, "utf8");
-    const info = pythonProjectInfo(source, { parse, stringify });
+    const info = pythonProjectInfo(source, { parse });
     return {
       directory: basename(resolve(path, "..")),
       name: info.name,
@@ -51,7 +50,6 @@ function pythonProjects(root: string): PythonProjectFile[] {
       path,
       private: info.private,
       source,
-      uniffi: info.uniffi,
     };
   });
   return allProjects;
@@ -91,8 +89,8 @@ export function publishPythonProjects(options: {
 /** Build the exact Python release distributions without publishing them. */
 export function buildPythonProjects(options: {
   readonly allowEmpty?: boolean;
-  readonly excludeUniFFI?: boolean;
   readonly output: string;
+  readonly packages?: readonly string[];
   readonly root: string;
   readonly version: string;
 }): void {
@@ -100,8 +98,13 @@ export function buildPythonProjects(options: {
   const output = resolve(options.output);
   rmSync(output, { recursive: true, force: true });
   const allProjects = pythonProjects(root);
+  const requested = new Set(options.packages ?? []);
+  const missing = [...requested].filter(
+    (directory) => !allProjects.some((project) => project.directory === directory),
+  );
+  if (missing.length > 0) throw new Error(`Unknown Python packages: ${missing.join(", ")}`);
   const projects = allProjects.filter(
-    (project) => !project.private && (!options.excludeUniFFI || !project.uniffi),
+    (project) => !project.private && (requested.size === 0 || requested.has(project.directory)),
   );
   if (projects.length === 0 && options.allowEmpty) {
     mkdirSync(output, { recursive: true });
@@ -141,9 +144,10 @@ if (import.meta.main) {
   const program = new Command();
   program
     .argument("<version>", "Python package version")
-    .requiredOption("--index-url <url>", "devpi Simple API URL")
-    .requiredOption("--publish-url <url>", "devpi writable index URL")
+    .option("--index-url <url>", "devpi Simple API URL")
+    .option("--publish-url <url>", "devpi writable index URL")
     .option("--root <path>", "Python workspace package root", "packages/py")
+    .option("--package <directory...>", "Build only selected package directories")
     .option("--output <path>", "Build distributions into a directory without publishing")
     .option("--dry-run", "build and inspect distributions without uploading")
     .action(
@@ -151,8 +155,9 @@ if (import.meta.main) {
         version: string,
         options: {
           dryRun?: boolean;
-          indexUrl: string;
-          publishUrl: string;
+          indexUrl?: string;
+          package?: string[];
+          publishUrl?: string;
           root: string;
           output?: string;
         },
@@ -160,11 +165,21 @@ if (import.meta.main) {
         if (options.output) {
           buildPythonProjects({
             output: options.output,
+            packages: options.package,
             root: options.root,
             version,
           });
         } else {
-          publishPythonProjects({ ...options, version });
+          if (!options.indexUrl || !options.publishUrl) {
+            throw new Error("--index-url and --publish-url are required when publishing");
+          }
+          publishPythonProjects({
+            dryRun: options.dryRun,
+            indexUrl: options.indexUrl,
+            publishUrl: options.publishUrl,
+            root: options.root,
+            version,
+          });
         }
       },
     );

package/tasks/publish.ts

@@ -1,22 +1,20 @@
 #!/usr/bin/env -S bun
 /**
  * `bun tasks/publish.ts <version> [--registry <url>] [--exclude <dir>] [--dry-run] [--skip-compile]`
- * - verify every workspace member and the Bun lock carry the release version,
- * then pack and publish each package owned by the standard Node release.
+ * - verify every workspace member carries the release version, then pack and
+ * publish each package owned by the standard Node release.
  *
  * Bun has no `pnpm -r publish`, so this loop is the recursive-publish stand-in.
  * It leans on native bun for everything bun already does:
  *
  *   - **version validation** fails when any member differs from the reviewed
  *     `VERSION`; publication never repairs committed release metadata;
- *   - **`workspace:` / `catalog:` rewriting** is NOT done here - `bun publish`
+ *   - **`workspace:` / `catalog:` rewriting** is NOT done here - `bun pm pack`
  *     strips both protocols in the PACKED tarball, resolving `workspace:*` to the
  *     sibling's version and `catalog:` to the root catalog entry. (Verified: a
  *     packed manifest shows `"@scope/x": "<version>"` and the real catalog range,
  *     while the on-disk manifest keeps the protocols.) Setting each member's
  *     version first is the only prerequisite, so a sibling resolves the release
- *     version. When Bun's workspace lock already carries the version, its
- *     refresh is skipped;
  *   - **`publishConfig` substitution** (compiled `lib/` entry points) is done
  *     inside the temporary archive, NOT in the checkout: unlike pnpm/npm, `bun
  *     publish`/`bun pm pack` do NOT fold `publishConfig`'s `main`/`types`/`bin`/
@@ -29,27 +27,26 @@
  *   - **compiled output** is emitted once, before packing, by one root-level
  *     filtered `bun run` that fans out to every publishable member in parallel.
  *     Every package is then packed once with lifecycle scripts disabled. The
- *     exact validated archive is passed to `bun publish`, so upload never
+ *     exact validated archive is passed to `npm publish`, so upload never
  *     repacks or repeats a member's `prepack`. Packages retain `prepack` for
  *     standalone publishes.
  *   - **release recovery** compares each packed archive's integrity and
  *     repository identity with registry metadata. A matching immutable version
  *     is skipped, while any mismatch fails the retry.
  *
- * `--dry-run` forwards to `bun publish`: it packs + validates
- * but uploads nothing, so the `release` workflow is testable end-to-end via a
- * `workflow_dispatch` run without anything reaching npm. `--registry` targets a
- * non-default registry (a local verdaccio); `--exclude <dir>` (repeatable,
- * repo-relative) skips a member owned by another publication flow.
+ * `--dry-run` forwards to `npm publish`: it packs + validates
+ * but uploads nothing. `--registry` targets a non-default registry (a local
+ * verdaccio); `--exclude <dir>` (repeatable, repo-relative) skips a member owned
+ * by another publication flow.
  *
- * The checkout remains byte-for-byte unchanged. A stale manifest or lockfile is
- * a release error rather than something publication repairs.
+ * The checkout remains byte-for-byte unchanged. A stale manifest is a release
+ * error rather than something publication repairs. Local lockfiles are ignored
+ * because configured registries can leak into them.
  */
 import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { delimiter, dirname, join, resolve } from "node:path";
 import { asyncUtils, log } from "@dbx-tools/shared-core";
-import ts from "typescript";
 import { parse } from "yaml";
 import {
   npmReleaseMatches,
@@ -58,7 +55,6 @@ import {
   readNpmArchiveIdentity,
 } from "./publish-npm.ts";
 import { runTaskCommand, runTaskCommandAsync } from "../src/_task-command.ts";
-import { toPosix } from "../src/packages.ts";
 
 const logger = log.logger("projen:publish");
 
@@ -85,28 +81,6 @@ function manifestsMatchVersion(members: readonly string[], version: string): boo
   });
 }
 
-/** Whether Bun's workspace lock records each live workspace manifest version. */
-function lockfileMatchesManifestVersions(root: string, members: readonly string[]): boolean {
-  const lockfile = join(root, "bun.lock");
-  if (!existsSync(lockfile)) return false;
-  try {
-    const parsed = ts.parseConfigFileTextToJson(lockfile, readFileSync(lockfile, "utf8"));
-    if (parsed.error) return false;
-    const lock = parsed.config as {
-      workspaces?: Record<string, { version?: string }>;
-    };
-    return members.every((dir) => {
-      const manifest = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")) as {
-        version?: string;
-      };
-      const relative = toPosix(dir.slice(resolve(root).length + 1));
-      return lock.workspaces?.[relative]?.version === manifest.version;
-    });
-  } catch {
-    return false;
-  }
-}
-
 /** Entry-point fields projen writes as `.ts` source in-repo and rewrites to `lib/` for publish. */
 const PUBLISH_CONFIG_ENTRY_FIELDS = ["main", "types", "bin", "exports"] as const;
 
@@ -183,11 +157,6 @@ if (!manifestsMatchVersion(members, version)) {
 }
 logger.info(`validated ${members.length} selected member manifests`);
 
-if (!lockfileMatchesManifestVersions(root, allMembers)) {
-  throw new Error("bun.lock does not match workspace versions; run bun install and commit it");
-}
-logger.info("validated workspace lock versions");
-
 const publishArgs = [
   "--ignore-scripts",
   ...(registry ? ["--registry", registry] : []),
@@ -206,17 +175,12 @@ for (const dir of members) {
     name?: string;
     version?: string;
     private?: boolean;
-    dbxToolsConfig?: { uniffi?: boolean };
     publishConfig?: { access?: unknown };
   };
   if (pkg.private) {
     logger.info(`skip private ${pkg.name ?? dirname(dir)}`);
     continue;
   }
-  if (pkg.dbxToolsConfig?.uniffi === true) {
-    logger.info(`skip UniFFI ${pkg.name ?? dirname(dir)}`);
-    continue;
-  }
   if (!pkg.version) throw new Error(`Missing package version for ${pkg.name ?? dirname(dir)}`);
   const access = pkg.publishConfig?.access;
   if (access !== undefined && access !== "public" && access !== "restricted") {
@@ -303,7 +267,7 @@ await asyncUtils.mapConcurrent(
       logger.info(`${dryRun ? "dry-run publishing" : "publishing"} ${name} @ ${packageVersion}`);
       await runTaskCommandAsync(
         dir,
-        "bun",
+        "npm",
         ["publish", ...(access ? ["--access", access] : []), ...publishArgs, archive],
         { env: { ...process.env, PATH: path } },
       );

package/tasks/python-node-bindings-watch.ts

@@ -0,0 +1,47 @@
+#!/usr/bin/env -S bun
+import { dirname, isAbsolute, resolve, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+import { log } from "@dbx-tools/shared-core";
+import { runTaskCommand } from "../src/_task-command.ts";
+import { pythonNodeBindingProjects, repoRoot } from "../src/packages.ts";
+import {
+  pythonNodeBindingWatchInputs,
+  resolvePythonNodeBindings,
+  type ResolvedPythonNodeBindings,
+} from "../src/python-node-bindings.ts";
+import { watchLoop } from "../src/watch.ts";
+
+const logger = log.logger("projen:python-node-bindings");
+const task = resolve(dirname(fileURLToPath(import.meta.url)), "python-node-bindings.ts");
+const { values } = parseArgs({ options: { project: { type: "string" } } });
+const projects = values.project ? [values.project] : pythonNodeBindingProjects(repoRoot);
+const configs = projects.map((project) => resolvePythonNodeBindings(repoRoot, project));
+
+if (configs.length > 0) {
+  watchLoop(
+    "python-node-bindings",
+    [...new Set(configs.flatMap(pythonNodeBindingWatchInputs))],
+    (changed) => {
+      for (const config of configs.filter((candidate) => affected(candidate, changed))) {
+        logger.start(`generating ${config.project} Node bindings`);
+        runTaskCommand(repoRoot, "bun", [task, "--project", config.project]);
+        logger.success(`generated ${config.project} Node bindings`);
+      }
+    },
+    {
+      check: (changed) => configs.some((config) => affected(config, changed)),
+    },
+  );
+}
+
+function affected(config: ResolvedPythonNodeBindings, changed: readonly string[]): boolean {
+  const inputs = pythonNodeBindingWatchInputs(config);
+  return changed.some((path) => inputs.some((input) => contains(input, path)));
+}
+
+function contains(input: string, candidate: string): boolean {
+  const root = isAbsolute(input) ? input : resolve(repoRoot, input);
+  const path = isAbsolute(candidate) ? candidate : resolve(repoRoot, candidate);
+  return path === root || path.startsWith(root + sep);
+}