@dbx-tools/projen 0.9.47 → 0.9.51

package/tasks/{lib/python-registry.ts → python-registry.ts}

@@ -4,7 +4,7 @@ import { homedir } from "node:os";
 import { resolve } from "node:path";
 import { net } from "@dbx-tools/shared-core";
 import { parse } from "smol-toml";
-import { captureTaskCommand } from "../../src/_task-command.ts";
+import { captureTaskCommand } from "../src/_task-command.ts";
 
 /** Paired Python package index and upload endpoints for a local registry. */
 export interface LocalPythonRegistry {
@@ -23,7 +23,9 @@ interface UvIndex {
 function uvIndexes(source: string): UvIndex[] {
   const document = parse(source) as { index?: unknown };
   return Array.isArray(document.index)
-    ? document.index.filter((value): value is UvIndex => Boolean(value && typeof value === "object"))
+    ? document.index.filter((value): value is UvIndex =>
+        Boolean(value && typeof value === "object"),
+      )
     : [];
 }
 

package/tasks/python-sync.ts

@@ -0,0 +1,248 @@
+#!/usr/bin/env -S bun
+/**
+ * Synchronize pinned Git source subsets into Python generated package trees.
+ */
+
+import { execFileSync } from "node:child_process";
+import { createHash } from "node:crypto";
+import {
+  chmodSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import { withFileLock } from "@dbx-tools/core/file-lock";
+import { parse } from "smol-toml";
+
+interface SyncConfig {
+  readonly name?: string;
+  readonly source: string;
+  readonly include?: readonly string[];
+  readonly exclude?: readonly string[];
+  readonly replace?: Readonly<Record<string, string>>;
+}
+
+interface SyncManifest {
+  readonly optionsHash: string;
+  readonly resolvedCommit: string;
+  readonly source: string;
+}
+
+interface ParsedSource {
+  readonly repository: string;
+  readonly reference: string;
+  readonly subdirectory?: string;
+}
+
+export async function main(args: readonly string[] = process.argv.slice(2)): Promise<void> {
+  const projectIndex = args.indexOf("--project");
+  if (projectIndex < 0 || !args[projectIndex + 1]) {
+    throw new Error("python-sync requires --project <directory>");
+  }
+  const project = resolve(args[projectIndex + 1]!);
+  const force = args.includes("--force");
+  const check = args.includes("--check");
+  const pyproject = parse(readFileSync(join(project, "pyproject.toml"), "utf8")) as {
+    tool?: {
+      uv?: { "build-backend"?: { "module-name"?: string; "module-root"?: string } };
+      dbx_tools?: { sync?: SyncConfig | SyncConfig[] };
+    };
+  };
+  const configured = pyproject.tool?.dbx_tools?.sync;
+  const configs = configured ? (Array.isArray(configured) ? configured : [configured]) : [];
+  const build = pyproject.tool?.uv?.["build-backend"];
+  const moduleName = build?.["module-name"];
+  if (!moduleName) throw new Error(`${project}/pyproject.toml has no uv module-name`);
+  const moduleRoot = resolve(project, build?.["module-root"] ?? "src");
+  const generatedRoot = join(moduleRoot, ...moduleName.split("."), "_generated", "sync");
+  for (const config of configs) await synchronize(config, generatedRoot, { check, force });
+}
+
+async function synchronize(
+  config: SyncConfig,
+  generatedRoot: string,
+  options: { readonly check: boolean; readonly force: boolean },
+): Promise<void> {
+  const source = parseSource(config.source);
+  const name = config.name ?? defaultName(source);
+  const target = join(generatedRoot, name);
+  const optionsHash = hash({
+    exclude: [...(config.exclude ?? [])].sort(),
+    include: [...(config.include ?? ["**/*"])].sort(),
+    name,
+    replace: config.replace ?? {},
+    source: config.source,
+  });
+  const initialManifest = readManifest(target);
+  if (
+    !options.force &&
+    initialManifest?.optionsHash === optionsHash &&
+    /^[a-f0-9]{40}$/i.test(source.reference) &&
+    initialManifest.resolvedCommit === source.reference.toLowerCase()
+  ) {
+    return;
+  }
+  const initialCommit = resolveCommit(source);
+  if (
+    !options.force &&
+    initialManifest?.optionsHash === optionsHash &&
+    initialManifest.resolvedCommit === initialCommit
+  ) {
+    return;
+  }
+  await withFileLock(["python-sync", target], async () => {
+    const resolvedCommit = resolveCommit(source);
+    const manifest = readManifest(target);
+    if (
+      !options.force &&
+      manifest?.optionsHash === optionsHash &&
+      manifest.resolvedCommit === resolvedCommit
+    ) {
+      return;
+    }
+    if (options.check) throw new Error(`Generated Python sync is stale: ${target}`);
+    replaceTarget(config, source, target, { optionsHash, resolvedCommit });
+  });
+}
+
+function replaceTarget(
+  config: SyncConfig,
+  source: ParsedSource,
+  target: string,
+  manifest: Omit<SyncManifest, "source">,
+): void {
+  const temporary = mkdtempSync(join(tmpdir(), "dbx-tools-python-sync-"));
+  const checkout = join(temporary, "checkout");
+  const staging = join(temporary, "staging");
+  try {
+    execFileSync("git", ["init", "--quiet", checkout]);
+    execFileSync("git", [
+      "-C",
+      checkout,
+      "fetch",
+      "--quiet",
+      "--depth=1",
+      source.repository,
+      source.reference,
+    ]);
+    execFileSync("git", ["-C", checkout, "checkout", "--quiet", "FETCH_HEAD"]);
+    const root = resolve(checkout, source.subdirectory ?? ".");
+    mkdirSync(staging, { recursive: true });
+    const included = new Set<string>();
+    const replacements = new Map(Object.keys(config.replace ?? {}).map((value) => [value, 0]));
+    for (const pattern of config.include ?? ["**/*"]) {
+      for (const relative of new Bun.Glob(pattern).scanSync({ cwd: root, onlyFiles: true })) {
+        included.add(relative);
+      }
+    }
+    const excluded = (config.exclude ?? []).map((pattern) => new Bun.Glob(pattern));
+    for (const relative of [...included].sort()) {
+      if (excluded.some((glob) => glob.match(relative))) continue;
+      const destination = join(staging, relative);
+      mkdirSync(dirname(destination), { recursive: true });
+      cpSync(join(root, relative), destination);
+      if (relative.endsWith(".py") && config.replace) {
+        let contents = readFileSync(destination, "utf8");
+        for (const [from, to] of Object.entries(config.replace)) {
+          const matches = contents.split(from).length - 1;
+          if (matches > 0) replacements.set(from, (replacements.get(from) ?? 0) + matches);
+          contents = contents.replaceAll(from, to);
+        }
+        writeFileSync(destination, contents);
+      }
+    }
+    const missing = [...replacements]
+      .filter(([, matches]) => matches === 0)
+      .map(([value]) => value);
+    if (missing.length > 0) {
+      throw new Error(
+        `Python sync replacements did not match ${source.repository}: ${missing.join(", ")}`,
+      );
+    }
+    const license = join(checkout, "LICENSE");
+    if (existsSync(license)) cpSync(license, join(staging, "LICENSE.upstream"));
+    writeFileSync(
+      join(staging, ".sync.json"),
+      `${JSON.stringify({ ...manifest, source: config.source }, null, 2)}\n`,
+    );
+    makeWritable(target);
+    rmSync(target, { recursive: true, force: true });
+    mkdirSync(dirname(target), { recursive: true });
+    renameSync(staging, target);
+    makeReadonly(target);
+  } finally {
+    rmSync(temporary, { recursive: true, force: true });
+  }
+}
+
+function parseSource(value: string): ParsedSource {
+  const match = /^(?:(.*?)\s+@\s+)?git\+(.+?)@([^#]+)(?:#subdirectory=(.+))?$/.exec(value);
+  if (!match) throw new Error(`Invalid pip Git source: ${value}`);
+  return {
+    repository: match[2]!,
+    reference: match[3]!,
+    ...(match[4] ? { subdirectory: match[4] } : {}),
+  };
+}
+
+function resolveCommit(source: ParsedSource): string {
+  if (/^[a-f0-9]{40}$/i.test(source.reference)) return source.reference.toLowerCase();
+  const output = execFileSync("git", ["ls-remote", source.repository, source.reference], {
+    encoding: "utf8",
+  }).trim();
+  const commit = output.split(/\s+/, 1)[0];
+  if (!commit) throw new Error(`Could not resolve ${source.reference} from ${source.repository}`);
+  return commit.toLowerCase();
+}
+
+function defaultName(source: ParsedSource): string {
+  const repository = basename(source.repository, ".git");
+  const owner = basename(dirname(source.repository));
+  return `${safeName(owner)}-${safeName(repository)}-${hash(source.repository).slice(0, 8)}`;
+}
+
+function safeName(value: string): string {
+  return value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "");
+}
+
+function hash(value: unknown): string {
+  return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}
+
+function readManifest(target: string): SyncManifest | undefined {
+  const path = join(target, ".sync.json");
+  return existsSync(path) ? (JSON.parse(readFileSync(path, "utf8")) as SyncManifest) : undefined;
+}
+
+function makeWritable(path: string): void {
+  if (!existsSync(path)) return;
+  if (statSync(path).isDirectory()) {
+    chmodSync(path, 0o755);
+    for (const entry of readdirSync(path)) makeWritable(join(path, entry));
+  } else {
+    chmodSync(path, 0o644);
+  }
+}
+
+function makeReadonly(path: string): void {
+  if (statSync(path).isDirectory()) {
+    for (const entry of readdirSync(path)) makeReadonly(join(path, entry));
+    chmodSync(path, 0o555);
+  } else {
+    chmodSync(path, 0o444);
+  }
+}
+
+if (import.meta.main) await main();

package/tasks/release-version.ts

@@ -1,6 +1,68 @@
 #!/usr/bin/env -S bun
-import { main } from "./lib/release-version.ts";
+/** Validate one stable release version against the canonical workspace policy. */
+import { resolve } from "node:path";
+import { parseArgs } from "node:util";
+import * as projectUtils from "@dbx-tools/core/project-utils";
+import { captureTaskCommand } from "../src/_task-command.ts";
+import { compareSemver, parseSemver, type Semver } from "../src/workspace-version.ts";
 
-export * from "./lib/release-version.ts";
+function publishedVersion(root: string, prefixes: readonly string[]): string | undefined {
+  let best: Semver | undefined;
+  for (const prefix of prefixes) {
+    const tags = captureTaskCommand(root, "git", [
+      "-c",
+      "versionsort.suffix=-",
+      "tag",
+      "--sort=-version:refname",
+      "--list",
+      `${prefix}*`,
+    ]);
+    for (const tag of tags.split("\n")) {
+      if (!tag.startsWith(prefix)) continue;
+      const version = parseSemver(tag.slice(prefix.length));
+      if (version && (!best || compareSemver(version, best) > 0)) best = version;
+    }
+  }
+  return best?.join(".");
+}
+
+export function assertReleaseVersion(
+  version: string,
+  options: { root?: string; prefixes?: readonly string[]; assertNext?: boolean } = {},
+): void {
+  const parsed = parseSemver(version);
+  if (!parsed) {
+    throw new Error(
+      `release version must be an exact stable x.y.z, got ${JSON.stringify(version)}`,
+    );
+  }
+  if (!options.assertNext) return;
+  const root = resolve(options.root ?? projectUtils.root() ?? process.cwd());
+  const prefixes = options.prefixes?.length ? options.prefixes : ["v"];
+  const current = publishedVersion(root, prefixes);
+  const currentParsed = current ? parseSemver(current) : undefined;
+  if (currentParsed && compareSemver(parsed, currentParsed) <= 0) {
+    throw new Error(`release version ${version} must be greater than published ${current}`);
+  }
+}
+
+export function main(args: string[] = process.argv.slice(2)): void {
+  const { values } = parseArgs({
+    args,
+    options: {
+      version: { type: "string" },
+      root: { type: "string" },
+      prefix: { type: "string", multiple: true },
+      "assert-next": { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+  if (!values.version) throw new Error("--version is required");
+  assertReleaseVersion(values.version, {
+    ...(values.root ? { root: values.root } : {}),
+    ...(values.prefix ? { prefixes: values.prefix } : {}),
+    assertNext: values["assert-next"],
+  });
+}
 
 if (import.meta.main) main();

package/tasks/release.ts

@@ -1,6 +1,276 @@
 #!/usr/bin/env -S bun
-import { main } from "./lib/release.ts";
+/** Run the configured release transaction. */
+import * as projectUtils from "@dbx-tools/core/project-utils";
+import { log } from "@dbx-tools/shared-core";
+import { Command, Option } from "commander";
+import { publishLocalRelease } from "./local-publish.ts";
+import { assertReleaseVersion } from "./release-version.ts";
+import { captureTaskCommand, runTaskCommand, taskCommandSucceeds } from "../src/_task-command.ts";
+import {
+  RELEASE_INSTALL_MODES,
+  RELEASE_PUBLISH_TARGETS,
+  releasePublishesLocally,
+  releaseStepSelection,
+  releaseTagAnnotation,
+  type ReleaseInstallMode,
+  type ReleaseSelectionOptions,
+} from "../src/release-options.ts";
+import { readWorkspaceVersion } from "../src/workspace-version.ts";
 
-export { createReleaseCommand, main, runRelease } from "./lib/release.ts";
+const logger = log.logger("projen:release");
+
+function gitHead(root: string, ref: string): string | undefined {
+  if (!taskCommandSucceeds(root, "git", ["rev-parse", "--verify", ref])) return undefined;
+  return captureTaskCommand(root, "git", ["rev-parse", ref], { check: true });
+}
+
+function assertAncestor(root: string, ancestor: string, descendant: string, message: string): void {
+  if (!taskCommandSucceeds(root, "git", ["merge-base", "--is-ancestor", ancestor, descendant])) {
+    throw new Error(message);
+  }
+}
+
+function prepareReleaseBranch(options: {
+  readonly branch: string;
+  readonly remote: string;
+  readonly root: string;
+}): void {
+  const { branch, remote, root } = options;
+  const sourceBranch = captureTaskCommand(root, "git", ["branch", "--show-current"], {
+    check: true,
+  });
+  if (!sourceBranch) throw new Error("release cannot run from a detached HEAD");
+
+  const status = captureTaskCommand(
+    root,
+    "git",
+    ["status", "--porcelain=v1", "--untracked-files=all"],
+    { check: true },
+  );
+  if (status) {
+    runTaskCommand(root, "git", ["add", "--all"]);
+    runTaskCommand(root, "git", ["commit", "-m", "chore: prepare release"]);
+  }
+
+  runTaskCommand(root, "git", ["fetch", "--prune", remote, "--tags"]);
+  const sourceHead = captureTaskCommand(root, "git", ["rev-parse", "HEAD"], { check: true });
+  const remoteSourceHead = gitHead(root, `refs/remotes/${remote}/${sourceBranch}`);
+  if (remoteSourceHead) {
+    assertAncestor(
+      root,
+      remoteSourceHead,
+      sourceHead,
+      `cannot fast-forward ${remote}/${sourceBranch} to ${sourceBranch}`,
+    );
+  }
+  runTaskCommand(root, "git", ["push", remote, `HEAD:${sourceBranch}`]);
+
+  if (sourceBranch === branch) return;
+
+  const remoteMain = gitHead(root, `refs/remotes/${remote}/${branch}`);
+  if (!remoteMain) throw new Error(`release branch ${remote}/${branch} does not exist`);
+  assertAncestor(
+    root,
+    remoteMain,
+    sourceHead,
+    `cannot safely merge ${sourceBranch} into ${branch}; ${branch} is not an ancestor`,
+  );
+
+  const localMain = gitHead(root, `refs/heads/${branch}`);
+  if (localMain) {
+    assertAncestor(
+      root,
+      localMain,
+      sourceHead,
+      `cannot safely fast-forward local ${branch} to ${sourceBranch}`,
+    );
+  } else {
+    runTaskCommand(root, "git", ["branch", branch, `${remote}/${branch}`]);
+  }
+  runTaskCommand(root, "git", ["switch", branch]);
+  runTaskCommand(root, "git", ["merge", "--ff-only", sourceBranch]);
+  runTaskCommand(root, "git", ["push", remote, `HEAD:${branch}`]);
+}
+
+export async function runRelease(
+  options: ReleaseSelectionOptions & {
+    readonly root: string;
+    readonly branch: string;
+    readonly bump?: boolean;
+    readonly prefix: string;
+    readonly remote: string;
+    readonly localPublish?: boolean;
+    readonly localRegistry?: string;
+    readonly localPypi?: string;
+    readonly install?: ReleaseInstallMode;
+    readonly pythonRoot?: string;
+    readonly validationTasks?: readonly string[];
+  },
+): Promise<string> {
+  const { branch, prefix, remote, root } = options;
+  const selection = releaseStepSelection(options);
+  const install = options.install ?? "auto";
+
+  if (install === "always") runTaskCommand(root, "bun", ["install"]);
+
+  prepareReleaseBranch({ branch, remote, root });
+
+  runTaskCommand(root, "git", ["fetch", remote, branch, "--tags"]);
+  if (options.bump ?? true) {
+    runTaskCommand(root, "bun", ["run", "bump"], {
+      env: { ...process.env, DBX_TOOLS_RELEASE_INSTALL: install === "auto" ? "auto" : "never" },
+    });
+  }
+
+  const version = readWorkspaceVersion(root);
+  const tag = `${prefix}${version}`;
+  assertReleaseVersion(version, { root, prefixes: [prefix], assertNext: true });
+
+  if (taskCommandSucceeds(root, "git", ["rev-parse", "--verify", `refs/tags/${tag}`])) {
+    throw new Error(`release tag ${tag} already exists`);
+  }
+
+  runTaskCommand(root, "bun", ["run", "version:check"]);
+  for (const task of selection.validation ? (options.validationTasks ?? []) : []) {
+    runTaskCommand(root, "bun", ["run", task]);
+  }
+  const status = captureTaskCommand(
+    root,
+    "git",
+    ["status", "--porcelain=v1", "--untracked-files=all"],
+    { check: true },
+  );
+  if (status) {
+    if (!(options.bump ?? true)) {
+      throw new Error("--no-bump requires an already committed synchronized version");
+    }
+    if (taskCommandSucceeds(root, "git", ["diff", "--quiet", "HEAD", "--", "VERSION"])) {
+      throw new Error("release changes must include the local VERSION bump");
+    }
+    runTaskCommand(root, "git", ["add", "--all"]);
+    runTaskCommand(root, "git", ["commit", "-m", `chore(release): ${version}`]);
+  }
+
+  const remoteHead = captureTaskCommand(root, "git", ["rev-parse", `${remote}/${branch}`], {
+    check: true,
+  });
+  const head = captureTaskCommand(root, "git", ["rev-parse", "HEAD"], { check: true });
+  if (!taskCommandSucceeds(root, "git", ["merge-base", "--is-ancestor", remoteHead, head])) {
+    throw new Error(
+      `cannot fast-forward ${remote}/${branch} to HEAD; ${branch} has diverged (rebase before release)`,
+    );
+  }
+
+  runTaskCommand(root, "git", [
+    "tag",
+    "--annotate",
+    tag,
+    "--message",
+    releaseTagAnnotation(tag, selection),
+  ]);
+  try {
+    runTaskCommand(root, "git", ["push", "--atomic", remote, `HEAD:${branch}`, `refs/tags/${tag}`]);
+  } catch (error) {
+    runTaskCommand(root, "git", ["tag", "--delete", tag]);
+    throw error;
+  }
+  logger.success(`released ${tag}`, { sha: head });
+
+  // Restore local deploys: when a non-standard (loopback) npm or uv registry is
+  // configured, publish the freshly tagged version to it directly. Detection and
+  // publishing are owned by local-publish.ts; this no-ops on standard registries.
+  if ((options.localPublish ?? true) && releasePublishesLocally(options.publish)) {
+    await publishLocalRelease({
+      localPypi: options.pypi === false ? "false" : (options.localPypi ?? "auto"),
+      localRegistry: options.npm === false ? "false" : (options.localRegistry ?? "auto"),
+      pythonRoot: options.pythonRoot ?? "packages/py",
+      root,
+      version,
+    });
+  }
+
+  return tag;
+}
+
+/** Build the release task's native parser without executing git or registry operations. */
+export function createReleaseCommand(): Command {
+  return new Command()
+    .name("release")
+    .description("Prepare an annotated release and select its build and publication steps")
+    .option("--root <path>", "repository root")
+    .option("--branch <name>", "release branch", "main")
+    .option("--prefix <prefix>", "release tag prefix", "v")
+    .option("--remote <name>", "git remote", "origin")
+    .option("--python-root <path>", "Python package root for local publish", "packages/py")
+    .option(
+      "--validate <task>",
+      "task to run before pushing",
+      (task, tasks: string[]) => [...tasks, task],
+      [],
+    )
+    .option("--no-bump", "use an existing synchronized local version bump")
+    .addOption(
+      new Option("--publish <target>", "publication scope")
+        .choices([...RELEASE_PUBLISH_TARGETS])
+        .default("auto"),
+    )
+    .addOption(
+      new Option("--install <mode>", "local workspace dependency installation")
+        .choices([...RELEASE_INSTALL_MODES])
+        .default("auto"),
+    )
+    .option("--no-npm", "skip npm build and publication, including local npm publication")
+    .option("--no-pypi", "skip Python build and publication, including local Python publication")
+    .option("--docs", "build and deploy docs for a selected scope")
+    .option("--no-docs", "skip documentation build and deployment")
+    .option(
+      "--no-validation",
+      "skip optional release validation tasks; version/source checks remain mandatory",
+    )
+    .option("--no-local-publish", "skip publishing to configured local registries")
+    .option("--local-registry <auto|false|url>", "local npm registry selection", "auto")
+    .option("--local-pypi <auto|false|url>", "local devpi registry selection", "auto")
+    .action(
+      async (
+        options: ReleaseSelectionOptions & {
+          root?: string;
+          branch: string;
+          bump: boolean;
+          prefix: string;
+          remote: string;
+          pythonRoot: string;
+          localPublish: boolean;
+          localRegistry: string;
+          localPypi: string;
+          install: ReleaseInstallMode;
+          validate: string[];
+        },
+      ) => {
+        await runRelease({
+          root: options.root ?? projectUtils.root() ?? process.cwd(),
+          branch: options.branch,
+          bump: options.bump,
+          prefix: options.prefix,
+          remote: options.remote,
+          pythonRoot: options.pythonRoot,
+          localPublish: options.localPublish,
+          localRegistry: options.localRegistry,
+          localPypi: options.localPypi,
+          install: options.install,
+          publish: options.publish,
+          npm: options.npm,
+          pypi: options.pypi,
+          docs: options.docs,
+          validation: options.validation,
+          validationTasks: options.validate,
+        });
+      },
+    );
+}
+
+/** Execute the native release task parser. */
+export async function main(): Promise<void> {
+  await createReleaseCommand().parseAsync();
+}
 
 if (import.meta.main) await main();