@dbx-tools/projen 0.9.153 → 0.9.157

package/tasks/release-version.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env -S bun
 /** Validate one stable release version against the canonical workspace policy. */
 import { resolve } from "node:path";
-import * as projectUtils from "@dbx-tools/core/project-utils";
+import { projectUtils } from "@dbx-tools/core";
 import { z } from "zod";
 import { runTaskMain, taskCommand, taskOptions } from "./cli.ts";
 import { TaskRootOptionSchema } from "./options.ts";

package/tasks/release.ts

@@ -2,8 +2,8 @@
 /** Run the configured release transaction. */
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import * as projectUtils from "@dbx-tools/core/project-utils";
-import { errorUtils, json, log } from "@dbx-tools/shared-core";
+import { projectUtils, type ProjectGhAccount, resolveProjectGhAccount } from "@dbx-tools/core";
+import { log, toError, parse } from "@dbx-tools/shared-core";
 import type { Command } from "commander";
 import { z } from "zod";
 import { parsedTaskOptions, runTaskMain, taskCommand } from "./cli.ts";
@@ -67,7 +67,7 @@ const GithubReleaseRunViewSchema = z
 
 /** Inputs required to locate and watch one tag-triggered release workflow. */
 export interface WatchReleaseGithubActionsOptions {
-  readonly account: projectUtils.ProjectGhAccount;
+  readonly account: ProjectGhAccount;
   readonly head: string;
   readonly root: string;
   readonly tag: string;
@@ -80,7 +80,7 @@ export type ReleaseGithubActionsWatcher = (
   options: WatchReleaseGithubActionsOptions,
 ) => Promise<void>;
 
-function githubRepository(account: projectUtils.ProjectGhAccount): string {
+function githubRepository(account: ProjectGhAccount): string {
   return account.host === "github.com"
     ? account.remote.repository
     : `${account.host}/${account.remote.repository}`;
@@ -194,7 +194,7 @@ export async function watchReleaseGithubActions(
       ],
       { check: true, env: account.env, stderr: "inherit" },
     );
-    const parsed = z.array(GithubReleaseRunSchema).safeParse(json.parse(output, []));
+    const parsed = z.array(GithubReleaseRunSchema).safeParse(parse(output, []));
     if (!parsed.success) throw new Error("GitHub returned invalid release workflow metadata");
     run = parsed.data.find(
       (candidate) => candidate.headSha === head && candidate.headBranch === tag,
@@ -234,7 +234,7 @@ export async function watchReleaseGithubActions(
         ],
         { check: true, env: account.env, stderr: "inherit" },
       );
-      const parsed = GithubReleaseRunViewSchema.safeParse(json.parse(output));
+      const parsed = GithubReleaseRunViewSchema.safeParse(parse(output));
       if (!parsed.success) throw new Error("GitHub returned invalid release progress metadata");
       reportGithubProgress(parsed.data, progress);
       completed = parsed.data.status === "completed";
@@ -251,7 +251,7 @@ export async function watchReleaseGithubActions(
     }
   } catch (error) {
     logger.warn("GitHub Actions release watch stopped; the remote run was not cancelled", {
-      error: errorUtils.toError(error),
+      error: toError(error),
       runId: run.databaseId,
       url: run.url,
     });
@@ -305,7 +305,7 @@ function pushAnnotatedReleaseTag(options: {
 }): void {
   const { annotation, branch, head, remote, root, tag } = options;
   runTaskCommand(root, "git", ["tag", "--annotate", tag, "--message", annotation]);
-  const githubAccount = projectUtils.resolveProjectGhAccount(root, { remote });
+  const githubAccount = resolveProjectGhAccount(root, { remote });
   if (!githubAccount) {
     logger.warn(
       "no authenticated GitHub account can access the release repository; using git push",
@@ -382,7 +382,7 @@ function pushAnnotatedReleaseTag(options: {
   } catch (error) {
     if (remoteRefCreated) throw error;
     logger.warn("GitHub API tag creation failed; using git push", {
-      error: errorUtils.toError(error),
+      error: toError(error),
       repository: githubAccount.remote.repository,
     });
     runTaskCommand(root, "git", ["push", remote, `refs/tags/${tag}`]);
@@ -480,7 +480,7 @@ export async function runRelease(
     /** After local release actions, wait for and stream the matching GitHub Actions run. */
     readonly watchGithubActions?: boolean;
     readonly githubActionsWatcher?: ReleaseGithubActionsWatcher;
-    readonly resolveGithubAccount?: typeof projectUtils.resolveProjectGhAccount;
+    readonly resolveGithubAccount?: typeof resolveProjectGhAccount;
   },
 ): Promise<string> {
   const { branch, prefix, remote, root } = options;
@@ -606,7 +606,7 @@ export async function runRelease(
   }
 
   if (shouldWatchReleaseGithubActions(options, selection)) {
-    const account = (options.resolveGithubAccount ?? projectUtils.resolveProjectGhAccount)(root, {
+    const account = (options.resolveGithubAccount ?? resolveProjectGhAccount)(root, {
       remote,
     });
     if (account) {

package/tasks/sync.ts

@@ -1,7 +1,7 @@
 #!/usr/bin/env -S bun
 /** Synthesize once or supervise Projen, barrels, and registered focused watchers. */
 import { fileURLToPath } from "node:url";
-import { errorUtils, log } from "@dbx-tools/shared-core";
+import { log, toError } from "@dbx-tools/shared-core";
 import concurrently from "concurrently";
 import { z } from "zod";
 import { runTaskMain, taskCommand, taskOptions } from "./cli.ts";
@@ -84,7 +84,7 @@ export async function main(args: string[] = process.argv.slice(2)): Promise<void
     } catch (err) {
       // A tree that doesn't synth is precisely when the watcher is most useful: the edit
       // that repairs it is the one the projenrc watcher is sitting there waiting for.
-      logger.error("initial sync failed - watching anyway", { error: errorUtils.toError(err) });
+      logger.error("initial sync failed - watching anyway", { error: toError(err) });
     }
 
     const watchers = [

package/tasks/test-workspace.ts

@@ -1,8 +1,8 @@
 import { readFileSync, writeFileSync } from "node:fs";
 import { basename, join, relative, resolve, sep } from "node:path";
-import * as exec from "@dbx-tools/core/exec";
-import { findFilesSync } from "@dbx-tools/path/find";
-import { log, object } from "@dbx-tools/shared-core";
+import { spawnSync, spawn } from "@dbx-tools/core";
+import { findFilesSync } from "@dbx-tools/path";
+import { log, sequence } from "@dbx-tools/shared-core";
 import { z } from "zod";
 import { runTaskMain, taskCommand, taskOptions, taskPositionals, taskRoot } from "./cli.ts";
 import { TaskRootOptionSchema } from "./options.ts";
@@ -137,7 +137,7 @@ function changedFiles(root: string, base?: string): string[] {
   const candidates = base ? [base] : ["origin/main", "main", "HEAD~1"];
   let committed: string[] = [];
   for (const candidate of candidates) {
-    const result = exec.spawnSync("git", ["diff", "--name-only", `${candidate}...HEAD`], {
+    const result = spawnSync("git", ["diff", "--name-only", `${candidate}...HEAD`], {
       cwd: root,
       stdin: "ignore",
       stdout: "capture",
@@ -148,14 +148,14 @@ function changedFiles(root: string, base?: string): string[] {
       break;
     }
   }
-  const working = exec.spawnSync("git", ["diff", "--name-only", "HEAD"], {
+  const working = spawnSync("git", ["diff", "--name-only", "HEAD"], {
     cwd: root,
     stdin: "ignore",
     stdout: "capture",
     stderr: "ignore",
     check: true,
   }).stdoutLines;
-  return [...object.sequence(committed, working).filter(Boolean).distinct()].sort();
+  return [...sequence(committed, working).filter(Boolean).distinct()].sort();
 }
 
 function selectedTests(
@@ -203,13 +203,13 @@ function runnerGroup(root: string, path: string): string {
   if (relativePath.startsWith("packages/js/shared/")) return "browser-shared";
   if (
     relativePath.startsWith("packages/js/ui/") ||
-    relativePath.startsWith("packages/example/app/")
+    relativePath.startsWith("packages/demo/client/")
   ) {
     return "react-ui";
   }
   if (relativePath.startsWith("projen/")) return "projen";
   if (relativePath.startsWith("docs/scripts/")) return "docs";
-  if (relativePath.startsWith("packages/example/")) return "examples";
+  if (relativePath.startsWith("packages/demo/")) return "demo";
   return "node";
 }
 
@@ -228,7 +228,7 @@ async function runTests(root: string, files: readonly string[]): Promise<void> {
   const started = performance.now();
   const jobs = [...groups].map(async ([name, paths]) => {
     const groupStarted = performance.now();
-    const result = await exec.spawn("bun", ["test", ...paths], {
+    const result = await spawn("bun", ["test", ...paths], {
       cwd: root,
       stdin: "ignore",
       stdout: "inherit",

package/tasks/verify-publications.ts

@@ -0,0 +1,273 @@
+#!/usr/bin/env -S bun
+/** Wait for one release batch to become available from public package registries. */
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+import { log, poll, mapConcurrent, toError } from "@dbx-tools/shared-core";
+import { z } from "zod";
+import { runTaskMain, taskCommand, taskOptions } from "./cli.ts";
+import {
+  npmReleaseMatches,
+  publishedNpmRelease,
+  readNpmArchiveIdentity,
+} from "../src/npm-release.ts";
+import { PUBLIC_NPM_REGISTRY_URL, PUBLIC_PYPI_REGISTRY_URL } from "../src/release-context.ts";
+
+const DEFAULT_CONCURRENCY = 8;
+const DEFAULT_INTERVAL_SECONDS = 5;
+const DEFAULT_TIMEOUT_SECONDS = 10 * 60;
+const logger = log.logger("projen:verify-publications");
+
+const PypiReleaseSchema = z
+  .object({
+    urls: z
+      .array(
+        z
+          .object({
+            filename: z.string().describe("Published distribution filename"),
+            digests: z
+              .object({
+                sha256: z.string().describe("Published distribution SHA-256 digest"),
+              })
+              .describe("Published distribution digests"),
+          })
+          .describe("One published Python distribution"),
+      )
+      .describe("Published Python distributions"),
+  })
+  .describe("PyPI release metadata");
+
+interface PypiReleaseIdentity {
+  readonly files: ReadonlyMap<string, string>;
+  readonly name: string;
+  readonly version: string;
+}
+
+/** One public artifact group checked during the shared availability barrier. */
+export interface PublicationProbe {
+  /** Human-readable package identity used in progress and timeout messages. */
+  readonly label: string;
+  /** Return true only when the exact expected public artifact is available. */
+  available(signal: AbortSignal): boolean | PromiseLike<boolean>;
+}
+
+/** Timing and concurrency controls for the shared publication barrier. */
+export interface PublicationWaitOptions {
+  readonly concurrency?: number;
+  readonly intervalMs?: number;
+  readonly timeoutMs?: number;
+}
+
+/** Poll every unresolved publication concurrently and stop after one shared timeout. */
+export async function waitForPublications(
+  probes: readonly PublicationProbe[],
+  options: PublicationWaitOptions = {},
+): Promise<void> {
+  if (probes.length === 0) throw new Error("No public publications were provided");
+  const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+  const errors = new Map<string, Error>();
+  let pending = [...probes];
+  let previousCount: number | undefined;
+  try {
+    for await (const unavailable of poll<PublicationProbe[]>(
+      async ({ signal }) => {
+        const results = await mapConcurrent(
+          pending,
+          async (probe) => {
+            try {
+              const available = await probe.available(signal);
+              if (available) errors.delete(probe.label);
+              return { available, probe };
+            } catch (error) {
+              errors.set(probe.label, toError(error));
+              return { available: false, probe };
+            }
+          },
+          { concurrency },
+        );
+        return results.filter(({ available }) => !available).map(({ probe }) => probe);
+      },
+      {
+        intervalMs: options.intervalMs ?? DEFAULT_INTERVAL_SECONDS * 1_000,
+        timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_SECONDS * 1_000,
+        predicate: (unavailable) => unavailable.length > 0,
+      },
+    )) {
+      pending = unavailable;
+      if (pending.length > 0 && pending.length !== previousCount) {
+        logger.info(
+          `waiting for ${pending.length} public publication${pending.length === 1 ? "" : "s"}`,
+        );
+      }
+      previousCount = pending.length;
+    }
+  } catch (error) {
+    if (pending.length === 0) throw error;
+    const details = pending
+      .map((probe) => {
+        const failure = errors.get(probe.label);
+        return failure ? `${probe.label}: ${failure.message}` : probe.label;
+      })
+      .join("\n");
+    throw new Error(`Publications did not become available:\n${details}`, { cause: error });
+  }
+  logger.info(`verified ${probes.length} public publication${probes.length === 1 ? "" : "s"}`);
+}
+
+function npmPublicationProbes(directory: string, version: string): PublicationProbe[] {
+  return readdirSync(directory)
+    .filter((file) => file.endsWith(".tgz"))
+    .sort()
+    .map((file) => {
+      const local = readNpmArchiveIdentity(join(directory, file));
+      if (local.version !== version) {
+        throw new Error(`${local.name} archive carries ${local.version}, expected ${version}`);
+      }
+      return {
+        label: `${local.name}@${local.version}`,
+        available: async (signal: AbortSignal) =>
+          npmReleaseMatches(
+            local,
+            await publishedNpmRelease(local.name, local.version, PUBLIC_NPM_REGISTRY_URL, signal),
+          ),
+      };
+    });
+}
+
+function distributionFiles(directory: string): string[] {
+  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) return distributionFiles(path);
+    return entry.name.endsWith(".whl") || entry.name.endsWith(".tar.gz") ? [path] : [];
+  });
+}
+
+function pythonDistributionName(file: string, version: string): string {
+  const name = basename(file);
+  const sourceSuffix = `-${version}.tar.gz`;
+  if (name.endsWith(sourceSuffix)) {
+    return name.slice(0, -sourceSuffix.length).replaceAll("_", "-").toLowerCase();
+  }
+  if (name.endsWith(".whl")) {
+    const marker = `-${version}-`;
+    const markerIndex = name.lastIndexOf(marker);
+    if (markerIndex > 0) {
+      return name.slice(0, markerIndex).replaceAll("_", "-").toLowerCase();
+    }
+  }
+  throw new Error(`Python distribution ${name} does not carry release version ${version}`);
+}
+
+function pypiReleaseIdentities(directory: string, version: string): PypiReleaseIdentity[] {
+  const releases = new Map<string, Map<string, string>>();
+  for (const path of distributionFiles(directory)) {
+    const name = pythonDistributionName(path, version);
+    const files = releases.get(name) ?? new Map<string, string>();
+    const filename = basename(path);
+    files.set(filename, createHash("sha256").update(readFileSync(path)).digest("hex"));
+    releases.set(name, files);
+  }
+  return [...releases]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([name, files]) => ({ files, name, version }));
+}
+
+function pypiPublicationProbes(directory: string, version: string): PublicationProbe[] {
+  return pypiReleaseIdentities(directory, version).map((local) => ({
+    label: `${local.name}==${local.version}`,
+    available: async (signal: AbortSignal) => {
+      const response = await fetch(
+        `${PUBLIC_PYPI_REGISTRY_URL}/${encodeURIComponent(local.name)}/${encodeURIComponent(local.version)}/json`,
+        {
+          headers: { accept: "application/json" },
+          signal,
+        },
+      );
+      if (response.status === 404) return false;
+      if (!response.ok) {
+        throw new Error(
+          `PyPI lookup failed for ${local.name}==${local.version}: ${response.status}`,
+        );
+      }
+      const published = PypiReleaseSchema.parse(await response.json());
+      const publishedFiles = new Map(
+        published.urls.map(({ filename, digests }) => [filename, digests.sha256]),
+      );
+      for (const [filename, digest] of local.files) {
+        const publishedDigest = publishedFiles.get(filename);
+        if (!publishedDigest) return false;
+        if (publishedDigest !== digest) {
+          throw new Error(
+            `PyPI content does not match ${local.name}==${local.version} ${filename}`,
+          );
+        }
+      }
+      return true;
+    },
+  }));
+}
+
+export const VerifyPublicationsOptionsSchema = z
+  .object({
+    version: z.string().trim().min(1).describe("Expected public release version"),
+    npmDirectory: z
+      .string()
+      .trim()
+      .min(1)
+      .optional()
+      .describe("Directory containing released npm archives"),
+    pypiDirectory: z
+      .string()
+      .trim()
+      .min(1)
+      .optional()
+      .describe("Directory containing released Python distributions"),
+    concurrency: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(DEFAULT_CONCURRENCY)
+      .describe("Maximum concurrent public registry checks"),
+    intervalSeconds: z.coerce
+      .number()
+      .positive()
+      .default(DEFAULT_INTERVAL_SECONDS)
+      .describe("Delay between shared publication checks"),
+    timeoutSeconds: z.coerce
+      .number()
+      .positive()
+      .default(DEFAULT_TIMEOUT_SECONDS)
+      .describe("Maximum time to wait for the complete publication batch"),
+  })
+  .refine((options) => options.npmDirectory || options.pypiDirectory, {
+    message: "At least one publication directory is required",
+  })
+  .describe("Options for the public package availability barrier");
+
+/** Parse expected release artifacts and wait for their public registry records. */
+export async function main(args: readonly string[] = process.argv.slice(2)): Promise<void> {
+  const options = await taskOptions(
+    taskCommand(
+      import.meta.url,
+      "Wait for released packages to become publicly available",
+      VerifyPublicationsOptionsSchema,
+    ),
+    VerifyPublicationsOptionsSchema,
+    args,
+  );
+  const probes = [
+    ...(options.npmDirectory
+      ? npmPublicationProbes(resolve(options.npmDirectory), options.version)
+      : []),
+    ...(options.pypiDirectory
+      ? pypiPublicationProbes(resolve(options.pypiDirectory), options.version)
+      : []),
+  ];
+  await waitForPublications(probes, {
+    concurrency: options.concurrency,
+    intervalMs: options.intervalSeconds * 1_000,
+    timeoutMs: options.timeoutSeconds * 1_000,
+  });
+}
+
+await runTaskMain(import.meta, main);

package/tasks/version-check.ts

@@ -2,9 +2,9 @@
 /** Verify that every committed package version matches the root VERSION file. */
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
-import * as projectUtils from "@dbx-tools/core/project-utils";
-import { find } from "@dbx-tools/path";
-import { json, log } from "@dbx-tools/shared-core";
+import { projectUtils } from "@dbx-tools/core";
+import { findFiles } from "@dbx-tools/path";
+import { log, parseRecord } from "@dbx-tools/shared-core";
 import { parse } from "smol-toml";
 import { runTaskMain } from "./cli.ts";
 import { recordedPackages } from "../src/packages.ts";
@@ -13,7 +13,7 @@ import { readWorkspaceVersion } from "../src/workspace-version.ts";
 const logger = log.logger("projen:version-check");
 
 function manifestVersion(path: string): string | undefined {
-  const manifest = json.parseRecord(readFileSync(path, "utf8"));
+  const manifest = parseRecord(readFileSync(path, "utf8"));
   return typeof manifest?.version === "string" ? manifest.version : undefined;
 }
 
@@ -37,7 +37,7 @@ export async function main(): Promise<void> {
     check(`${pkg.path}/package.json`, manifestVersion(join(pkg.dir, "package.json")));
   }
 
-  for await (const path of find.findFiles("**/pyproject.toml", { cwd: root })) {
+  for await (const path of findFiles("**/pyproject.toml", { cwd: root })) {
     check(path, pythonVersion(join(root, path)));
   }
   if (mismatches.length > 0) {