@dbx-tools/projen 0.9.47 → 0.9.51

package/README.md

@@ -184,17 +184,56 @@ polyfills. Python-host-backed adapters for process execution, files, crypto,
 OS values, and process state are included automatically, so packages do not
 configure a shim directory.
 
-The workspace creates `<name>:python-runtime` and
-`<name>:python-runtime:check`. Full synthesis regenerates bindings and removes
-stale `_generated/node` trees by reconciling them with the current
-`pyproject.toml` files, including the resynthesis triggered by a `.projenrc.ts`
-change.
+The workspace creates `<name>:python-runtime`, `<name>:python-runtime:check`,
+and `<name>:python-runtime:watch`. The focused watcher regenerates a Python
+project when its bound Node package, a transitive workspace dependency, its
+binding configuration, a function override, or a shared shim changes.
+`bun run sync --watch` supervises this watcher alongside Projen and barrel
+generation. Full synthesis regenerates bindings and removes stale
+`_generated/node` trees by reconciling them with the current `pyproject.toml`
+files, including the resynthesis triggered by a `.projenrc.ts` change.
 Built-in shims can make ordinary Node imports work under PythonMonkey.
 `functionOverrides` replaces named exports only in the generated runtime, so the
 Node package does not gain Python callbacks or alternate source files. A
 replacement may use a different export name through `handlerExport`; every
 other export continues to come from the original module.
 
+### Synchronize Pinned Python Sources
+
+Use a package's `sync` field when published Python code must include a reviewed
+subset of a Git repository instead of exposing a direct URL dependency:
+
+```ts
+{
+  directory: "service",
+  description: "Python service with synchronized upstream code",
+  sync: [
+    {
+      name: "upstream_driver",
+      source:
+        "driver @ git+https://github.com/example/project.git@0123456789abcdef0123456789abcdef01234567#subdirectory=src/driver",
+      include: ["driver.py", "operations/**/*.py"],
+      exclude: ["**/test_*.py"],
+      replace: {
+        "from driver.": "from example.service._generated.sync.upstream_driver.",
+      },
+    },
+  ],
+}
+```
+
+Each source is written beneath
+`<python-module>/_generated/sync/<name-or-owner-repository-hash>`. The generated
+package task parses pip-style Git sources, filters files with the configured
+globs, verifies every replacement matched, records the options and resolved
+commit in `.sync.json`, and recursively marks the result read-only.
+
+Synchronization uses a check-lock-check sequence and an atomic directory
+replacement. A current pinned commit performs no network request. Branches and
+tags use `git ls-remote` to detect upstream movement. Run the generated
+`<package>:python-sync:check` task in validation, or pass `--force` to refresh a
+current source intentionally.
+
 ## Customize Packages With Mixins
 
 ```ts

package/index.ts

@@ -3,7 +3,7 @@
 // Hand edits are overwritten on the next watch; this file is read-only.
 
 export const PACKAGE_IDENTIFIER = "@dbx-tools/projen";
-export const PACKAGE_VERSION = "0.9.47";
+export const PACKAGE_VERSION = "0.9.51";
 export * as barrels from "./src/barrels.ts";
 export * as bunApp from "./src/bun-app.ts";
 export * as bunWorkflow from "./src/bun-workflow.ts";
@@ -50,7 +50,7 @@ export type { DBXToolsProjectLanguage, DBXToolsProjectOptions, DBXToolsProject,
 export { PackageIdentifier, LICENSE, DBXToolsNodeProject, ROOT_INSTALL_ONLY_MIXIN, DBXToolsTypeScriptProject } from "./src/project-js.ts";
 export type { DBXToolsJavaScriptProject, DBXToolsReleaseMode, DiscoveredTypeScriptPackageOptions, DBXToolsJavaScriptProjectOptions, DBXToolsTypeScriptProjectOptions } from "./src/project-js.ts";
 export { DBXToolsPythonProject, DBXToolsPythonWorkspace } from "./src/project-py.ts";
-export type { PythonRepositoryOptions, PythonPackageOptions, DBXToolsPythonProjectOptions, PythonReleaseOptions, DBXToolsPythonWorkspaceOptions } from "./src/project-py.ts";
+export type { PythonRepositoryOptions, PythonPackageOptions, PythonSourceSyncOptions, DBXToolsPythonProjectOptions, PythonReleaseOptions, DBXToolsPythonWorkspaceOptions } from "./src/project-py.ts";
 export { PROJEN_VERSION } from "./src/projen-version.ts";
 export { COMPILED_DIR, COMPILED_COMPILER_OPTIONS } from "./src/publish.ts";
 export { PYTHON_NODE_SHIM_ROOT } from "./src/python-node-bindings.ts";

package/package.json

@@ -30,9 +30,9 @@
   },
   "dependencies": {
     "@clack/prompts": "^1.7.0",
-    "@dbx-tools/core": "^0.9.47",
-    "@dbx-tools/path": "^0.9.47",
-    "@dbx-tools/shared-core": "^0.9.47",
+    "@dbx-tools/core": "^0.9.51",
+    "@dbx-tools/path": "^0.9.51",
+    "@dbx-tools/shared-core": "^0.9.51",
     "commander": "^15.0.0",
     "concurrently": "^10.0.3",
     "constructs": "^10.6.0",
@@ -54,12 +54,12 @@
   },
   "main": "index.ts",
   "license": "Apache-2.0",
-  "version": "0.9.47",
+  "version": "0.9.51",
   "types": "index.ts",
   "type": "module",
   "exports": {
     ".": "./index.ts",
-    "./release-packaging": "./tasks/lib/publish-npm.ts",
+    "./release-packaging": "./tasks/publish-npm.ts",
     "./package.json": "./package.json"
   },
   "files": [

package/src/project-py.ts

@@ -3,7 +3,7 @@ import * as projectUtils from "@dbx-tools/core/project-utils";
 import { stringUtils, type OneOrMany } from "@dbx-tools/shared-core";
 import { Component, License, TextFile, type Project, github, javascript, python } from "projen";
 import { JobPermission, type Job, type JobStep } from "projen/lib/github/workflows-model";
-import { LICENSE, projectRepositoryUrl } from "./project-js.ts";
+import { LICENSE, projectRepositoryUrl, taskCommand } from "./project-js.ts";
 import { isDBXToolsJavaScriptProject } from "./project-predicate.ts";
 import type { DBXToolsProject, DBXToolsProjectOptions } from "./project.ts";
 import { PythonNodeBundle, type PythonNodeBindingsOptions } from "./python-node-bundle.ts";
@@ -33,6 +33,8 @@ export interface PythonPackageOptions extends DBXToolsProjectOptions {
   readonly moduleRoot?: string;
   readonly description: string;
   readonly dependencies?: readonly string[];
+  /** Development dependencies emitted only in this package's dependency group. */
+  readonly devDependencies?: readonly string[];
   /** Workspace package directories rendered as standalone Git dependencies. */
   readonly internalDependencies?: readonly string[];
   readonly scripts?: Readonly<Record<string, string>>;
@@ -40,6 +42,17 @@ export interface PythonPackageOptions extends DBXToolsProjectOptions {
   readonly releaseEnvironment?: string;
   /** One or more build-time Node packages embedded through PythonMonkey. */
   readonly nodeBindings?: OneOrMany<PythonNodeBindingsOptions>;
+  /** Pinned Git source subsets synchronized into this package's generated tree. */
+  readonly sync?: readonly PythonSourceSyncOptions[];
+}
+
+/** One pip-style Git source synchronized into a generated Python package tree. */
+export interface PythonSourceSyncOptions {
+  readonly name?: string;
+  readonly source: string;
+  readonly include?: readonly string[];
+  readonly exclude?: readonly string[];
+  readonly replace?: Readonly<Record<string, string>>;
 }
 
 interface ResolvedPythonPackageOptions extends PythonPackageOptions {
@@ -245,7 +258,11 @@ export class DBXToolsPythonProject extends python.PythonProject implements DBXTo
     this.tasks.removeTask("publish");
     this.tasks.removeTask("publish:test");
     this.uv.file.addDeletionOverride("project.authors");
-    this.uv.file.addDeletionOverride("dependency-groups");
+    if (pkg.devDependencies?.length) {
+      this.uv.file.addOverride("dependency-groups.dev", [...pkg.devDependencies]);
+    } else {
+      this.uv.file.addDeletionOverride("dependency-groups");
+    }
     if (nodeBindings.length > 0) {
       const rendered = nodeBindings.map(renderPythonNodeBindings);
       this.uv.file.addOverride(
@@ -253,6 +270,9 @@ export class DBXToolsPythonProject extends python.PythonProject implements DBXTo
         rendered.length === 1 ? rendered[0] : rendered,
       );
     }
+    if (pkg.sync?.length) {
+      this.uv.file.addOverride("tool.dbx_tools.sync", [...pkg.sync]);
+    }
     this.uv.file.readonly = true;
 
     for (const path of [".gitattributes", ".gitignore"]) {
@@ -345,6 +365,21 @@ export class DBXToolsPythonWorkspace extends Component {
         projectDirectory: pythonPackagePath(this.repository, pkg.packageOptions.directory),
       });
     }
+    for (const pkg of this.packages) {
+      if (!pkg.packageOptions.sync?.length) continue;
+      const directory = pythonPackagePath(this.repository, pkg.packageOptions.directory);
+      const command = taskCommand("python-sync.ts", "--project", directory);
+      const sync = project.addTask(`${pkg.packageOptions.directory}:python-sync`, {
+        description: `Synchronize generated Git sources for ${pkg.packageOptions.name}`,
+        execArgs: command,
+      });
+      const check = project.addTask(`${pkg.packageOptions.directory}:python-sync:check`, {
+        description: `Verify generated Git sources for ${pkg.packageOptions.name}`,
+        execArgs: [...command, "--check"],
+      });
+      project.preCompileTask.spawn(sync);
+      project.testTask.spawn(check);
+    }
     for (const pkg of this.packages) {
       const pyproject = `/${pythonPackagePath(this.repository, pkg.packageOptions.directory)}/pyproject.toml`;
       project.gitignore.include(pyproject);

package/src/python-node-bindings.ts

@@ -1,6 +1,6 @@
 /** Shared pyproject-backed configuration for PythonMonkey Node bindings. */
 import { readFileSync, rmSync } from "node:fs";
-import { dirname, join, relative, resolve } from "node:path";
+import { dirname, join, relative, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { find } from "@dbx-tools/path";
 import { object, stringUtils } from "@dbx-tools/shared-core";
@@ -58,7 +58,8 @@ export function generatePythonNodeBindings(projectRoot: string = resolveRepoRoot
   }
 }
 
-function pythonNodeBindingProjects(projectRoot: string): string[] {
+/** Repository-relative Python projects that configure generated Node bindings. */
+export function pythonNodeBindingProjects(projectRoot: string): string[] {
   return [
     ...find
       .findFiles("**/pyproject.toml", { cwd: projectRoot })
@@ -169,6 +170,35 @@ export function pythonNodeBindingWatchInputs(config: ResolvedPythonNodeBindings)
   ];
 }
 
+/**
+ * Python projects whose generated runtimes depend on at least one changed path.
+ *
+ * The project list is computed once per watch batch and reused by the lock preflight
+ * and generator callback. This keeps unrelated edits on the no-lock fast path while
+ * ensuring each affected Python project is regenerated only once.
+ */
+export function affectedPythonNodeBindingProjects(
+  root: string,
+  configs: readonly ResolvedPythonNodeBindings[],
+  changed: readonly string[],
+): string[] {
+  const affected = new Set<string>();
+  for (const config of configs) {
+    const inputs = pythonNodeBindingWatchInputs(config);
+    if (changed.some((candidate) => inputs.some((input) => containsPath(root, input, candidate)))) {
+      affected.add(config.project);
+    }
+  }
+  return [...affected].sort();
+}
+
+/** Whether `candidate` is an input path or one of its descendants. */
+function containsPath(root: string, input: string, candidate: string): boolean {
+  const inputPath = resolve(root, input);
+  const candidatePath = resolve(root, candidate);
+  return candidatePath === inputPath || candidatePath.startsWith(`${inputPath}${sep}`);
+}
+
 function validateBindings(configs: readonly ResolvedPythonNodeBindings[]): void {
   const names = configs.map(({ bindingName }) => bindingName);
   if (new Set(names).size !== names.length) {

package/src/python-node-bundle.ts

@@ -45,6 +45,7 @@ export interface PythonNodeBundleOptions {
 export class PythonNodeBundle extends Component {
   readonly buildTask: Task;
   readonly checkTask: Task;
+  readonly watchTask: Task;
 
   constructor(project: Project, options: PythonNodeBundleOptions) {
     super(project);
@@ -57,6 +58,10 @@ export class PythonNodeBundle extends Component {
       description: `Verify generated Node bindings for Python package ${options.name}`,
       execArgs: [...command, "--check"],
     });
+    this.watchTask = project.addTask(`${options.name}:python-runtime:watch`, {
+      description: `Watch and regenerate Node bindings for Python package ${options.name}`,
+      execArgs: taskCommand("python-node-bindings-watch.ts", "--project", options.projectDirectory),
+    });
     if (options.test ?? true) project.testTask.spawn(this.checkTask);
   }
 }

package/tasks/compile-workspace.ts

@@ -1,6 +1,145 @@
 #!/usr/bin/env -S bun
-import { main } from "./lib/compile-workspace.ts";
+/** Batch package type-checks and run package-owned compile lifecycles. */
+import { existsSync, readFileSync } from "node:fs";
+import { availableParallelism } from "node:os";
+import { join, resolve } from "node:path";
+import * as exec from "@dbx-tools/core/exec";
+import { log } from "@dbx-tools/shared-core";
 
-export { compilePlan, main } from "./lib/compile-workspace.ts";
+const logger = log.logger("projen:compile");
+
+interface PackageManifest {
+  readonly name?: string;
+  readonly scripts?: Record<string, string>;
+  readonly workspaces?: string[];
+}
+
+interface TaskManifest {
+  readonly tasks?: {
+    readonly "pre-compile"?: {
+      readonly steps?: Array<{ readonly exec?: string; readonly execArgs?: string[] }>;
+    };
+    readonly compile?: {
+      readonly steps?: Array<{ readonly exec?: string; readonly execArgs?: string[] }>;
+    };
+    readonly "post-compile"?: {
+      readonly steps?: Array<{ readonly exec?: string; readonly execArgs?: string[] }>;
+    };
+  };
+}
+
+export interface CompilePlan {
+  readonly typescriptConfigs: string[];
+  readonly customPackages: Array<{ readonly name: string; readonly directory: string }>;
+}
+
+function readJson<T>(path: string): T {
+  return JSON.parse(readFileSync(path, "utf8")) as T;
+}
+
+function plainTypeScriptBuild(steps: unknown): boolean {
+  if (!Array.isArray(steps) || steps.length !== 1) return false;
+  const step = steps[0] as { execArgs?: string[] };
+  if (!step || typeof step !== "object") return false;
+  return (
+    Object.keys(step).length === 1 &&
+    step.execArgs?.length === 2 &&
+    step.execArgs[0] === "tsc" &&
+    step.execArgs[1] === "--build"
+  );
+}
+
+function hasSteps(steps: unknown): boolean {
+  return Array.isArray(steps) && steps.length > 0;
+}
+
+/** Read the root's concrete workspace paths on every run, including newly added members. */
+export function compilePlan(root: string): CompilePlan {
+  const workspace = readJson<PackageManifest>(join(root, "package.json"));
+  const typescriptConfigs: string[] = [];
+  const customPackages: CompilePlan["customPackages"] = [];
+  for (const member of workspace.workspaces ?? []) {
+    const directory = resolve(root, member);
+    const manifest = readJson<PackageManifest>(join(directory, "package.json"));
+    const script = manifest.scripts?.compile;
+    if (!script) continue;
+    const taskPath = join(directory, ".projen", "tasks.json");
+    const tasks = existsSync(taskPath) ? readJson<TaskManifest>(taskPath).tasks : undefined;
+    const plainCompile = plainTypeScriptBuild(tasks?.compile?.steps);
+    if (
+      script === "projen compile" &&
+      plainCompile &&
+      !hasSteps(tasks?.["pre-compile"]?.steps) &&
+      !hasSteps(tasks?.["post-compile"]?.steps)
+    ) {
+      typescriptConfigs.push(join(directory, "tsconfig.json"));
+    } else {
+      customPackages.push({ name: manifest.name ?? member, directory });
+    }
+  }
+  return { typescriptConfigs, customPackages };
+}
+
+const emoji = /[\p{Extended_Pictographic}\uFE0F\u200D]/gu;
+
+export async function main(): Promise<void> {
+  const root = process.cwd();
+  const plan = compilePlan(root);
+  const jobs: Array<{ name: string; result: Promise<exec.ExecResult> }> = [];
+  if (plan.typescriptConfigs.length > 0) {
+    const batchCount = Math.min(4, availableParallelism(), plan.typescriptConfigs.length);
+    const batches = Array.from({ length: batchCount }, () => [] as string[]);
+    plan.typescriptConfigs.forEach((config, index) => batches[index % batchCount]!.push(config));
+    logger.info(
+      `type-checking ${plan.typescriptConfigs.length} workspaces in ${batchCount} batches`,
+    );
+    for (const [index, batch] of batches.entries()) {
+      jobs.push({
+        name: `TypeScript batch ${index + 1}`,
+        result: exec.spawn("bunx", ["tsc", "--build", ...batch], {
+          cwd: root,
+          stdin: "ignore",
+          stdout: "inherit",
+          stderr: "inherit",
+        }),
+      });
+    }
+  }
+  for (const pkg of plan.customPackages) {
+    jobs.push({
+      name: pkg.name,
+      result: compileCustomPackage(pkg),
+    });
+  }
+  const results = await Promise.allSettled(jobs.map((job) => job.result));
+  let failed = false;
+  for (const [index, result] of results.entries()) {
+    if (result.status === "rejected") {
+      logger.error(`${jobs[index]!.name} compile failed:`, result.reason);
+      failed = true;
+    } else if (result.value.exitCode !== 0) {
+      logger.error(`${jobs[index]!.name} compile exited with ${result.value.exitCode}`);
+      failed = true;
+    }
+  }
+  if (failed) process.exitCode = 1;
+}
+
+async function compileCustomPackage(pkg: {
+  readonly name: string;
+  readonly directory: string;
+}): Promise<exec.ExecResult> {
+  let result: exec.ExecResult | undefined;
+  for (const task of ["pre-compile", "compile", "post-compile"]) {
+    result = await exec.spawn("bun", ["run", task], {
+      cwd: pkg.directory,
+      stdin: "ignore",
+      stdout: (line) => logger.info(`${pkg.name}: ${line.replace(emoji, "")}`),
+      stderr: (line) => logger.error(`${pkg.name}: ${line.replace(emoji, "")}`),
+    });
+    if (result.exitCode !== 0) return result;
+  }
+  return result!;
+}
 
 if (import.meta.main) await main();

package/tasks/{lib/local-publish.ts → local-publish.ts}

@@ -4,7 +4,7 @@ import { fileURLToPath } from "node:url";
 import * as projectUtils from "@dbx-tools/core/project-utils";
 import { log, net } from "@dbx-tools/shared-core";
 import { activePythonIndexes, resolveLocalPypi } from "./python-registry.ts";
-import { runTaskCommandAsync } from "../../src/_task-command.ts";
+import { runTaskCommandAsync } from "../src/_task-command.ts";
 
 const logger = log.logger("projen:local-publish");
 
@@ -33,6 +33,11 @@ export interface LocalPublishResult {
   readonly python: boolean;
 }
 
+/** Resolve a sibling release task after task implementations are flattened. */
+export function localPublishTaskPath(task: "publish.ts" | "publish-python.ts"): string {
+  return fileURLToPath(new URL(task, import.meta.url));
+}
+
 /** Build and publish the current workspace directly to local registries. */
 export async function publishLocalRelease(
   options: LocalPublishOptions,
@@ -54,7 +59,7 @@ export async function publishLocalRelease(
   }
 
   if (localRegistry) {
-    const publishNpmScript = fileURLToPath(new URL("../publish.ts", import.meta.url));
+    const publishNpmScript = localPublishTaskPath("publish.ts");
     logger.info(`building and publishing npm workspace to ${localRegistry}`);
     publishes.push(
       runTaskCommandAsync(root, process.execPath, [
@@ -67,7 +72,7 @@ export async function publishLocalRelease(
   }
 
   if (localPypi) {
-    const publishPythonScript = fileURLToPath(new URL("../publish-python.ts", import.meta.url));
+    const publishPythonScript = localPublishTaskPath("publish-python.ts");
     logger.info(`building and publishing Python workspace to ${localPypi.publishUrl}`);
     publishes.push(
       runTaskCommandAsync(root, process.execPath, [