@dbx-tools/projen 0.9.93 → 0.9.99

package/src/python-node-bindings.ts

@@ -1,14 +1,58 @@
 /** Shared pyproject-backed configuration for PythonMonkey Node bindings. */
-import { readFileSync, realpathSync, rmSync } from "node:fs";
-import { dirname, join, relative, resolve, sep } from "node:path";
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, readdirSync, realpathSync, rmSync } from "node:fs";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { find } from "@dbx-tools/path";
 import { object, stringUtils } from "@dbx-tools/shared-core";
 import { parse } from "smol-toml";
 import ts from "typescript";
 import { runTaskCommand } from "./_task-command.ts";
+import { makeReadonly } from "./generated.ts";
+import { publicFunctionExports, publicNamespaceExports } from "./module-exports.ts";
 import { resolveRepoRoot, workspaceDependencyDirectories } from "./packages.ts";
 
+const FINGERPRINT_VERSION = 5;
+const INPUT_FINGERPRINT_LABEL = "dbx-tools binding inputs sha256";
+const CONTENT_FINGERPRINT_LABEL = "dbx-tools binding content sha256";
+const PYTHON_KEYWORDS = new Set([
+  "and",
+  "as",
+  "assert",
+  "async",
+  "await",
+  "break",
+  "class",
+  "continue",
+  "def",
+  "del",
+  "elif",
+  "else",
+  "except",
+  "False",
+  "finally",
+  "for",
+  "from",
+  "global",
+  "if",
+  "import",
+  "in",
+  "is",
+  "lambda",
+  "None",
+  "nonlocal",
+  "not",
+  "or",
+  "pass",
+  "raise",
+  "return",
+  "True",
+  "try",
+  "while",
+  "with",
+  "yield",
+]);
+
 /** Resolved function replacement consumed while generating one PythonMonkey bundle. */
 export interface ResolvedPythonNodeFunctionOverride {
   readonly handlerExport: string;
@@ -38,6 +82,8 @@ export interface ResolvedPythonNodeBindings {
 export interface GeneratePythonNodeBindingsOptions {
   /** Verify generated output without changing it. */
   readonly check?: boolean;
+  /** Regenerate outputs without consulting fingerprints. */
+  readonly force?: boolean;
 }
 
 /** Generate or verify every pyproject-configured Node binding in a repository. */
@@ -46,10 +92,14 @@ export function generatePythonNodeBindings(
   options: GeneratePythonNodeBindingsOptions = {},
 ): void {
   const task = resolve(dirname(fileURLToPath(import.meta.url)), "../tasks/python-node-bindings.ts");
-  const configuredProjects = pythonNodeBindingProjects(projectRoot);
+  const force = options.force ?? process.env.CI === "true";
+  const configuredProjects = pythonNodeBindingProjects(projectRoot).map((project) => ({
+    project,
+    configs: resolvePythonNodeBindings(projectRoot, project).map(resolvePythonNodeBindingModules),
+  }));
   const directories = new Set<string>();
-  for (const project of configuredProjects) {
-    for (const config of resolvePythonNodeBindings(projectRoot, project)) {
+  for (const { configs } of configuredProjects) {
+    for (const config of configs) {
       directories.add(relative(projectRoot, join(config.moduleDirectory, "_generated", "node")));
     }
   }
@@ -64,7 +114,12 @@ export function generatePythonNodeBindings(
   for (const directory of stale) {
     rmSync(resolve(projectRoot, directory), { recursive: true, force: true });
   }
-  for (const project of configuredProjects) {
+  for (const { configs, project } of configuredProjects) {
+    const inputFingerprint = pythonNodeBindingInputFingerprint(projectRoot, configs);
+    if (!force && pythonNodeBindingOutputsCurrent(configs, inputFingerprint)) {
+      for (const output of pythonNodeBindingOutputs(configs)) makeReadonly(output);
+      continue;
+    }
     runTaskCommand(projectRoot, "bun", [
       task,
       "--root",
@@ -72,10 +127,194 @@ export function generatePythonNodeBindings(
       "--project",
       project,
       ...(options.check ? ["--check"] : []),
+      ...(force ? ["--force"] : []),
     ]);
   }
 }
 
+/** Resolve automatically discovered bindable namespaces for one configured package. */
+export function resolvePythonNodeBindingModules(
+  config: ResolvedPythonNodeBindings,
+): ResolvedPythonNodeBindings {
+  if (config.modules.length > 0) return config;
+  const packageEntrypoint = Bun.resolveSync(config.entrypoint, config.projectDirectory);
+  const modules = publicNamespaceExports(packageEntrypoint).filter(
+    (module) =>
+      publicFunctionExports(resolvePythonNodeBindingModule(packageEntrypoint, module)).length > 0,
+  );
+  return { ...config, modules };
+}
+
+/** Map a JavaScript export or namespace to its generated Python identifier. */
+export function pythonNodeBindingFunctionName(javascriptName: string): string {
+  const name = stringUtils.toIdentifierWithOptions({ delimiter: "_" }, javascriptName);
+  if (!isPythonNodeBindingIdentifier(name)) {
+    throw new Error(
+      `JavaScript export ${javascriptName} does not map to a valid Python function name`,
+    );
+  }
+  return name;
+}
+
+/** Whether a value is a legal non-keyword Python identifier. */
+export function isPythonNodeBindingIdentifier(value: string): boolean {
+  return /^[_A-Za-z]\w*$/.test(value) && !PYTHON_KEYWORDS.has(value);
+}
+
+/** Content fingerprint for every source and toolchain input affecting generated bindings. */
+export function pythonNodeBindingInputFingerprint(
+  root: string,
+  configs: readonly ResolvedPythonNodeBindings[],
+): string {
+  const canonicalRoot = canonicalPath(root);
+  const fingerprintPath = (path: string): string => relative(canonicalRoot, canonicalPath(path));
+  const sourceFile = canonicalPath(fileURLToPath(import.meta.url));
+  const sourceDirectory = dirname(sourceFile);
+  const files = new Set<string>([
+    sourceFile,
+    canonicalPath(resolve(sourceDirectory, "../tasks/python-node-bindings.ts")),
+    canonicalPath(join(sourceDirectory, "generated.ts")),
+    canonicalPath(join(sourceDirectory, "module-exports.ts")),
+    canonicalPath(join(sourceDirectory, "packages.ts")),
+    canonicalPath(join(sourceDirectory, "python-node-runtime.ts")),
+  ]);
+  const versionFile = join(root, "VERSION");
+  if (existsSync(versionFile)) files.add(canonicalPath(versionFile));
+  for (const config of configs) {
+    for (const input of config.watchInputs) files.add(canonicalPath(resolve(root, input)));
+    for (const { handlerFile } of config.functionOverrides) files.add(canonicalPath(handlerFile));
+    const packageEntrypoint = canonicalPath(
+      Bun.resolveSync(config.entrypoint, config.projectDirectory),
+    );
+    files.add(packageEntrypoint);
+    const packageManifest = nearestPackageManifest(packageEntrypoint);
+    if (packageManifest) files.add(canonicalPath(packageManifest));
+    for (const module of config.modules) {
+      files.add(canonicalPath(resolvePythonNodeBindingModule(packageEntrypoint, module)));
+    }
+  }
+
+  const hash = createHash("sha256");
+  hash.update(
+    JSON.stringify({
+      configs: configs.map((config) => ({
+        bindingDirectory: fingerprintPath(config.bindingDirectory),
+        bindingName: config.bindingName,
+        entrypoint: config.entrypoint,
+        functionOverrides: config.functionOverrides.map((override) => ({
+          handlerExport: override.handlerExport,
+          handlerFile: fingerprintPath(override.handlerFile),
+          targetExport: override.targetExport,
+          targetModule: override.targetModule,
+        })),
+        moduleDirectory: fingerprintPath(config.moduleDirectory),
+        modules: config.modules,
+        package: config.package,
+        project: config.project,
+        runtimeOutput: fingerprintPath(config.runtimeOutput),
+      })),
+      format: FINGERPRINT_VERSION,
+    }),
+  );
+  for (const { file, path } of [...files]
+    .map((file) => ({ file, path: fingerprintPath(file) }))
+    .sort((left, right) => left.path.localeCompare(right.path))) {
+    hash.update("\0file\0");
+    hash.update(path);
+    hash.update("\0");
+    hash.update(existsSync(file) ? readFileSync(file) : "<missing>");
+  }
+  return hash.digest("hex");
+}
+
+/** Complete generated output set for one Python project's Node bindings. */
+export function pythonNodeBindingOutputs(configs: readonly ResolvedPythonNodeBindings[]): string[] {
+  const nodeDirectory = pythonNodeBindingDirectory(configs);
+  return [
+    join(nodeDirectory, "_runtime.js"),
+    join(nodeDirectory, "_runtime.py"),
+    ...configs.flatMap((config) =>
+      config.modules.length > 0
+        ? config.modules.map((module) =>
+            join(config.bindingDirectory, `${pythonNodeBindingFunctionName(module)}.py`),
+          )
+        : [join(config.bindingDirectory, "index.py")],
+    ),
+  ].sort();
+}
+
+/** Whether all expected outputs exactly match their input and body fingerprints. */
+export function pythonNodeBindingOutputsCurrent(
+  configs: readonly ResolvedPythonNodeBindings[],
+  inputFingerprint: string,
+): boolean {
+  const generatedPackage = join(configs[0]!.moduleDirectory, "_generated", "__init__.py");
+  if (existsSync(generatedPackage)) return false;
+  const expected = pythonNodeBindingOutputs(configs);
+  const actual = filesBelow(pythonNodeBindingDirectory(configs)).sort();
+  if (actual.length !== expected.length || actual.some((file, index) => file !== expected[index])) {
+    return false;
+  }
+  return expected.every((output) => pythonNodeBindingFingerprintCurrent(output, inputFingerprint));
+}
+
+/** Prefix generated output with its input fingerprint and a self-validating body digest. */
+export function fingerprintPythonNodeBindingOutput(
+  contents: string,
+  inputFingerprint: string,
+  output: string,
+): string {
+  const prefix = output.endsWith(".py") ? "#" : "//";
+  const contentFingerprint = createHash("sha256").update(contents).digest("hex");
+  return [
+    `${prefix} ${INPUT_FINGERPRINT_LABEL}: ${inputFingerprint}`,
+    `${prefix} ${CONTENT_FINGERPRINT_LABEL}: ${contentFingerprint}`,
+    contents,
+  ].join("\n");
+}
+
+function pythonNodeBindingDirectory(configs: readonly ResolvedPythonNodeBindings[]): string {
+  const moduleDirectory = configs[0]?.moduleDirectory;
+  if (!moduleDirectory) throw new Error("Expected at least one Node binding configuration");
+  return join(moduleDirectory, "_generated", "node");
+}
+
+function pythonNodeBindingFingerprintCurrent(output: string, inputFingerprint: string): boolean {
+  try {
+    const [inputLine, contentLine, ...bodyLines] = readFileSync(output, "utf8").split("\n");
+    const prefix = output.endsWith(".py") ? "#" : "//";
+    if (inputLine !== `${prefix} ${INPUT_FINGERPRINT_LABEL}: ${inputFingerprint}`) return false;
+    const escapedPrefix = prefix === "//" ? "\\/\\/" : "#";
+    const expectedContent = contentLine?.match(
+      new RegExp(`^${escapedPrefix} ${CONTENT_FINGERPRINT_LABEL}: ([a-f0-9]{64})$`),
+    )?.[1];
+    if (!expectedContent) return false;
+    return createHash("sha256").update(bodyLines.join("\n")).digest("hex") === expectedContent;
+  } catch {
+    return false;
+  }
+}
+
+function filesBelow(directory: string): string[] {
+  if (!existsSync(directory)) return [];
+  return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => {
+    if (entry.name === "__pycache__" || entry.name.endsWith(".pyc")) return [];
+    const path = join(directory, entry.name);
+    return entry.isDirectory() ? filesBelow(path) : [path];
+  });
+}
+
+function nearestPackageManifest(path: string): string | undefined {
+  let directory = dirname(path);
+  while (true) {
+    const manifest = join(directory, "package.json");
+    if (existsSync(manifest)) return manifest;
+    const parent = dirname(directory);
+    if (parent === directory) return undefined;
+    directory = parent;
+  }
+}
+
 /** Resolve every configured binding in a repository. */
 export function resolveAllPythonNodeBindings(
   projectRoot: string = resolveRepoRoot(),
@@ -102,16 +341,28 @@ export function pythonNodeBindingProjects(projectRoot: string): string[] {
 
 function existingBindingDirectories(projectRoot: string): string[] {
   const directories = new Set<string>();
-  for (const file of find.findFiles("**/_generated/node/**/*", {
-    cwd: projectRoot,
-    ignore: () => false,
-  })) {
-    const segments = file.split("/");
-    const generated = segments.lastIndexOf("_generated");
-    if (generated >= 0 && segments[generated + 1] === "node") {
-      directories.add(segments.slice(0, generated + 2).join("/"));
+  const ignored = new Set([
+    ".git",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".venv",
+    "__pycache__",
+    "dist",
+    "node_modules",
+  ]);
+  const visit = (directory: string): void => {
+    for (const entry of readdirSync(directory, { withFileTypes: true })) {
+      if (!entry.isDirectory() || ignored.has(entry.name)) continue;
+      const path = join(directory, entry.name);
+      if (entry.name === "node" && basename(directory) === "_generated") {
+        directories.add(relative(projectRoot, path));
+        continue;
+      }
+      visit(path);
     }
-  }
+  };
+  visit(projectRoot);
   return [...directories].sort();
 }
 
@@ -192,16 +443,23 @@ type BindingSourceConfig = Omit<ResolvedPythonNodeBindings, "watchInputs">;
 
 /** Transitive workspace source files reachable from configured binding modules. */
 function bindingSourceInputs(root: string, config: BindingSourceConfig): string[] {
-  const allowed = config.workspaceDirectories.map(canonicalPath);
   const packageEntrypoint = Bun.resolveSync(config.entrypoint, config.projectDirectory);
   const entrypoints =
     config.modules.length > 0
       ? config.modules.map((module) => resolvePythonNodeBindingModule(packageEntrypoint, module))
       : [packageEntrypoint];
-  const pending = [
-    ...entrypoints,
-    ...config.functionOverrides.map(({ handlerFile }) => handlerFile),
-  ];
+  return transitiveSourceFiles(
+    [...entrypoints, ...config.functionOverrides.map(({ handlerFile }) => handlerFile)],
+    config.workspaceDirectories,
+  ).map((path) => relative(root, path));
+}
+
+function transitiveSourceFiles(
+  entrypoints: readonly string[],
+  allowedDirectories: readonly string[],
+): string[] {
+  const allowed = allowedDirectories.map(canonicalPath);
+  const pending = [...entrypoints];
   const visited = new Set<string>();
   while (pending.length > 0) {
     const candidate = canonicalPath(pending.pop()!);
@@ -227,7 +485,7 @@ function bindingSourceInputs(root: string, config: BindingSourceConfig): string[
       }
     }
   }
-  return [...visited].sort().map((path) => relative(root, path));
+  return [...visited].sort();
 }
 
 /** Resolve one configured namespace from a package entrypoint barrel. */

package/src/release.ts

@@ -51,8 +51,6 @@ export interface DBXToolsReleaseOptions {
   readonly validationTasks?: readonly string[];
   /** Repository prerequisites installed before release validation and artifact builds. */
   readonly setupSteps?: readonly JobStep[];
-  /** Repository-specific synthesis commands run before the root Projen synthesis check. */
-  readonly synthesisCommands?: readonly string[];
 }
 
 /** Locate the unified workflow when release generation is enabled. */
@@ -204,14 +202,6 @@ function releaseBuildJob(
           // ============================================================================
         ),
       },
-      {
-        name: "Verify generated sources",
-        run: [
-          ...(options.synthesisCommands ?? []),
-          "bunx projen",
-          "git diff --ignore-space-at-eol --exit-code",
-        ].join("\n"),
-      },
       { name: "Verify workspace versions", run: "bun run version:check" },
       ...(options.setupSteps ?? []),
       ...releaseBuildSteps(

package/src/tags.ts

@@ -163,15 +163,11 @@ export const PACKAGE_TAG_MIXINS = {
     addPackageFiles(p, "bin");
   }),
   server: create(projectPredicate.hasTag("server"), (p) => {
-    // A Node/Express service. `dev` and `start` run `src/server.ts` directly
-    // through Bun, while package compilation remains native TypeScript.
+    // A Node/Express service. The root's generic `dev:watch` task owns watched
+    // execution because this tag cannot assume an entrypoint or command shape.
     p.addDeps("express@catalog:");
     p.addDevDeps("@types/node@catalog:", "@types/express@catalog:");
     applyCompilerOptions(p, NODE_COMPILER_OPTIONS);
-    // bun runs the server `.ts` directly (native TS, no tsx). `--watch` restarts
-    // on change - the tsx-watch replacement.
-    (p.tasks.tryFind("dev") ?? p.addTask("dev")).reset("bun --watch src/server.ts");
-    (p.tasks.tryFind("start") ?? p.addTask("start")).reset("bun src/server.ts");
   }),
   node: create(projectPredicate.hasTag("node"), (p) => {
     p.addDevDeps("@types/node@catalog:");

package/src/tsconfig.ts

@@ -55,7 +55,7 @@ export class DBXToolsRootTsconfig extends Component {
       extends: javascript.TypescriptConfigExtends.fromTypescriptConfigs([this.base]),
       compilerOptions: {
         lib: ["ESNext"],
-        types: ["node"],
+        types: ["node", "bun"],
       },
       include: [".projenrc.ts"],
       exclude: ["node_modules", "**/dist", "**/node_modules"],