@dbx-tools/projen 0.9.50 → 0.9.54

package/tasks/python-sync.ts

@@ -20,8 +20,13 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "node:path";
+import * as exec from "@dbx-tools/core/exec";
 import { withFileLock } from "@dbx-tools/core/file-lock";
 import { parse } from "smol-toml";
+import { PYTHON_GENERATED_PACKAGE, PYTHON_SYNC_PACKAGE } from "../src/generated.ts";
+
+/** AST rewriter that points absolute imports of synchronized modules at their generated package. */
+const LOCALIZER = join(import.meta.dirname, "python-localize-imports.py");
 
 interface SyncConfig {
   readonly name?: string;
@@ -29,6 +34,7 @@ interface SyncConfig {
   readonly include?: readonly string[];
   readonly exclude?: readonly string[];
   readonly replace?: Readonly<Record<string, string>>;
+  readonly localize_imports?: boolean | readonly string[];
 }
 
 interface SyncManifest {
@@ -63,21 +69,33 @@ export async function main(args: readonly string[] = process.argv.slice(2)): Pro
   const moduleName = build?.["module-name"];
   if (!moduleName) throw new Error(`${project}/pyproject.toml has no uv module-name`);
   const moduleRoot = resolve(project, build?.["module-root"] ?? "src");
-  const generatedRoot = join(moduleRoot, ...moduleName.split("."), "_generated", "sync");
-  for (const config of configs) await synchronize(config, generatedRoot, { check, force });
+  // The import package and the output directory derive from the same segments.
+  const generatedPackage = [moduleName, PYTHON_GENERATED_PACKAGE, PYTHON_SYNC_PACKAGE].join(".");
+  const generatedRoot = join(moduleRoot, ...generatedPackage.split("."));
+  for (const config of configs) {
+    await synchronize(config, generatedRoot, generatedPackage, { check, force });
+  }
 }
 
 async function synchronize(
   config: SyncConfig,
   generatedRoot: string,
+  generatedPackage: string,
   options: { readonly check: boolean; readonly force: boolean },
 ): Promise<void> {
   const source = parseSource(config.source);
   const name = config.name ?? defaultName(source);
   const target = join(generatedRoot, name);
+  const localize = localizedModules(config, name);
+  const importPackage = `${generatedPackage}.${name}`;
   const optionsHash = hash({
     exclude: [...(config.exclude ?? [])].sort(),
     include: [...(config.include ?? ["**/*"])].sort(),
+    localize: localize && {
+      localizer: hash(readFileSync(LOCALIZER, "utf8")),
+      modules: [...localize].sort(),
+      package: importPackage,
+    },
     name,
     replace: config.replace ?? {},
     source: config.source,
@@ -110,7 +128,13 @@ async function synchronize(
       return;
     }
     if (options.check) throw new Error(`Generated Python sync is stale: ${target}`);
-    replaceTarget(config, source, target, { optionsHash, resolvedCommit });
+    replaceTarget(
+      config,
+      source,
+      target,
+      localize && { modules: localize, package: importPackage },
+      { optionsHash, resolvedCommit },
+    );
   });
 }
 
@@ -118,6 +142,7 @@ function replaceTarget(
   config: SyncConfig,
   source: ParsedSource,
   target: string,
+  localize: { readonly modules: readonly string[]; readonly package: string } | undefined,
   manifest: Omit<SyncManifest, "source">,
 ): void {
   const temporary = mkdtempSync(join(tmpdir(), "dbx-tools-python-sync-"));
@@ -168,6 +193,7 @@ function replaceTarget(
         `Python sync replacements did not match ${source.repository}: ${missing.join(", ")}`,
       );
     }
+    if (localize) localizeImports(staging, source, localize.package, localize.modules);
     const license = join(checkout, "LICENSE");
     if (existsSync(license)) cpSync(license, join(staging, "LICENSE.upstream"));
     writeFileSync(
@@ -184,6 +210,62 @@ function replaceTarget(
   }
 }
 
+/**
+ * Upstream modules whose imports are rewritten: empty to detect every synchronized
+ * module, or `undefined` when localization is off. A sync whose name is not a Python
+ * identifier is not importable as a package, so it only localizes when asked to.
+ */
+function localizedModules(config: SyncConfig, name: string): readonly string[] | undefined {
+  const option = config.localize_imports;
+  if (option === false) return undefined;
+  if (Array.isArray(option) && option.length === 0) {
+    throw new Error(`Python sync ${name} lists no localize_imports modules; use false instead`);
+  }
+  if (!PYTHON_IDENTIFIER.test(name)) {
+    if (option === undefined) return undefined;
+    throw new Error(`Python sync name ${name} must be a Python identifier to localize imports`);
+  }
+  return Array.isArray(option) ? option : [];
+}
+
+const PYTHON_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+function localizeImports(
+  staging: string,
+  source: ParsedSource,
+  importPackage: string,
+  modules: readonly string[],
+): void {
+  // Upstream code may import its modules under the subdirectory's dotted path.
+  const segments = (source.subdirectory ?? "").split("/").filter(Boolean);
+  const sourcePackage = segments.every((segment) => PYTHON_IDENTIFIER.test(segment))
+    ? segments.join(".")
+    : "";
+  // Captured stderr carries the localizer's reason into the thrown error.
+  const { stdout } = exec.spawnSync(
+    "uv",
+    [
+      "run",
+      "--no-project",
+      "--quiet",
+      "python",
+      LOCALIZER,
+      staging,
+      importPackage,
+      sourcePackage,
+      ...modules,
+    ],
+    { cwd: staging, stdout: "capture", stderr: "capture", stdin: "ignore", check: true },
+  );
+  const counts = JSON.parse(stdout ?? "{}") as Record<string, number>;
+  const missing = modules.filter((module) => !counts[module]);
+  if (missing.length > 0) {
+    throw new Error(
+      `Python sync localize_imports did not match ${source.repository}: ${missing.join(", ")}`,
+    );
+  }
+}
+
 function parseSource(value: string): ParsedSource {
   const match = /^(?:(.*?)\s+@\s+)?git\+(.+?)@([^#]+)(?:#subdirectory=(.+))?$/.exec(value);
   if (!match) throw new Error(`Invalid pip Git source: ${value}`);

package/tasks/sync.ts

@@ -1,9 +1,9 @@
 #!/usr/bin/env -S bun
-/** Synthesize once or supervise the focused Projen and barrel watch loops. */
+/** Synthesize once or supervise Projen, barrels, and registered focused watchers. */
 import { fileURLToPath } from "node:url";
 import { log } from "@dbx-tools/shared-core";
 import concurrently from "concurrently";
-import { repoRoot } from "../src/packages.ts";
+import { repoRoot, syncWatchTasks } from "../src/packages.ts";
 import { runSynth } from "../src/scaffold.ts";
 import { withWorkspaceMutationLock } from "../src/workspace-lock.ts";
 
@@ -36,6 +36,15 @@ function watcher(script: string, name: string, prefixColor: string, ...args: str
   };
 }
 
+/** One Projen task registered by a component as a focused sync watcher. */
+function taskWatcher(task: string) {
+  return {
+    command: ["bun", "run", JSON.stringify(task)].join(" "),
+    name: task.replace(/:watch$/, ""),
+    prefixColor: "yellow",
+  };
+}
+
 export async function main(args: string[] = process.argv.slice(2)): Promise<void> {
   if (!args.includes("--watch")) {
     // One-shot: full synth (+install + barrels via the post-synth component). This is
@@ -48,7 +57,9 @@ export async function main(args: string[] = process.argv.slice(2)): Promise<void
     // watchers under `concurrently`. The projenrc watcher is the intelligent stand-in
     // for stock `projen --watch` - it re-synths (+install) ONLY when `.projenrc.ts` or
     // a configured `syncResynthPaths` entry changes, while the barrel watcher keeps
-    // generated package exports fresh on source edits with no full synth.
+    // generated package exports fresh on source edits with no full synth. Components
+    // can register other repository-wide watchers, such as Python Node bindings and
+    // the shared Python Node runtime, through the generated dbxToolsConfig record.
     //
     // VS Code auto-runs this on folder open, so from here down nothing is allowed to be
     // fatal: errors are logged and retried, and only a stop signal ends the task.
@@ -68,6 +79,7 @@ export async function main(args: string[] = process.argv.slice(2)): Promise<void
     const watchers = [
       watcher("projenrc.ts", "projenrc", "magenta"),
       watcher("barrels.ts", "barrels", "cyan", "--watch"),
+      ...syncWatchTasks().map(taskWatcher),
     ];
     const { result } = concurrently(watchers, {
       prefix: "name",

package/shims/python-node/bootstrap.ts

@@ -1,129 +0,0 @@
-import { Buffer } from "node:buffer";
-
-import { PythonHeaders } from "./headers.ts";
-import { installPythonGlobals, pythonHost } from "./host.ts";
-
-installPythonGlobals();
-
-(globalThis as typeof globalThis & { Buffer?: typeof Buffer }).Buffer = Buffer;
-
-const globals = globalThis as typeof globalThis & {
-  AbortController?: typeof AbortController;
-  Headers?: typeof Headers;
-  Request?: typeof Request;
-  Response?: typeof Response;
-  structuredClone?: typeof structuredClone;
-  TextDecoder?: typeof TextDecoder;
-  TextEncoder?: typeof TextEncoder;
-  fetch?: typeof fetch;
-};
-globals.AbortController ??= class AbortController {
-  readonly signal: AbortSignal;
-  readonly #listeners = new Set<() => void>();
-
-  constructor() {
-    this.signal = {
-      aborted: false,
-      addEventListener: (name: string, listener: EventListenerOrEventListenerObject) => {
-        if (name === "abort") {
-          this.#listeners.add(
-            typeof listener === "function"
-              ? () => listener(new Event("abort"))
-              : () => listener.handleEvent(new Event("abort")),
-          );
-        }
-      },
-      removeEventListener: () => {},
-    } as AbortSignal;
-  }
-
-  abort(): void {
-    if (this.signal.aborted) return;
-    Object.defineProperty(this.signal, "aborted", { value: true });
-    for (const listener of this.#listeners) listener();
-    this.#listeners.clear();
-  }
-} as typeof AbortController;
-
-globals.Headers ??= PythonHeaders as unknown as typeof Headers;
-
-class PythonResponse {
-  readonly headers: Headers;
-  readonly ok: boolean;
-  readonly status: number;
-  readonly #body: Uint8Array;
-
-  constructor(body: Uint8Array, init: ResponseInit = {}) {
-    this.#body = body;
-    this.status = init.status ?? 200;
-    this.ok = this.status >= 200 && this.status < 300;
-    this.headers = new Headers(init.headers);
-  }
-
-  async arrayBuffer(): Promise<ArrayBuffer> {
-    return this.#body.slice().buffer;
-  }
-
-  async text(): Promise<string> {
-    return new TextDecoder().decode(this.#body);
-  }
-
-  async json(): Promise<unknown> {
-    return JSON.parse(await this.text());
-  }
-}
-
-globals.Response ??= PythonResponse as unknown as typeof Response;
-globals.structuredClone ??= ((value: unknown) => cloneStructured(value)) as typeof structuredClone;
-globals.TextEncoder ??= class TextEncoder {
-  encode(value: string): Uint8Array {
-    const encoded = unescape(encodeURIComponent(String(value)));
-    return Uint8Array.from(encoded, (character) => character.charCodeAt(0));
-  }
-} as typeof TextEncoder;
-globals.TextDecoder ??= class TextDecoder {
-  decode(value?: ArrayBuffer | ArrayBufferView): string {
-    const bytes =
-      value instanceof ArrayBuffer
-        ? new Uint8Array(value)
-        : value
-          ? new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
-          : new Uint8Array();
-    const encoded = Array.from(bytes, (byte) => String.fromCharCode(byte)).join("");
-    return decodeURIComponent(escape(encoded));
-  }
-} as typeof TextDecoder;
-globals.fetch ??= (async (input: string | URL | Request, init: RequestInit = {}) => {
-  const request = typeof input === "object" && "url" in input ? input : undefined;
-  const headers = init.headers ?? request?.headers;
-  const response = await pythonHost().http.fetch(
-    request?.url ?? String(input),
-    init.method ?? request?.method,
-    headers ? Object.fromEntries(new Headers(headers).entries()) : undefined,
-    typeof init.body === "string" ? init.body : undefined,
-  );
-  return new Response(Uint8Array.from(response.body), {
-    status: response.status,
-    headers: response.headers,
-  });
-}) as typeof fetch;
-
-function cloneStructured(value: unknown, seen = new Map<object, unknown>()): unknown {
-  if (value === null || typeof value !== "object") return value;
-  const cached = seen.get(value);
-  if (cached !== undefined) return cached;
-  if (value instanceof Date) return new Date(value.getTime());
-  if (Array.isArray(value)) {
-    const clone: unknown[] = [];
-    seen.set(value, clone);
-    for (const item of value) clone.push(cloneStructured(item, seen));
-    return clone;
-  }
-  if (Object.getPrototypeOf(value) === Object.prototype) {
-    const clone: Record<string, unknown> = {};
-    seen.set(value, clone);
-    for (const [key, item] of Object.entries(value)) clone[key] = cloneStructured(item, seen);
-    return clone;
-  }
-  throw new TypeError(`PythonMonkey structuredClone does not support ${value.constructor.name}`);
-}

package/shims/python-node/child-process.ts

@@ -1,96 +0,0 @@
-import { EventEmitter } from "node:events";
-import { Readable, Writable } from "node:stream";
-
-import { pythonHost, type PythonProcessHandle } from "./host.ts";
-
-interface SpawnOptions {
-  env?: Record<string, string>;
-  signal?: AbortSignal;
-}
-
-class PythonChildProcess extends EventEmitter {
-  readonly pid = undefined;
-  readonly stdin = new Writable({ write: (_chunk, _encoding, callback) => callback() });
-  readonly stdout = new Readable({ read() {} });
-  readonly stderr = new Readable({ read() {} });
-  private active = true;
-  private readonly handle: PythonProcessHandle;
-
-  constructor(command: string, args: string[], options: SpawnOptions) {
-    super();
-    this.handle = pythonHost().process.start(command, args, options.env, undefined);
-    options.signal?.addEventListener("abort", () => {
-      if (!this.active) return;
-      this.active = false;
-      // Actually terminate the Python child so an aborted call leaks nothing.
-      this.handle.kill();
-      this.stdout.push(null);
-      this.stderr.push(null);
-      this.emit(
-        "error",
-        Object.assign(new Error("Process execution was aborted"), { code: "ABORT_ERR" }),
-      );
-    });
-    void this.handle.wait().then(
-      (result) => {
-        if (!this.active) return;
-        this.active = false;
-        if (result.stdout) this.stdout.push(result.stdout);
-        if (result.stderr) this.stderr.push(result.stderr);
-        this.stdout.push(null);
-        this.stderr.push(null);
-        this.emit("close", result.exitCode);
-        this.emit("exit", result.exitCode);
-      },
-      (error) => {
-        if (!this.active) return;
-        this.active = false;
-        this.stdout.push(null);
-        this.stderr.push(null);
-        this.emit("error", error);
-      },
-    );
-  }
-
-  kill(): boolean {
-    this.handle.kill();
-    return true;
-  }
-}
-
-export function spawn(command: string, args: string[] = [], options: SpawnOptions = {}) {
-  return new PythonChildProcess(command, args, options);
-}
-
-export function execFile(
-  command: string,
-  args: string[],
-  _options: object,
-  callback: (error: Error | null, result?: { stdout: string; stderr: string }) => void,
-): void {
-  void pythonHost()
-    .process.start(command, args, undefined, undefined)
-    .wait()
-    .then(
-      (result) => {
-        if (result.exitCode === 0) {
-          callback(null, { stdout: result.stdout ?? "", stderr: result.stderr ?? "" });
-        } else {
-          callback(
-            Object.assign(new Error(result.stderr || `Command exited ${result.exitCode}`), {
-              code: result.exitCode,
-            }),
-          );
-        }
-      },
-      (error) => callback(error instanceof Error ? error : new Error(String(error))),
-    );
-}
-
-export function spawnSync(): never {
-  throw Object.assign(new Error("Synchronous subprocesses are unavailable in PythonMonkey"), {
-    code: "ENOSYS",
-  });
-}
-
-export default { execFile, spawn, spawnSync };

package/shims/python-node/crypto.ts

@@ -1,60 +0,0 @@
-import { pythonHost } from "./host.ts";
-
-class PythonHash {
-  private readonly content: number[] = [];
-
-  constructor(private readonly algorithm: string) {}
-
-  update(value: string | ArrayBuffer | ArrayBufferView): this {
-    const bytes =
-      typeof value === "string"
-        ? new TextEncoder().encode(value)
-        : value instanceof ArrayBuffer
-          ? new Uint8Array(value)
-          : new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
-    this.content.push(...bytes);
-    return this;
-  }
-
-  digest(encoding?: "hex" | "base64"): string | Uint8Array {
-    const digest = pythonHost().crypto.digest(this.algorithm, this.content, encoding);
-    if (typeof digest === "string") return digest;
-    const bytes = Uint8Array.from(digest) as Uint8Array & {
-      readBigInt64BE(offset: number): bigint;
-    };
-    bytes.readBigInt64BE = (offset) => {
-      let value = 0n;
-      for (const byte of bytes.slice(offset, offset + 8)) {
-        value = (value << 8n) | BigInt(byte);
-      }
-      return BigInt.asIntN(64, value);
-    };
-    return bytes;
-  }
-}
-
-export function createHash(algorithm: string): PythonHash {
-  const normalized = algorithm.toLowerCase();
-  if (!getHashes().includes(normalized)) {
-    throw new Error(`Unsupported hash algorithm: ${algorithm}`);
-  }
-  return new PythonHash(normalized);
-}
-
-export function getHashes(): string[] {
-  return pythonHost().crypto.hashes();
-}
-
-export function randomBytes(length: number): Uint8Array {
-  return Uint8Array.from(pythonHost().crypto.randomBytes(length));
-}
-
-export function randomUUID(): string {
-  const bytes = randomBytes(16);
-  bytes[6] = (bytes[6]! & 0x0f) | 0x40;
-  bytes[8] = (bytes[8]! & 0x3f) | 0x80;
-  const hex = Array.from(bytes, (value) => value.toString(16).padStart(2, "0")).join("");
-  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
-}
-
-export default { createHash, getHashes, randomBytes, randomUUID };

package/shims/python-node/fs-promises.ts

@@ -1,179 +0,0 @@
-import { Buffer } from "node:buffer";
-
-import { pythonHost } from "./host.ts";
-
-function nodeError(error: unknown, code: string, path: string): Error {
-  return Object.assign(error instanceof Error ? error : new Error(String(error)), { code, path });
-}
-
-function translatePythonError(cause: unknown, path: string): never {
-  const message = String(cause);
-  if (/FileNotFoundError|Errno 2/.test(message)) throw nodeError(cause, "ENOENT", path);
-  if (/FileExistsError|Errno 17/.test(message)) throw nodeError(cause, "EEXIST", path);
-  if (/PermissionError|Errno 13/.test(message)) throw nodeError(cause, "EACCES", path);
-  throw cause;
-}
-
-export async function access(path: string): Promise<void> {
-  try {
-    await pythonHost().file.stat(String(path));
-  } catch (cause) {
-    translatePythonError(cause, path);
-  }
-}
-
-export async function chmod(path: string, mode: number): Promise<void> {
-  await pythonHost().file.chmod(String(path), mode);
-}
-
-export async function appendFile(
-  path: string,
-  content: string | ArrayBuffer | ArrayBufferView,
-): Promise<void> {
-  const bytes =
-    typeof content === "string"
-      ? new TextEncoder().encode(content)
-      : content instanceof ArrayBuffer
-        ? new Uint8Array(content)
-        : new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
-  await pythonHost().file.appendBytes(String(path), Array.from(bytes));
-}
-
-export async function copyFile(source: string, destination: string): Promise<void> {
-  await pythonHost().file.copy(String(source), String(destination));
-}
-
-export async function mkdir(
-  path: string,
-  options: { recursive?: boolean } = {},
-): Promise<string | undefined> {
-  let created: boolean;
-  try {
-    created = await pythonHost().file.mkdir(String(path), options.recursive === true);
-  } catch (cause) {
-    if (/FileExistsError|Errno 17/.test(String(cause))) {
-      if (options.recursive) return undefined;
-      throw nodeError(cause, "EEXIST", path);
-    }
-    throw cause;
-  }
-  if (!created && !options.recursive) throw nodeError(new Error(`EEXIST: ${path}`), "EEXIST", path);
-  return created ? String(path) : undefined;
-}
-
-export async function mkdtemp(prefix: string): Promise<string> {
-  return pythonHost().file.mkdtemp(String(prefix));
-}
-
-export async function readdir(
-  path: string,
-  options: { withFileTypes?: boolean } = {},
-): Promise<unknown[]> {
-  const entries = await pythonHost().file.readDirectory(String(path));
-  if (!options.withFileTypes) return entries.map((entry) => entry.name);
-  return entries.map((entry) => ({
-    name: entry.name,
-    isDirectory: () => entry.directory,
-    isFile: () => entry.file,
-    isSymbolicLink: () => false,
-  }));
-}
-
-export async function readFile(
-  path: string,
-  options?: string | { encoding?: string | null },
-): Promise<string | Uint8Array> {
-  try {
-    const bytes = Uint8Array.from(await pythonHost().file.readBytes(String(path)));
-    const encoding = typeof options === "string" ? options : options?.encoding;
-    return encoding ? new TextDecoder(encoding).decode(bytes) : Buffer.from(bytes);
-  } catch (cause) {
-    translatePythonError(cause, path);
-  }
-}
-
-export async function realpath(path: string): Promise<string> {
-  try {
-    return await pythonHost().file.realpath(String(path));
-  } catch (cause) {
-    translatePythonError(cause, path);
-  }
-}
-
-export async function rename(source: string, destination: string): Promise<void> {
-  await pythonHost().file.rename(String(source), String(destination));
-}
-
-export async function rm(
-  path: string,
-  options: { recursive?: boolean; force?: boolean } = {},
-): Promise<void> {
-  await pythonHost().file.remove(String(path), options.recursive === true, options.force === true);
-}
-
-export async function unlink(path: string): Promise<void> {
-  // No synchronous existence precheck: let the async remove run and treat a
-  // missing file as success, so this never blocks on a sync Python call.
-  try {
-    await pythonHost().file.remove(String(path), false, true);
-  } catch (cause) {
-    if (/FileNotFoundError|Errno 2|ENOENT/.test(String(cause))) return;
-    translatePythonError(cause, path);
-  }
-}
-
-export async function stat(path: string): Promise<unknown> {
-  let value: Awaited<ReturnType<ReturnType<typeof pythonHost>["file"]["stat"]>>;
-  try {
-    value = await pythonHost().file.stat(String(path));
-  } catch (cause) {
-    translatePythonError(cause, path);
-  }
-  return {
-    mode: value.mode,
-    mtime: new Date(value.mtimeMs),
-    mtimeMs: value.mtimeMs,
-    size: value.size,
-    isDirectory: () => value.directory,
-    isFile: () => value.file,
-    isSymbolicLink: () => false,
-  };
-}
-
-export async function writeFile(
-  path: string,
-  content: string | ArrayBuffer | ArrayBufferView,
-  options: { mode?: number } = {},
-): Promise<void> {
-  const bytes =
-    typeof content === "string"
-      ? new TextEncoder().encode(content)
-      : content instanceof ArrayBuffer
-        ? new Uint8Array(content)
-        : new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
-  await pythonHost().file.writeBytes(String(path), Array.from(bytes), options.mode);
-}
-
-export async function open(): Promise<never> {
-  throw Object.assign(new Error("File descriptors are unavailable in PythonMonkey"), {
-    code: "ENOSYS",
-  });
-}
-
-export default {
-  access,
-  appendFile,
-  chmod,
-  copyFile,
-  mkdir,
-  mkdtemp,
-  open,
-  readFile,
-  readdir,
-  realpath,
-  rename,
-  rm,
-  stat,
-  unlink,
-  writeFile,
-};