@dbx-tools/projen 0.9.43 → 0.9.47

package/shims/python-node/host.ts

@@ -0,0 +1,341 @@
+interface PythonBridge {
+  eval(source: string): unknown;
+}
+
+interface PythonCompletedProcess {
+  returncode: number;
+  stdout: string | null;
+  stderr: string | null;
+}
+
+interface PythonHttpResponse {
+  status_code: number;
+  headers: Record<string, string>;
+  content: Iterable<number>;
+}
+
+export interface PythonFileHost {
+  appendBytes(path: string, content: number[]): Promise<void>;
+  chmod(path: string, mode: number): Promise<void>;
+  copy(source: string, destination: string): Promise<void>;
+  exists(path: string): boolean;
+  mkdir(path: string, recursive: boolean): Promise<boolean>;
+  mkdtemp(prefix: string): Promise<string>;
+  readBytes(path: string): Promise<number[]>;
+  readDirectory(path: string): Promise<{ name: string; directory: boolean; file: boolean }[]>;
+  readTextSync(path: string): string;
+  readDirectorySync(path: string): { name: string; directory: boolean; file: boolean }[];
+  readLinkSync(path: string): string;
+  realpath(path: string): Promise<string>;
+  realpathSync(path: string): string;
+  remove(path: string, recursive: boolean, force: boolean): Promise<void>;
+  rename(source: string, destination: string): Promise<void>;
+  stat(path: string): Promise<{
+    directory: boolean;
+    file: boolean;
+    mode: number;
+    mtimeMs: number;
+    size: number;
+  }>;
+  statSync(path: string): {
+    directory: boolean;
+    file: boolean;
+    mode: number;
+    mtimeMs: number;
+    size: number;
+  };
+  touch(path: string, atimeMs: number, mtimeMs: number): Promise<void>;
+  writeBytes(path: string, content: number[], mode?: number): Promise<void>;
+}
+
+export interface PythonPathHost {
+  basename(path: string): string;
+  dirname(path: string): string;
+  fileUrlToPath(url: string): string;
+  isAbsolute(path: string): boolean;
+  join(parts: string[]): string;
+  relative(from: string, to: string): string;
+  resolve(parts: string[]): string;
+}
+
+export interface PythonProcessResult {
+  exitCode: number;
+  stdout?: string;
+  stderr?: string;
+}
+
+/** A started subprocess: await {@link wait} for its result, or {@link kill} it. */
+export interface PythonProcessHandle {
+  /** Terminate the child (SIGKILL). No-op once it has already exited. */
+  kill(): void;
+  /** Resolve with the child's result once it exits (runs off the main thread). */
+  wait(): Promise<PythonProcessResult>;
+}
+
+export interface PythonRuntimeHost {
+  crypto: {
+    digest(algorithm: string, content: number[], encoding?: "hex" | "base64"): string | number[];
+    hashes(): string[];
+    randomBytes(length: number): number[];
+  };
+  file: PythonFileHost;
+  os: {
+    homedir(): string;
+    tmpdir(): string;
+  };
+  path: PythonPathHost;
+  http: {
+    fetch(
+      url: string,
+      method?: string,
+      headers?: Record<string, string>,
+      body?: string,
+      timeoutMs?: number,
+    ): Promise<{ status: number; headers: Record<string, string>; body: number[] }>;
+  };
+  process: {
+    start(
+      command: string,
+      args: string[],
+      environment: Record<string, string> | undefined,
+      input: string | undefined,
+    ): PythonProcessHandle;
+  };
+}
+
+type PythonFunction = (...args: any[]) => any;
+
+const python = (globalThis as typeof globalThis & { python?: PythonBridge }).python;
+if (!python) throw new Error("PythonMonkey globalThis.python is unavailable");
+
+function evaluate<T extends PythonFunction>(source: string): T {
+  return python!.eval(source) as T;
+}
+
+const toThread = evaluate<PythonFunction>("__import__('asyncio').to_thread");
+const osPath = {
+  basename: evaluate<PythonFunction>("__import__('os').path.basename"),
+  dirname: evaluate<PythonFunction>("__import__('os').path.dirname"),
+  exists: evaluate<PythonFunction>("__import__('os').path.exists"),
+  isAbsolute: evaluate<PythonFunction>("__import__('os').path.isabs"),
+  join: evaluate<PythonFunction>("lambda parts: __import__('os').path.join(*list(parts))"),
+  realpath: evaluate<PythonFunction>("__import__('os').path.realpath"),
+  relative: evaluate<PythonFunction>("__import__('os').path.relpath"),
+  resolve: evaluate<PythonFunction>(
+    "lambda parts: __import__('os').path.abspath(__import__('os').path.join(*list(parts))) if list(parts) else __import__('os').getcwd()",
+  ),
+};
+const readBytes = evaluate<PythonFunction>("lambda path: list(open(path, 'rb').read())");
+const readText = evaluate<PythonFunction>("lambda path: open(path, encoding='utf-8').read()");
+const writeBytes = evaluate<PythonFunction>(
+  "lambda path, content: open(path, 'wb').write(bytes(int(value) for value in content))",
+);
+const appendBytes = evaluate<PythonFunction>(
+  "lambda path, content: open(path, 'ab').write(bytes(int(value) for value in content))",
+);
+const mkdir = evaluate<PythonFunction>(
+  "lambda path, recursive: __import__('os').makedirs(path, exist_ok=recursive) if recursive else __import__('os').mkdir(path)",
+);
+const listDirectory = evaluate<PythonFunction>(
+  "lambda path: [{'name': entry.name, 'directory': entry.is_dir(), 'file': entry.is_file()} for entry in __import__('pathlib').Path(path).iterdir()]",
+);
+const readLink = evaluate<PythonFunction>("__import__('os').readlink");
+const statPath = evaluate<PythonFunction>(
+  "lambda path: {'directory': __import__('os').path.isdir(path), 'file': __import__('os').path.isfile(path), 'mode': __import__('os').stat(path).st_mode, 'mtimeMs': __import__('os').stat(path).st_mtime * 1000, 'size': __import__('os').stat(path).st_size}",
+);
+const popen = evaluate<PythonFunction>(
+  "lambda command, args, environment, input_text: __import__('subprocess').Popen([command, *list(args)], env=dict(environment) if environment is not None else None, stdin=(__import__('subprocess').PIPE if input_text is not None else None), stdout=__import__('subprocess').PIPE, stderr=__import__('subprocess').PIPE, text=True)",
+);
+const communicate = evaluate<PythonFunction>(
+  "lambda proc, input_text: (lambda out: {'returncode': proc.returncode, 'stdout': out[0], 'stderr': out[1]})(proc.communicate(input=input_text))",
+);
+const killProcess = evaluate<PythonFunction>("lambda proc: proc.kill()");
+const requestHttp = evaluate<PythonFunction>(
+  "lambda url, method, headers, body, timeout: __import__('httpx').request(method or 'GET', url, headers=dict(headers) if headers is not None else None, content=body, timeout=(timeout / 1000) if timeout is not None else 30, follow_redirects=True)",
+);
+const chmod = evaluate<PythonFunction>(
+  "lambda path, mode: __import__('os').chmod(path, int(mode))",
+);
+const isDir = evaluate<PythonFunction>("__import__('os').path.isdir");
+const rmtree = evaluate<PythonFunction>("__import__('shutil').rmtree");
+const rmdir = evaluate<PythonFunction>("__import__('os').rmdir");
+const unlink = evaluate<PythonFunction>("__import__('os').unlink");
+
+const host: PythonRuntimeHost = {
+  crypto: {
+    digest: evaluate<PythonFunction>(
+      "lambda algorithm, content, encoding: (lambda value: value.hexdigest() if encoding == 'hex' else __import__('base64').b64encode(value.digest()).decode('ascii') if encoding == 'base64' else list(value.digest()))(__import__('hashlib').new(algorithm, bytes(int(item) for item in content)))",
+    ),
+    hashes: evaluate<PythonFunction>("lambda: sorted(__import__('hashlib').algorithms_available)"),
+    randomBytes: evaluate<PythonFunction>(
+      "lambda length: list(__import__('os').urandom(int(length)))",
+    ),
+  },
+  file: {
+    async appendBytes(path, content) {
+      await toThread(appendBytes, path, content);
+    },
+    async chmod(path, mode) {
+      await toThread(chmod, path, mode);
+    },
+    async copy(source, destination) {
+      await toThread(
+        evaluate<PythonFunction>("__import__('shutil').copyfile"),
+        source,
+        destination,
+      );
+    },
+    exists: (path) => Boolean(osPath.exists(path)),
+    async mkdir(path, recursive) {
+      if (await toThread(osPath.exists, path)) return false;
+      await toThread(mkdir, path, recursive);
+      return true;
+    },
+    async mkdtemp(prefix) {
+      return toThread(
+        evaluate<PythonFunction>("__import__('tempfile').mkdtemp"),
+        undefined,
+        prefix,
+      );
+    },
+    async readBytes(path) {
+      return toThread(readBytes, path);
+    },
+    async readDirectory(path) {
+      return toThread(listDirectory, path);
+    },
+    readDirectorySync: (path) => listDirectory(path),
+    readLinkSync: (path) => String(readLink(path)),
+    readTextSync: (path) => String(readText(path)),
+    async realpath(path) {
+      return String(await toThread(osPath.realpath, path));
+    },
+    realpathSync: (path) => String(osPath.realpath(path)),
+    async remove(path, recursive, force) {
+      if (!(await toThread(osPath.exists, path))) {
+        if (!force) throw Object.assign(new Error(`ENOENT: ${path}`), { code: "ENOENT" });
+        return;
+      }
+      const isDirectory = await toThread(isDir, path);
+      const operation = isDirectory ? (recursive ? rmtree : rmdir) : unlink;
+      await toThread(operation, path);
+    },
+    async rename(source, destination) {
+      await toThread(evaluate<PythonFunction>("__import__('os').replace"), source, destination);
+    },
+    async stat(path) {
+      return toThread(statPath, path);
+    },
+    statSync: (path) => statPath(path),
+    async touch(path, atimeMs, mtimeMs) {
+      await toThread(
+        evaluate<PythonFunction>(
+          "lambda path, atime, mtime: __import__('os').utime(path, (atime / 1000, mtime / 1000))",
+        ),
+        path,
+        atimeMs,
+        mtimeMs,
+      );
+    },
+    async writeBytes(path, content, mode) {
+      await toThread(writeBytes, path, content);
+      if (mode !== undefined) await toThread(chmod, path, mode);
+    },
+  },
+  http: {
+    async fetch(url, method, headers, body, timeoutMs) {
+      const response = (await toThread(
+        requestHttp,
+        url,
+        method,
+        headers,
+        body,
+        timeoutMs,
+      )) as PythonHttpResponse;
+      return {
+        status: response.status_code,
+        headers: Object.fromEntries(Object.entries(response.headers)),
+        body: Array.from(response.content),
+      };
+    },
+  },
+  os: {
+    homedir: evaluate<PythonFunction>("lambda: __import__('pathlib').Path.home().as_posix()"),
+    tmpdir: evaluate<PythonFunction>("__import__('tempfile').gettempdir"),
+  },
+  path: {
+    basename: (path) => String(osPath.basename(path)),
+    dirname: (path) => String(osPath.dirname(path)),
+    fileUrlToPath: evaluate<PythonFunction>(
+      "lambda url: __import__('urllib.parse', fromlist=['urlparse']).unquote(__import__('urllib.parse', fromlist=['urlparse']).urlparse(url).path)",
+    ),
+    isAbsolute: (path) => Boolean(osPath.isAbsolute(path)),
+    join: (parts) => String(osPath.join(parts)),
+    relative: (from, to) => String(osPath.relative(to, from)),
+    resolve: (parts) => String(osPath.resolve(parts)),
+  },
+  process: {
+    start(command, args, environment, input) {
+      // Popen (not subprocess.run) so the child is killable: an aborted caller
+      // can terminate it instead of leaking a background process. There is no
+      // imposed timeout - the child runs to completion (an interactive login may
+      // legitimately take minutes); the caller owns cancellation.
+      const proc = popen(command, args, environment, input);
+      let killed = false;
+      return {
+        kill() {
+          if (killed) return;
+          killed = true;
+          try {
+            killProcess(proc);
+          } catch {
+            // The child already exited; there is nothing to signal.
+          }
+        },
+        async wait() {
+          // communicate() blocks until exit, so run it off the main thread.
+          const result = (await toThread(communicate, proc, input)) as PythonCompletedProcess;
+          return {
+            exitCode: result.returncode,
+            ...(result.stdout?.trim() ? { stdout: result.stdout.trim() } : {}),
+            ...(result.stderr?.trim() ? { stderr: result.stderr.trim() } : {}),
+          };
+        },
+      };
+    },
+  },
+};
+
+export function installPythonGlobals(): void {
+  const globals = globalThis as typeof globalThis & {
+    global?: typeof globalThis;
+    self?: typeof globalThis;
+    window?: typeof globalThis;
+    process?: Record<string, unknown>;
+  };
+  globals.global = globalThis;
+  globals.self = globalThis;
+  globals.window = globalThis;
+  globals.process ??= {
+    arch: String(evaluate<PythonFunction>("__import__('platform').machine")())
+      .replace("aarch64", "arm64")
+      .replace("x86_64", "x64"),
+    argv: [],
+    browser: true,
+    cwd: evaluate<PythonFunction>("__import__('os').getcwd"),
+    env: evaluate<PythonFunction>("lambda: dict(__import__('os').environ)")(),
+    nextTick: (callback: (...args: unknown[]) => void, ...args: unknown[]) =>
+      Promise.resolve().then(() => callback(...args)),
+    platform: { Darwin: "darwin", Linux: "linux", Windows: "win32" }[
+      String(evaluate<PythonFunction>("__import__('platform').system")())
+    ],
+    version: "v22.0.0",
+    versions: {},
+  };
+}
+
+installPythonGlobals();
+
+export function pythonHost(): PythonRuntimeHost {
+  return host;
+}

package/shims/python-node/os.ts

@@ -0,0 +1,20 @@
+import { pythonHost } from "./host.ts";
+
+export const constants = {
+  errno: {
+    EEXIST: 17,
+    ENOTDIR: 20,
+    EISDIR: 21,
+    EINVAL: 22,
+  },
+} as const;
+
+export function homedir(): string {
+  return pythonHost().os.homedir();
+}
+
+export function tmpdir(): string {
+  return pythonHost().os.tmpdir();
+}
+
+export default { constants, homedir, tmpdir };

package/shims/python-node/process.ts

@@ -0,0 +1,13 @@
+import { installPythonGlobals } from "./host.ts";
+
+installPythonGlobals();
+
+export default process;
+export const arch = process.arch;
+export const argv = process.argv;
+export const cwd = process.cwd;
+export const env = process.env;
+export const nextTick = process.nextTick;
+export const platform = process.platform;
+export const version = process.version;
+export const versions = process.versions;

package/shims/python-node/readline.ts

@@ -0,0 +1,37 @@
+/**
+ * Async line reader used by subprocess output capture in PythonMonkey bundles.
+ *
+ * @module
+ */
+
+interface ReadLineOptions {
+  readonly input: AsyncIterable<unknown>;
+}
+
+class PythonReadLine implements AsyncIterable<string> {
+  private closed = false;
+
+  constructor(private readonly input: AsyncIterable<unknown>) {}
+
+  async *[Symbol.asyncIterator](): AsyncIterator<string> {
+    let buffered = "";
+    for await (const chunk of this.input) {
+      if (this.closed) break;
+      buffered += String(chunk);
+      const lines = buffered.split(/\r?\n/);
+      buffered = lines.pop() ?? "";
+      for (const line of lines) yield line;
+    }
+    if (!this.closed && buffered) yield buffered.replace(/\r$/, "");
+  }
+
+  close(): void {
+    this.closed = true;
+  }
+}
+
+export function createInterface(options: ReadLineOptions): PythonReadLine {
+  return new PythonReadLine(options.input);
+}
+
+export default { createInterface };

package/shims/python-node/tsconfig.json

@@ -0,0 +1,24 @@
+// ~~ Generated by projen. To modify, edit .projenrc.js and run "bunx projen".
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "lib": [
+      "ES2022",
+      "DOM"
+    ],
+    "types": [
+      "node"
+    ],
+    "noEmit": true,
+    "skipLibCheck": true,
+    "strict": false
+  },
+  "include": [
+    "*.ts"
+  ],
+  "exclude": [
+    "node_modules"
+  ]
+}

package/src/_lazy-require.ts

@@ -1,30 +1,14 @@
 /**
- * Loading heavy generator tools out of the CONSUMING workspace.
+ * Load heavy generator tools only when code generation has work to do.
  *
- * The generators (`openapi.ts`, `codegen.ts`) each drive a large toolchain that
- * only matters when that generator actually has work to do, so none of them are
- * imported at module scope and several are not engine dependencies at all. Node
- * resolves a bare specifier from the engine's own location by walking up, and under
- * pnpm that walk passes through `node_modules/.pnpm/node_modules` - the hidden
- * directory holding every package installed ANYWHERE in the workspace. So a tool
- * declared by some member package (`tsoa`, via the `server` tag) or by the root
- * (`typescript`) resolves fine from here without the engine shipping its own copy.
- *
- * The failure mode is a consumer whose workspace never installed the tool, which
- * surfaces as a bare MODULE_NOT_FOUND naming a package they never asked for. The
- * helper below turns that into the install command instead.
+ * Code generation drives a large toolchain only when a package declares inputs,
+ * so those tools are not imported at module scope. This helper identifies a
+ * broken engine installation instead of exposing a bare MODULE_NOT_FOUND from a
+ * lazy require.
  */
 
-/** How to obtain each lazily-loaded tool, for the error message. */
-const INSTALL_HINTS: Record<string, string> = {
-  tsoa: "pnpm add tsoa",
-  typescript: "pnpm add -D typescript",
-  "ts-to-zod": "pnpm add -D ts-to-zod",
-};
-
 /**
- * `require` a generator tool from the workspace, reporting a missing one as an
- * actionable error rather than a raw MODULE_NOT_FOUND.
+ * `require` an engine-owned generator dependency only when codegen needs it.
  *
  * @param require - a `createRequire(import.meta.url)` bound to the calling module,
  *   so the resolution walk starts at the engine and reaches the consumer's install.
@@ -36,10 +20,8 @@ export function lazyRequire<T>(require: NodeJS.Require, name: string, reason: st
     return require(name) as T;
   } catch (cause) {
     if ((cause as NodeJS.ErrnoException)?.code !== "MODULE_NOT_FOUND") throw cause;
-    const hint = INSTALL_HINTS[name] ?? `pnpm add -D ${name}`;
     throw new Error(
-      `${reason} needs the \`${name}\` package, which is not installed in this workspace. ` +
-        `Add it to the package that needs it (\`${hint}\`) and re-run.`,
+      `${reason} requires the \`${name}\` dependency shipped by @dbx-tools/projen; reinstall workspace dependencies and re-run`,
       { cause },
     );
   }

package/src/bun-app.ts

@@ -11,8 +11,8 @@
  *   imports `index.html` and serves the SPA with React hot module reloading.
  * - {@link BunBuildFile} - `build.ts`: a Databricks-safe `Bun.build()` production
  *   bundle of `index.html` (the `bun-plugin-tailwind` plugin must run through the
- *   JS API, NOT the `bun build` CLI, which does not load it), plus Vite-compatible
- *   staging of an optional `public/` directory.
+ *   JS API, NOT the `bun build` CLI, which does not load it), plus copying an
+ *   optional `public/` directory.
  *
  * The dev and build scripts support unmanaged `bun-dev.override.ts` and
  * `bun-build.override.ts` modules beside them. Each script imports the override's

package/src/bun-workflow.ts

@@ -1,12 +1,4 @@
 /** Shared Bun setup for generated workflows. */
 
+/** Bun version used by package manifests, generated workflows, and CLI services. */
 export const BUN_VERSION = "1.3.14";
-
-/** Install the pinned Bun version. */
-export function bunSetupStep(): Readonly<Record<string, unknown>> {
-  return {
-    name: "Setup Bun",
-    uses: "oven-sh/setup-bun@v2",
-    with: { "bun-version": "${{ env.BUN_VERSION }}" },
-  };
-}

package/src/codegen.ts

@@ -1,20 +1,20 @@
 /**
  * Codegen generator (ts-to-zod based).
  *
- * Scans every package whose `package.json` declares a `codegen`
- * field and turns the listed upstream `.d.ts` inputs into read-only `src/`
+ * Scans every package whose `dbxToolsConfig.codegenInputs` declares upstream
+ * `.d.ts` inputs and turns them into read-only `src/`
  * modules of zod schemas plus matching inferred TypeScript types. Each input
  * emits one `src/<name>.ts` (schemas + `export type X = z.infer<typeof
  * xSchema>` lines); the barrel generator then namespaces it into the package's
  * root barrel like any other `src/` module (`sdkModel.dashboards.genieMessageSchema`).
  *
  * The single source of truth for "which packages get generated content, from
- * which inputs" is each consumer's own `package.json`:
+ * which inputs" is the typed config serialized into each package manifest:
  *
  *   {
  *     "name": "@dbx-tools/shared-sdk-model",
- *     "codegen": {
- *       "inputs": [
+ *     "dbxToolsConfig": {
+ *       "codegenInputs": [
  *         "node_modules/@databricks/sdk-experimental/dist/apis/dashboards/model.d.ts"
  *       ]
  *     }
@@ -46,18 +46,23 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { createRequire } from "node:module";
 import { basename, dirname, join, resolve } from "node:path";
-import { log, object } from "@dbx-tools/shared-core";
+import { log } from "@dbx-tools/shared-core";
 import type * as ts from "typescript";
 import { lazyRequire } from "./_lazy-require.ts";
 import { header, makeReadonly, makeWritable } from "./generated.ts";
-import { readPackageManifest, recordedPackages, resolveRepoRoot } from "./packages.ts";
+import {
+  readDbxToolsConfig,
+  readPackageManifest,
+  recordedPackages,
+  resolveRepoRoot,
+} from "./packages.ts";
 
 const logger = log.logger("projen:codegen");
 
 /** Do-not-edit banner stamped on every generated codegen module. */
 const HEADER = header({
   tool: "projen synth (codegen: ts-to-zod)",
-  source: "the upstream .d.ts declared in package.json codegen.inputs",
+  source: "the upstream .d.ts declared in dbxToolsConfig.codegenInputs",
 });
 
 interface CodegenInput {
@@ -77,11 +82,9 @@ function sourceLabel(source: string): string {
   return source.startsWith(nodeModulesPrefix) ? source.slice(nodeModulesPrefix.length) : source;
 }
 
-/** Read a package's `package.json` `codegen.inputs`, or `undefined` if absent. */
+/** Read a package's serialized `dbxToolsConfig.codegenInputs`, or `undefined` if absent. */
 function codegenInputs(dir: string): string[] | undefined {
-  const codegen = readPackageManifest(dir)?.codegen;
-  if (!object.isRecord(codegen)) return undefined;
-  const inputs = codegen.inputs;
+  const inputs = readDbxToolsConfig(dir)?.codegenInputs;
   return Array.isArray(inputs) && inputs.length > 0 ? (inputs as string[]) : undefined;
 }
 
@@ -100,7 +103,7 @@ function parseInputArg(value: string): CodegenInput {
 }
 
 /**
- * The `src/` modules a set of `codegen.inputs` specs will emit, as package-relative
+ * The `src/` modules a set of codegen input specs will emit, as package-relative
  * posix paths - WITHOUT resolving or reading any input.
  *
  * This exists so callers that only need to know WHICH files are generated (the
@@ -344,9 +347,9 @@ function generatePackage(
 }
 
 /**
- * Regenerate the `generated/` tree for every package declaring a
- * `codegen` field. Returns the package dirs it wrote so the caller can rebuild
- * their barrels. `ts-to-zod` + `typescript` are lazy-loaded.
+ * Regenerate source modules for every package declaring `dbxToolsConfig.codegenInputs`.
+ * Returns the package dirs it wrote so the caller can rebuild their barrels.
+ * `ts-to-zod` and `typescript` are lazy-loaded.
  */
 export function generateCodegen(
   projectRoot: string = resolveRepoRoot(),
@@ -365,7 +368,7 @@ export function generateCodegen(
   // than printing a line about work that was never asked for. `debug` keeps it
   // reachable when someone is actually chasing why codegen produced nothing.
   if (targets.length === 0) {
-    logger.debug("no packages declare a `codegen` field");
+    logger.debug("no packages declare dbxToolsConfig.codegenInputs");
     return [];
   }