@dbx-tools/projen 0.6.57 → 0.6.59

package/README.md

@@ -124,6 +124,35 @@ package is attempted even if one fails; the failures are re-thrown together as a
 `AggregateError` naming each package, rather than the first one abandoning the
 rest of the sweep.
 
+### Generate Node And Python From Rosetta Sources
+
+The experimental `rs-packages` convention keeps TypeScript as the canonical
+Node implementation while colocating an explicit Python implementation in the
+same file:
+
+```ts
+export function capitalize(value: string): string {
+  return value ? value[0]!.toUpperCase() + value.slice(1) : value;
+}
+
+/* @rs-python
+def capitalize(value: str) -> str:
+    return value[:1].upper() + value[1:] if value else value
+@rs-end */
+```
+
+By default, `rs-packages/shared/core/src/string.ts` generates:
+
+- `packages/shared/core/src/string.ts`
+- `python-packages/shared-core/src/shared_core/string.py`
+
+Use `// @rs-node <path>` or `// @rs-python-path <path>` only when the inferred
+paths are unsuitable. Every Rosetta source must live below a package's `src/`
+folder and include at least one Python block; generation fails rather than
+silently emitting an empty Python module. `DBXToolsNodeProject` enables the
+`rs-packages` root by default, accepts `rsPackageRoots` for custom roots, and
+accepts `rsPackageRoots: false` to disable the experiment.
+
 ## Generate OpenAPI Clients
 
 ```ts
@@ -188,6 +217,7 @@ file contract as the CLI.
 - `pnpmWorkspace` - generated pnpm workspace file and catalog model.
 - `barrels` / `moduleExports` - public entrypoint generation.
 - `codegen` - `.d.ts` to zod schema generation.
+- `rsPackages` - annotated TypeScript source to Node/Python generation.
 - `openapi` - tsoa/OpenAPI package generation.
 - `bunApp` / `tsconfig` / `vscode` - generated support files/components.
 - `generated` / `clean` / `watch` / `scaffold` - read-only file stamping,
@@ -196,8 +226,9 @@ file contract as the CLI.
 - `engineRoot` - engine package root resolution for bootstrapped repos.
 
 The engine registers its commands as projen tasks on the workspace root, so run
-them with `bun run <task>` - `sync` (add `--watch`), `barrels`, `openapi`, and
-`clean`. [`@dbx-tools/cli`](../packages/cli/dbx-tools) is only needed to
+them with `bun run <task>` - `sync` (add `--watch`), `barrels`, `openapi`,
+`rs-packages` (also supports `--watch`), and `clean`.
+[`@dbx-tools/cli`](../packages/cli/dbx-tools) is only needed to
 bootstrap a folder that has no `.projenrc.ts` or toolchain yet.
 
 ## Run Tasks From The ROOT
@@ -206,14 +237,15 @@ Every repo-wide task lives on the root, and the root's `compile` / `test`
 delegate with `bun run --filter '*'` rather than emitting a step per member - so
 a new package is covered without a re-synth. Work from the root:
 
-| Task              | What it does                                       |
-| ----------------- | -------------------------------------------------- |
-| `bun run build`   | `compile` + `test` + `package` across every member |
-| `bun run compile` | `tsc --build` in each member, in parallel          |
-| `bun run test`    | `eslint` once, then each member's tests            |
-| `bun run sync`    | re-synth (`--watch` to keep synthing)              |
-| `bun run barrels` | regenerate the read-only `index.ts` barrels        |
-| `bun run bump`    | version, tag, and publish                          |
+| Task                  | What it does                                       |
+| --------------------- | -------------------------------------------------- |
+| `bun run build`       | `compile` + `test` + `package` across every member |
+| `bun run compile`     | `tsc --build` in each member, in parallel          |
+| `bun run test`        | `eslint` once, then each member's tests            |
+| `bun run sync`        | re-synth (`--watch` to keep synthing)              |
+| `bun run barrels`     | regenerate the read-only `index.ts` barrels        |
+| `bun run rs-packages` | regenerate Node/Python Rosetta outputs             |
+| `bun run bump`        | version, tag, and publish                          |
 
 Members intentionally keep only the tasks that something OTHER than a human
 invokes, so there is no second place to run the same thing:

package/index.ts

@@ -18,11 +18,13 @@ export * as project from "./src/project.ts";
 export * as projectPredicate from "./src/project-predicate.ts";
 export * as publish from "./src/publish.ts";
 export * as release from "./src/release.ts";
+export * as rsPackages from "./src/rs-packages.ts";
 export * as scaffold from "./src/scaffold.ts";
 export * as tags from "./src/tags.ts";
 export * as tsconfig from "./src/tsconfig.ts";
 export * as vscode from "./src/vscode.ts";
 export * as watch from "./src/watch.ts";
+export { BUN_DEV_OVERRIDE, BUN_BUILD_OVERRIDE, BUN_APP_OVERRIDES, RootBunfigFile, BunfigFile, BunDevServerFile, BunBuildFile } from "./src/bun-app.ts";
 export { DBXToolsConfig } from "./src/dbx-tools-config.ts";
 export type { DBXToolsConfigOptions } from "./src/dbx-tools-config.ts";
 export { resolvePkgRoot } from "./src/engine-root.ts";
@@ -38,9 +40,10 @@ export type { DBXToolsProject, DBXToolsProjectOptions, DBXToolsTypeScriptProject
 export { COMPILED_DIR, COMPILED_COMPILER_OPTIONS } from "./src/publish.ts";
 export { DBXToolsRelease } from "./src/release.ts";
 export type { StandaloneRelease, DBXToolsReleaseOptions } from "./src/release.ts";
+export { DEFAULT_RS_PACKAGE_ROOTS, RsPackages } from "./src/rs-packages.ts";
+export type { RsPackageOutput, GenerateRsPackagesOptions } from "./src/rs-packages.ts";
 export { AGNOSTIC_COMPILER_OPTIONS, PACKAGE_TAG_MIXINS } from "./src/tags.ts";
 export type { PackageTag } from "./src/tags.ts";
 export { DBXToolsRootTsconfig } from "./src/tsconfig.ts";
-export { BUN_DEV_OVERRIDE, BUN_BUILD_OVERRIDE, BUN_APP_OVERRIDES, BunfigFile, BunDevServerFile, BunBuildFile } from "./src/bun-app.ts";
 export { DBXToolsVsCode } from "./src/vscode.ts";
 export type { IgnoreGroupOptions } from "./src/watch.ts";

package/package.json

@@ -26,9 +26,9 @@
   },
   "dependencies": {
     "@clack/prompts": "^1.7.0",
-    "@dbx-tools/core": "0.6.57",
-    "@dbx-tools/path": "0.6.57",
-    "@dbx-tools/shared-core": "0.6.57",
+    "@dbx-tools/core": "0.6.59",
+    "@dbx-tools/path": "0.6.59",
+    "@dbx-tools/shared-core": "0.6.59",
     "commander": "^15.0.0",
     "concurrently": "^10.0.3",
     "constructs": "^10.6.0",
@@ -47,7 +47,7 @@
   },
   "main": "index.ts",
   "license": "Apache-2.0",
-  "version": "0.6.57",
+  "version": "0.6.59",
   "types": "index.ts",
   "type": "module",
   "exports": {

package/src/project.ts

@@ -30,6 +30,7 @@ import {
   RootBunfigFile,
 } from "./bun-app.ts";
 import { DBXToolsVsCode } from "./vscode.ts";
+import { DEFAULT_RS_PACKAGE_ROOTS, RsPackages } from "./rs-packages.ts";
 import {
   DEFAULT_PACKAGE_ROOTS,
   type DiscoveredPackage,
@@ -519,6 +520,11 @@ export interface DBXToolsProjectOptions
    * is a member of the single bun workspace, so the root links it from source.
    */
   readonly extraWorkspaceMembers?: readonly string[];
+  /**
+   * Experimental TypeScript-to-Node/Python source roots. Defaults to
+   * `["rs-packages"]`; set `false` to disable the generator.
+   */
+  readonly rsPackageRoots?: readonly string[] | false;
 }
 
 /** Options for {@link DBXToolsTypeScriptProject} (a package, or a compiling root). */
@@ -841,6 +847,7 @@ function registerRootTasks(project: javascript.NodeProject): void {
   applyTasks(project, {
     barrels: { exec: taskScript(project, "barrels.ts") },
     openapi: { exec: taskScript(project, "openapi.ts") },
+    "rs-packages": { exec: taskScript(project, "rs-packages.ts"), receiveArgs: true },
     clean: { exec: taskScript(project, "clean.ts"), receiveArgs: true },
     // `receiveArgs` forwards `--watch`, so `bun run sync -- --watch` syncs once
     // then starts the single node-path watcher loop.
@@ -925,6 +932,13 @@ function initProject(
   project.vsCode = new DBXToolsVsCode(project);
 
   registerRootTasks(project);
+  project.dbxToolsConfig.rsPackageRoots =
+    options.rsPackageRoots === false
+      ? false
+      : [...(options.rsPackageRoots ?? DEFAULT_RS_PACKAGE_ROOTS)];
+  if (options.rsPackageRoots !== false) {
+    new RsPackages(project, options.rsPackageRoots ?? DEFAULT_RS_PACKAGE_ROOTS);
+  }
   if (options.prettier || project.prettier) {
     const formatTask = project.tasks.tryFind("format") ?? project.addTask("format");
     formatTask.prependExec("prettier . --write", { receiveArgs: true });

package/src/rs-packages.ts

@@ -0,0 +1,237 @@
+/**
+ * Experimental Rosetta-source generation.
+ *
+ * A TypeScript file under `rs-packages/<package>/src/` is the canonical Node
+ * implementation. Projen copies its TypeScript surface to the matching
+ * `packages/<package>/src/` path and extracts one or more `@rs-python` blocks
+ * into a conventional Python package path.
+ */
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { dirname, extname, join, relative, resolve } from "node:path";
+import { json, object } from "@dbx-tools/shared-core";
+import { Component, javascript } from "projen";
+import { header, makeReadonly, makeWritable } from "./generated.ts";
+import { toPosix } from "./packages.ts";
+
+export const DEFAULT_RS_PACKAGE_ROOTS = ["rs-packages"] as const;
+
+const SOURCE_EXTENSIONS = new Set([".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"]);
+const PYTHON_BLOCK = /\/\*\s*@rs-python\s*\n([\s\S]*?)\n\s*@rs-end\s*\*\//g;
+const NODE_OVERRIDE = /^\s*\/\/\s*@rs-node\s+(.+?)\s*$/m;
+const PYTHON_OVERRIDE = /^\s*\/\/\s*@rs-python-path\s+(.+?)\s*$/m;
+const RS_METADATA = /^\s*\/\/\s*@rs-(?:node|python-path)\s+.+?\s*$\n?/gm;
+const NODE_MARKER = "// GENERATED by projen synth (rs-packages)";
+const PYTHON_MARKER = "# GENERATED by projen synth (rs-packages)";
+
+export interface RsPackageOutput {
+  readonly source: string;
+  readonly node: string;
+  readonly python: string;
+}
+
+export interface GenerateRsPackagesOptions {
+  readonly projectRoot?: string;
+  readonly roots?: readonly string[];
+}
+
+function sourceFiles(root: string): string[] {
+  if (!existsSync(root)) return [];
+  const files: string[] = [];
+  const visit = (dir: string): void => {
+    for (const entry of readdirSync(dir)) {
+      const path = join(dir, entry);
+      if (statSync(path).isDirectory()) visit(path);
+      else if (SOURCE_EXTENSIONS.has(extname(entry))) files.push(path);
+    }
+  };
+  visit(root);
+  return files.sort();
+}
+
+function dedent(value: string): string {
+  const lines = value.replace(/^\n+|\n+$/g, "").split("\n");
+  const indents = lines
+    .filter((line) => line.trim())
+    .map((line) => line.match(/^\s*/)?.[0].length ?? 0);
+  const width = indents.length ? Math.min(...indents) : 0;
+  return lines.map((line) => line.slice(width)).join("\n");
+}
+
+function defaultOutputPaths(
+  projectRoot: string,
+  root: string,
+  source: string,
+): Pick<RsPackageOutput, "node" | "python"> {
+  const sourceRelative = toPosix(relative(resolve(projectRoot, root), source));
+  const segments = sourceRelative.split("/");
+  const srcIndex = segments.indexOf("src");
+  if (srcIndex < 1) {
+    throw new Error(
+      `Rosetta source must live under <package>/src/: ${toPosix(relative(projectRoot, source))}`,
+    );
+  }
+
+  const packageSegments = segments.slice(0, srcIndex);
+  const moduleSegments = segments.slice(srcIndex + 1);
+  const moduleFile = moduleSegments.pop();
+  if (!moduleFile) throw new Error(`Rosetta source has no module name: ${source}`);
+
+  const pythonDistribution = packageSegments.join("-");
+  const pythonImport = packageSegments.join("_").replace(/-/g, "_");
+  const pythonModule = `${moduleFile.slice(0, -extname(moduleFile).length).replace(/-/g, "_")}.py`;
+
+  return {
+    node: resolve(
+      projectRoot,
+      "packages",
+      ...packageSegments,
+      "src",
+      ...moduleSegments,
+      moduleFile,
+    ),
+    python: resolve(
+      projectRoot,
+      "python-packages",
+      pythonDistribution,
+      "src",
+      pythonImport,
+      ...moduleSegments,
+      pythonModule,
+    ),
+  };
+}
+
+function resolveOutput(
+  projectRoot: string,
+  override: RegExpMatchArray | null,
+  fallback: string,
+): string {
+  const value = override?.[1]?.trim();
+  return value ? resolve(projectRoot, value) : fallback;
+}
+
+function assertWritableOutput(projectRoot: string, path: string, marker: string): void {
+  if (!existsSync(path)) return;
+  if (readFileSync(path, "utf8").startsWith(marker)) return;
+  throw new Error(
+    `Refusing to overwrite hand-authored Rosetta output: ${toPosix(relative(projectRoot, path))}`,
+  );
+}
+
+function renderSource(projectRoot: string, root: string, source: string): RsPackageOutput {
+  const input = readFileSync(source, "utf8");
+  const defaults = defaultOutputPaths(projectRoot, root, source);
+  const node = resolveOutput(projectRoot, input.match(NODE_OVERRIDE), defaults.node);
+  const python = resolveOutput(projectRoot, input.match(PYTHON_OVERRIDE), defaults.python);
+  const pythonBlocks = [...input.matchAll(PYTHON_BLOCK)].map((match) => dedent(match[1] ?? ""));
+
+  if (!pythonBlocks.length) {
+    throw new Error(
+      `Rosetta source needs at least one /* @rs-python ... @rs-end */ block: ${toPosix(relative(projectRoot, source))}`,
+    );
+  }
+
+  const nodeBody = input.replace(PYTHON_BLOCK, "").replace(RS_METADATA, "").trim();
+  const nodeContent = `${header({
+    tool: "projen synth (rs-packages)",
+    source: toPosix(relative(projectRoot, source)),
+  })}\n${nodeBody}\n`;
+  const pythonContent = [
+    "# GENERATED by projen synth (rs-packages) - DO NOT EDIT.",
+    `# Regenerated from ${toPosix(relative(projectRoot, source))}.`,
+    "# Hand edits are overwritten; edit the Rosetta source instead.",
+    "",
+    pythonBlocks.join("\n\n"),
+    "",
+  ].join("\n");
+
+  for (const [path, content] of [
+    [node, nodeContent],
+    [python, pythonContent],
+  ] as const) {
+    assertWritableOutput(projectRoot, path, path === node ? NODE_MARKER : PYTHON_MARKER);
+    mkdirSync(dirname(path), { recursive: true });
+    makeWritable(path);
+    writeFileSync(path, content);
+    makeReadonly(path);
+  }
+
+  return { source, node, python };
+}
+
+/** Read the roots recorded by synth for standalone tasks and watchers. */
+export function configuredRsPackageRoots(projectRoot: string): readonly string[] | false {
+  const manifest = json.parseRecord(readFileSync(resolve(projectRoot, "package.json"), "utf8"));
+  const config = object.isRecord(manifest?.dbxToolsConfig) ? manifest.dbxToolsConfig : undefined;
+  const configured = config?.rsPackageRoots;
+  if (configured === false) return false;
+  if (Array.isArray(configured)) {
+    const roots = configured.filter(
+      (value): value is string => typeof value === "string" && !!value,
+    );
+    if (roots.length) return roots;
+  }
+  return DEFAULT_RS_PACKAGE_ROOTS;
+}
+
+export function discoverRsPackageOutputs(
+  options: GenerateRsPackagesOptions = {},
+): RsPackageOutput[] {
+  const projectRoot = resolve(options.projectRoot ?? process.cwd());
+  const roots = options.roots ?? DEFAULT_RS_PACKAGE_ROOTS;
+  return roots.flatMap((root) =>
+    sourceFiles(resolve(projectRoot, root)).map((source) => {
+      const defaults = defaultOutputPaths(projectRoot, root, source);
+      const input = readFileSync(source, "utf8");
+      return {
+        source,
+        node: resolveOutput(projectRoot, input.match(NODE_OVERRIDE), defaults.node),
+        python: resolveOutput(projectRoot, input.match(PYTHON_OVERRIDE), defaults.python),
+      };
+    }),
+  );
+}
+
+export function generateRsPackages(options: GenerateRsPackagesOptions = {}): RsPackageOutput[] {
+  const projectRoot = resolve(options.projectRoot ?? process.cwd());
+  const roots = options.roots ?? DEFAULT_RS_PACKAGE_ROOTS;
+  return roots.flatMap((root) =>
+    sourceFiles(resolve(projectRoot, root)).map((source) =>
+      renderSource(projectRoot, root, source),
+    ),
+  );
+}
+
+/** Projen lifecycle component that regenerates Rosetta outputs after synth. */
+export class RsPackages extends Component {
+  private readonly outputs: RsPackageOutput[];
+
+  constructor(
+    project: javascript.NodeProject,
+    private readonly roots: readonly string[] = DEFAULT_RS_PACKAGE_ROOTS,
+  ) {
+    super(project);
+    // Generate during construction so a brand-new Node package exists before the
+    // root's package scan later in the same synth. postSynthesize repeats this to
+    // restore outputs after projen has written the rest of the tree.
+    this.outputs = generateRsPackages({ projectRoot: project.outdir, roots });
+    for (const output of this.outputs) {
+      const node = `/${toPosix(relative(project.outdir, output.node))}`;
+      const python = `/${toPosix(relative(project.outdir, output.python))}`;
+      project.annotateGenerated(node);
+      project.annotateGenerated(python);
+    }
+  }
+
+  public override preSynthesize(): void {
+    const eslint = javascript.Eslint.of(this.project);
+    if (!eslint) return;
+    for (const output of this.outputs) {
+      eslint.addIgnorePattern(toPosix(relative(this.project.outdir, output.node)));
+    }
+  }
+
+  public override postSynthesize(): void {
+    generateRsPackages({ projectRoot: this.project.outdir, roots: this.roots });
+  }
+}

package/tasks/rs-packages.ts

@@ -0,0 +1,26 @@
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { log } from "@dbx-tools/shared-core";
+import { configuredRsPackageRoots, generateRsPackages } from "../src/rs-packages.ts";
+import { repoRoot } from "../src/packages.ts";
+import { watchLoop } from "../src/watch.ts";
+
+const logger = log.logger("projen:rs-packages");
+const configuredRoots = configuredRsPackageRoots(repoRoot);
+const roots = configuredRoots === false ? [] : configuredRoots;
+const watchRoots = roots.map((root) => resolve(repoRoot, root)).filter(existsSync);
+
+function generate(): void {
+  const outputs = generateRsPackages({ projectRoot: repoRoot, roots });
+  logger.success(`generated ${outputs.length} Rosetta module${outputs.length === 1 ? "" : "s"}`);
+}
+
+generate();
+
+if (process.argv.includes("--watch")) {
+  if (watchRoots.length) watchLoop("rs-packages", watchRoots, generate);
+  else {
+    logger.info("no Rosetta roots configured; watcher is idle");
+    setInterval(() => {}, 2_147_483_647);
+  }
+}

package/tasks/sync.ts

@@ -31,7 +31,7 @@ if (!process.argv.includes("--watch")) {
   runSynth({ post: true });
   logger.success("synced");
 } else {
-  // Watch: one initial full synth to bring the tree up to date, then three focused
+  // Watch: one initial full synth to bring the tree up to date, then focused
   // watchers under `concurrently`. The projenrc watcher is the intelligent stand-in
   // for stock `projen --watch` - it re-synths (+install) ONLY when `.projenrc.ts` or
   // a configured `syncResynthPaths` entry changes, while barrels/openapi keep generated
@@ -57,6 +57,11 @@ if (!process.argv.includes("--watch")) {
       { command: `bun "${taskPath("projenrc.ts")}"`, name: "projenrc", prefixColor: "magenta" },
       { command: `bun "${taskPath("barrels.ts")}" --watch`, name: "barrels", prefixColor: "cyan" },
       { command: `bun "${taskPath("openapi.ts")}" --watch`, name: "openapi", prefixColor: "green" },
+      {
+        command: `bun "${taskPath("rs-packages.ts")}" --watch`,
+        name: "rs-packages",
+        prefixColor: "yellow",
+      },
     ],
     {
       prefix: "name",