@backstage/cli 0.29.5-next.1 → 0.30.0-next.0

package/CHANGELOG.md

@@ -1,5 +1,57 @@
 # @backstage/cli
 
+## 0.30.0-next.0
+
+### Minor Changes
+
+- cb76663: **BREAKING**: Add support for native ESM in Node.js code. This changes the behavior of dynamic import expressions in Node.js code. Typically this can be fixed by replacing `import(...)` with `require(...)`, with an `as typeof import(...)` cast if needed for types. This is because dynamic imports will no longer be transformed to `require(...)` calls, but instead be left as-is. This in turn allows you to load ESM modules from CommonJS code using `import(...)`.
+
+  This change adds support for the following in Node.js packages, across type checking, package builds, runtime transforms and Jest tests:
+
+  - Dynamic imports that load ESM modules from CommonJS code.
+  - Both `.mjs` and `.mts` files as explicit ESM files, as well as `.cjs` and `.cts` as explicit CommonJS files.
+  - Support for the `"type": "module"` field in `package.json` to indicate that the package is an ESM package.
+
+  There are a few caveats to be aware of:
+
+  - To enable support for native ESM in tests, you need to run the tests with the `--experimental-vm-modules` flag enabled, typically via `NODE_OPTIONS='--experimental-vm-modules'`.
+  - Declaring a package as `"type": "module"` in `package.json` is supported, but in tests it will cause all local transitive dependencies to also be treated as ESM, regardless of whether they declare `"type": "module"` or not.
+  - Node.js has an [ESM interoperability layer with CommonJS](https://nodejs.org/docs/latest-v22.x/api/esm.html#interoperability-with-commonjs) that allows for imports from ESM to identify named exports in CommonJS packages. This interoperability layer is **only** enabled when importing packages with a `.cts` or `.cjs` extension. This is because the interoperability layer is not fully compatible with the NPM ecosystem, and would break package if it was enabled for `.js` files.
+  - Dynamic imports of CommonJS packages will vary in shape depending on the runtime, i.e. test vs local development, etc. It is therefore recommended to avoid dynamic imports of CommonJS packages and instead use `require`, or to use the explicit CommonJS extensions as mentioned above. If you do need to dynamically import CommonJS packages, avoid using `default` exports, as the shape of them vary across different environments and you would otherwise need to manually unwrap the import based on the shape of the module object.
+
+### Patch Changes
+
+- f21b125: Ensure that both global-agent and undici agents are enabled when proxying is enabled.
+- Updated dependencies
+  - @backstage/cli-node@0.2.13-next.0
+  - @backstage/config-loader@1.9.6-next.0
+  - @backstage/catalog-model@1.7.3
+  - @backstage/cli-common@0.1.15
+  - @backstage/config@1.3.2
+  - @backstage/errors@1.2.7
+  - @backstage/eslint-plugin@0.1.10
+  - @backstage/integration@1.16.1
+  - @backstage/release-manifests@0.0.12
+  - @backstage/types@1.2.1
+
+## 0.29.5
+
+### Patch Changes
+
+- e937ce0: Fixed incompatible `@typescript-eslint` versions with current `eslint@8.x.x`
+- 8557e09: Removed the `EXPERIMENTAL_VITE` flag for using Vite as a dev server. If you were using this feature, we recommend switching to Rspack via the `EXPERIMENTAL_RSPACK` flag.
+- Updated dependencies
+  - @backstage/types@1.2.1
+  - @backstage/config-loader@1.9.5
+  - @backstage/integration@1.16.1
+  - @backstage/catalog-model@1.7.3
+  - @backstage/cli-common@0.1.15
+  - @backstage/cli-node@0.2.12
+  - @backstage/config@1.3.2
+  - @backstage/errors@1.2.7
+  - @backstage/eslint-plugin@0.1.10
+  - @backstage/release-manifests@0.0.12
+
 ## 0.29.5-next.1
 
 ### Patch Changes

package/config/jest.js

@@ -31,6 +31,14 @@ const FRONTEND_ROLES = [
   'frontend-plugin-module',
 ];
 
+const NODE_ROLES = [
+  'backend',
+  'cli',
+  'node-library',
+  'backend-plugin',
+  'backend-plugin-module',
+];
+
 const envOptions = {
   oldTests: Boolean(process.env.BACKSTAGE_OLD_TESTS),
 };
@@ -130,11 +138,97 @@ const transformIgnorePattern = [
 ].join('|');
 
 // Provides additional config that's based on the role of the target package
-function getRoleConfig(role) {
+function getRoleConfig(role, pkgJson) {
+  // Only Node.js package roles support native ESM modules, frontend and common
+  // packages are always transpiled to CommonJS.
+  const moduleOpts = NODE_ROLES.includes(role)
+    ? {
+        module: {
+          ignoreDynamic: true,
+          exportInteropAnnotation: true,
+        },
+      }
+    : undefined;
+
+  const transform = {
+    '\\.(mjs|cjs|js)$': [
+      require.resolve('./jestSwcTransform'),
+      {
+        ...moduleOpts,
+        jsc: {
+          parser: {
+            syntax: 'ecmascript',
+          },
+        },
+      },
+    ],
+    '\\.jsx$': [
+      require.resolve('./jestSwcTransform'),
+      {
+        jsc: {
+          parser: {
+            syntax: 'ecmascript',
+            jsx: true,
+          },
+          transform: {
+            react: {
+              runtime: 'automatic',
+            },
+          },
+        },
+      },
+    ],
+    '\\.(mts|cts|ts)$': [
+      require.resolve('./jestSwcTransform'),
+      {
+        ...moduleOpts,
+        jsc: {
+          parser: {
+            syntax: 'typescript',
+          },
+        },
+      },
+    ],
+    '\\.tsx$': [
+      require.resolve('./jestSwcTransform'),
+      {
+        jsc: {
+          parser: {
+            syntax: 'typescript',
+            tsx: true,
+          },
+          transform: {
+            react: {
+              runtime: 'automatic',
+            },
+          },
+        },
+      },
+    ],
+    '\\.(bmp|gif|jpg|jpeg|png|ico|webp|frag|xml|svg|eot|woff|woff2|ttf)$':
+      require.resolve('./jestFileTransform.js'),
+    '\\.(yaml)$': require.resolve('./jestYamlTransform'),
+  };
   if (FRONTEND_ROLES.includes(role)) {
-    return { testEnvironment: require.resolve('jest-environment-jsdom') };
+    return {
+      testEnvironment: require.resolve('jest-environment-jsdom'),
+      transform,
+    };
   }
-  return { testEnvironment: require.resolve('jest-environment-node') };
+  return {
+    testEnvironment: require.resolve('jest-environment-node'),
+    moduleFileExtensions: [...SRC_EXTS, 'json', 'node'],
+    // Jest doesn't let us dynamically detect type=module per transformed file,
+    // so we have to assume that if the entry point is ESM, all TS files are
+    // ESM.
+    //
+    // This means you can't switch a package to type=module until all of its
+    // monorepo dependencies are also type=module or does not contain any .ts
+    // files.
+    extensionsToTreatAsEsm:
+      pkgJson.type === 'module' ? ['.ts', '.mts'] : ['.mts'],
+    transform,
+  };
 }
 
 async function getProjectConfig(targetPath, extraConfig, extraOptions) {
@@ -160,64 +254,6 @@ async function getProjectConfig(targetPath, extraConfig, extraOptions) {
       '\\.(css|less|scss|sss|styl)$': require.resolve('jest-css-modules'),
     },
 
-    transform: {
-      '\\.(mjs|cjs|js)$': [
-        require.resolve('./jestSwcTransform'),
-        {
-          jsc: {
-            parser: {
-              syntax: 'ecmascript',
-            },
-          },
-        },
-      ],
-      '\\.jsx$': [
-        require.resolve('./jestSwcTransform'),
-        {
-          jsc: {
-            parser: {
-              syntax: 'ecmascript',
-              jsx: true,
-            },
-            transform: {
-              react: {
-                runtime: 'automatic',
-              },
-            },
-          },
-        },
-      ],
-      '\\.ts$': [
-        require.resolve('./jestSwcTransform'),
-        {
-          jsc: {
-            parser: {
-              syntax: 'typescript',
-            },
-          },
-        },
-      ],
-      '\\.tsx$': [
-        require.resolve('./jestSwcTransform'),
-        {
-          jsc: {
-            parser: {
-              syntax: 'typescript',
-              tsx: true,
-            },
-            transform: {
-              react: {
-                runtime: 'automatic',
-              },
-            },
-          },
-        },
-      ],
-      '\\.(bmp|gif|jpg|jpeg|png|ico|webp|frag|xml|svg|eot|woff|woff2|ttf)$':
-        require.resolve('./jestFileTransform.js'),
-      '\\.(yaml)$': require.resolve('./jestYamlTransform'),
-    },
-
     // A bit more opinionated
     testMatch: [`**/*.test.{${SRC_EXTS.join(',')}}`],
 
@@ -226,7 +262,7 @@ async function getProjectConfig(targetPath, extraConfig, extraOptions) {
       : require.resolve('./jestCachingModuleLoader'),
 
     transformIgnorePatterns: [`/node_modules/(?:${transformIgnorePattern})/`],
-    ...getRoleConfig(pkgJson.backstage?.role),
+    ...getRoleConfig(pkgJson.backstage?.role, pkgJson),
   };
 
   options.setupFilesAfterEnv = options.setupFilesAfterEnv || [];

package/config/jestCachingModuleLoader.js

@@ -19,6 +19,11 @@ const { default: JestRuntime } = require('jest-runtime');
 const scriptTransformCache = new Map();
 
 module.exports = class CachingJestRuntime extends JestRuntime {
+  constructor(config, ...restAgs) {
+    super(config, ...restAgs);
+    this.allowLoadAsEsm = config.extensionsToTreatAsEsm.includes('.mts');
+  }
+
   // This may or may not be a good idea. Theoretically I don't know why this would impact
   // test correctness and flakiness, but it seems like it may introduce flakiness and strange failures.
   // It does seem to speed up test execution by a fair amount though.
@@ -33,4 +38,13 @@ module.exports = class CachingJestRuntime extends JestRuntime {
     }
     return script;
   }
+
+  // Unfortunately we need to use this unstable API to make sure that .js files
+  // are only loaded as modules where ESM is supported, i.e. Node.js packages.
+  unstable_shouldLoadAsEsm(path, ...restArgs) {
+    if (!this.allowLoadAsEsm) {
+      return false;
+    }
+    return super.unstable_shouldLoadAsEsm(path, ...restArgs);
+  }
 };

package/config/jestSwcTransform.js

@@ -23,10 +23,16 @@ function createTransformer(config) {
     ...config,
   });
   const process = (source, filePath, jestOptions) => {
+    // Skip transformation of .js files without ESM syntax, we never transform from CJS to ESM
     if (filePath.endsWith('.js') && !ESM_REGEX.test(source)) {
       return { code: source };
     }
 
+    // Skip transformation of .mjs files, they should only be used if ESM support is available
+    if (filePath.endsWith('.mjs')) {
+      return { code: source };
+    }
+
     return swcTransformer.process(source, filePath, jestOptions);
   };
 

package/config/nodeTransform.cjs

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+const { pathToFileURL } = require('url');
 const { transformSync } = require('@swc/core');
 const { addHook } = require('pirates');
 const { Module } = require('module');
@@ -55,7 +56,10 @@ addHook(
     const transformed = transformSync(code, {
       filename,
       sourceMaps: 'inline',
-      module: { type: 'commonjs' },
+      module: {
+        type: 'commonjs',
+        ignoreDynamic: true,
+      },
       jsc: {
         target: 'es2022',
         parser: {
@@ -76,3 +80,8 @@ addHook(
   },
   { extensions: ['.js', '.cjs'], ignoreNodeModules: true },
 );
+
+// Register module hooks, used by "type": "module" in package.json, .mjs and
+// .mts files, as well as dynamic import(...)s, although dynamic imports will be
+// handled be the CommonJS hooks in this file if what it points to is CommonJS.
+Module.register('./nodeTransformHooks.mjs', pathToFileURL(__filename));

package/config/nodeTransformHooks.mjs

@@ -0,0 +1,282 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { dirname, extname, resolve as resolvePath } from 'path';
+import { fileURLToPath } from 'url';
+import { transformFile } from '@swc/core';
+import { isBuiltin } from 'node:module';
+import { readFile } from 'fs/promises';
+import { existsSync } from 'fs';
+
+// @ts-check
+
+// No explicit file extension, no type in package.json
+const DEFAULT_MODULE_FORMAT = 'commonjs';
+
+// Source file extensions to look for when using bundle resolution strategy
+const SRC_EXTS = ['.ts', '.js'];
+const TS_EXTS = ['.ts', '.mts', '.cts'];
+const moduleTypeTable = {
+  '.mjs': 'module',
+  '.mts': 'module',
+  '.cjs': 'commonjs',
+  '.cts': 'commonjs',
+  '.ts': undefined,
+  '.js': undefined,
+};
+
+/** @type {import('module').ResolveHook} */
+export async function resolve(specifier, context, nextResolve) {
+  // Built-in modules are handled by the default resolver
+  if (isBuiltin(specifier)) {
+    return nextResolve(specifier, context);
+  }
+
+  const ext = extname(specifier);
+
+  // Unless there's an explicit import attribute, JSON files are loaded with our custom loader that's defined below.
+  if (ext === '.json' && !context.importAttributes?.type) {
+    const jsonResult = await nextResolve(specifier, context);
+    return {
+      ...jsonResult,
+      format: 'commonjs',
+      importAttributes: { type: 'json' },
+    };
+  }
+
+  // Anything else with an explicit extension is handled by the default
+  // resolver, except that we help determine the module type where needed.
+  if (ext !== '') {
+    return withDetectedModuleType(await nextResolve(specifier, context));
+  }
+
+  // Other external modules are handled by the default resolver, but again we
+  // help determine the module type where needed.
+  if (!specifier.startsWith('.')) {
+    return withDetectedModuleType(await nextResolve(specifier, context));
+  }
+
+  // The rest of this function handles the case of resolving imports that do not
+  // specify any extension and might point to a directory with an `index.*`
+  // file. We resolve those using the same logic as most JS bundlers would, with
+  // the addition of checking if there's an explicit module format listed in the
+  // closest `package.json` file.
+  //
+  // We use a bundle resolution strategy in order to keep code consistent across
+  // Backstage codebases that contains code both for Web and Node.js, and to
+  // support packages with common code that can be used in both environments.
+  try {
+    // This is expected to throw, but in the event that this module specifier is
+    // supported we prefer to use the default resolver.
+    return await nextResolve(specifier, context);
+  } catch (error) {
+    if (error.code === 'ERR_UNSUPPORTED_DIR_IMPORT') {
+      const spec = `${specifier}${specifier.endsWith('/') ? '' : '/'}index`;
+      const resolved = await resolveWithoutExt(spec, context, nextResolve);
+      if (resolved) {
+        return withDetectedModuleType(resolved);
+      }
+    } else if (error.code === 'ERR_MODULE_NOT_FOUND') {
+      const resolved = await resolveWithoutExt(specifier, context, nextResolve);
+      if (resolved) {
+        return withDetectedModuleType(resolved);
+      }
+    }
+
+    // Unexpected error or no resolution found
+    throw error;
+  }
+}
+
+/**
+ * Populates the `format` field in the resolved object based on the closest `package.json` file.
+ *
+ * @param {import('module').ResolveFnOutput} resolved
+ * @returns {Promise<import('module').ResolveFnOutput>}
+ */
+async function withDetectedModuleType(resolved) {
+  // Already has an explicit format
+  if (resolved.format) {
+    return resolved;
+  }
+  // Happens in Node.js v22 when there's a package.json without an explicit "type" field. Use the default.
+  if (resolved.format === null) {
+    return { ...resolved, format: DEFAULT_MODULE_FORMAT };
+  }
+
+  const ext = extname(resolved.url);
+
+  const explicitFormat = moduleTypeTable[ext];
+  if (explicitFormat) {
+    return {
+      ...resolved,
+      format: explicitFormat,
+    };
+  }
+
+  // TODO(Rugvip): Afaik this should never happen and we can remove this check, but want it here for a little while to verify.
+  if (ext === '.js') {
+    throw new Error('Unexpected .js file without explicit format');
+  }
+
+  // TODO(Rugvip): Does this need caching? kept it simple for now but worth exploring
+  const packageJsonPath = await findPackageJSON(fileURLToPath(resolved.url));
+  if (!packageJsonPath) {
+    return resolved;
+  }
+
+  const packageJson = JSON.parse(await readFile(packageJsonPath, 'utf8'));
+  return {
+    ...resolved,
+    format: packageJson.type ?? DEFAULT_MODULE_FORMAT,
+  };
+}
+
+/**
+ * Find the closest package.json file from the given path.
+ *
+ * TODO(Rugvip): This can be replaced with the Node.js built-in with the same name once it is stable.
+ * @param {string} startPath
+ * @returns {Promise<string | undefined>}
+ */
+async function findPackageJSON(startPath) {
+  let path = startPath;
+
+  // Some confidence check to avoid infinite loop
+  for (let i = 0; i < 1000; i++) {
+    const packagePath = resolvePath(path, 'package.json');
+    if (existsSync(packagePath)) {
+      return packagePath;
+    }
+
+    const newPath = dirname(path);
+    if (newPath === path) {
+      return undefined;
+    }
+    path = newPath;
+  }
+
+  throw new Error(
+    `Iteration limit reached when searching for package.json at ${startPath}`,
+  );
+}
+
+/** @type {import('module').ResolveHook} */
+async function resolveWithoutExt(specifier, context, nextResolve) {
+  for (const tryExt of SRC_EXTS) {
+    try {
+      const resolved = await nextResolve(specifier + tryExt, {
+        ...context,
+        format: 'commonjs',
+      });
+      return {
+        ...resolved,
+        format: moduleTypeTable[tryExt] ?? resolved.format,
+      };
+    } catch {
+      /* ignore */
+    }
+  }
+  return undefined;
+}
+
+/** @type {import('module').LoadHook} */
+export async function load(url, context, nextLoad) {
+  // Non-file URLs are handled by the default loader
+  if (!url.startsWith('file://')) {
+    return nextLoad(url, context);
+  }
+
+  // JSON files loaded as CommonJS are handled by this custom loader, because
+  // the default one doesn't work. For JSON loading to work we'd need the
+  // synchronous hooks that aren't supported yet, or avoid using the CommonJS
+  // compatibility.
+  if (
+    context.format === 'commonjs' &&
+    context.importAttributes?.type === 'json'
+  ) {
+    try {
+      // TODO(Rugvip): Make sure this is valid JSON
+      const content = await readFile(fileURLToPath(url), 'utf8');
+      return {
+        source: `module.exports = (${content})`,
+        format: 'commonjs',
+        shortCircuit: true,
+      };
+    } catch {
+      // Let the default loader generate the error
+      return nextLoad(url, context);
+    }
+  }
+
+  const ext = extname(url);
+
+  // Non-TS files are handled by the default loader
+  if (!TS_EXTS.includes(ext)) {
+    return nextLoad(url, context);
+  }
+
+  const format = context.format ?? DEFAULT_MODULE_FORMAT;
+
+  // We have two choices at this point, we can either transform CommonJS files
+  // and return the transformed source code, or let the default loader handle
+  // them. If we transform them ourselves we will enter CommonJS compatibility
+  // mode in the new module system in Node.js, this effectively means all
+  // CommonJS loaded via `require` calls from this point will all be treated as
+  // if it was loaded via `import` calls from modules.
+  //
+  // The CommonJS compatibility layer will try to identify named exports and
+  // make them available directly, which is convenient as it avoids things like
+  // `import(...).then(m => m.default.foo)`, allowing you to instead write
+  // `import(...).then(m => m.foo)`. The compatibility layer doesn't always work
+  // all that well though, and can lead to module loading issues in many cases,
+  // especially for older code.
+
+  // This `if` block opts-out of using CommonJS compatibility mode by default,
+  // and instead leaves it to our existing loader to transform CommonJS. We do
+  // however use compatibility mode for the more explicit .cts file extension,
+  // allows for a way to opt-in to the new behavior.
+  //
+  // TODO(Rugvip): Once the synchronous hooks API is available for us to use, we might be able to adopt that instead
+  if (format === 'commonjs' && ext !== '.cts') {
+    return nextLoad(url, { ...context, format });
+  }
+
+  const transformed = await transformFile(fileURLToPath(url), {
+    sourceMaps: 'inline',
+    module: {
+      type: format === 'module' ? 'es6' : 'commonjs',
+      ignoreDynamic: true,
+
+      // This helps the Node.js CommonJS compat layer identify named exports.
+      exportInteropAnnotation: true,
+    },
+    jsc: {
+      target: 'es2022',
+      parser: {
+        syntax: 'typescript',
+      },
+    },
+  });
+
+  return {
+    ...context,
+    shortCircuit: true,
+    source: transformed.code,
+    format,
+    responseURL: url,
+  };
+}

package/config/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "allowImportingTsExtensions": true,
     "allowJs": true,
     "declaration": true,
     "declarationMap": false,