@jsenv/plugin-bundling 1.0.0

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@jsenv/plugin-bundling",
+  "version": "1.0.0",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/jsenv/jsenv-core",
+    "directory": "packages/jsenv-plugin-bundling"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./src/main.js"
+    },
+    "./*": "./*"
+  },
+  "files": [
+    "/src/"
+  ],
+  "scripts": {
+    "test": "node --conditions=development ./scripts/test.mjs"
+  },
+  "dependencies": {
+    "@jsenv/ast": "1.4.8",
+    "@jsenv/babel-plugins": "1.1.0",
+    "@jsenv/log": "3.3.2",
+    "@jsenv/url-meta": "7.0.0",
+    "@jsenv/urls": "1.2.8",
+    "@jsenv/sourcemap": "1.0.8",
+    "acorn-import-assertions": "1.8.0",
+    "rollup": "3.7.4"
+  },
+  "peerDependencies": {
+    "@jsenv/core": "*"
+  },
+  "devDependencies": {
+    "@jsenv/core": "../../"
+  }
+}

package/src/css/bundle_css.js

@@ -0,0 +1,140 @@
+/*
+ * Each @import found in css is replaced by the file content
+ * - There is no need to worry about urls (such as background-image: url())
+ *   because they are absolute (file://*) and will be made relative again by jsenv build
+ * - The sourcemap are not generated but ideally they should be
+ *   It can be quite challenging, see "bundle_sourcemap.js"
+ */
+
+import { createMagicSource } from "@jsenv/sourcemap"
+import { applyPostCss, postCssPluginUrlVisitor } from "@jsenv/ast"
+
+import { sortByDependencies } from "./sort_by_dependencies.js"
+
+// Do not use until https://github.com/parcel-bundler/parcel-css/issues/181
+export const bundleCss = async ({ cssUrlInfos, context }) => {
+  const bundledCssUrlInfos = {}
+  const cssBundleInfos = await performCssBundling({
+    cssEntryUrlInfos: cssUrlInfos,
+    context,
+  })
+  cssUrlInfos.forEach((cssUrlInfo) => {
+    bundledCssUrlInfos[cssUrlInfo.url] = {
+      data: {
+        bundlerName: "parcel",
+      },
+      contentType: "text/css",
+      content: cssBundleInfos[cssUrlInfo.url].bundleContent,
+    }
+  })
+  return bundledCssUrlInfos
+}
+
+const performCssBundling = async ({ cssEntryUrlInfos, context }) => {
+  const cssBundleInfos = await loadCssUrls({
+    cssEntryUrlInfos,
+    context,
+  })
+  const cssUrlsSorted = sortByDependencies(cssBundleInfos)
+  cssUrlsSorted.forEach((cssUrl) => {
+    const cssBundleInfo = cssBundleInfos[cssUrl]
+    const magicSource = createMagicSource(cssBundleInfo.content)
+    cssBundleInfo.cssUrls.forEach((cssUrl) => {
+      if (cssUrl.type === "@import") {
+        magicSource.replace({
+          start: cssUrl.atRuleStart,
+          end: cssUrl.atRuleEnd,
+          replacement: cssBundleInfos[cssUrl.url].bundleContent,
+        })
+      }
+    })
+    const { content } = magicSource.toContentAndSourcemap()
+    cssBundleInfo.bundleContent = content.trim()
+  })
+  return cssBundleInfos
+}
+
+const parseCssUrls = async ({ css, url }) => {
+  const cssUrls = []
+  await applyPostCss({
+    sourcemaps: false,
+    plugins: [
+      postCssPluginUrlVisitor({
+        urlVisitor: ({
+          type,
+          specifier,
+          specifierStart,
+          specifierEnd,
+          atRuleStart,
+          atRuleEnd,
+        }) => {
+          cssUrls.push({
+            type,
+            url: new URL(specifier, url).href,
+            specifierStart,
+            specifierEnd,
+            atRuleStart,
+            atRuleEnd,
+          })
+        },
+      }),
+    ],
+    url,
+    content: css,
+  })
+
+  return cssUrls
+}
+
+const loadCssUrls = async ({ cssEntryUrlInfos, context }) => {
+  const cssBundleInfos = {}
+  const promises = []
+  const promiseMap = new Map()
+
+  const load = (cssUrlInfo) => {
+    const promiseFromData = promiseMap.get(cssUrlInfo.url)
+    if (promiseFromData) return promiseFromData
+    const promise = _load(cssUrlInfo)
+    promises.push(promise)
+    promiseMap.set(cssUrlInfo.url, promise)
+    return promise
+  }
+
+  const _load = async (cssUrlInfo) => {
+    const cssUrls = await parseCssUrls({
+      css: cssUrlInfo.content,
+      url: cssUrlInfo.url,
+    })
+    const cssBundleInfo = {
+      content: cssUrlInfo.content,
+      cssUrls,
+      dependencies: [],
+    }
+    cssBundleInfos[cssUrlInfo.url] = cssBundleInfo
+    cssUrls.forEach((cssUrl) => {
+      if (cssUrl.type === "@import") {
+        cssBundleInfo.dependencies.push(cssUrl.url)
+        const importedCssUrlInfo = context.urlGraph.getUrlInfo(cssUrl.url)
+        load(importedCssUrlInfo)
+      }
+    })
+  }
+
+  cssEntryUrlInfos.forEach((cssEntryUrlInfo) => {
+    load(cssEntryUrlInfo)
+  })
+
+  const waitAll = async () => {
+    if (promises.length === 0) {
+      return
+    }
+    const promisesToWait = promises.slice()
+    promises.length = 0
+    await Promise.all(promisesToWait)
+    await waitAll()
+  }
+  await waitAll()
+  promiseMap.clear()
+
+  return cssBundleInfos
+}

package/src/css/sort_by_dependencies.js

@@ -0,0 +1,28 @@
+export const sortByDependencies = (nodes) => {
+  const visited = []
+  const sorted = []
+  const circular = []
+  const visit = (url) => {
+    const isSorted = sorted.includes(url)
+    if (isSorted) {
+      return
+    }
+    const isVisited = visited.includes(url)
+    if (isVisited) {
+      if (!circular.includes(url)) {
+        circular.push(url)
+      }
+    } else {
+      visited.push(url)
+      nodes[url].dependencies.forEach((dependencyUrl) => {
+        visit(dependencyUrl, url)
+      })
+      sorted.push(url)
+    }
+  }
+  Object.keys(nodes).forEach((url) => {
+    visit(url)
+  })
+  sorted.circular = circular
+  return sorted
+}

package/src/js_classic_workers/bundle_js_classic_workers.js

@@ -0,0 +1,13 @@
+/*
+ * TODO:
+ * for each js_classic where subtype is a worker
+ * take the url info and find importScripts calls
+ * and replace them with the corresponding url info file content
+ * we'll ikely need to save the importScripts node location to be able to do that
+ */
+
+// import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+
+export const bundleJsClassicWorkers = () => {
+  return {}
+}

package/src/js_module/bundle_js_modules.js

@@ -0,0 +1,386 @@
+import { pathToFileURL } from "node:url"
+
+import { URL_META } from "@jsenv/url-meta"
+import { isFileSystemPath } from "@jsenv/urls"
+import { createDetailedMessage } from "@jsenv/log"
+import { babelHelperNameFromUrl } from "@jsenv/babel-plugins"
+import { sourcemapConverter } from "@jsenv/sourcemap"
+
+import { fileUrlConverter } from "./file_url_converter.js"
+
+export const bundleJsModules = async ({
+  jsModuleUrlInfos,
+  context,
+  options,
+}) => {
+  const {
+    signal,
+    logger,
+    rootDirectoryUrl,
+    buildDirectoryUrl,
+    assetsDirectory,
+    urlGraph,
+    runtimeCompat,
+    sourcemaps,
+  } = context
+  const {
+    babelHelpersChunk = true,
+    include,
+    preserveDynamicImport = false,
+    strictExports = false,
+    rollup,
+    rollupInput = {},
+    rollupOutput = {},
+    rollupPlugins = [],
+  } = options
+
+  const resultRef = { current: null }
+  try {
+    await applyRollupPlugins({
+      rollup,
+      rollupPlugins: [
+        ...rollupPlugins,
+        rollupPluginJsenv({
+          signal,
+          logger,
+          rootDirectoryUrl,
+          buildDirectoryUrl,
+          assetsDirectory,
+          urlGraph,
+          jsModuleUrlInfos,
+
+          runtimeCompat,
+          sourcemaps,
+          include,
+          babelHelpersChunk,
+          preserveDynamicImport,
+          strictExports,
+          resultRef,
+        }),
+      ],
+      rollupInput: {
+        input: [],
+        onwarn: (warning) => {
+          if (warning.code === "CIRCULAR_DEPENDENCY") {
+            return
+          }
+          if (warning.code === "THIS_IS_UNDEFINED") {
+            const urlInfo = urlGraph.getUrlInfo(pathToFileURL(warning.id).href)
+            if (urlInfo && urlInfo.data.isBabelClientFile) {
+              return
+            }
+          }
+          if (warning.code === "EVAL") {
+            // ideally we should disable only for jsenv files
+            return
+          }
+          logger.warn(String(warning))
+        },
+        ...rollupInput,
+      },
+      rollupOutput: {
+        ...rollupOutput,
+      },
+    })
+    return resultRef.current.jsModuleBundleUrlInfos
+  } catch (e) {
+    if (e.code === "MISSING_EXPORT") {
+      const detailedMessage = createDetailedMessage(e.message, {
+        frame: e.frame,
+      })
+      throw new Error(detailedMessage, { cause: e })
+    }
+    throw e
+  }
+}
+
+const rollupPluginJsenv = ({
+  // logger,
+  rootDirectoryUrl,
+  buildDirectoryUrl,
+  assetsDirectory,
+  urlGraph,
+  jsModuleUrlInfos,
+  sourcemaps,
+
+  include,
+  babelHelpersChunk,
+  preserveDynamicImport,
+  strictExports,
+
+  resultRef,
+}) => {
+  let _rollupEmitFile = () => {
+    throw new Error("not implemented")
+  }
+  const format = jsModuleUrlInfos.some((jsModuleUrlInfo) =>
+    jsModuleUrlInfo.filename.endsWith(".cjs"),
+  )
+    ? "cjs"
+    : "esm"
+  const emitChunk = (chunk) => {
+    return _rollupEmitFile({
+      type: "chunk",
+      ...chunk,
+    })
+  }
+  let importCanBeBundled = () => true
+  if (include) {
+    const associations = URL_META.resolveAssociations(
+      { bundle: include },
+      rootDirectoryUrl,
+    )
+    importCanBeBundled = (url) => {
+      return URL_META.applyAssociations({ url, associations }).bundle
+    }
+  }
+  const urlImporters = {}
+
+  return {
+    name: "jsenv",
+    async buildStart() {
+      _rollupEmitFile = (...args) => this.emitFile(...args)
+      let previousNonEntryPointModuleId
+      jsModuleUrlInfos.forEach((jsModuleUrlInfo) => {
+        const id = jsModuleUrlInfo.url
+        if (jsModuleUrlInfo.isEntryPoint) {
+          emitChunk({
+            id,
+          })
+          return
+        }
+        emitChunk({
+          id,
+          implicitlyLoadedAfterOneOf: previousNonEntryPointModuleId
+            ? [previousNonEntryPointModuleId]
+            : null,
+          preserveSignature: strictExports
+            ? "strict"
+            : jsModuleUrlInfo.dependents.size < 2
+            ? "allow-extension"
+            : "strict",
+        })
+        previousNonEntryPointModuleId = id
+      })
+    },
+    async generateBundle(outputOptions, rollupResult) {
+      _rollupEmitFile = (...args) => this.emitFile(...args)
+
+      const jsModuleBundleUrlInfos = {}
+      Object.keys(rollupResult).forEach((fileName) => {
+        const rollupFileInfo = rollupResult[fileName]
+        // there is 3 types of file: "placeholder", "asset", "chunk"
+        if (rollupFileInfo.type === "chunk") {
+          const sourceUrls = Object.keys(rollupFileInfo.modules).map((id) =>
+            fileUrlConverter.asFileUrl(id),
+          )
+
+          let url
+          let originalUrl
+          if (rollupFileInfo.facadeModuleId) {
+            url = fileUrlConverter.asFileUrl(rollupFileInfo.facadeModuleId)
+            originalUrl = url
+          } else {
+            url = new URL(rollupFileInfo.fileName, buildDirectoryUrl).href
+            if (rollupFileInfo.isDynamicEntry) {
+              originalUrl = sourceUrls[sourceUrls.length - 1]
+            } else {
+              originalUrl = url
+            }
+          }
+
+          const jsModuleBundleUrlInfo = {
+            url,
+            originalUrl,
+            type: format === "esm" ? "js_module" : "common_js",
+            data: {
+              bundlerName: "rollup",
+              bundleRelativeUrl: rollupFileInfo.fileName,
+              usesImport:
+                rollupFileInfo.imports.length > 0 ||
+                rollupFileInfo.dynamicImports.length > 0,
+              isDynamicEntry: rollupFileInfo.isDynamicEntry,
+            },
+            sourceUrls,
+            contentType: "text/javascript",
+            content: rollupFileInfo.code,
+            sourcemap: rollupFileInfo.map,
+          }
+          jsModuleBundleUrlInfos[url] = jsModuleBundleUrlInfo
+        }
+      })
+      resultRef.current = {
+        jsModuleBundleUrlInfos,
+      }
+    },
+    outputOptions: (outputOptions) => {
+      // const sourcemapFile = buildDirectoryUrl
+      Object.assign(outputOptions, {
+        format,
+        dir: fileUrlConverter.asFilePath(buildDirectoryUrl),
+        sourcemap: sourcemaps === "file" || sourcemaps === "inline",
+        // sourcemapFile,
+        sourcemapPathTransform: (relativePath) => {
+          return new URL(relativePath, buildDirectoryUrl).href
+        },
+        entryFileNames: () => {
+          return `[name].js`
+        },
+        chunkFileNames: (chunkInfo) => {
+          const insideJs = willBeInsideJsDirectory({
+            chunkInfo,
+            fileUrlConverter,
+            jsModuleUrlInfos,
+          })
+          let nameFromUrlInfo
+          if (chunkInfo.facadeModuleId) {
+            const url = fileUrlConverter.asFileUrl(chunkInfo.facadeModuleId)
+            const urlInfo = jsModuleUrlInfos.find(
+              (jsModuleUrlInfo) => jsModuleUrlInfo.url === url,
+            )
+            if (urlInfo) {
+              nameFromUrlInfo = urlInfo.filename
+            }
+          }
+          const name = nameFromUrlInfo || `${chunkInfo.name}.js`
+          return insideJs ? `${assetsDirectory}js/${name}` : `${name}`
+        },
+        manualChunks: (id) => {
+          if (babelHelpersChunk) {
+            const fileUrl = fileUrlConverter.asFileUrl(id)
+            if (
+              fileUrl.endsWith(
+                "babel-plugin-transform-async-to-promises/helpers.mjs",
+              )
+            ) {
+              return "babel_helpers"
+            }
+            if (babelHelperNameFromUrl(fileUrl)) {
+              return "babel_helpers"
+            }
+            const urlInfo = urlGraph.getUrlInfo(fileUrl)
+            if (urlInfo && urlInfo.data.isBabelClientFile) {
+              return "babel_helpers"
+            }
+          }
+          return null
+        },
+        // https://rollupjs.org/guide/en/#outputpaths
+        // paths: (id) => {
+        //   return id
+        // },
+      })
+    },
+    // https://rollupjs.org/guide/en/#resolvedynamicimport
+    resolveDynamicImport: (specifier, importer) => {
+      if (preserveDynamicImport) {
+        let urlObject
+        if (specifier[0] === "/") {
+          urlObject = new URL(specifier.slice(1), rootDirectoryUrl)
+        } else {
+          if (isFileSystemPath(importer)) {
+            importer = fileUrlConverter.asFileUrl(importer)
+          }
+          urlObject = new URL(specifier, importer)
+        }
+        urlObject.searchParams.set("as_js_classic_library", "")
+        return { external: true, id: urlObject.href }
+      }
+      return null
+    },
+    resolveId: (specifier, importer = rootDirectoryUrl) => {
+      if (isFileSystemPath(importer)) {
+        importer = fileUrlConverter.asFileUrl(importer)
+      }
+      let url
+      if (specifier[0] === "/") {
+        url = new URL(specifier.slice(1), rootDirectoryUrl).href
+      } else {
+        url = new URL(specifier, importer).href
+      }
+      const existingImporter = urlImporters[url]
+      if (!existingImporter) {
+        urlImporters[url] = importer
+      }
+      if (!url.startsWith("file:")) {
+        return { id: url, external: true }
+      }
+      if (!importCanBeBundled(url)) {
+        return { id: url, external: true }
+      }
+      const urlInfo = urlGraph.getUrlInfo(url)
+      if (!urlInfo) {
+        // happen when excluded by urlAnalysis.include
+        return { id: url, external: true }
+      }
+      if (!urlInfo.shouldHandle) {
+        return { id: url, external: true }
+      }
+      const filePath = fileUrlConverter.asFilePath(url)
+      return filePath
+    },
+    async load(rollupId) {
+      const fileUrl = fileUrlConverter.asFileUrl(rollupId)
+      const urlInfo = urlGraph.getUrlInfo(fileUrl)
+      return {
+        code: urlInfo.content,
+        map:
+          (sourcemaps === "file" || sourcemaps === "inline") &&
+          urlInfo.sourcemap
+            ? sourcemapConverter.toFilePaths(urlInfo.sourcemap)
+            : null,
+      }
+    },
+  }
+}
+
+const applyRollupPlugins = async ({
+  rollup,
+  rollupPlugins,
+  rollupInput,
+  rollupOutput,
+}) => {
+  if (!rollup) {
+    const rollupModule = await import("rollup")
+    rollup = rollupModule.rollup
+  }
+  const { importAssertions } = await import("acorn-import-assertions")
+  const rollupReturnValue = await rollup({
+    ...rollupInput,
+    plugins: rollupPlugins,
+    acornInjectPlugins: [
+      importAssertions,
+      ...(rollupInput.acornInjectPlugins || []),
+    ],
+  })
+  const rollupOutputArray = await rollupReturnValue.generate(rollupOutput)
+  return rollupOutputArray
+}
+
+const willBeInsideJsDirectory = ({
+  chunkInfo,
+  fileUrlConverter,
+  jsModuleUrlInfos,
+}) => {
+  // if the chunk is generated dynamically by rollup
+  // for an entry point jsenv will put that file inside js/ directory
+  // if it's generated dynamically for a file already in js/ directory
+  // both will be inside the js/ directory
+  if (!chunkInfo.facadeModuleId) {
+    // generated by rollup
+    return true
+  }
+  const url = fileUrlConverter.asFileUrl(chunkInfo.facadeModuleId)
+  const jsModuleUrlInfo = jsModuleUrlInfos.find(
+    (jsModuleUrlInfo) => jsModuleUrlInfo.url === url,
+  )
+  if (!jsModuleUrlInfo) {
+    // generated by rollup
+    return true
+  }
+  if (!jsModuleUrlInfo.isEntryPoint) {
+    // not an entry point, jsenv will put it inside js/ directory
+    return true
+  }
+  return false
+}

package/src/js_module/file_url_converter.js

@@ -0,0 +1,21 @@
+import { fileSystemPathToUrl, urlToFileSystemPath } from "@jsenv/urls"
+
+export const fileUrlConverter = {
+  asFilePath: (fileUrl) => {
+    const filePath = urlToFileSystemPath(fileUrl)
+    const urlObject = new URL(fileUrl)
+    const { searchParams } = urlObject
+    return `${filePath}${stringifyQuery(searchParams)}`
+  },
+  asFileUrl: (filePath) => {
+    return decodeURIComponent(fileSystemPathToUrl(filePath)).replace(
+      /[=](?=&|$)/g,
+      "",
+    )
+  },
+}
+
+const stringifyQuery = (searchParams) => {
+  const search = searchParams.toString()
+  return search ? `?${search}` : ""
+}

package/src/jsenv_plugin_bundling.js

@@ -0,0 +1,43 @@
+import { bundleCss } from "./css/bundle_css.js"
+import { bundleJsClassicWorkers } from "./js_classic_workers/bundle_js_classic_workers.js"
+import { bundleJsModules } from "./js_module/bundle_js_modules.js"
+
+export const jsenvPluginBundling = ({
+  css = {},
+  js_classic = {},
+  js_module = {},
+} = {}) => {
+  return {
+    name: "jsenv:bundling",
+    appliesDuring: "build",
+    bundle: {
+      css: css
+        ? (cssUrlInfos, context) => {
+            return bundleCss({
+              cssUrlInfos,
+              context,
+              options: css === true ? {} : css,
+            })
+          }
+        : undefined,
+      js_classic: js_classic
+        ? (jsClassicUrlInfos, context) => {
+            return bundleJsClassicWorkers({
+              jsClassicUrlInfos,
+              context,
+              options: js_classic === true ? {} : js_classic,
+            })
+          }
+        : undefined,
+      js_module: js_module
+        ? (jsModuleUrlInfos, context) => {
+            return bundleJsModules({
+              jsModuleUrlInfos,
+              context,
+              options: js_module === true ? {} : js_module,
+            })
+          }
+        : undefined,
+    },
+  }
+}

package/src/main.js

@@ -0,0 +1,2 @@
+export { jsenvPluginBundling } from "./jsenv_plugin_bundling.js"
+export { bundleJsModules } from "./js_module/bundle_js_modules.js"