@jsenv/core 27.0.0-alpha.5 → 27.0.0-alpha.50

package/src/plugins/url_analysis/html/html_urls.js

@@ -0,0 +1,273 @@
+import {
+  parseHtmlString,
+  stringifyHtmlAst,
+  getHtmlNodeAttributeByName,
+  htmlNodePosition,
+  visitHtmlAst,
+} from "@jsenv/utils/html_ast/html_ast.js"
+import { htmlAttributeSrcSet } from "@jsenv/utils/html_ast/html_attribute_src_set.js"
+
+export const parseAndTransformHtmlUrls = async (urlInfo, context) => {
+  const url = urlInfo.data.rawUrl || urlInfo.url
+  const content = urlInfo.content
+  const { scenario, referenceUtils } = context
+  const htmlAst = parseHtmlString(content, {
+    storeOriginalPositions: scenario !== "build",
+  })
+  const actions = []
+  visitHtmlUrls({
+    url,
+    htmlAst,
+    onUrl: ({
+      type,
+      subtype,
+      expectedType,
+      line,
+      column,
+      originalLine,
+      originalColumn,
+      specifier,
+      attribute,
+    }) => {
+      const isRessourceHint = [
+        "preconnect",
+        "dns-prefetch",
+        "prefetch",
+        "preload",
+        "modulepreload",
+      ].includes(subtype)
+      const [reference] = referenceUtils.found({
+        type,
+        expectedType,
+        originalLine,
+        originalColumn,
+        specifier,
+        specifierLine: line,
+        specifierColumn: column,
+        isRessourceHint,
+      })
+      actions.push(async () => {
+        attribute.value = await referenceUtils.readGeneratedSpecifier(reference)
+      })
+    },
+  })
+  await Promise.all(actions.map((action) => action()))
+  return {
+    content: stringifyHtmlAst(htmlAst),
+  }
+}
+
+const visitHtmlUrls = ({ url, htmlAst, onUrl }) => {
+  const addDependency = ({
+    type,
+    subtype,
+    expectedType,
+    node,
+    attribute,
+    specifier,
+  }) => {
+    const generatedFromInlineContent = Boolean(
+      getHtmlNodeAttributeByName(node, "generated-from-inline-content"),
+    )
+    let position
+    if (generatedFromInlineContent) {
+      // when generated from inline content,
+      // line, column is not "src" nor "generated-from-src" but "original-position"
+      position = htmlNodePosition.readNodePosition(node)
+    } else {
+      position = htmlNodePosition.readAttributePosition(node, attribute.name)
+    }
+    const {
+      line,
+      column,
+      // originalLine, originalColumn
+    } = position
+    onUrl({
+      type,
+      subtype,
+      expectedType,
+      line,
+      column,
+      // originalLine, originalColumn
+      specifier,
+      attribute,
+      // injected:Boolean(getHtmlNodeAttributeByName(node, "injected-by"))
+      // srcGeneratedFromInlineContent
+      ...readFetchMetas(node),
+    })
+  }
+  const onNode = (node) => {
+    if (node.nodeName === "link") {
+      const relAttribute = getHtmlNodeAttributeByName(node, "rel")
+      const rel = relAttribute ? relAttribute.value : undefined
+      const typeAttribute = getHtmlNodeAttributeByName(node, "type")
+      const type = typeAttribute ? typeAttribute.value : undefined
+      visitAttributeAsUrlSpecifier({
+        type: "link_href",
+        subtype: rel,
+        node,
+        attributeName: "href",
+        expectedContentType: type,
+        expectedType: {
+          manifest: "webmanifest",
+          modulepreload: "js_module",
+          stylesheet: "css",
+        }[rel],
+      })
+      return
+    }
+    // if (node.nodeName === "style") {
+    //   // styles.push(node)
+    //   return
+    // }
+    if (node.nodeName === "script") {
+      const typeAttributeNode = getHtmlNodeAttributeByName(node, "type")
+      visitAttributeAsUrlSpecifier({
+        type: "script_src",
+        expectedType: {
+          "undefined": "js_classic",
+          "text/javascript": "js_classic",
+          "module": "js_module",
+          "importmap": "importmap",
+        }[typeAttributeNode ? typeAttributeNode.value : undefined],
+        node,
+        attributeName: "src",
+      })
+      return
+    }
+    if (node.nodeName === "a") {
+      visitAttributeAsUrlSpecifier({
+        type: "a_href",
+        node,
+        attributeName: "href",
+      })
+    }
+    if (node.nodeName === "iframe") {
+      visitAttributeAsUrlSpecifier({
+        type: "iframe_src",
+        node,
+        attributeName: "src",
+      })
+    }
+    if (node.nodeName === "img") {
+      visitAttributeAsUrlSpecifier({
+        type: "img_src",
+        node,
+        attributeName: "src",
+      })
+      visitSrcset({
+        type: "img_srcset",
+        node,
+      })
+      return
+    }
+    if (node.nodeName === "source") {
+      visitAttributeAsUrlSpecifier({
+        type: "source_src",
+        node,
+        attributeName: "src",
+      })
+      visitSrcset({
+        type: "source_srcset",
+        node,
+      })
+      return
+    }
+    // svg <image> tag
+    if (node.nodeName === "image") {
+      visitAttributeAsUrlSpecifier({
+        type: "image_href",
+        node,
+        attributeName: "href",
+      })
+      return
+    }
+    if (node.nodeName === "use") {
+      visitAttributeAsUrlSpecifier({
+        type: "use_href",
+        node,
+        attributeName: "href",
+      })
+      return
+    }
+  }
+  const visitAttributeAsUrlSpecifier = ({
+    type,
+    subtype,
+    expectedType,
+    node,
+    attributeName,
+  }) => {
+    const attribute = getHtmlNodeAttributeByName(node, attributeName)
+    const value = attribute ? attribute.value : undefined
+    if (value) {
+      const generatedBy = getHtmlNodeAttributeByName(node, "generated-by")
+      if (generatedBy) {
+        // during build the importmap is inlined
+        // and shoud not be considered as a dependency anymore
+        return
+      }
+      addDependency({
+        type,
+        subtype,
+        expectedType,
+        node,
+        attribute,
+        specifier:
+          attributeName === "generated-from-src" ||
+          attributeName === "generated-from-href"
+            ? new URL(value, url).href
+            : value,
+      })
+    } else if (attributeName === "src") {
+      visitAttributeAsUrlSpecifier({
+        type,
+        subtype,
+        expectedType,
+        node,
+        attributeName: "generated-from-src",
+      })
+    } else if (attributeName === "href") {
+      visitAttributeAsUrlSpecifier({
+        type,
+        subtype,
+        expectedType,
+        node,
+        attributeName: "generated-from-href",
+      })
+    }
+  }
+  const visitSrcset = ({ type, node }) => {
+    const srcsetAttribute = getHtmlNodeAttributeByName(node, "srcset")
+    const srcset = srcsetAttribute ? srcsetAttribute.value : undefined
+    if (srcset) {
+      const srcCandidates = htmlAttributeSrcSet.parse(srcset)
+      srcCandidates.forEach((srcCandidate) => {
+        addDependency({
+          type,
+          node,
+          attribute: srcsetAttribute,
+          specifier: srcCandidate.specifier,
+        })
+      })
+    }
+  }
+  visitHtmlAst(htmlAst, onNode)
+}
+
+const crossOriginCompatibleTagNames = ["script", "link", "img", "source"]
+const integrityCompatibleTagNames = ["script", "link", "img", "source"]
+const readFetchMetas = (node) => {
+  const meta = {}
+  if (crossOriginCompatibleTagNames.includes(node.nodeName)) {
+    const crossoriginAttribute = getHtmlNodeAttributeByName(node, "crossorigin")
+    meta.crossorigin = crossoriginAttribute
+      ? crossoriginAttribute.value
+      : undefined
+  }
+  if (integrityCompatibleTagNames.includes(node.nodeName)) {
+    const integrityAttribute = getHtmlNodeAttributeByName(node, "integrity")
+    meta.integrity = integrityAttribute ? integrityAttribute.value : undefined
+  }
+  return meta
+}

package/src/plugins/url_analysis/js/js_urls.js

@@ -0,0 +1,67 @@
+import { parseJsUrls } from "@jsenv/utils/js_ast/parse_js_urls.js"
+import { createMagicSource } from "@jsenv/utils/sourcemap/magic_source.js"
+import { isWebWorkerUrlInfo } from "@jsenv/core/src/omega/web_workers.js"
+
+export const parseAndTransformJsUrls = async (urlInfo, context) => {
+  const jsMentions = await parseJsUrls({
+    js: urlInfo.content,
+    url: (urlInfo.data && urlInfo.data.rawUrl) || urlInfo.url,
+    isJsModule: urlInfo.type === "js_module",
+    isWebWorker: isWebWorkerUrlInfo(urlInfo),
+  })
+  const { rootDirectoryUrl, referenceUtils } = context
+  const actions = []
+  const magicSource = createMagicSource(urlInfo.content)
+  urlInfo.data.usesImport = false
+  urlInfo.data.usesExport = false
+  urlInfo.data.usesImportAssertion = false
+  jsMentions.forEach((jsMention) => {
+    if (jsMention.assert) {
+      urlInfo.data.usesImportAssertion = true
+    }
+    if (
+      jsMention.subtype === "import_static" ||
+      jsMention.subtype === "import_dynamic"
+    ) {
+      urlInfo.data.usesImport = true
+    }
+    if (jsMention.subtype === "export") {
+      urlInfo.data.usesExport = true
+    }
+    const [reference] = referenceUtils.found({
+      type: jsMention.type,
+      subtype: jsMention.subtype,
+      expectedType: jsMention.expectedType,
+      expectedSubtype: jsMention.expectedSubtype || urlInfo.subtype,
+      specifier: jsMention.specifier,
+      specifierStart: jsMention.specifierStart,
+      specifierEnd: jsMention.specifierEnd,
+      specifierLine: jsMention.specifierLine,
+      specifierColumn: jsMention.specifierColumn,
+      data: jsMention.data,
+      baseUrl: {
+        "StringLiteral": jsMention.baseUrl,
+        "window.origin": rootDirectoryUrl,
+        "import.meta.url": urlInfo.url,
+        "context.meta.url": urlInfo.url,
+        "document.currentScript.src": urlInfo.url,
+      }[jsMention.baseUrlType],
+      assert: jsMention.assert,
+      assertNode: jsMention.assertNode,
+      typePropertyNode: jsMention.typePropertyNode,
+    })
+    actions.push(async () => {
+      magicSource.replace({
+        start: jsMention.specifierStart,
+        end: jsMention.specifierEnd,
+        replacement: await referenceUtils.readGeneratedSpecifier(reference),
+      })
+      if (reference.mutation) {
+        reference.mutation(magicSource)
+      }
+    })
+  })
+  await Promise.all(actions.map((action) => action()))
+  const { content, sourcemap } = magicSource.toContentAndSourcemap()
+  return { content, sourcemap }
+}

package/src/plugins/url_analysis/jsenv_plugin_url_analysis.js

@@ -0,0 +1,18 @@
+import { parseAndTransformHtmlUrls } from "./html/html_urls.js"
+import { parseAndTransformCssUrls } from "./css/css_urls.js"
+import { parseAndTransformJsUrls } from "./js/js_urls.js"
+import { parseAndTransformWebmanifestUrls } from "./webmanifest/webmanifest_urls.js"
+
+export const jsenvPluginUrlAnalysis = () => {
+  return {
+    name: "jsenv:url_analysis",
+    appliesDuring: "*",
+    transformUrlContent: {
+      html: parseAndTransformHtmlUrls,
+      css: parseAndTransformCssUrls,
+      js_classic: parseAndTransformJsUrls,
+      js_module: parseAndTransformJsUrls,
+      webmanifest: parseAndTransformWebmanifestUrls,
+    },
+  }
+}

package/src/plugins/url_analysis/webmanifest/webmanifest_urls.js

@@ -0,0 +1,17 @@
+export const parseAndTransformWebmanifestUrls = async (urlInfo, context) => {
+  const content = urlInfo.content
+  const manifest = JSON.parse(content)
+  const actions = []
+  const { icons = [] } = manifest
+  icons.forEach((icon) => {
+    const [reference] = context.referenceUtils.found({
+      type: "webmanifest_icon_src",
+      specifier: icon.src,
+    })
+    actions.push(async () => {
+      icon.src = await context.referenceUtils.readGeneratedSpecifier(reference)
+    })
+  })
+  await Promise.all(actions.map((action) => action()))
+  return JSON.stringify(manifest, null, "  ")
+}

package/src/{omega/core_plugins → plugins}/url_resolution/jsenv_plugin_url_resolution.js

@@ -1,13 +1,18 @@
 export const jsenvPluginUrlResolution = () => {
-  const urlResolver = ({ parentUrl, specifier }) => {
-    return new URL(specifier, parentUrl).href
+  const urlResolver = (reference) => {
+    if (reference.specifier[0] === "#") {
+      reference.external = true
+    }
+    return new URL(
+      reference.specifier,
+      reference.baseUrl || reference.parentUrl,
+    ).href
   }
   return {
     name: "jsenv:url_resolution",
     appliesDuring: "*",
-    resolve: {
+    resolveUrl: {
       "entry_point": urlResolver,
-      "js_import_export": urlResolver,
       "link_href": urlResolver,
       "script_src": urlResolver,
       "a_href": urlResolver,
@@ -21,8 +26,10 @@ export const jsenvPluginUrlResolution = () => {
       "css_@import": urlResolver,
       "css_url": urlResolver,
       "sourcemap_comment": urlResolver,
-      "js_import_meta_url_pattern": urlResolver,
+      "js_import_export": urlResolver,
+      "js_url_specifier": urlResolver,
       "js_inline_content": urlResolver,
+      "webmanifest_icon_src": urlResolver,
     },
   }
 }

package/src/{omega/core_plugins → plugins}/url_version/jsenv_plugin_url_version.js

@@ -1,40 +1,37 @@
-export const jsenvPluginUrlVersion = ({
-  longTermCache = false, // will become true once things get more stable
-} = {}) => {
+export const jsenvPluginUrlVersion = ({ longTermCache = true } = {}) => {
   return {
     name: "jsenv:url_version",
     appliesDuring: "*", // maybe only during dev?
-    normalize: ({ url }) => {
+    normalizeUrl: (reference) => {
       // "v" search param goal is to enable long-term cache
       // for server response headers
       // it is also used by hmr to bypass browser cache
       // this goal is achieved when we reach this part of the code
       // We get rid of this params so that urlGraph and other parts of the code
       // recognize the url (it is not considered as a different url)
-      const urlObject = new URL(url)
+      const urlObject = new URL(reference.url)
       urlObject.searchParams.delete("v")
       return urlObject.href
     },
-    transformReferencedUrl: ({ url, data }) => {
-      if (!data.version) {
+    transformUrlSearchParams: (reference) => {
+      if (!reference.data.version) {
         return null
       }
-      const urlObject = new URL(url)
-      if (urlObject.searchParams.has("v")) {
+      if (reference.searchParams.has("v")) {
         return null
       }
-      urlObject.searchParams.set("v", data.version)
-      return urlObject.href
+      return {
+        v: reference.data.version,
+      }
     },
-    augmentResponse: ({ url }) => {
+    augmentResponse: ({ reference }) => {
       if (!longTermCache) {
         return null
       }
-      const urlObject = new URL(url)
-      if (!urlObject.searchParams.has("v")) {
+      if (!reference.searchParams.has("v")) {
         return null
       }
-      if (urlObject.searchParams.has("hmr")) {
+      if (reference.searchParams.has("hmr")) {
         return null
       }
       // When url is versioned put it in browser cache for 30 days

package/src/test/execute_plan.js

@@ -9,20 +9,16 @@ import {
   ensureEmptyDirectory,
   normalizeStructuredMetaMap,
   urlToMeta,
-  writeFile,
+  writeFileSync,
 } from "@jsenv/filesystem"
-import {
-  createLogger,
-  createDetailedMessage,
-  loggerToLevels,
-} from "@jsenv/logger"
+import { createDetailedMessage, loggerToLevels } from "@jsenv/logger"
 import { createLog, startSpinner } from "@jsenv/log"
 import { Abort, raceProcessTeardownEvents } from "@jsenv/abort"
 
 import { babelPluginInstrument } from "@jsenv/utils/coverage/babel_plugin_instrument.js"
 import { reportToCoverage } from "@jsenv/utils/coverage/report_to_coverage.js"
 import { createUrlGraph } from "@jsenv/core/src/omega/url_graph.js"
-import { getCorePlugins } from "@jsenv/core/src/omega/core_plugins.js"
+import { getCorePlugins } from "@jsenv/core/src/plugins/plugins.js"
 import { createKitchen } from "@jsenv/core/src/omega/kitchen.js"
 import { startOmegaServer } from "@jsenv/core/src/omega/omega_server.js"
 import { run } from "@jsenv/core/src/execute/run.js"
@@ -62,9 +58,13 @@ export const executePlan = async (
     coverageV8ConflictWarning,
     coverageTempDirectoryRelativeUrl,
 
-    plugins = [],
-    scenario = "test",
-    sourcemaps = "inline",
+    scenario,
+    sourcemaps,
+    plugins,
+    injectedGlobals,
+    nodeEsmResolution,
+    fileSystemMagicResolution,
+    transpilation,
 
     protocol,
     privateKey,
@@ -132,12 +132,27 @@ export const executePlan = async (
         logger,
         rootDirectoryUrl,
         urlGraph,
+        scenario,
+        sourcemaps,
+        writeOnFileSystem: false,
         plugins: [
           ...plugins,
           ...getCorePlugins({
-            babel: {
-              getCustomBabelPlugins: ({ runtimeSupport }) => {
-                if (coverage && Object.keys(runtimeSupport)[0] !== "chrome") {
+            rootDirectoryUrl,
+            urlGraph,
+            scenario,
+
+            htmlSupervisor: true,
+            nodeEsmResolution,
+            fileSystemMagicResolution,
+            injectedGlobals,
+            transpilation: {
+              ...transpilation,
+              getCustomBabelPlugins: ({ clientRuntimeCompat }) => {
+                if (
+                  coverage &&
+                  Object.keys(clientRuntimeCompat)[0] !== "chrome"
+                ) {
                   return {
                     "transform-instrument": [
                       babelPluginInstrument,
@@ -153,13 +168,10 @@ export const executePlan = async (
             },
           }),
         ],
-        scenario,
-        sourcemaps,
       })
-      const serverLogger = createLogger({ logLevel: "warn" })
       const server = await startOmegaServer({
         signal: multipleExecutionsOperation.signal,
-        logger: serverLogger,
+        logLevel: "warn",
         rootDirectoryUrl,
         urlGraph,
         kitchen,
@@ -433,7 +445,7 @@ export const executePlan = async (
     }
     if (summary.counters.total !== summary.counters.completed) {
       const logFileUrl = new URL(logFileRelativeUrl, rootDirectoryUrl).href
-      writeFile(logFileUrl, rawOutput)
+      writeFileSync(logFileUrl, rawOutput)
       logger.info(`-> ${urlToFileSystemPath(logFileUrl)}`)
     }
     const result = await transformReturnValue({

package/src/test/execute_test_plan.js

@@ -13,6 +13,14 @@ import { generateCoverageHtmlDirectory } from "@jsenv/utils/coverage/coverage_re
 import { generateCoverageTextLog } from "@jsenv/utils/coverage/coverage_reporter_text_log.js"
 import { executePlan } from "./execute_plan.js"
 
+export const defaultCoverageConfig = {
+  "./index.js": true,
+  "./main.js": true,
+  "./src/**/*.js": true,
+  "./**/*.test.*": false, // contains .test. -> nope
+  "./**/test/": false, // inside a test folder -> nope,
+}
+
 /**
  * Execute a list of files and log how it goes
  * @param {Object} testPlanParameters
@@ -57,13 +65,7 @@ export const executeTestPlan = async ({
   coverage = process.argv.includes("--cover") ||
     process.argv.includes("--coverage"),
   coverageTempDirectoryRelativeUrl = "./.coverage/tmp/",
-  coverageConfig = {
-    "./index.js": true,
-    "./main.js": true,
-    "./src/**/*.js": true,
-    "./**/*.test.*": false, // contains .test. -> nope
-    "./**/test/": false, // inside a test folder -> nope,
-  },
+  coverageConfig = defaultCoverageConfig,
   coverageIncludeMissing = true,
   coverageAndExecutionAllowed = false,
   coverageForceIstanbul = false,
@@ -80,6 +82,12 @@ export const executeTestPlan = async ({
   // skip full means file with 100% coverage won't appear in coverage reports (log and html)
   coverageSkipFull = false,
 
+  sourcemaps = "inline",
+  plugins = [],
+  injectedGlobals,
+  nodeEsmResolution,
+  fileSystemMagicResolution,
+
   protocol,
   privateKey,
   certificate,
@@ -143,8 +151,8 @@ export const executeTestPlan = async ({
     logFileRelativeUrl,
     completedExecutionLogMerging,
     completedExecutionLogAbbreviation,
-
     rootDirectoryUrl,
+
     maxExecutionsInParallel,
     defaultMsAllocatedPerExecution,
     failFast,
@@ -159,6 +167,13 @@ export const executeTestPlan = async ({
     coverageV8ConflictWarning,
     coverageTempDirectoryRelativeUrl,
 
+    scenario: "test",
+    sourcemaps,
+    plugins,
+    injectedGlobals,
+    nodeEsmResolution,
+    fileSystemMagicResolution,
+
     protocol,
     privateKey,
     certificate,

package/src/test/logs_file_execution.js

@@ -38,7 +38,8 @@ export const formatExecutionResult = (
   { completedExecutionLogAbbreviation, counters, memoryHeap },
 ) => {
   const { status } = executionResult
-  const description = descriptionFormatters[status]({
+  const descriptionFormatter = descriptionFormatters[status]
+  const description = descriptionFormatter({
     index: executionIndex,
     total: counters.total,
     executionParams,
@@ -52,7 +53,7 @@ export const formatExecutionResult = (
     return `${description} ${summary}`
   }
   const { consoleCalls = [], error, duration } = executionResult
-  const console = formatConsoleCalls(consoleCalls)
+  const consoleOutput = formatConsoleCalls(consoleCalls)
   return formatExecution({
     label: `${description} ${summary}`,
     details: {
@@ -61,7 +62,7 @@ export const formatExecutionResult = (
       duration: msAsDuration(duration),
       ...(error ? { error: error.stack } : {}),
     },
-    console,
+    consoleOutput,
   })
 }
 
@@ -203,21 +204,21 @@ const formatConsoleCalls = (consoleCalls) => {
   if (consoleOutputTrimmed === "") {
     return ""
   }
-  return `${ANSI.color(`-------- console --------`, ANSI.GREY)}
+  return `${ANSI.color(`-------- console output --------`, ANSI.GREY)}
 ${consoleOutputTrimmed}
 ${ANSI.color(`-------------------------`, ANSI.GREY)}`
 }
 
-const formatExecution = ({ label, details = {}, console }) => {
+const formatExecution = ({ label, details = {}, consoleOutput }) => {
   let message = ``
   message += label
   Object.keys(details).forEach((key) => {
     message += `
 ${key}: ${details[key]}`
   })
-  if (console) {
+  if (consoleOutput) {
     message += `
-${console}`
+${consoleOutput}`
   }
   return message
 }