@jsenv/core 26.0.1 → 27.0.0-alpha.2

package/src/internal/compile_server/jsenv_directory/reuse_or_create_compiled_file.js

@@ -1,269 +0,0 @@
-import { timeStart, timeFunction } from "@jsenv/server"
-import { urlToFileSystemPath, readFile } from "@jsenv/filesystem"
-import { createDetailedMessage } from "@jsenv/logger"
-
-import { validateCompileCache } from "./validate_compile_cache.js"
-import { getMetaJsonFileUrl } from "./compile_asset.js"
-import { createLockRegistry } from "./file_lock_registry.js"
-
-const { lockForRessource } = createLockRegistry()
-
-export const reuseOrCreateCompiledFile = async ({
-  logger,
-
-  projectDirectoryUrl,
-  originalFileUrl,
-  compiledFileUrl = originalFileUrl,
-  jsenvRemoteDirectory,
-
-  compileCacheStrategy,
-  compileCacheSourcesValidation,
-  compileCacheAssetsValidation,
-  request,
-  compile,
-}) => {
-  if (typeof projectDirectoryUrl !== "string") {
-    throw new TypeError(
-      `projectDirectoryUrl must be a string, got ${projectDirectoryUrl}`,
-    )
-  }
-  if (typeof originalFileUrl !== "string") {
-    throw new TypeError(
-      `originalFileUrl must be a string, got ${originalFileUrl}`,
-    )
-  }
-  if (!originalFileUrl.startsWith(projectDirectoryUrl)) {
-    throw new Error(
-      createDetailedMessage(`origin file must be inside project`, {
-        ["original file url"]: originalFileUrl,
-        ["project directory url"]: projectDirectoryUrl,
-      }),
-    )
-  }
-  if (typeof compiledFileUrl !== "string") {
-    throw new TypeError(
-      `compiledFileUrl must be a string, got ${compiledFileUrl}`,
-    )
-  }
-  if (!compiledFileUrl.startsWith(projectDirectoryUrl)) {
-    throw new Error(
-      createDetailedMessage(`compiled file must be inside project`, {
-        ["compiled file url"]: compiledFileUrl,
-        ["project directory url"]: projectDirectoryUrl,
-      }),
-    )
-  }
-  if (typeof compile !== "function") {
-    throw new TypeError(`compile must be a function, got ${compile}`)
-  }
-
-  const lockTimeEnd = timeStart("lock")
-  return startAsap(
-    async () => {
-      const lockTiming = lockTimeEnd()
-      const { meta, compileResult, compileResultStatus, timing } =
-        await computeCompileReport({
-          projectDirectoryUrl,
-          originalFileUrl,
-          compiledFileUrl,
-          jsenvRemoteDirectory,
-
-          compile,
-          compileCacheStrategy,
-          compileCacheSourcesValidation,
-          compileCacheAssetsValidation,
-          request,
-          logger,
-        })
-
-      return {
-        meta,
-        compileResult,
-        compileResultStatus,
-        timing: {
-          ...lockTiming,
-          ...timing,
-        },
-      }
-    },
-    {
-      compiledFileUrl,
-      logger,
-    },
-  )
-}
-
-const computeCompileReport = async ({
-  // projectDirectoryUrl,
-  logger,
-  originalFileUrl,
-  compiledFileUrl,
-  jsenvRemoteDirectory,
-
-  compile,
-  compileCacheStrategy,
-  compileCacheSourcesValidation,
-  compileCacheAssetsValidation,
-  request,
-}) => {
-  const [readCacheTiming, cacheValidity] = await timeFunction(
-    "read cache",
-    () => {
-      // if (!useFilesystemAsCache) {
-      //   return {
-      //     isValid: false,
-      //     code: "META_FILE_NOT_FOUND",
-      //     meta: {
-      //       isValid: false,
-      //       code: "META_FILE_NOT_FOUND",
-      //     },
-      //   }
-      // }
-      return validateCompileCache({
-        logger,
-        compiledFileUrl,
-        compileCacheStrategy,
-        compileCacheSourcesValidation,
-        compileCacheAssetsValidation,
-        request,
-      })
-    },
-  )
-
-  if (!cacheValidity.isValid) {
-    if (cacheValidity.code === "SOURCES_EMPTY") {
-      logger.warn(`WARNING: meta.sources is empty for ${compiledFileUrl}`)
-    }
-
-    const metaIsValid = cacheValidity.meta ? cacheValidity.meta.isValid : false
-
-    const fetchOriginalFile = async () => {
-      // The original file might be behind an http url.
-      // In that case jsenv try first to read file from filesystem
-      // in ".jsenv/.http/" directory. If not found, the url
-      // is fetched and file is written in that ".jsenv/.http/" directory.
-      // After that the only way to re-fetch this ressource is
-      // to delete the content of ".jsenv/.http/"
-      try {
-        const code = await readFile(originalFileUrl)
-        return { code }
-      } catch (e) {
-        // when file is not found and the file is referenced with an http url
-        if (
-          e &&
-          e.code === "ENOENT" &&
-          jsenvRemoteDirectory.isFileUrlForRemoteUrl(originalFileUrl)
-        ) {
-          const responseBodyAsBuffer =
-            await jsenvRemoteDirectory.loadFileUrlFromRemote(
-              originalFileUrl,
-              request,
-            )
-          const code = String(responseBodyAsBuffer)
-          return { code }
-        }
-        throw e
-      }
-    }
-    const { code } = await fetchOriginalFile()
-    const [compileTiming, compileResult] = await timeFunction("compile", () =>
-      callCompile({
-        logger,
-        originalFileUrl,
-        code,
-        compile,
-      }),
-    )
-
-    return {
-      meta: metaIsValid ? cacheValidity.meta.data : null,
-      compileResult,
-      compileResultStatus: metaIsValid ? "updated" : "created",
-      timing: {
-        ...readCacheTiming,
-        ...compileTiming,
-      },
-    }
-  }
-
-  const meta = cacheValidity.meta.data
-  const { contentType, sources, assets, dependencies } = meta
-  return {
-    meta,
-    compileResult: {
-      compiledSource: String(
-        cacheValidity.compiledFile.data.compiledSourceBuffer,
-      ),
-      compiledEtag: cacheValidity.compiledFile.data.compiledEtag,
-      compiledMtime: cacheValidity.compiledFile.data.compiledMtime,
-      contentType,
-      sources,
-      assets,
-      dependencies,
-    },
-    compileResultStatus: "cached",
-    timing: {
-      ...readCacheTiming,
-    },
-  }
-}
-
-const callCompile = async ({ logger, code, originalFileUrl, compile }) => {
-  logger.debug(`compile ${originalFileUrl}`)
-  const compileReturnValue = await compile({ code })
-  if (typeof compileReturnValue !== "object" || compileReturnValue === null) {
-    throw new TypeError(
-      `compile must return an object, got ${compileReturnValue}`,
-    )
-  }
-  const {
-    contentType,
-    compiledSource,
-    sources = [],
-    sourcesContent = [],
-    assets = [],
-    assetsContent = [],
-    dependencies = [],
-    responseHeaders,
-  } = compileReturnValue
-  if (typeof contentType !== "string") {
-    throw new TypeError(
-      `compile must return a contentType string, got ${contentType}`,
-    )
-  }
-  if (typeof compiledSource !== "string") {
-    throw new TypeError(
-      `compile must return a compiledSource string, got ${compiledSource}`,
-    )
-  }
-  return {
-    contentType,
-    compiledSource,
-    sources,
-    sourcesContent,
-    assets,
-    assetsContent,
-    dependencies,
-    responseHeaders,
-  }
-}
-
-const startAsap = async (fn, { logger, compiledFileUrl }) => {
-  const metaJsonFileUrl = getMetaJsonFileUrl(compiledFileUrl)
-  const metaJsonFilePath = urlToFileSystemPath(metaJsonFileUrl)
-
-  logger.debug(`lock ${metaJsonFilePath}`)
-  // in case this process try to concurrently access meta we wait for previous to be done
-  const unlockLocal = await lockForRessource(metaJsonFilePath)
-
-  let unlockInterProcessLock = () => {}
-
-  try {
-    return await fn()
-  } finally {
-    // we want to unlock in case of error too
-    logger.debug(`unlock ${metaJsonFilePath}`)
-    unlockLocal()
-    unlockInterProcessLock()
-  }
-}

package/src/internal/compile_server/jsenv_directory/update_compile_cache.js

@@ -1,131 +0,0 @@
-import {
-  urlToRelativeUrl,
-  urlToFileSystemPath,
-  bufferToEtag,
-} from "@jsenv/filesystem"
-import { utimesSync } from "node:fs"
-
-import { writeFileContent, testFilePresence } from "./fs_optimized_for_cache.js"
-import { getMetaJsonFileUrl } from "./compile_asset.js"
-
-export const updateCompileCache = async ({
-  logger,
-  jsenvDirectory,
-  meta,
-  compiledFileUrl,
-  compileResult,
-  compileResultStatus,
-}) => {
-  const isNew = compileResultStatus === "created"
-  const isUpdated = compileResultStatus === "updated"
-  if (!isNew && !isUpdated) {
-    return
-  }
-  const {
-    compiledSource,
-    contentType,
-    sources,
-    sourcesContent,
-    assets,
-    assetsContent,
-    dependencies,
-  } = compileResult
-  const promises = []
-  // ensure source that does not leads to concrete files are not capable to invalidate the cache
-  const sourcesToRemove = sources.filter((sourceFileUrl) => {
-    return (
-      sourceFileUrl.startsWith("file://") && !testFilePresence(sourceFileUrl)
-    )
-  })
-  const sourceNotFoundCount = sourcesToRemove.length
-  if (sourceNotFoundCount > 0) {
-    logger.warn(`SOURCE_META_NOT_FOUND: ${sourceNotFoundCount} source file(s) not found.
---- consequence ---
-cache will be reused even if one of the source file is modified
---- source files not found ---
-${sourcesToRemove.join(`\n`)}`)
-    sourcesToRemove.forEach((url) => {
-      const sourceIndex = sources.indexOf(url)
-      if (sourceIndex) {
-        sources.splice(sourceIndex, 1)
-        sourcesContent.splice(sourceIndex, 1)
-      }
-    })
-  }
-  const { writeCompiledSourceFile = true, writeAssetsFile = true } =
-    compileResult
-  if (writeCompiledSourceFile) {
-    logger.debug(
-      `write compiled file at ${urlToFileSystemPath(compiledFileUrl)}`,
-    )
-    promises.push(
-      writeFileContent(compiledFileUrl, compiledSource, {
-        fileLikelyNotFound: isNew,
-      }).then(() => {
-        const mtime = compileResult.compiledMtime
-        // when compileResult.compiledMtime do not exists it means
-        // the client is not interested in it so
-        // -> moment we write the file is not important
-        // -> There is no need to update mtime
-        if (mtime) {
-          utimesSync(
-            urlToFileSystemPath(compiledFileUrl),
-            new Date(mtime),
-            new Date(mtime),
-          )
-        }
-      }),
-    )
-  }
-  if (writeAssetsFile) {
-    promises.push(
-      ...assets.map((assetFileUrl, index) => {
-        logger.debug(
-          `write compiled file asset at ${urlToFileSystemPath(assetFileUrl)}`,
-        )
-        return writeFileContent(assetFileUrl, assetsContent[index], {
-          fileLikelyNotFound: isNew,
-        })
-      }),
-    )
-  }
-  const metaJsonFileUrl = getMetaJsonFileUrl(compiledFileUrl)
-  let latestMeta
-  const sourceAndAssetProps = {
-    sources: sources.map((source) => urlToRelativeUrl(source, metaJsonFileUrl)),
-    sourcesEtag: sourcesContent.map((sourceContent) =>
-      bufferToEtag(Buffer.from(sourceContent)),
-    ),
-    assets: assets.map((asset) => urlToRelativeUrl(asset, metaJsonFileUrl)),
-    assetsEtag: assetsContent.map((assetContent) =>
-      bufferToEtag(Buffer.from(assetContent)),
-    ),
-    dependencies: dependencies.filter((dep) => {
-      return !dep.startsWith("data:")
-    }),
-  }
-  if (isNew) {
-    latestMeta = {
-      contentType,
-      ...sourceAndAssetProps,
-      createdMs: Number(Date.now()),
-      lastModifiedMs: Number(Date.now()),
-    }
-  } else if (isUpdated) {
-    latestMeta = {
-      ...meta,
-      ...sourceAndAssetProps,
-      lastModifiedMs: Number(Date.now()),
-    }
-  }
-  logger.debug(
-    `write compiled file meta at ${urlToFileSystemPath(metaJsonFileUrl)}`,
-  )
-  promises.push(
-    writeFileContent(metaJsonFileUrl, JSON.stringify(latestMeta, null, "  "), {
-      fileLikelyNotFound: isNew,
-    }),
-  )
-  promises.push(jsenvDirectory.compiledFileWriteSignal.onwrite())
-  await Promise.all(promises)
-}

package/src/internal/compile_server/jsenv_directory/validate_compile_cache.js

@@ -1,264 +0,0 @@
-import { fileURLToPath } from "node:url"
-import { readFileSync, statSync } from "node:fs"
-import { resolveUrl, bufferToEtag } from "@jsenv/filesystem"
-
-export const validateCompileCache = async ({
-  compiledFileUrl,
-  compileCacheStrategy,
-  compileCacheSourcesValidation = true,
-  // When "compileCacheAssetsValidation" is enabled, code ensures that
-  // - asset file exists
-  // - asset file content matches an etag generated when the file was compiled
-  // In practice a compiled file asset is a sourcemap file or a coverage.json file.
-  // It's unlikely that something or someone would update an asset file
-  // and even when it happens it would be a bit strict to invalidate the cache
-  // so by default "compileCacheAssetsValidation" is disabled
-  // to avoid checking things for nothing
-  compileCacheAssetsValidation = false,
-  request,
-}) => {
-  const validity = { isValid: true }
-
-  const metaJsonFileUrl = `${compiledFileUrl}__asset__meta.json`
-  const metaValidity = await validateMetaFile(metaJsonFileUrl)
-  mergeValidity(validity, "meta", metaValidity)
-  if (!validity.isValid) {
-    return validity
-  }
-
-  const compiledFileValidation = await validateCompiledFile({
-    compiledFileUrl,
-    request,
-    compileCacheStrategy,
-  })
-  mergeValidity(validity, "compiledFile", compiledFileValidation)
-  if (!validity.isValid) {
-    return validity
-  }
-
-  const meta = metaValidity.data
-  const [sourcesValidity, assetsValidity] = await Promise.all([
-    compileCacheSourcesValidation
-      ? validateSources({
-          meta,
-          metaJsonFileUrl,
-        })
-      : { isValid: true, code: "SOURCES_VALIDATION_DISABLED" },
-    compileCacheAssetsValidation
-      ? validateAssets({
-          meta,
-          metaJsonFileUrl,
-        })
-      : { isValid: true, code: "ASSETS_VALIDATION_DISABLED" },
-  ])
-  mergeValidity(validity, "sources", sourcesValidity)
-  if (!validity.valid) {
-    return validity
-  }
-  mergeValidity(validity, "assets", assetsValidity)
-
-  return validity
-}
-
-const mergeValidity = (parentValidity, childValidityName, childValidity) => {
-  parentValidity.isValid = childValidity.isValid
-  if (childValidity.code) parentValidity.code = childValidity.code
-  parentValidity[childValidityName] = childValidity
-}
-
-const validateMetaFile = async (metaJsonFileUrl) => {
-  const validity = { isValid: true, data: {} }
-
-  let metaJsonBuffer
-  try {
-    metaJsonBuffer = readFileSync(fileURLToPath(metaJsonFileUrl))
-  } catch (error) {
-    if (error && error.code === "ENOENT") {
-      validity.isValid = false
-      validity.code = "META_FILE_NOT_FOUND"
-      return validity
-    }
-    throw error
-  }
-
-  const metaJsonString = String(metaJsonBuffer)
-  let meta
-  try {
-    meta = JSON.parse(metaJsonString)
-  } catch (error) {
-    if (error && error.name === "SyntaxError") {
-      validity.isValid = false
-      validity.code = "META_FILE_SYNTAX_ERROR"
-      return validity
-    }
-    throw error
-  }
-
-  validity.data = meta
-  if (meta.sources.length === 0) {
-    validity.isValid = false
-    validity.code = "SOURCES_EMPTY"
-    return validity
-  }
-
-  return validity
-}
-
-const validateCompiledFile = async ({
-  compiledFileUrl,
-  compileCacheStrategy,
-  request,
-}) => {
-  const validity = { isValid: true, data: {} }
-
-  const clientCacheDisabled = request.headers["cache-control"] === "no-cache"
-
-  try {
-    const compiledSourceBuffer = readFileSync(fileURLToPath(compiledFileUrl))
-    validity.data.compiledSourceBuffer = compiledSourceBuffer
-
-    if (!clientCacheDisabled && compileCacheStrategy === "etag") {
-      const compiledEtag = bufferToEtag(compiledSourceBuffer)
-      validity.data.compiledEtag = compiledEtag
-      const ifNoneMatch = request.headers["if-none-match"]
-      if (ifNoneMatch && ifNoneMatch !== compiledEtag) {
-        validity.isValid = false
-        validity.code = "COMPILED_FILE_ETAG_MISMATCH"
-        return validity
-      }
-    }
-
-    if (!clientCacheDisabled && compileCacheStrategy === "mtime") {
-      const stats = statSync(fileURLToPath(compiledFileUrl))
-      const compiledMtime = Math.floor(stats.mtimeMs)
-      validity.data.compiledMtime = compiledMtime
-
-      const ifModifiedSince = request.headers["if-modified-since"]
-      let ifModifiedSinceDate
-      try {
-        ifModifiedSinceDate = new Date(ifModifiedSince)
-      } catch (e) {
-        ifModifiedSinceDate = null
-        // ideally we should rather respond with
-        // 400 "if-modified-since header is not a valid date"
-      }
-
-      if (
-        ifModifiedSinceDate &&
-        ifModifiedSinceDate < dateToSecondsPrecision(compiledMtime)
-      ) {
-        validity.isValid = false
-        validity.code = "COMPILED_FILE_MTIME_OUTDATED"
-        return validity
-      }
-    }
-
-    return validity
-  } catch (error) {
-    if (error && error.code === "ENOENT") {
-      validity.isValid = false
-      validity.code = "COMPILED_FILE_NOT_FOUND"
-      return validity
-    }
-    throw error
-  }
-}
-
-const validateSources = async ({ meta, metaJsonFileUrl }) => {
-  const sourcesValidity = { isValid: true }
-  await Promise.all(
-    meta.sources.map(async (source, index) => {
-      const sourceValidity = await validateSource({
-        metaJsonFileUrl,
-        source,
-        eTag: meta.sourcesEtag[index],
-      })
-      mergeValidity(sourcesValidity, source, sourceValidity)
-    }),
-  )
-  return sourcesValidity
-}
-
-const validateSource = async ({ metaJsonFileUrl, source, eTag }) => {
-  const validity = { isValid: true, data: {} }
-  const sourceFileUrl = resolveUrl(source, metaJsonFileUrl)
-
-  try {
-    const sourceBuffer = readFileSync(fileURLToPath(sourceFileUrl))
-    const sourceETag = bufferToEtag(sourceBuffer)
-    validity.data.sourceBuffer = sourceBuffer
-    validity.data.sourceETag = sourceETag
-
-    if (sourceETag !== eTag) {
-      validity.isValid = false
-      validity.code = "SOURCE_ETAG_MISMATCH"
-      return validity
-    }
-
-    return validity
-  } catch (e) {
-    if (e && e.code === "ENOENT") {
-      // missing source invalidates the cache because
-      // we cannot check its validity
-      // HOWEVER inside writeMeta we will check if a source can be found
-      // when it cannot we will not put it as a dependency
-      // to invalidate the cache.
-      // It is important because some files are constructed on other files
-      // which are not truly on the filesystem
-      // (IN theory the above happens only for convertCommonJsWithRollup because jsenv
-      // always have a concrete file especially to avoid that kind of thing)
-      validity.isValid = false
-      validity.code = "SOURCE_NOT_FOUND"
-      return validity
-    }
-    throw e
-  }
-}
-
-const validateAssets = async ({ metaJsonFileUrl, meta }) => {
-  const assetsValidity = { isValid: true }
-  await Promise.all(
-    meta.assets.map(async (asset, index) => {
-      const assetValidity = await validateAsset({
-        asset,
-        metaJsonFileUrl,
-        eTag: meta.assetsEtag[index],
-      })
-      mergeValidity(assetsValidity, asset, assetValidity)
-    }),
-  )
-  return assetsValidity
-}
-
-const validateAsset = async ({ asset, metaJsonFileUrl, eTag }) => {
-  const validity = { isValid: true, data: {} }
-  const assetFileUrl = resolveUrl(asset, metaJsonFileUrl)
-
-  try {
-    const assetBuffer = readFileSync(fileURLToPath(assetFileUrl))
-    const assetContentETag = bufferToEtag(assetBuffer)
-    validity.data.buffer = assetBuffer
-    validity.data.etag = assetContentETag
-
-    if (eTag !== assetContentETag) {
-      validity.isValid = false
-      validity.code = "ASSET_ETAG_MISMATCH"
-      return validity
-    }
-
-    return validity
-  } catch (error) {
-    if (error && error.code === "ENOENT") {
-      validity.isValid = false
-      validity.code = "ASSET_FILE_NOT_FOUND"
-      return validity
-    }
-    throw error
-  }
-}
-
-const dateToSecondsPrecision = (date) => {
-  const dateWithSecondsPrecision = new Date(date)
-  dateWithSecondsPrecision.setMilliseconds(0)
-  return dateWithSecondsPrecision
-}