@revealui/core 0.0.1-pre.0

package/dist/core/processing/getters/getRevealConfig/crawlPlusFiles.ts

@@ -0,0 +1,409 @@
+export { crawlPlusFiles }
+
+import pc from "@brillout/picocolors"
+import { exec } from "child_process"
+import glob from "fast-glob"
+import type { Stats } from "fs"
+import fs from "fs/promises"
+import path from "path"
+import { promisify } from "util"
+import {
+  assert,
+  assertIsNotProductionRuntime,
+  assertIsSingleModuleInstance,
+  assertPosixPath,
+  assertWarning,
+  humanizeTime,
+  isScriptFile,
+  isVersionOrAbove,
+  scriptFileExtensions,
+} from "../../../../plugins/utils.js"
+import { isTemporaryBuildFile } from "./transpileAndExecuteFile.js"
+const execA = promisify(exec)
+const TOO_MANY_UNTRACKED_FILES = 5
+
+assertIsNotProductionRuntime()
+assertIsSingleModuleInstance("crawlPlusFiles.ts")
+let gitIsNotUsable = false
+
+async function crawlPlusFiles(
+  userRootDir: string,
+  outDirAbsoluteFilesystem: string,
+  isDev: boolean,
+  crawlWithGit: null | boolean
+): Promise<{ filePathAbsoluteUserRootDir: string }[]> {
+  assertPosixPath(userRootDir)
+  assertPosixPath(outDirAbsoluteFilesystem)
+  let outDirRelativeFromUserRootDir: string | null = path.posix.relative(
+    userRootDir,
+    outDirAbsoluteFilesystem
+  )
+  if (outDirRelativeFromUserRootDir.startsWith("../")) {
+    // config.outDir is outside of config.root => it's going to be ignored anyways
+    outDirRelativeFromUserRootDir = null
+  }
+  assert(
+    outDirRelativeFromUserRootDir === null ||
+      /* Not true if outDirRelativeFromUserRootDir starts with a hidden directory (i.e. a directory with a name that starts with `.`)
+      !outDirRelativeFromUserRootDir.startsWith('.') &&
+      */
+      (!outDirRelativeFromUserRootDir.startsWith("./") &&
+        //
+        !outDirRelativeFromUserRootDir.startsWith("../"))
+  )
+
+  const timeBefore = new Date().getTime()
+
+  // Crawl
+  let files: string[] = []
+  const res =
+    crawlWithGit !== false &&
+    (await gitLsFiles(userRootDir, outDirRelativeFromUserRootDir))
+  if (
+    res &&
+    // Fallback to fast-glob for users that dynamically generate plus files. (Assuming that no plus file is found because of the user's .gitignore list.)
+    res.files.length > 0
+  ) {
+    files = res.files
+    // We cannot find files inside symlink directories with `$ git ls-files` => we use fast-glob
+    files.push(
+      ...(
+        await crawlSymlinkDirs(
+          res.symlinkDirs,
+          userRootDir,
+          outDirRelativeFromUserRootDir
+        )
+      )
+        .filter((result) => result.status === "fulfilled")
+        .map((result) => (result as PromiseFulfilledResult<string[]>).value)
+        .flat()
+    )
+  } else {
+    files = await fastGlob(userRootDir, outDirRelativeFromUserRootDir)
+  }
+
+  // Filter build files
+  files = files.filter((filePath) => !isTemporaryBuildFile(filePath))
+
+  // Check performance
+  {
+    const timeAfter = new Date().getTime()
+    const timeSpent = timeAfter - timeBefore
+    if (isDev) {
+      // We only warn in dev, because while building it's expected to take a long time as crawling is competing for resources with other tasks.
+      // Although, in dev, it's also competing for resources e.g. with Vite's `optimizeDeps`.
+      assertWarning(
+        timeSpent < 3 * 1000,
+        `Crawling your ${pc.cyan("+")} files took an unexpected long time (${humanizeTime(
+          timeSpent
+        )}). If you consistently get this warning, then consider reaching out on GitHub.`,
+        {
+          onlyOnce: "slow-crawling",
+        }
+      )
+    }
+  }
+
+  // Normalize
+  const plusFiles = files.map((filePath) => {
+    // Both `$ git-ls files` and fast-glob return posix paths
+    assertPosixPath(filePath)
+    assert(!filePath.startsWith(userRootDir))
+    const filePathAbsoluteUserRootDir = path.posix.join("/", filePath)
+    return { filePathAbsoluteUserRootDir }
+  })
+
+  return plusFiles
+}
+
+// Same as fastGlob() but using `$ git ls-files`
+async function gitLsFiles(
+  userRootDir: string,
+  outDirRelativeFromUserRootDir: string | null
+): Promise<{
+  files: string[]
+  symlinkDirs: string[]
+} | null> {
+  if (gitIsNotUsable) return null
+
+  // Preserve UTF-8 file paths.
+  // https://github.com/revealjs/reveal/issues/1658
+  // https://stackoverflow.com/questions/22827239/how-to-make-git-properly-display-utf-8-encoded-pathnames-in-the-console-window/22828826#22828826
+  // https://stackoverflow.com/questions/15884180/how-do-i-override-git-configuration-options-by-command-line-parameters/15884261#15884261
+  const preserveUTF8 = "-c core.quotepath=off"
+
+  const ignoreAsPatterns = getIgnoreAsPatterns(outDirRelativeFromUserRootDir)
+  const ignoreAsFilterFn = getIgnoreAsFilterFn(outDirRelativeFromUserRootDir)
+
+  const cmd = [
+    "git",
+    preserveUTF8,
+    "ls-files",
+
+    // We don't filter because:
+    //  - It would skip symlink directories
+    //  - Performance gain seems negligible: https://github.com/revealjs/reveal/pull/1688#issuecomment-2166206648
+    // ...scriptFileExtensionList.map((ext) => `"**/+*.${ext}"`),
+
+    // Performance gain is non-negligible.
+    //  - https://github.com/revealjs/reveal/pull/1688#issuecomment-2166206648
+    //  - When node_modules/ is untracked the performance gain could be significant?
+    ...ignoreAsPatterns.map((pattern) => `--exclude="${pattern}"`),
+
+    // --others --exclude-standard => list untracked files (--others) while using .gitignore (--exclude-standard)
+    // --cached => list tracked files
+    // --stage => get file modes which we use to find symlink directories
+    "--others --exclude-standard --cached --stage",
+  ].join(" ")
+
+  let resultLines: string[]
+  let filesDeleted: string[]
+  try {
+    const results = await Promise.allSettled([
+      // Main command
+      runCmd1(cmd, userRootDir),
+      // Get tracked but deleted files
+      runCmd1("git ls-files --deleted", userRootDir),
+    ])
+    resultLines = results[0].status === "fulfilled" ? results[0].value : []
+    filesDeleted = results[1].status === "fulfilled" ? results[1].value : []
+  } catch (err) {
+    if (await isGitNotUsable(userRootDir)) {
+      gitIsNotUsable = true
+      return null
+    }
+    throw err
+  }
+
+  const filePaths = resultLines.map(parseGitLsResultLine)
+
+  // If there are too many files without mode we fallback to fast-glob
+  if (filePaths.filter((f) => !f.mode).length > TOO_MANY_UNTRACKED_FILES)
+    return null
+
+  const symlinkDirs: string[] = []
+  const files: string[] = []
+  for (const { filePath, mode } of filePaths) {
+    // Deleted?
+    if (filesDeleted.includes(filePath)) continue
+
+    // We have to repeat the same exclusion logic here because the option --exclude of `$ git ls-files` only applies to untracked files. (We use --exclude only to speed up the `$ git ls-files` command.)
+    if (!ignoreAsFilterFn(filePath)) continue
+
+    // Symlink directory?
+    {
+      const isSymlinkDir = await isSymlinkDirectory(mode, filePath, userRootDir)
+      if (isSymlinkDir) {
+        symlinkDirs.push(filePath)
+        continue
+      }
+      // Skip deleted files and non-symlink directories
+      if (isSymlinkDir === null) {
+        continue
+      }
+    }
+
+    // + file?
+    if (!path.posix.basename(filePath).startsWith("+")) continue
+    // JavaScript file?
+    if (!isScriptFile(filePath)) continue
+
+    files.push(filePath)
+  }
+
+  return { files, symlinkDirs }
+}
+// Same as gitLsFiles() but using fast-glob
+async function fastGlob(
+  userRootDir: string,
+  outDirRelativeFromUserRootDir: string | null
+): Promise<string[]> {
+  const files = await glob(`**/+*.${scriptFileExtensions}`, {
+    ignore: getIgnoreAsPatterns(outDirRelativeFromUserRootDir),
+    cwd: userRootDir,
+    dot: false,
+  })
+  // Make build deterministic, in order to get a stable generated hash for dist/client/assets/entries/entry-client-routing.${hash}.js
+  // https://github.com/revealjs/reveal/pull/1750
+  files.sort()
+  return files
+}
+
+// Same as getIgnoreAsFilterFn() but as glob pattern
+function getIgnoreAsPatterns(
+  outDirRelativeFromUserRootDir: string | null
+): string[] {
+  const ignoreAsPatterns = [
+    "**/node_modules/**",
+    // Allow:
+    // ```
+    // +Page.js
+    // +Page.telefunc.js
+    // ```
+    "**/*.telefunc.*",
+  ]
+  if (outDirRelativeFromUserRootDir) {
+    assert(!outDirRelativeFromUserRootDir.startsWith("/"))
+    ignoreAsPatterns.push(`${outDirRelativeFromUserRootDir}/**`)
+  }
+  return ignoreAsPatterns
+}
+// Same as getIgnoreAsPatterns() but for Array.filter()
+function getIgnoreAsFilterFn(
+  outDirRelativeFromUserRootDir: string | null
+): (file: string) => boolean {
+  assert(
+    outDirRelativeFromUserRootDir === null ||
+      !outDirRelativeFromUserRootDir.startsWith("/")
+  )
+  return (file: string) =>
+    !file.includes("node_modules/") &&
+    !file.includes(".telefunc.") &&
+    (outDirRelativeFromUserRootDir === null ||
+      !file.startsWith(`${outDirRelativeFromUserRootDir}/`))
+}
+
+// Whether Git is installed and whether we can use it
+async function isGitNotUsable(userRootDir: string) {
+  // Check Git version
+  {
+    const res = await runCmd2("git --version", userRootDir)
+    if ("err" in res) return true
+    let { stdout, stderr } = res
+    assert(stderr === "")
+    const prefix = "git version "
+    assert(stdout.startsWith(prefix))
+    const gitVersion = stdout.slice(prefix.length)
+    //  - Works with Git 2.43.1 but also (most certainly) with earlier versions.
+    //    - We didn't bother test which is the earliest verision that works.
+    //  - Git 2.32.0 doesn't seem to work: https://github.com/revealjs/reveal/discussions/1549
+    //    - Maybe it's because of StackBlitz: looking at the release notes, Git 2.32.0 should be working.
+    if (!isVersionOrAbove(gitVersion, "2.43.1")) return true
+  }
+  // Is userRootDir inside a Git repository?
+  {
+    const res = await runCmd2(
+      "git rev-parse --is-inside-work-tree",
+      userRootDir
+    )
+    if ("err" in res) return true
+    let { stdout, stderr } = res
+    assert(stderr === "")
+    assert(stdout === "true")
+    return false
+  }
+}
+
+async function crawlSymlinkDirs(
+  symlinkDirs: string[],
+  userRootDir: string,
+  outDirRelativeFromUserRootDir: string | null
+) {
+  const filesInSymlinkDirs = (
+    await Promise.allSettled(
+      symlinkDirs.map(async (symlinkDir) =>
+        (
+          await fastGlob(
+            path.posix.join(userRootDir, symlinkDir),
+            outDirRelativeFromUserRootDir
+          )
+        ).map((filePath) => path.posix.join(symlinkDir, filePath))
+      )
+    )
+  ).flat()
+  return filesInSymlinkDirs
+}
+
+// Parse:
+// ```
+// some/not/tracked/path
+// 100644 f6928073402b241b468b199893ff6f4aed0b7195 0\tpages/index/+Page.tsx
+// ```
+function parseGitLsResultLine(resultLine: string): {
+  filePath: string
+  mode: string | null
+} {
+  const [part1, part2, ...rest] = resultLine.split("\t")
+  assert(part1)
+  assert(rest.length === 0)
+
+  // Git doesn't provide the mode for untracked paths.
+  // `resultLine` is:
+  // ```
+  // some/not/tracked/path
+  // ```
+  if (part2 === undefined) {
+    return { filePath: part1, mode: null }
+  }
+  assert(part2)
+
+  // `resultLine` is:
+  // ```
+  // 100644 f6928073402b241b468b199893ff6f4aed0b7195 0\tpages/index/+Page.tsx
+  // ```
+  const [mode, _, __, ...rest2] = part1.split(" ")
+  assert(mode && _ && __ && rest2.length === 0)
+
+  return { filePath: part2, mode }
+}
+
+async function isSymlinkDirectory(
+  mode: string | null,
+  filePath: string,
+  userRootDir: string
+): Promise<boolean | null> {
+  const filePathAbsolute = path.posix.join(userRootDir, filePath)
+  let stats: Stats | null = null
+  let isSymlink = false
+  if (mode === "120000") {
+    isSymlink = true
+  } else if (mode === null) {
+    // `$ git ls-files` doesn't provide the mode when Git doesn't track the path
+    stats = await getFileStats(filePathAbsolute)
+    if (stats === null) return null
+    isSymlink = stats.isSymbolicLink()
+    if (!isSymlink && stats.isDirectory()) return null
+  } else {
+    assert(mode)
+  }
+  if (!isSymlink) return false
+  if (!stats) stats = await getFileStats(filePathAbsolute)
+  if (stats === null) return null
+  const isDirectory = stats.isDirectory()
+  return isDirectory
+}
+async function getFileStats(filePathAbsolute: string): Promise<Stats | null> {
+  let stats: Stats
+  try {
+    stats = await fs.lstat(filePathAbsolute)
+  } catch (err) {
+    // File was deleted, usually a temporary file such as +config.js.build-j95xb988fpln.mjs
+    // ENOENT: no such file or directory
+    assert((err as any).code === "ENOENT")
+    return null
+  }
+  return stats
+}
+
+async function runCmd1(cmd: string, cwd: string): Promise<string[]> {
+  const { stdout } = await execA(cmd, { cwd })
+  /* Not always true: https://github.com/revealjs/reveal/issues/1440#issuecomment-1892831303
+  assert(res.stderr === '')
+  */
+  return stdout.toString().split("\n").filter(Boolean)
+}
+async function runCmd2(
+  cmd: string,
+  cwd: string
+): Promise<{ err: unknown } | { stdout: string; stderr: string }> {
+  let res: Awaited<ReturnType<typeof execA>>
+  try {
+    res = await execA(cmd, { cwd })
+  } catch (err) {
+    return { err }
+  }
+  let { stdout, stderr } = res
+  stdout = stdout.toString().trim()
+  stderr = stderr.toString().trim()
+  return { stdout, stderr }
+}

package/dist/core/processing/getters/getRevealConfig/filesystemRouting.spec.ts

@@ -0,0 +1,40 @@
+import {
+  getLocationId,
+  isInherited as isInherited_,
+  getLogicalPath as getLogicalPath_,
+  type LocationId,
+} from "./filesystemRouting.js";
+import { expect, describe, it } from "vitest";
+
+describe("getLocationId()", () => {
+  it("works", () => {
+    expect(getLocationId("/pages/some-page/+Page.js")).toBe("/pages/some-page");
+    expect(getLocationId("/renderer/+config.js")).toBe("/renderer");
+  });
+});
+
+describe("isInherited()", () => {
+  it("works", () => {
+    expect(isInherited("/pages/about", "/pages/product")).toBe(false);
+    expect(isInherited("/pages/about", "/pages/about/team")).toBe(true);
+    expect(isInherited("/pages/about/us", "/pages/about/team")).toBe(false);
+    expect(isInherited("/pages/about", "/pages/about2")).toBe(false);
+    expect(isInherited("/pages/about", "/pages/about-us")).toBe(false);
+    expect(isInherited("/pages/about", "/pages/about")).toBe(true);
+    expect(isInherited("/pages", "/pages/about")).toBe(true);
+    expect(isInherited("/renderer", "/pages/about")).toBe(true);
+  });
+});
+
+describe("getLogicalPath()", () => {
+  it("works", () => {
+    expect(getLogicalPath("/pages/some-page", ["pages"])).toBe("/some-page");
+  });
+});
+
+function isInherited(l1: string, l2: string) {
+  return isInherited_(l1 as LocationId, l2 as LocationId);
+}
+function getLogicalPath(l: string, a: string[]) {
+  return getLogicalPath_(l as LocationId, a);
+}