@lightnet/cli 4.2.0 → 4.4.0

package/src/check-links.js

@@ -0,0 +1,456 @@
+// @ts-check
+
+import { posix, resolve } from "node:path"
+import { cwd as processCwd, stdin, stdout } from "node:process"
+
+import { intro, isCancel, log, outro, text } from "@clack/prompts"
+
+import {
+  cliConfigFileName,
+  readCliConfig,
+  writeCliConfig,
+} from "./support/cli-config.js"
+import { CliError } from "./support/cli-error.js"
+import { contentCollections } from "./support/content-collections.js"
+import { pathExists } from "./support/filesystem.js"
+
+const mediaDir = "src/content/media"
+const defaultConcurrency = 24
+const defaultTimeoutMs = 10_000
+const headFallbackStatuses = new Set([405, 501])
+const requestHeaders = {
+  accept: "*/*",
+  "user-agent": "@lightnet/cli link checker",
+}
+
+/** @typedef {import("./support/content-collections.js").MediaItem} MediaItem */
+
+/**
+ * @typedef {{
+ *   timeout?: string
+ * }} CheckLinksOptions
+ */
+
+/**
+ * @typedef {{
+ *   cwd?: string
+ *   fetch?: typeof fetch
+ *   isInteractive?: boolean
+ *   promptText?: (message: string) => Promise<string>
+ * }} CheckLinksRuntime
+ */
+
+/**
+ * @typedef {{
+ *   displayUrl: string
+ *   resolvedUrl: string
+ *   sources: Set<string>
+ * }} LinkReference
+ */
+
+/**
+ * @typedef {{
+ *   ok: boolean
+ *   reason?: string
+ *   status?: number
+ * }} LinkCheckResult
+ */
+
+/**
+ * @param {CheckLinksOptions} options
+ * @param {CheckLinksRuntime} [runtime]
+ */
+export async function checkLinks(options, runtime = {}) {
+  const cwd = runtime.cwd ?? processCwd()
+  const interactive =
+    runtime.isInteractive ?? Boolean(stdin.isTTY && stdout.isTTY)
+  const promptText =
+    runtime.promptText ?? (async (message) => defaultPromptText(message))
+  const runFetch = runtime.fetch ?? fetch
+  const timeoutMs = parsePositiveInteger(
+    options.timeout,
+    "--timeout",
+    defaultTimeoutMs,
+  )
+
+  intro("check-links")
+
+  await assertLightNetSiteRoot(cwd)
+
+  const collections = contentCollections(cwd)
+  const mediaItems = await collections.getMediaItems()
+  const siteUrl = await initSiteUrl({
+    cwd,
+    interactive,
+    mediaItems,
+    promptText,
+  })
+  const references = collectLinkReferences(mediaItems, siteUrl)
+  log.message(
+    `Checking ${references.length} unique links from ${mediaItems.length} media items.`,
+  )
+
+  if (references.length === 0) {
+    log.warn("No media content URLs found.")
+    outro("No issues found. 🎉")
+    return true
+  }
+
+  const results = await mapConcurrent(
+    references,
+    defaultConcurrency,
+    (reference) =>
+      checkLink(reference.resolvedUrl, { fetch: runFetch, timeoutMs }),
+  )
+  const checkedLinks = references
+    .map((reference, index) => ({ reference, result: results[index] }))
+    .sort((a, b) =>
+      a.reference.displayUrl.localeCompare(b.reference.displayUrl),
+    )
+  const protectedLinks = checkedLinks.filter((item) =>
+    isProtectedLink(item.result),
+  )
+  const unavailableLinks = checkedLinks.filter(
+    (item) => !item.result.ok && !isProtectedLink(item.result),
+  )
+
+  printProtectedLinks(protectedLinks)
+
+  if (unavailableLinks.length === 0) {
+    if (protectedLinks.length > 0) {
+      outro("No unavailable links found. Some links could not be verified. ⚠️")
+      return true
+    }
+    outro("No issues found. 🎉")
+    return true
+  }
+
+  log.error(`Unavailable links (${unavailableLinks.length})`)
+  for (const { reference, result } of unavailableLinks) {
+    log.message(
+      `· ${reference.displayUrl} (${formatFailure(result)}, referenced by ${[
+        ...reference.sources,
+      ]
+        .toSorted()
+        .join(", ")})`,
+    )
+  }
+
+  outro("Issues found. 🚧")
+
+  return false
+}
+
+/**
+ * @param {{
+ *   cwd: string
+ *   interactive: boolean
+ *   mediaItems: MediaItem[]
+ *   promptText: (message: string) => Promise<string>
+ * }} args
+ */
+async function initSiteUrl({ cwd, interactive, mediaItems, promptText }) {
+  const needsSiteUrl = mediaItems.some((item) =>
+    item.content.some((contentItem) => isRootRelativeUrl(contentItem.url)),
+  )
+  if (!needsSiteUrl) {
+    return undefined
+  }
+
+  const config = await readCliConfig(cwd)
+  const configuredSiteUrl = parseSiteUrl(config)
+  if (configuredSiteUrl) {
+    return configuredSiteUrl
+  }
+
+  if (!interactive) {
+    throw new CliError(
+      `Root-relative media content URLs require a site URL. Add "siteUrl" to "${cliConfigFileName}".`,
+    )
+  }
+
+  log.error(`Missing siteUrl in "${cliConfigFileName}".`)
+  while (true) {
+    const value = (await promptText("Site URL: ")).trim()
+    if (!value) {
+      continue
+    }
+    const siteUrl = normalizeSiteUrl(value, "Site URL")
+    await writeCliConfig(cwd, {
+      ...(isPlainObject(config) ? config : {}),
+      siteUrl,
+    })
+    return siteUrl
+  }
+}
+
+/**
+ * @param {unknown} config
+ */
+function parseSiteUrl(config) {
+  if (!isPlainObject(config)) {
+    return undefined
+  }
+  const siteUrl = /** @type {Record<string, unknown>} */ (config).siteUrl
+  if (typeof siteUrl !== "string" || !siteUrl.trim()) {
+    return undefined
+  }
+  return normalizeSiteUrl(siteUrl, `"${cliConfigFileName}" siteUrl`)
+}
+
+/**
+ * @param {MediaItem[]} mediaItems
+ * @param {string|undefined} siteUrl
+ */
+function collectLinkReferences(mediaItems, siteUrl) {
+  /** @type {Map<string, LinkReference>} */
+  const references = new Map()
+  for (const item of mediaItems) {
+    const sourceFileName = posix.basename(toPosixPath(item.path))
+    for (const contentItem of item.content) {
+      const resolvedUrl = resolveUrl(contentItem.url, siteUrl)
+      const current = references.get(resolvedUrl)
+      if (current) {
+        current.sources.add(sourceFileName)
+        if (contentItem.url.length < current.displayUrl.length) {
+          current.displayUrl = contentItem.url
+        }
+      } else {
+        references.set(resolvedUrl, {
+          displayUrl: contentItem.url,
+          resolvedUrl,
+          sources: new Set([sourceFileName]),
+        })
+      }
+    }
+  }
+  return [...references.values()].sort((a, b) =>
+    a.displayUrl.localeCompare(b.displayUrl),
+  )
+}
+
+/**
+ * @param {string} url
+ * @param {string|undefined} siteUrl
+ */
+function resolveUrl(url, siteUrl) {
+  if (isRootRelativeUrl(url)) {
+    if (!siteUrl) {
+      throw new CliError(`Root-relative URL "${url}" requires a site URL.`)
+    }
+    const parsed = new URL(url, siteUrl)
+    parsed.hash = ""
+    return parsed.toString()
+  }
+
+  let parsed
+  try {
+    parsed = new URL(url)
+  } catch {
+    throw new CliError(`Invalid media content URL "${url}".`)
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new CliError(
+      `Unsupported media content URL "${url}". Expected http or https.`,
+    )
+  }
+  parsed.hash = ""
+  return parsed.toString()
+}
+
+/**
+ * @param {string} url
+ * @param {{fetch: typeof fetch, timeoutMs: number}} options
+ * @returns {Promise<LinkCheckResult>}
+ */
+async function checkLink(url, options) {
+  const headResult = await fetchStatus(url, {
+    ...options,
+    headers: requestHeaders,
+    method: "HEAD",
+  })
+  if (headResult.ok || !headFallbackStatuses.has(headResult.status ?? 0)) {
+    return headResult
+  }
+  return fetchStatus(url, {
+    ...options,
+    headers: {
+      ...requestHeaders,
+      "accept-encoding": "identity",
+      range: "bytes=0-0",
+    },
+    method: "GET",
+  })
+}
+
+/**
+ * @param {{reference: LinkReference, result: LinkCheckResult}[]} links
+ */
+function printProtectedLinks(links) {
+  if (links.length === 0) {
+    return
+  }
+
+  log.warn(`Protected or forbidden links (${links.length})`)
+  for (const { reference, result } of links) {
+    log.message(
+      `· ${reference.displayUrl} (${formatFailure(result)}, referenced by ${[
+        ...reference.sources,
+      ]
+        .toSorted()
+        .join(", ")})`,
+    )
+  }
+}
+
+/**
+ * @param {string} url
+ * @param {{
+ *   fetch: typeof fetch
+ *   headers?: Record<string, string>
+ *   method: "GET"|"HEAD"
+ *   timeoutMs: number
+ * }} options
+ * @returns {Promise<LinkCheckResult>}
+ */
+async function fetchStatus(url, options) {
+  const controller = new AbortController()
+  const timeout = setTimeout(() => controller.abort(), options.timeoutMs)
+  try {
+    const response = await options.fetch(url, {
+      headers: options.headers,
+      method: options.method,
+      redirect: "follow",
+      signal: controller.signal,
+    })
+    await response.body?.cancel()
+    const ok = response.status >= 200 && response.status < 400
+    return { ok, status: response.status }
+  } catch (error) {
+    return {
+      ok: false,
+      reason: error instanceof Error ? error.message : "Request failed",
+    }
+  } finally {
+    clearTimeout(timeout)
+  }
+}
+
+/**
+ * @template T
+ * @template R
+ * @param {T[]} items
+ * @param {number} concurrency
+ * @param {(item: T) => Promise<R>} mapper
+ * @returns {Promise<R[]>}
+ */
+async function mapConcurrent(items, concurrency, mapper) {
+  /** @type {R[]} */
+  const results = []
+  let nextIndex = 0
+  const workerCount = Math.min(concurrency, items.length)
+  await Promise.all(
+    Array.from({ length: workerCount }, async () => {
+      while (true) {
+        const index = nextIndex
+        nextIndex += 1
+        const item = items[index]
+        if (item === undefined) {
+          return
+        }
+        results[index] = await mapper(item)
+      }
+    }),
+  )
+  return results
+}
+
+/**
+ * @param {string|undefined} rawValue
+ * @param {string} optionName
+ * @param {number} defaultValue
+ */
+function parsePositiveInteger(rawValue, optionName, defaultValue) {
+  if (rawValue === undefined) {
+    return defaultValue
+  }
+  const value = Number(rawValue)
+  if (!Number.isInteger(value) || value < 1) {
+    throw new CliError(`Expected "${optionName}" to be a positive integer.`)
+  }
+  return value
+}
+
+/**
+ * @param {string} value
+ * @param {string} label
+ */
+function normalizeSiteUrl(value, label) {
+  let parsed
+  try {
+    parsed = new URL(value)
+  } catch {
+    throw new CliError(`Invalid ${label}: expected an absolute http(s) URL.`)
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new CliError(`Invalid ${label}: expected an absolute http(s) URL.`)
+  }
+  return parsed.toString().replace(/\/+$/, "")
+}
+
+/**
+ * @param {string} cwd
+ */
+async function assertLightNetSiteRoot(cwd) {
+  if (!(await pathExists(resolve(cwd, mediaDir)))) {
+    throw new CliError(
+      'Expected a LightNet site root with "src/content/media". Run this command inside a LightNet site project.',
+    )
+  }
+}
+
+/**
+ * @param {string} url
+ */
+function isRootRelativeUrl(url) {
+  return url.startsWith("/") && !url.startsWith("//")
+}
+
+/**
+ * @param {LinkCheckResult} result
+ */
+function formatFailure(result) {
+  if (result.status !== undefined) {
+    return `status ${result.status}`
+  }
+  return result.reason ?? "request failed"
+}
+
+/**
+ * @param {LinkCheckResult} result
+ */
+function isProtectedLink(result) {
+  return !result.ok && result.status === 403
+}
+
+/**
+ * @param {string} filePath
+ */
+function toPosixPath(filePath) {
+  return filePath.split("\\").join("/")
+}
+
+/**
+ * @param {string} message
+ */
+async function defaultPromptText(message) {
+  const answer = await text({ message })
+  return isCancel(answer) ? "" : String(answer)
+}
+
+/**
+ * @param {unknown} value
+ */
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}

package/src/check-translations.js

@@ -1,9 +1,12 @@
 // @ts-check
 
+import { spawn } from "node:child_process"
 import { readFile } from "node:fs/promises"
 import { resolve } from "node:path"
 import { cwd } from "node:process"
 
+import { confirm, intro, log, outro, taskLog } from "@clack/prompts"
+
 /**
  * @typedef {{
  * type: "lightnet" | "user" | "map"
@@ -19,29 +22,46 @@ import { cwd } from "node:process"
  * }} Languages
  */
 
+/**
+ * @typedef {{
+ *   build?: boolean
+ * }} CheckTranslationsOptions
+ */
+
 const lightnetCachePath = resolve(cwd(), "node_modules", ".cache", "lightnet")
 
 /** @type {{type:Translation["type"], title:string, action:string}[]} */
 const translationSources = [
   {
     type: "lightnet",
-    title: "LightNet built-in translations",
+    title: "Missing LightNet built-in translations",
     action: "Add the missing entries in your /src/translations/*.yaml files.",
   },
   {
     type: "user",
-    title: "User translation files",
+    title: "Incomplete user translations",
     action: "Add the missing entries in your /src/translations/*.yaml files.",
   },
   {
     type: "map",
-    title: "Inline translation maps",
+    title: "Incomplete inline translation maps",
     action:
       "Update the inline translation map to include values for every configured site language.",
   },
 ]
 
-export async function checkTranslations() {
+/**
+ * @param {CheckTranslationsOptions} [options]
+ */
+export async function checkTranslations(options = {}) {
+  intro("check-translations")
+
+  const buildAvailable = await runBuild(options.build)
+  if (!buildAvailable) {
+    outro("Build failed. 🚧")
+    return false
+  }
+
   const translations = await readTranslations()
   const languages = await readLanguages()
   if (!translations || !languages || translations.length === 0) {
@@ -55,6 +75,7 @@ export async function checkTranslations() {
     .filter((translation) => translation.missingLocales.length > 0)
 
   if (incompleteTranslations.length === 0) {
+    outro("No issues found. 🎉")
     return true
   }
 
@@ -63,15 +84,80 @@ export async function checkTranslations() {
     (translation) => translation.type,
   )
 
-  console.log("Translation check failed")
+  log.error("Translation check failed")
   for (const source of translationSources) {
     printMissingTranslations(source, grouped[source.type])
   }
 
-  console.log("\n\n")
+  outro("Issues found. 🚧")
+
   return false
 }
 
+/**
+ *
+ * @param {boolean|undefined} build
+ * @returns
+ */
+async function runBuild(build) {
+  const shouldRunBuild =
+    build ??
+    (await confirm({
+      message:
+        "Run pnpm build now? Command requires an up-to-date dist/ directory.",
+      initialValue: false,
+    }))
+  if (!shouldRunBuild) {
+    return true
+  }
+  const buildLog = taskLog({
+    title: "Running pnpm build",
+  })
+
+  const child = spawn("pnpm", ["build"], {
+    shell: process.platform === "win32",
+  })
+
+  child.stdout?.setEncoding("utf8")
+  child.stderr?.setEncoding("utf8")
+
+  child.stdout?.on("data", (chunk) => {
+    for (const line of chunk.trimEnd().split("\n")) {
+      if (line) {
+        buildLog.message(line)
+      }
+    }
+  })
+
+  child.stderr?.on("data", (chunk) => {
+    for (const line of chunk.trimEnd().split("\n")) {
+      if (line) {
+        buildLog.message(line)
+      }
+    }
+  })
+
+  try {
+    await new Promise((resolve, reject) => {
+      child.on("error", reject)
+
+      child.on("close", (code) => {
+        if (code === 0) {
+          resolve(0)
+        } else {
+          reject(new Error(`pnpm build failed with exit code ${code}`))
+        }
+      })
+    })
+    buildLog.success("Build completed")
+    return true
+  } catch (e) {
+    buildLog.message(`${e}`)
+    buildLog.error("pnpm build failed")
+    return false
+  }
+}
+
 /**
  *
  * @param {{title:string, action:string}} source
@@ -82,8 +168,7 @@ function printMissingTranslations(source, translations) {
     return
   }
 
-  console.log()
-  console.log(source.title)
+  log.warn(source.title)
   translations
     .toSorted(
       (t1, t2) =>
@@ -91,11 +176,10 @@ function printMissingTranslations(source, translations) {
         t1.key.localeCompare(t2.key),
     )
     .forEach(({ key, missingLocales }) => {
-      console.log(`- ${key}`)
-      console.log(`  Missing locales: ${missingLocales.join(", ")}`)
+      log.message(`• ${key} > Missing: ${missingLocales.join(", ")}`)
     })
 
-  console.log(`Action: ${source.action}`)
+  log.message(`Action: ${source.action}`)
 }
 
 /**
@@ -122,10 +206,8 @@ async function readTranslations() {
       .filter((line) => line.trim())
       .map((line) => JSON.parse(line))
   } catch {
-    console.error("No translation build cache found.")
-    console.error(
-      "Action: Run build and try lightnet check-translations again.",
-    )
+    log.error("No translation build cache found.")
+    log.error("Action: Run build and try lightnet check-translations again.")
     return undefined
   }
 }
@@ -141,10 +223,8 @@ async function readLanguages() {
     )
     return JSON.parse(languagesText)
   } catch {
-    console.error("No language manifest found from the last build.")
-    console.error(
-      "Action: Run build and try lightnet check-translations again.",
-    )
+    log.error("No language manifest found from the last build.")
+    log.error("Action: Run build and try lightnet check-translations again.")
     return undefined
   }
 }

package/src/index.js

@@ -5,6 +5,7 @@ import { Command } from "commander"
 
 import pkg from "../package.json" with { type: "json" }
 import { checkFiles } from "./check-files.js"
+import { checkLinks } from "./check-links.js"
 import { checkTranslations } from "./check-translations.js"
 import { CliError } from "./support/cli-error.js"
 const { version } = pkg
@@ -20,8 +21,10 @@ program
 program
   .command("check-translations")
   .description("check if last build has been missing any translations")
-  .action(async () => {
-    const checkSuccessful = await checkTranslations()
+  .option("--build", "run pnpm build before checking translations")
+  .option("--no-build", "skip the build prompt and use the latest dist/ output")
+  .action(async (options) => {
+    const checkSuccessful = await checkTranslations(options)
     commandExitCode = checkSuccessful ? 0 : 1
   })
 
@@ -31,7 +34,7 @@ program
     "check for missing and orphaned content files and thumbnails in a LightNet site",
   )
   .option("--fix", "remove orphaned files")
-  .option("--yes", "skip deletion confirmation when used with --fix")
+  .option("--no-confirm", "skip deletion confirmation when used with --fix")
   .option(
     "--r2",
     "validate remote content files in Cloudflare R2 instead of public/files",
@@ -54,5 +57,23 @@ program
     }
   })
 
+program
+  .command("check-links")
+  .description("check media content links in a LightNet site")
+  .option("--timeout <ms>", "request timeout per link in milliseconds")
+  .action(async (options) => {
+    try {
+      const checkSuccessful = await checkLinks(options)
+      commandExitCode = checkSuccessful ? 0 : 1
+    } catch (error) {
+      if (error instanceof CliError) {
+        console.error(error.message)
+        commandExitCode = 1
+        return
+      }
+      throw error
+    }
+  })
+
 await program.parseAsync()
 process.exitCode = commandExitCode