@lightnet/cli 4.1.1 → 4.3.0

package/CHANGELOG.md

@@ -1,5 +1,25 @@
 # @lightnet/cli
 
+## 4.3.0
+
+### Minor Changes
+
+- [#427](https://github.com/LightNetDev/LightNet/pull/427) [`d34539d`](https://github.com/LightNetDev/LightNet/commit/d34539dc21d8e37d25cf65233ba0837675062fd7) - Add a `lightnet check-links` command that validates media content URLs without downloading linked files.
+
+- [#427](https://github.com/LightNetDev/LightNet/pull/427) [`d34539d`](https://github.com/LightNetDev/LightNet/commit/d34539dc21d8e37d25cf65233ba0837675062fd7) - check-translations will provide option to run `pnpm build`
+
+### Patch Changes
+
+- [#427](https://github.com/LightNetDev/LightNet/pull/427) [`d34539d`](https://github.com/LightNetDev/LightNet/commit/d34539dc21d8e37d25cf65233ba0837675062fd7) - Refactor check-files content storage handling.
+
+- [#427](https://github.com/LightNetDev/LightNet/pull/427) [`d34539d`](https://github.com/LightNetDev/LightNet/commit/d34539dc21d8e37d25cf65233ba0837675062fd7) - Improve output of check-translations
+
+## 4.2.0
+
+### Minor Changes
+
+- [#423](https://github.com/LightNetDev/LightNet/pull/423) [`30fabef`](https://github.com/LightNetDev/LightNet/commit/30fabef4897506cf4d0194b8883257de513ec452) - Add a new `lightnet check-files` command to validate orphaned and missing content files and thumbnails in LightNet site projects.
+
 ## 4.1.1
 
 ### Patch Changes

package/README.md

@@ -1,11 +1,11 @@
 # LightNet CLI
 
-Command-line tools for working with LightNet sites. The CLI currently focuses on translation validation, with more commands planned over time.
+Command-line tools for working with LightNet sites. The CLI currently includes translation validation, file consistency checks, and media link availability checks.
 
 ## Install
 
 ```bash
-npm install --save-dev @lightnet/cli
+pnpm add --D @lightnet/cli
 ```
 
 ## LightNet CLI reference

package/package.json

@@ -3,7 +3,7 @@
   "description": "CLI for managing LightNet projects",
   "type": "module",
   "license": "MIT",
-  "version": "4.1.1",
+  "version": "4.3.0",
   "publishConfig": {
     "access": "public"
   },
@@ -25,7 +25,8 @@
     "CHANGELOG.md"
   ],
   "dependencies": {
-    "commander": "^15.0.0"
+    "commander": "^15.0.0",
+    "@clack/prompts": "^1.6.0"
   },
   "engines": {
     "node": ">=22"

package/src/check-files.js

@@ -0,0 +1,526 @@
+// @ts-check
+
+import { join, posix, resolve } from "node:path"
+import { cwd as processCwd, stdin, stdout } from "node:process"
+
+import { confirm, intro, isCancel, log, outro, text } from "@clack/prompts"
+
+import { CliError } from "./support/cli-error.js"
+import { contentCollections } from "./support/content-collections.js"
+import {
+  contentFiles,
+  files,
+  pathExists,
+  toLocalContentDisplayPath,
+} from "./support/filesystem.js"
+import { createR2FileStorage } from "./support/r2.js"
+
+const supportedScopes = new Set(["content-files", "thumbnails"])
+
+const mediaDir = "src/content/media"
+const mediaImagesDir = "src/content/media/images"
+const categoryImagesDir = "src/content/categories/images"
+
+/**
+ * @typedef {{
+ *   scope?: string
+ *   fix?: boolean
+ *   yes?: boolean
+ *   r2?: boolean
+ * }} CheckFilesOptions
+ */
+
+/** @typedef {import("./support/content-collections.js").MediaItem} MediaItem */
+/** @typedef {import("./support/content-collections.js").Category} Category */
+/** @typedef {import("./support/filesystem.js").FileStorage} FileStorage */
+
+/**
+ * @typedef {{
+ *   path: string
+ *   sources: string[]
+ * }} MissingReference
+ */
+
+/**
+ * @typedef {{
+ *   displayPath: string
+ *   sources: Set<string>
+ * }} FileReference
+ */
+
+/**
+ * @typedef {{
+ *   cwd?: string
+ *   isInteractive?: boolean
+ *   promptText?: (message: string) => Promise<string>
+ *   promptConfirm?: (message: string) => Promise<boolean>
+ * }} CheckFilesRuntime
+ */
+
+/**
+ * @param {CheckFilesOptions} options
+ * @param {CheckFilesRuntime} [runtime]
+ */
+export async function checkFiles(options, runtime = {}) {
+  const cwd = runtime.cwd ?? processCwd()
+  const interactive =
+    runtime.isInteractive ?? Boolean(stdin.isTTY && stdout.isTTY)
+  const promptText =
+    runtime.promptText ?? (async (message) => defaultPromptText(message))
+  const promptConfirm =
+    runtime.promptConfirm ?? (async (message) => defaultPromptConfirm(message))
+
+  const scopes = parseScopes(options.scope)
+  const includeContentFiles = scopes.has("content-files")
+  const includeThumbnails = scopes.has("thumbnails")
+
+  intro("check-files")
+
+  await assertLightNetSiteRoot(cwd)
+
+  const collections = contentCollections(cwd)
+  const mediaItems = await collections.getMediaItems()
+  const categories = await collections.getCategories()
+  log.message(
+    `Checking ${mediaItems.length} media items and ${categories.length} categories.`,
+  )
+
+  /** @type {MissingReference[]} */
+  let missingContentFiles = []
+  /** @type {string[]} */
+  let orphanedContentFiles = []
+  /** @type {string[]} */
+  let orphanedMediaThumbnails = []
+  /** @type {string[]} */
+  let orphanedCategoryThumbnails = []
+  /** @type {string[]} */
+  let removedItems = []
+  /** @type {string[]} */
+  const warnings = []
+
+  /** @type {FileStorage|undefined} */
+  let contentFileStorage = undefined
+  const mediaThumbnailStorage = files(cwd, { rootDir: mediaImagesDir })
+  const categoryThumbnailStorage = files(cwd, { rootDir: categoryImagesDir })
+
+  if (includeContentFiles) {
+    if (options.r2) {
+      contentFileStorage = createR2FileStorage({
+        cwd,
+        interactive,
+        promptConfirm,
+        promptText,
+      })
+      const r2Result = await validateContentFiles({
+        mediaItems,
+        storage: contentFileStorage,
+      })
+      missingContentFiles = r2Result.missingReferences
+      orphanedContentFiles = r2Result.orphanedFiles
+      if (r2Result.referenceCount === 0) {
+        warnings.push(
+          'No R2-backed content file references found. Use "--scope=thumbnails" or check your "--r2" setup.',
+        )
+      }
+    } else {
+      contentFileStorage = contentFiles(cwd)
+      const localContentResult = await validateContentFiles({
+        mediaItems,
+        storage: contentFileStorage,
+      })
+      missingContentFiles = localContentResult.missingReferences
+      orphanedContentFiles = localContentResult.orphanedFiles
+      if (localContentResult.referenceCount === 0) {
+        warnings.push(
+          'No local "/files" content file references found. Use "--scope=thumbnails" or run with "--r2".',
+        )
+      }
+    }
+  }
+
+  if (includeThumbnails) {
+    const thumbnailsResult = await validateThumbnails(
+      mediaItems,
+      categories,
+      mediaThumbnailStorage,
+      categoryThumbnailStorage,
+    )
+    orphanedMediaThumbnails = thumbnailsResult.orphanedMediaThumbnails
+    orphanedCategoryThumbnails = thumbnailsResult.orphanedCategoryThumbnails
+  }
+
+  if (options.fix) {
+    /** @type {{displayPath:string, target:string}[]} */
+    const deletions = []
+    /** @type {{displayPath:string, target:string}[]} */
+    const contentDeletions = []
+    /** @type {{displayPath:string, target:string}[]} */
+    const localDeletions = []
+    for (const filePath of orphanedContentFiles) {
+      if (!contentFileStorage) {
+        throw new CliError("Missing file storage for content deletion.")
+      }
+      const deletion = {
+        displayPath: formatContentFilePath(filePath, options),
+        target: filePath,
+      }
+      contentDeletions.push(deletion)
+      deletions.push(deletion)
+    }
+    for (const filePath of orphanedMediaThumbnails) {
+      const deletion = {
+        displayPath: filePath,
+        target: filePath,
+      }
+      localDeletions.push(deletion)
+      deletions.push(deletion)
+    }
+    for (const filePath of orphanedCategoryThumbnails) {
+      const deletion = {
+        displayPath: filePath,
+        target: filePath,
+      }
+      localDeletions.push(deletion)
+      deletions.push(deletion)
+    }
+
+    if (deletions.length > 0) {
+      if (!options.yes && !interactive) {
+        throw new CliError(
+          'Deletion requires confirmation. Re-run with "--fix --yes" in non-interactive environments.',
+        )
+      }
+      const shouldDelete =
+        options.yes ||
+        (await confirmDeletion({
+          deletions,
+          promptConfirm,
+        }))
+
+      if (shouldDelete) {
+        const deletedContentTargets = contentFileStorage
+          ? await contentFileStorage.delete(
+              contentDeletions.map((deletion) => deletion.target),
+            )
+          : []
+        const deletedLocalTargets = [
+          ...(await mediaThumbnailStorage.delete(
+            localDeletions
+              .filter((deletion) => deletion.target.startsWith(mediaImagesDir))
+              .map((deletion) => deletion.target),
+          )),
+          ...(await categoryThumbnailStorage.delete(
+            localDeletions
+              .filter((deletion) =>
+                deletion.target.startsWith(categoryImagesDir),
+              )
+              .map((deletion) => deletion.target),
+          )),
+        ]
+        const deletedTargets = new Set([
+          ...deletedContentTargets,
+          ...deletedLocalTargets,
+        ])
+        for (const deletion of [...contentDeletions, ...localDeletions]) {
+          if (deletedTargets.has(deletion.target)) {
+            removedItems.push(deletion.displayPath)
+          } else {
+            log.error(`Failed to delete "${deletion.displayPath}".`)
+          }
+        }
+        orphanedContentFiles = orphanedContentFiles.filter(
+          (item) => !deletedTargets.has(item),
+        )
+        orphanedMediaThumbnails = orphanedMediaThumbnails.filter(
+          (item) => !deletedTargets.has(item),
+        )
+        orphanedCategoryThumbnails = orphanedCategoryThumbnails.filter(
+          (item) => !deletedTargets.has(item),
+        )
+      }
+    }
+  }
+
+  const hasIssues =
+    warnings.length > 0 ||
+    missingContentFiles.length > 0 ||
+    orphanedContentFiles.length > 0 ||
+    orphanedMediaThumbnails.length > 0 ||
+    orphanedCategoryThumbnails.length > 0
+
+  if (!hasIssues && removedItems.length === 0) {
+    outro("No issues found. 🎉")
+    return true
+  }
+
+  for (const warning of warnings) {
+    log.warn(warning)
+  }
+
+  printMissingReferenceSection(
+    "Missing referenced content files",
+    missingContentFiles,
+  )
+  printSection(
+    options.r2 ? "Orphaned R2 objects" : "Orphaned local content files",
+    orphanedContentFiles.map((filePath) =>
+      formatContentFilePath(filePath, options),
+    ),
+  )
+  printSection("Orphaned media thumbnails", orphanedMediaThumbnails)
+  printSection("Orphaned category thumbnails", orphanedCategoryThumbnails)
+  printSection("Removed items", removedItems)
+
+  outro(hasIssues ? "Issues found. 🚧" : "Cleanup complete. 🧹")
+
+  return !hasIssues
+}
+
+/**
+ * @param {string} title
+ * @param {string[]} items
+ */
+function printSection(title, items) {
+  if (items.length === 0) {
+    return
+  }
+  log.warn(`${title} (${items.length})`)
+  for (const item of items) {
+    log.message(`• ${item}`)
+  }
+}
+
+/**
+ * @param {string} title
+ * @param {MissingReference[]} items
+ */
+function printMissingReferenceSection(title, items) {
+  if (items.length === 0) {
+    return
+  }
+  log.error(`${title} (${items.length})`)
+  for (const item of items) {
+    log.message(
+      `• ${item.path} (referenced by ${item.sources.toSorted().join(", ")})`,
+    )
+  }
+}
+
+/**
+ * @param {string|undefined} rawScope
+ */
+function parseScopes(rawScope) {
+  const values = (rawScope ?? "content-files,thumbnails")
+    .split(",")
+    .map((value) => value.trim())
+    .filter(Boolean)
+
+  if (values.length === 0) {
+    throw new CliError(
+      // intentionally kept as a CLI-facing validation error
+      'Expected "--scope" to include at least one of "content-files" or "thumbnails".',
+    )
+  }
+
+  const scopes = new Set(values)
+  for (const scope of scopes) {
+    if (!supportedScopes.has(scope)) {
+      throw new CliError(
+        `Unsupported scope "${scope}". Allowed values: content-files, thumbnails.`,
+      )
+    }
+  }
+
+  return scopes
+}
+
+/**
+ * @param {string} cwd
+ */
+async function assertLightNetSiteRoot(cwd) {
+  if (!(await pathExists(resolve(cwd, mediaDir)))) {
+    throw new CliError(
+      'Expected a LightNet site root with "src/content/media". Run this command inside a LightNet site project.',
+    )
+  }
+}
+
+/**
+ * @param {MediaItem[]} mediaItems
+ * @param {Category[]} categories
+ * @param {FileStorage} mediaThumbnailStorage
+ * @param {FileStorage} categoryThumbnailStorage
+ */
+async function validateThumbnails(
+  mediaItems,
+  categories,
+  mediaThumbnailStorage,
+  categoryThumbnailStorage,
+) {
+  const mediaReferences = new Set(
+    mediaItems
+      .map((item) => toThumbnailPath("media", item.image))
+      .filter((value) => value !== undefined),
+  )
+  const categoryReferences = new Set(
+    categories
+      .map((item) => toThumbnailPath("categories", item.image))
+      .filter((value) => value !== undefined),
+  )
+
+  const initializedMediaStorage = await mediaThumbnailStorage.init()
+  const initializedCategoryStorage = await categoryThumbnailStorage.init()
+  const mediaFiles = await initializedMediaStorage.list()
+  const categoryFiles = await initializedCategoryStorage.list()
+
+  return {
+    orphanedMediaThumbnails: mediaFiles.filter(
+      (file) => !mediaReferences.has(file),
+    ),
+    orphanedCategoryThumbnails: categoryFiles.filter(
+      (file) => !categoryReferences.has(file),
+    ),
+  }
+}
+
+/**
+ * @param {{
+ *   mediaItems: MediaItem[]
+ *   storage: FileStorage
+ * }} args
+ */
+async function validateContentFiles({ mediaItems, storage }) {
+  const initializedStorage = await storage.init()
+  const references = collectContentReferences(mediaItems, initializedStorage)
+  if (references.size === 0) {
+    return {
+      missingReferences: /** @type {MissingReference[]} */ ([]),
+      orphanedFiles: /** @type {string[]} */ ([]),
+      referenceCount: 0,
+    }
+  }
+
+  const files = await initializedStorage.list()
+  const fileSet = new Set(files)
+
+  const missingReferences = [...references.entries()]
+    .filter(([path]) => !fileSet.has(path))
+    .map(([, reference]) => ({
+      path: reference.displayPath,
+      sources: [...reference.sources].toSorted(),
+    }))
+    .sort((a, b) => a.path.localeCompare(b.path))
+
+  const orphanedFiles = files
+    .filter((filePath) => !references.has(filePath))
+    .sort()
+
+  return {
+    missingReferences,
+    orphanedFiles,
+    referenceCount: references.size,
+  }
+}
+
+/**
+ * @param {MediaItem[]} mediaItems
+ * @param {FileStorage} storage
+ */
+function collectContentReferences(mediaItems, storage) {
+  const references = new Map()
+  for (const item of mediaItems) {
+    const sourceFileName = posix.basename(toPosixPath(item.path))
+    for (const contentItem of item.content) {
+      if (contentItem.type !== "upload") {
+        continue
+      }
+      const filePath = storage.toPath(contentItem.url)
+      if (!filePath) {
+        continue
+      }
+      const current = references.get(filePath)
+      if (current) {
+        current.sources.add(sourceFileName)
+      } else {
+        references.set(filePath, {
+          displayPath: contentItem.url,
+          sources: new Set([sourceFileName]),
+        })
+      }
+    }
+  }
+  return references
+}
+
+/**
+ * @param {"media"|"categories"} kind
+ * @param {string|undefined} image
+ */
+function toThumbnailPath(kind, image) {
+  if (typeof image !== "string") {
+    return undefined
+  }
+  const relativePath = image.startsWith("./images/")
+    ? image.slice("./images/".length)
+    : image.startsWith("images/")
+      ? image.slice("images/".length)
+      : undefined
+  if (
+    !relativePath ||
+    relativePath.startsWith("/") ||
+    relativePath.includes("../")
+  ) {
+    return undefined
+  }
+  return kind === "media"
+    ? toPosixPath(join(mediaImagesDir, relativePath))
+    : toPosixPath(join(categoryImagesDir, relativePath))
+}
+
+/**
+ * @param {string} path
+ * @param {CheckFilesOptions} options
+ */
+function formatContentFilePath(path, options) {
+  return options.r2 ? path : toLocalContentDisplayPath(path)
+}
+
+/**
+ * @param {{
+ *   deletions: {displayPath:string, target:string}[]
+ *   promptConfirm: (message:string)=>Promise<boolean>
+ * }} args
+ */
+async function confirmDeletion({ deletions, promptConfirm }) {
+  log.warn(`Files to remove (${deletions.length})`)
+  for (const deletion of deletions) {
+    log.message(`• ${deletion.displayPath}`)
+  }
+  return promptConfirm("Delete these files? [y/N] ")
+}
+
+/**
+ * @param {string} filePath
+ */
+function toPosixPath(filePath) {
+  return filePath.split("\\").join("/")
+}
+
+/**
+ * @param {string} message
+ */
+async function defaultPromptText(message) {
+  const answer = await text({ message })
+  return isCancel(answer) ? "" : String(answer)
+}
+
+/**
+ * @param {string} message
+ */
+async function defaultPromptConfirm(message) {
+  const answer = await confirm({
+    message,
+    initialValue: false,
+  })
+  return isCancel(answer) ? false : answer
+}