@learnpack/learnpack 5.0.353 → 5.0.355

package/src/utils/s3/packageSourcesAudit.ts

@@ -0,0 +1,311 @@
+import { ListObjectsV2Command } from "@aws-sdk/client-s3"
+import { AwsClient, withRetry } from "./packageManifestBackfill"
+
+/**
+ * Inventory of the files a published package needs for its
+ * `package-manifest.json` to be generated at full fidelity.
+ *
+ * This reads **key names only** — one paginated `ListObjectsV2` over the bucket,
+ * zero `GetObject`. Cheap enough to run over the whole catalogue on demand, and
+ * safe at any time: it never writes.
+ *
+ * What it answers: which packages would *fail* the manifest backfill (no
+ * `config.json`, no exercises), and which ones only produce a *degraded*
+ * manifest (no `initialSyllabus.json` → every lesson typed `READ`, ids derived
+ * from the folder name, and descriptions permanently `null`, because the
+ * descriptions pipeline builds its work list from the syllabus keys and cannot
+ * see a course that has none).
+ *
+ * Companion of `legacyPackageRepair.ts`, which writes what this reports missing.
+ */
+
+export type PackageSourcesAuditEntry = {
+  slug: string;
+  /** A published package is a prefix holding an `index.html` at its root. */
+  hasIndexHtml: boolean;
+  /** Required: without it the manifest backfill reports `failed`. */
+  hasConfigJson: boolean;
+  configJsonKey: string | null;
+  /**
+   * Optional *as a key*: `config.json` may carry the same metadata under
+   * `config`. Only a `GetObject` could tell, so `false` here means "resolves
+   * through the fallback, or not at all" — not "broken".
+   */
+  hasLearnJson: boolean;
+  hasSyllabus: boolean;
+  hasSidebar: boolean;
+  sidebarKey: string | null;
+  hasManifest: boolean;
+  /** Folders directly under `exercises/` holding at least one object. */
+  exerciseFolders: number;
+  /** `README.md` / `README.xx.md` files found in those folders. */
+  readmeCount: number;
+  /** Missing inputs, named as in the docs, for quick reporting. */
+  missing: string[];
+  /** True when the manifest backfill cannot produce a manifest at all. */
+  blocking: boolean;
+};
+
+export type PackageSourcesAuditSummary = {
+  missingConfigJson: number;
+  missingLearnJson: number;
+  missingSyllabus: number;
+  missingSidebar: number;
+  missingManifest: number;
+  noExerciseFolders: number;
+  blocking: number;
+};
+
+export type PackageSourcesAuditReport = {
+  generatedAt: string;
+  bucket: string;
+  scannedKeys: number;
+  totalPackages: number;
+  summary: PackageSourcesAuditSummary;
+  packages: PackageSourcesAuditEntry[];
+};
+
+export type AuditOptions = {
+  /** Restrict the scan to a single package (uses a prefixed listing). */
+  slug?: string;
+  /** Keep prefixes with no `index.html` (drafts, leftovers). Default false. */
+  includeUnpublished?: boolean;
+};
+
+const README_FILENAME_PATTERN = /^readme(?:\.[a-z]{2})?\.md$/i
+
+const ROOT_FILES = new Set([
+  "index.html",
+  "learn.json",
+  "config.json",
+  ".learn/config.json",
+  ".learn/initialSyllabus.json",
+  ".learn/sidebar.json",
+  "sidebar.json",
+  ".learn/package-manifest.json",
+])
+
+type PackageAccumulator = {
+  slug: string;
+  hasIndexHtml: boolean;
+  configJsonKey: string | null;
+  hasLearnJson: boolean;
+  hasSyllabus: boolean;
+  sidebarKey: string | null;
+  hasManifest: boolean;
+  exerciseFolders: Set<string>;
+  readmeCount: number;
+};
+
+function emptyAccumulator(slug: string): PackageAccumulator {
+  return {
+    slug,
+    hasIndexHtml: false,
+    configJsonKey: null,
+    hasLearnJson: false,
+    hasSyllabus: false,
+    sidebarKey: null,
+    hasManifest: false,
+    exerciseFolders: new Set<string>(),
+    readmeCount: 0,
+  }
+}
+
+/** Fold one object key into the accumulator map. */
+function classifyKey(
+  entries: Map<string, PackageAccumulator>,
+  key: string
+): void {
+  const separatorIndex = key.indexOf("/")
+  if (separatorIndex <= 0) {
+    // Object at the bucket root: not part of any package.
+    return
+  }
+
+  const slug = key.slice(0, separatorIndex)
+  const rest = key.slice(separatorIndex + 1)
+  if (!rest) {
+    return
+  }
+
+  const exerciseParts = rest.startsWith("exercises/") ? rest.split("/") : []
+  // `exercises/<slug>/<file>`: shorter paths are the folder marker itself.
+  const exerciseSlug =
+    exerciseParts.length >= 3 && exerciseParts[1] ? exerciseParts[1] : null
+
+  if (!ROOT_FILES.has(rest) && !exerciseSlug) {
+    return
+  }
+
+  let entry = entries.get(slug)
+  if (!entry) {
+    entry = emptyAccumulator(slug)
+    entries.set(slug, entry)
+  }
+
+  if (exerciseSlug) {
+    entry.exerciseFolders.add(exerciseSlug)
+    const filename = exerciseParts[exerciseParts.length - 1]
+    if (exerciseParts.length === 3 && README_FILENAME_PATTERN.test(filename)) {
+      entry.readmeCount += 1
+    }
+
+    return
+  }
+
+  switch (rest) {
+    case "index.html":
+      entry.hasIndexHtml = true
+      break
+    case "learn.json":
+      entry.hasLearnJson = true
+      break
+    // The root copy wins over `.learn/config.json`, matching the order
+    // `fetchPackageSources` resolves them in. Same for the sidebar, inverted.
+    case "config.json":
+      entry.configJsonKey = key
+      break
+    case ".learn/config.json":
+      entry.configJsonKey = entry.configJsonKey || key
+      break
+    case ".learn/sidebar.json":
+      entry.sidebarKey = key
+      break
+    case "sidebar.json":
+      entry.sidebarKey = entry.sidebarKey || key
+      break
+    case ".learn/initialSyllabus.json":
+      entry.hasSyllabus = true
+      break
+    case ".learn/package-manifest.json":
+      entry.hasManifest = true
+      break
+    default:
+      break
+  }
+}
+
+function finalizeEntry(entry: PackageAccumulator): PackageSourcesAuditEntry {
+  const missing: string[] = []
+
+  if (!entry.configJsonKey) {
+    missing.push("config.json")
+  }
+
+  if (!entry.hasLearnJson) {
+    missing.push("learn.json")
+  }
+
+  if (!entry.hasSyllabus) {
+    missing.push("initialSyllabus.json")
+  }
+
+  if (!entry.sidebarKey) {
+    missing.push("sidebar.json")
+  }
+
+  if (entry.exerciseFolders.size === 0) {
+    missing.push("exercises/")
+  }
+
+  if (!entry.hasManifest) {
+    missing.push("package-manifest.json")
+  }
+
+  return {
+    slug: entry.slug,
+    hasIndexHtml: entry.hasIndexHtml,
+    hasConfigJson: Boolean(entry.configJsonKey),
+    configJsonKey: entry.configJsonKey,
+    hasLearnJson: entry.hasLearnJson,
+    hasSyllabus: entry.hasSyllabus,
+    hasSidebar: Boolean(entry.sidebarKey),
+    sidebarKey: entry.sidebarKey,
+    hasManifest: entry.hasManifest,
+    exerciseFolders: entry.exerciseFolders.size,
+    readmeCount: entry.readmeCount,
+    missing,
+    // A missing `learn.json` is not blocking on its own: `config.config` can
+    // stand in for it, and only a download would tell.
+    blocking: !entry.configJsonKey || entry.exerciseFolders.size === 0,
+  }
+}
+
+function count(
+  packages: PackageSourcesAuditEntry[],
+  predicate: (entry: PackageSourcesAuditEntry) => boolean
+): number {
+  return packages.filter(entry => predicate(entry)).length
+}
+
+export function summarize(
+  packages: PackageSourcesAuditEntry[]
+): PackageSourcesAuditSummary {
+  return {
+    missingConfigJson: count(packages, entry => !entry.hasConfigJson),
+    missingLearnJson: count(packages, entry => !entry.hasLearnJson),
+    missingSyllabus: count(packages, entry => !entry.hasSyllabus),
+    missingSidebar: count(packages, entry => !entry.hasSidebar),
+    missingManifest: count(packages, entry => !entry.hasManifest),
+    noExerciseFolders: count(packages, entry => entry.exerciseFolders === 0),
+    blocking: count(packages, entry => entry.blocking),
+  }
+}
+
+/**
+ * One paginated pass over the bucket, folding each page into the accumulator.
+ *
+ * Deliberately not built on `listObjectKeys`: that one materializes every key in
+ * the catalogue before the caller sees the first one.
+ */
+export async function auditPackageSources(
+  s3: AwsClient,
+  bucket: string,
+  options: AuditOptions = {}
+): Promise<PackageSourcesAuditReport> {
+  const entries = new Map<string, PackageAccumulator>()
+  const prefix = options.slug ? `${options.slug}/` : undefined
+  let continuationToken: string | undefined
+  let scannedKeys = 0
+
+  do {
+    // eslint-disable-next-line no-await-in-loop -- S3 pagination is sequential
+    const response = (await withRetry(() =>
+      s3.send(
+        new ListObjectsV2Command({
+          Bucket: bucket,
+          Prefix: prefix,
+          ContinuationToken: continuationToken,
+        })
+      )
+    )) as {
+      Contents?: Array<{ Key?: string }>;
+      NextContinuationToken?: string;
+    }
+
+    for (const item of response.Contents || []) {
+      if (!item.Key) {
+        continue
+      }
+
+      scannedKeys += 1
+      classifyKey(entries, item.Key)
+    }
+
+    continuationToken = response.NextContinuationToken
+  } while (continuationToken)
+
+  const packages = [...entries.values()]
+    .filter(entry => options.includeUnpublished || entry.hasIndexHtml)
+    .map(entry => finalizeEntry(entry))
+    .sort((a, b) => a.slug.localeCompare(b.slug))
+
+  return {
+    generatedAt: new Date().toISOString(),
+    bucket,
+    scannedKeys,
+    totalPackages: packages.length,
+    summary: summarize(packages),
+    packages,
+  }
+}