@schemasentry/cli 0.3.2 → 0.4.0

package/README.md

@@ -56,6 +56,31 @@ pnpm schemasentry audit \
   --output ./report.html
 ```
 
+### `collect`
+
+Collect JSON-LD blocks from built HTML output and emit schema data JSON:
+
+```bash
+pnpm schemasentry collect --root ./out --output ./schema-sentry.data.json
+```
+
+Check collected output against your current data file (CI drift guard):
+
+```bash
+pnpm schemasentry collect --root ./out --check --data ./schema-sentry.data.json
+```
+
+Collect and compare only selected routes, failing if any required route is missing:
+
+```bash
+pnpm schemasentry collect \
+  --root ./out \
+  --routes / /blog /faq \
+  --strict-routes \
+  --check \
+  --data ./schema-sentry.data.json
+```
+
 ## Options
 
 | Option | Description |
@@ -63,6 +88,10 @@ pnpm schemasentry audit \
 | `--format json\|html` | Output format |
 | `--annotations none\|github` | CI annotations |
 | `-o, --output <path>` | Write output to file |
+| `--root <path>` | Root directory to scan for HTML output (`collect`) |
+| `--routes <routes...>` | Collect only specific routes (`collect`) |
+| `--strict-routes` | Fail when any route passed to `--routes` is missing (`collect`) |
+| `--check` | Compare collected output with existing data and fail on drift (`collect`) |
 | `--recommended / --no-recommended` | Enable recommended field checks |
 
 ## Documentation

package/dist/index.js

@@ -4,8 +4,8 @@
 import { Command } from "commander";
 import { mkdir, readFile, writeFile } from "fs/promises";
 import { readFileSync } from "fs";
-import path4 from "path";
-import { stableStringify as stableStringify2 } from "@schemasentry/core";
+import path5 from "path";
+import { stableStringify as stableStringify3 } from "@schemasentry/core";
 
 // src/report.ts
 import {
@@ -654,6 +654,229 @@ var emitGitHubAnnotations = (report, commandLabel) => {
   }
 };
 
+// src/collect.ts
+import { promises as fs4 } from "fs";
+import path4 from "path";
+import { stableStringify as stableStringify2 } from "@schemasentry/core";
+var IGNORED_DIRECTORIES = /* @__PURE__ */ new Set([".git", "node_modules", ".pnpm-store"]);
+var SCRIPT_TAG_REGEX = /<script\b([^>]*)>([\s\S]*?)<\/script>/gi;
+var JSON_LD_TYPE_REGEX = /\btype\s*=\s*(?:"application\/ld\+json"|'application\/ld\+json'|application\/ld\+json)/i;
+var collectSchemaData = async (options) => {
+  const rootDir = path4.resolve(options.rootDir);
+  const requestedRoutes = normalizeRouteFilter(options.routes ?? []);
+  const htmlFiles = (await walkHtmlFiles(rootDir)).sort((a, b) => a.localeCompare(b));
+  const routes = {};
+  const warnings = [];
+  let blockCount = 0;
+  let invalidBlocks = 0;
+  for (const filePath of htmlFiles) {
+    const route = filePathToRoute(rootDir, filePath);
+    if (!route) {
+      continue;
+    }
+    const html = await fs4.readFile(filePath, "utf8");
+    const extracted = extractSchemaNodes(html, filePath);
+    if (extracted.nodes.length > 0) {
+      routes[route] = [...routes[route] ?? [], ...extracted.nodes];
+      blockCount += extracted.nodes.length;
+    }
+    invalidBlocks += extracted.invalidBlocks;
+    warnings.push(...extracted.warnings);
+  }
+  const missingRoutes = [];
+  const filteredRoutes = requestedRoutes.length > 0 ? filterRoutesByAllowlist(routes, requestedRoutes) : routes;
+  if (requestedRoutes.length > 0) {
+    for (const route of requestedRoutes) {
+      if (!Object.prototype.hasOwnProperty.call(filteredRoutes, route)) {
+        missingRoutes.push(route);
+      }
+    }
+  }
+  const filteredBlockCount = Object.values(filteredRoutes).reduce(
+    (total, nodes) => total + nodes.length,
+    0
+  );
+  return {
+    data: {
+      routes: sortRoutes(filteredRoutes)
+    },
+    stats: {
+      htmlFiles: htmlFiles.length,
+      routes: Object.keys(filteredRoutes).length,
+      blocks: filteredBlockCount,
+      invalidBlocks
+    },
+    warnings,
+    requestedRoutes,
+    missingRoutes
+  };
+};
+var compareSchemaData = (existing, collected) => {
+  const existingRoutes = existing.routes ?? {};
+  const collectedRoutes = collected.routes ?? {};
+  const existingKeys = Object.keys(existingRoutes);
+  const collectedKeys = Object.keys(collectedRoutes);
+  const addedRoutes = collectedKeys.filter((route) => !Object.prototype.hasOwnProperty.call(existingRoutes, route)).sort();
+  const removedRoutes = existingKeys.filter((route) => !Object.prototype.hasOwnProperty.call(collectedRoutes, route)).sort();
+  const changedRoutes = existingKeys.filter((route) => Object.prototype.hasOwnProperty.call(collectedRoutes, route)).filter(
+    (route) => stableStringify2(existingRoutes[route]) !== stableStringify2(collectedRoutes[route])
+  ).sort();
+  const changedRouteDetails = changedRoutes.map(
+    (route) => buildRouteDriftDetail(route, existingRoutes[route] ?? [], collectedRoutes[route] ?? [])
+  );
+  return {
+    hasChanges: addedRoutes.length > 0 || removedRoutes.length > 0 || changedRoutes.length > 0,
+    addedRoutes,
+    removedRoutes,
+    changedRoutes,
+    changedRouteDetails
+  };
+};
+var formatSchemaDataDrift = (drift, maxRoutes = 5) => {
+  if (!drift.hasChanges) {
+    return "No schema data drift detected.";
+  }
+  const lines = [
+    `Schema data drift detected: added_routes=${drift.addedRoutes.length} removed_routes=${drift.removedRoutes.length} changed_routes=${drift.changedRoutes.length}`
+  ];
+  if (drift.addedRoutes.length > 0) {
+    lines.push(formatRoutePreview("Added routes", drift.addedRoutes, maxRoutes));
+  }
+  if (drift.removedRoutes.length > 0) {
+    lines.push(formatRoutePreview("Removed routes", drift.removedRoutes, maxRoutes));
+  }
+  if (drift.changedRoutes.length > 0) {
+    lines.push(formatRoutePreview("Changed routes", drift.changedRoutes, maxRoutes));
+    const details = drift.changedRouteDetails.slice(0, maxRoutes).map((detail) => formatRouteDriftDetail(detail));
+    if (details.length > 0) {
+      lines.push("Changed route details:");
+      for (const detail of details) {
+        lines.push(`- ${detail}`);
+      }
+    }
+  }
+  return lines.join("\n");
+};
+var formatRoutePreview = (label, routes, maxRoutes) => {
+  const preview = routes.slice(0, maxRoutes);
+  const suffix = routes.length > maxRoutes ? ` (+${routes.length - maxRoutes} more)` : "";
+  return `${label}: ${preview.join(", ")}${suffix}`;
+};
+var formatRouteDriftDetail = (detail) => {
+  const added = detail.addedTypes.length > 0 ? detail.addedTypes.join(",") : "(none)";
+  const removed = detail.removedTypes.length > 0 ? detail.removedTypes.join(",") : "(none)";
+  return `${detail.route} blocks ${detail.beforeBlocks}->${detail.afterBlocks} | +types ${added} | -types ${removed}`;
+};
+var sortRoutes = (routes) => Object.fromEntries(
+  Object.entries(routes).sort(([a], [b]) => a.localeCompare(b))
+);
+var filterRoutesByAllowlist = (routes, allowlist) => {
+  const filtered = {};
+  for (const route of allowlist) {
+    if (Object.prototype.hasOwnProperty.call(routes, route)) {
+      filtered[route] = routes[route];
+    }
+  }
+  return filtered;
+};
+var normalizeRouteFilter = (input2) => {
+  const normalized = input2.flatMap((entry) => entry.split(",")).map((route) => route.trim()).filter((route) => route.length > 0);
+  return Array.from(new Set(normalized)).sort();
+};
+var walkHtmlFiles = async (rootDir) => {
+  const entries = await fs4.readdir(rootDir, { withFileTypes: true });
+  const files = [];
+  for (const entry of entries) {
+    if (entry.isDirectory() && IGNORED_DIRECTORIES.has(entry.name)) {
+      continue;
+    }
+    const resolved = path4.join(rootDir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...await walkHtmlFiles(resolved));
+      continue;
+    }
+    if (entry.isFile() && entry.name.endsWith(".html")) {
+      files.push(resolved);
+    }
+  }
+  return files;
+};
+var filePathToRoute = (rootDir, filePath) => {
+  const relative = path4.relative(rootDir, filePath).replace(/\\/g, "/");
+  if (relative === "index.html") {
+    return "/";
+  }
+  if (relative.endsWith("/index.html")) {
+    return `/${relative.slice(0, -"/index.html".length)}`;
+  }
+  if (relative.endsWith(".html")) {
+    return `/${relative.slice(0, -".html".length)}`;
+  }
+  return null;
+};
+var extractSchemaNodes = (html, filePath) => {
+  const nodes = [];
+  const warnings = [];
+  let invalidBlocks = 0;
+  let scriptIndex = 0;
+  for (const match of html.matchAll(SCRIPT_TAG_REGEX)) {
+    scriptIndex += 1;
+    const attributes = match[1] ?? "";
+    if (!JSON_LD_TYPE_REGEX.test(attributes)) {
+      continue;
+    }
+    const scriptBody = (match[2] ?? "").trim();
+    if (!scriptBody) {
+      continue;
+    }
+    let parsed;
+    try {
+      parsed = JSON.parse(scriptBody);
+    } catch {
+      invalidBlocks += 1;
+      warnings.push({
+        file: filePath,
+        message: `Invalid JSON-LD block at script #${scriptIndex}`
+      });
+      continue;
+    }
+    const normalized = normalizeParsedBlock(parsed);
+    nodes.push(...normalized);
+  }
+  return { nodes, invalidBlocks, warnings };
+};
+var normalizeParsedBlock = (value) => {
+  if (Array.isArray(value)) {
+    return value.filter(isJsonObject);
+  }
+  if (!isJsonObject(value)) {
+    return [];
+  }
+  const graph = value["@graph"];
+  if (Array.isArray(graph)) {
+    return graph.filter(isJsonObject);
+  }
+  return [value];
+};
+var isJsonObject = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
+var buildRouteDriftDetail = (route, beforeNodes, afterNodes) => {
+  const beforeTypes = new Set(beforeNodes.map((node) => schemaTypeLabel(node)));
+  const afterTypes = new Set(afterNodes.map((node) => schemaTypeLabel(node)));
+  const addedTypes = Array.from(afterTypes).filter((type) => !beforeTypes.has(type)).sort();
+  const removedTypes = Array.from(beforeTypes).filter((type) => !afterTypes.has(type)).sort();
+  return {
+    route,
+    beforeBlocks: beforeNodes.length,
+    afterBlocks: afterNodes.length,
+    addedTypes,
+    removedTypes
+  };
+};
+var schemaTypeLabel = (node) => {
+  const type = node["@type"];
+  return typeof type === "string" && type.trim().length > 0 ? type : "(unknown)";
+};
+
 // src/index.ts
 import { createInterface } from "readline/promises";
 import { stdin as input, stdout as output } from "process";
@@ -672,8 +895,8 @@ program.command("validate").description("Validate schema coverage and rules").op
   const format = resolveOutputFormat(options.format);
   const annotationsMode = resolveAnnotationsMode(options.annotations);
   const recommended = await resolveRecommendedOption(options.config);
-  const manifestPath = path4.resolve(process.cwd(), options.manifest);
-  const dataPath = path4.resolve(process.cwd(), options.data);
+  const manifestPath = path5.resolve(process.cwd(), options.manifest);
+  const dataPath = path5.resolve(process.cwd(), options.data);
   let raw;
   try {
     raw = await readFile(manifestPath, "utf8");
@@ -760,12 +983,12 @@ program.command("init").description("Interactive setup wizard").option(
   "Path to schema data JSON",
   "schema-sentry.data.json"
 ).option("-y, --yes", "Use defaults and skip prompts").option("-f, --force", "Overwrite existing files").option("--scan", "Scan the filesystem for routes and add WebPage entries").option("--root <path>", "Project root for scanning", ".").action(async (options) => {
-  const manifestPath = path4.resolve(process.cwd(), options.manifest);
-  const dataPath = path4.resolve(process.cwd(), options.data);
+  const manifestPath = path5.resolve(process.cwd(), options.manifest);
+  const dataPath = path5.resolve(process.cwd(), options.data);
   const force = options.force ?? false;
   const useDefaults = options.yes ?? false;
   const answers = useDefaults ? getDefaultAnswers() : await promptAnswers();
-  const scannedRoutes = options.scan ? await scanRoutes({ rootDir: path4.resolve(process.cwd(), options.root ?? ".") }) : [];
+  const scannedRoutes = options.scan ? await scanRoutes({ rootDir: path5.resolve(process.cwd(), options.root ?? ".") }) : [];
   if (options.scan && scannedRoutes.length === 0) {
     console.error("No routes found during scan.");
   }
@@ -794,7 +1017,7 @@ program.command("audit").description("Analyze schema health and report issues").
   const format = resolveOutputFormat(options.format);
   const annotationsMode = resolveAnnotationsMode(options.annotations);
   const recommended = await resolveRecommendedOption(options.config);
-  const dataPath = path4.resolve(process.cwd(), options.data);
+  const dataPath = path5.resolve(process.cwd(), options.data);
   let dataRaw;
   try {
     dataRaw = await readFile(dataPath, "utf8");
@@ -830,7 +1053,7 @@ program.command("audit").description("Analyze schema health and report issues").
   }
   let manifest;
   if (options.manifest) {
-    const manifestPath = path4.resolve(process.cwd(), options.manifest);
+    const manifestPath = path5.resolve(process.cwd(), options.manifest);
     let manifestRaw;
     try {
       manifestRaw = await readFile(manifestPath, "utf8");
@@ -864,7 +1087,7 @@ program.command("audit").description("Analyze schema health and report issues").
       return;
     }
   }
-  const requiredRoutes = options.scan ? await scanRoutes({ rootDir: path4.resolve(process.cwd(), options.root ?? ".") }) : [];
+  const requiredRoutes = options.scan ? await scanRoutes({ rootDir: path5.resolve(process.cwd(), options.root ?? ".") }) : [];
   if (options.scan && requiredRoutes.length === 0) {
     console.error("No routes found during scan.");
   }
@@ -883,6 +1106,125 @@ program.command("audit").description("Analyze schema health and report issues").
   printAuditSummary(report, Boolean(manifest), Date.now() - start);
   process.exit(report.ok ? 0 : 1);
 });
+program.command("collect").description("Collect JSON-LD blocks from built HTML output").option("--root <path>", "Root directory to scan for HTML files", ".").option("--routes <routes...>", "Only collect specific routes (repeat or comma-separated)").option("--strict-routes", "Fail when any route passed via --routes is missing").option("--format <format>", "Output format (json)", "json").option("-o, --output <path>", "Write collected schema data to file").option("--check", "Compare collected output with an existing schema data file").option(
+  "-d, --data <path>",
+  "Path to existing schema data JSON for --check",
+  "schema-sentry.data.json"
+).action(async (options) => {
+  const start = Date.now();
+  const format = resolveCollectOutputFormat(options.format);
+  const rootDir = path5.resolve(process.cwd(), options.root ?? ".");
+  const check = options.check ?? false;
+  const requestedRoutes = normalizeRouteFilter(options.routes ?? []);
+  const strictRoutes = options.strictRoutes ?? false;
+  let collected;
+  try {
+    collected = await collectSchemaData({ rootDir, routes: requestedRoutes });
+  } catch (error) {
+    const reason = error instanceof Error && error.message.length > 0 ? error.message : "Unknown file system error";
+    printCliError(
+      "collect.scan_failed",
+      `Could not scan HTML output at ${rootDir}: ${reason}`,
+      "Point --root to a directory containing built HTML output."
+    );
+    process.exit(1);
+    return;
+  }
+  if (collected.stats.htmlFiles === 0) {
+    printCliError(
+      "collect.no_html",
+      `No HTML files found under ${rootDir}`,
+      "Point --root to a static output directory (for example ./out)."
+    );
+    process.exit(1);
+    return;
+  }
+  if (strictRoutes && collected.missingRoutes.length > 0) {
+    printCliError(
+      "collect.missing_required_routes",
+      `Required routes were not found in collected HTML: ${collected.missingRoutes.join(", ")}`,
+      "Rebuild output, adjust --root, or update --routes."
+    );
+    process.exit(1);
+    return;
+  }
+  let driftDetected = false;
+  if (check) {
+    const existingPath = path5.resolve(process.cwd(), options.data);
+    let existingRaw;
+    try {
+      existingRaw = await readFile(existingPath, "utf8");
+    } catch (error) {
+      printCliError(
+        "data.not_found",
+        `Schema data not found at ${existingPath}`,
+        "Run `schemasentry collect --output ./schema-sentry.data.json` to generate it."
+      );
+      process.exit(1);
+      return;
+    }
+    let existingData;
+    try {
+      existingData = JSON.parse(existingRaw);
+    } catch (error) {
+      printCliError(
+        "data.invalid_json",
+        "Schema data is not valid JSON",
+        "Check the JSON syntax or regenerate with `schemasentry collect --output`."
+      );
+      process.exit(1);
+      return;
+    }
+    if (!isSchemaData(existingData)) {
+      printCliError(
+        "data.invalid_shape",
+        "Schema data must contain a 'routes' object with array values",
+        "Ensure each route maps to an array of JSON-LD blocks."
+      );
+      process.exit(1);
+      return;
+    }
+    const existingDataForCompare = requestedRoutes.length > 0 ? filterSchemaDataByRoutes(existingData, requestedRoutes) : existingData;
+    const drift = compareSchemaData(existingDataForCompare, collected.data);
+    driftDetected = drift.hasChanges;
+    if (driftDetected) {
+      console.error(formatSchemaDataDrift(drift));
+    } else {
+      console.error("collect | No schema data drift detected.");
+    }
+  }
+  const content = formatCollectOutput(collected.data, format);
+  if (options.output) {
+    const resolvedPath = path5.resolve(process.cwd(), options.output);
+    try {
+      await mkdir(path5.dirname(resolvedPath), { recursive: true });
+      await writeFile(resolvedPath, `${content}
+`, "utf8");
+      console.error(`Collected data written to ${resolvedPath}`);
+    } catch (error) {
+      const reason = error instanceof Error && error.message.length > 0 ? error.message : "Unknown file system error";
+      printCliError(
+        "output.write_failed",
+        `Could not write collected data to ${resolvedPath}: ${reason}`
+      );
+      process.exit(1);
+      return;
+    }
+  } else if (!check) {
+    console.log(content);
+  }
+  printCollectWarnings(collected.warnings);
+  printCollectSummary({
+    stats: collected.stats,
+    durationMs: Date.now() - start,
+    checked: check,
+    driftDetected,
+    requestedRoutes: collected.requestedRoutes,
+    missingRoutes: collected.missingRoutes,
+    strictRoutes
+  });
+  process.exit(driftDetected ? 1 : 0);
+});
 function isManifest(value) {
   if (!value || typeof value !== "object") {
     return false;
@@ -939,11 +1281,30 @@ function resolveAnnotationsMode(value) {
   process.exit(1);
   return "none";
 }
+function resolveCollectOutputFormat(value) {
+  const format = (value ?? "json").trim().toLowerCase();
+  if (format === "json") {
+    return format;
+  }
+  printCliError(
+    "output.invalid_format",
+    `Unsupported collect output format '${value ?? ""}'`,
+    "Use --format json."
+  );
+  process.exit(1);
+  return "json";
+}
 function formatReportOutput(report, format, title) {
   if (format === "html") {
     return renderHtmlReport(report, { title });
   }
-  return stableStringify2(report);
+  return stableStringify3(report);
+}
+function formatCollectOutput(data, format) {
+  if (format === "json") {
+    return stableStringify3(data);
+  }
+  return stableStringify3(data);
 }
 async function emitReport(options) {
   const { report, format, outputPath, title } = options;
@@ -952,9 +1313,9 @@ async function emitReport(options) {
     console.log(content);
     return;
   }
-  const resolvedPath = path4.resolve(process.cwd(), outputPath);
+  const resolvedPath = path5.resolve(process.cwd(), outputPath);
   try {
-    await mkdir(path4.dirname(resolvedPath), { recursive: true });
+    await mkdir(path5.dirname(resolvedPath), { recursive: true });
     await writeFile(resolvedPath, content, "utf8");
     console.error(`Report written to ${resolvedPath}`);
   } catch (error) {
@@ -974,7 +1335,7 @@ function emitAnnotations(report, mode, commandLabel) {
 }
 function printCliError(code, message, suggestion) {
   console.error(
-    stableStringify2({
+    stableStringify3({
       ok: false,
       errors: [
         {
@@ -1040,6 +1401,61 @@ function printAuditSummary(report, coverageEnabled, durationMs) {
     console.error("Coverage checks skipped (no manifest provided).");
   }
 }
+function printCollectWarnings(warnings) {
+  if (warnings.length === 0) {
+    return;
+  }
+  const maxPrinted = 10;
+  console.error(`collect | Warnings: ${warnings.length}`);
+  for (const warning of warnings.slice(0, maxPrinted)) {
+    console.error(`- ${warning.file}: ${warning.message}`);
+  }
+  if (warnings.length > maxPrinted) {
+    console.error(`- ... ${warnings.length - maxPrinted} more warning(s)`);
+  }
+}
+function printCollectSummary(options) {
+  const {
+    stats,
+    durationMs,
+    checked,
+    driftDetected,
+    requestedRoutes,
+    missingRoutes,
+    strictRoutes
+  } = options;
+  const parts = [
+    `HTML files: ${stats.htmlFiles}`,
+    `Routes: ${stats.routes}`,
+    `Blocks: ${stats.blocks}`,
+    `Invalid blocks: ${stats.invalidBlocks}`,
+    `Duration: ${formatDuration(durationMs)}`
+  ];
+  if (checked) {
+    parts.push(`Check: ${driftDetected ? "drift_detected" : "clean"}`);
+  }
+  if (requestedRoutes.length > 0) {
+    parts.push(`Route filter: ${requestedRoutes.length}`);
+  }
+  if (missingRoutes.length > 0) {
+    parts.push(`Missing filtered routes: ${missingRoutes.length}`);
+  }
+  if (strictRoutes) {
+    parts.push("Strict routes: enabled");
+  }
+  console.error(`collect | ${parts.join(" | ")}`);
+}
+function filterSchemaDataByRoutes(data, routes) {
+  const filteredRoutes = {};
+  for (const route of routes) {
+    if (Object.prototype.hasOwnProperty.call(data.routes, route)) {
+      filteredRoutes[route] = data.routes[route];
+    }
+  }
+  return {
+    routes: filteredRoutes
+  };
+}
 async function promptAnswers() {
   const defaults = getDefaultAnswers();
   const rl = createInterface({ input, output });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@schemasentry/cli",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "CLI for Schema Sentry validation and reporting.",
   "license": "MIT",
   "type": "module",
@@ -33,7 +33,7 @@
   },
   "dependencies": {
     "commander": "^12.0.0",
-    "@schemasentry/core": "0.3.2"
+    "@schemasentry/core": "0.4.0"
   },
   "scripts": {
     "build": "tsup src/index.ts --format esm --dts --clean --tsconfig tsconfig.build.json",