@topogram/cli 0.3.79 → 0.3.80

package/src/cli/help.js

@@ -38,7 +38,7 @@ export function printUsage(options = {}) {
   console.log("   or: topogram package update-cli <version|--latest> [--json]");
   console.log("   or: topogram copy <source> <target> [--version <version>] [--catalog <path-or-source>] [--json]");
   console.log("   or: topogram copy --list [--json] [--catalog <path-or-source>]");
-  console.log("   or: topogram extract <app-path> --out <target> [--from <track[,track]>] [--json]");
+  console.log("   or: topogram extract <app-path> --out <target> [--from <track[,track]>] [--extractor <id-or-package-or-path>] [--json]");
   console.log("   or: topogram extract refresh [path] [--from <app-path>] [--dry-run] [--json]");
   console.log("   or: topogram extract diff [path] [--json]");
   console.log("   or: topogram extract check [path] [--json]");
@@ -67,6 +67,14 @@ export function printUsage(options = {}) {
   console.log("   or: topogram generator policy check [path] [--json]");
   console.log("   or: topogram generator policy explain [path] [--json]");
   console.log("   or: topogram generator policy pin [package@version] [path] [--json]");
+  console.log("   or: topogram extractor list [--json]");
+  console.log("   or: topogram extractor show <id-or-package> [--json]");
+  console.log("   or: topogram extractor check <path-or-package> [--json]");
+  console.log("   or: topogram extractor policy init [path] [--json]");
+  console.log("   or: topogram extractor policy status [path] [--json]");
+  console.log("   or: topogram extractor policy check [path] [--json]");
+  console.log("   or: topogram extractor policy explain [path] [--json]");
+  console.log("   or: topogram extractor policy pin [package@version] [path] [--json]");
   console.log("   or: topogram init [path] [--with-sdlc] [--json]");
   console.log("");
   console.log("Common commands:");
@@ -98,6 +106,9 @@ export function printUsage(options = {}) {
   console.log("  topogram generator show @topogram/generator-react-web");
   console.log("  topogram generator check ./generator-package");
   console.log("  topogram generator policy check");
+  console.log("  topogram extractor list");
+  console.log("  topogram extractor check ./extractor-package");
+  console.log("  topogram extractor policy check");
   console.log("  topogram generate");
   console.log("  topogram extract ./existing-app --out ./extracted-topogram");
   console.log("  topogram extract diff ./extracted-topogram");

package/src/cli/options.js

@@ -22,6 +22,21 @@ export function optionValueIfPresent(args, flag) {
   return value && !value.startsWith("-") ? value : null;
 }
 
+/**
+ * @param {string[]} args
+ * @param {string} flag
+ * @returns {string[]}
+ */
+export function optionValues(args, flag) {
+  const values = [];
+  for (let index = 0; index < args.length; index += 1) {
+    if (args[index] === flag && args[index + 1] && !args[index + 1].startsWith("-")) {
+      values.push(args[index + 1]);
+    }
+  }
+  return values;
+}
+
 /**
  * @param {string[]} args
  * @param {string} flag
@@ -51,6 +66,8 @@ export function parseCliOptions(args, commandArgs) {
     workflowName: commandArgs?.workflowName || (!generateTarget && workflowFlagValue ? workflowFlagValue : null),
     workflowId: generateTarget ? workflowFlagValue : null,
     fromValue: optionValue(args, "--from"),
+    extractorSpecs: optionValues(args, "--extractor"),
+    extractorPolicyPath: optionValueIfPresent(args, "--extractor-policy"),
     adoptValue: commandArgs?.adoptValue || optionValue(args, "--adopt"),
     reasonValue: optionValueIfPresent(args, "--reason"),
     modeId: optionValue(args, "--mode"),

package/src/extractor/check.js

@@ -0,0 +1,155 @@
+// @ts-check
+
+import {
+  createExtractorSmokeContext,
+  loadExtractorPackageAdapterForSpec,
+  validateExtractorAdapter
+} from "./packages.js";
+
+/**
+ * @typedef {import("./registry.js").ExtractorManifest} ExtractorManifest
+ */
+
+/**
+ * @typedef {Object} ExtractorCheckResult
+ * @property {boolean} ok
+ * @property {string} sourceSpec
+ * @property {"path"|"package"} source
+ * @property {string|null} packageName
+ * @property {string|null} packageRoot
+ * @property {string|null} manifestPath
+ * @property {ExtractorManifest|null} manifest
+ * @property {Array<{ name: string, ok: boolean, message: string }>} checks
+ * @property {string[]} errors
+ * @property {{ extractors: number, findings: number, candidateKeys: number, diagnostics: number }|null} smoke
+ * @property {boolean} executesPackageCode
+ */
+
+/**
+ * @param {any} result
+ * @returns {{ ok: boolean, message: string, smoke: { findings: number, candidateKeys: number, diagnostics: number }|null }}
+ */
+function validateExtractResult(result) {
+  if (!result || typeof result !== "object" || Array.isArray(result)) {
+    return { ok: false, message: "extract(context) must return an object", smoke: null };
+  }
+  if (result.findings != null && !Array.isArray(result.findings)) {
+    return { ok: false, message: "extract(context) findings must be an array when present", smoke: null };
+  }
+  if (result.diagnostics != null && !Array.isArray(result.diagnostics)) {
+    return { ok: false, message: "extract(context) diagnostics must be an array when present", smoke: null };
+  }
+  if (!result.candidates || typeof result.candidates !== "object" || Array.isArray(result.candidates)) {
+    return { ok: false, message: "extract(context) result must include a candidates object", smoke: null };
+  }
+  for (const [key, value] of Object.entries(result.candidates)) {
+    if (!Array.isArray(value)) {
+      return { ok: false, message: `extract(context) candidates.${key} must be an array`, smoke: null };
+    }
+  }
+  return {
+    ok: true,
+    message: `extract(context) returned ${Object.keys(result.candidates).length} candidate bucket(s)`,
+    smoke: {
+      findings: Array.isArray(result.findings) ? result.findings.length : 0,
+      candidateKeys: Object.keys(result.candidates).length,
+      diagnostics: Array.isArray(result.diagnostics) ? result.diagnostics.length : 0
+    }
+  };
+}
+
+/**
+ * @param {string} sourceSpec
+ * @param {{ cwd?: string }} [options]
+ * @returns {ExtractorCheckResult}
+ */
+export function checkExtractorPack(sourceSpec, options = {}) {
+  /** @type {ExtractorCheckResult} */
+  const payload = {
+    ok: false,
+    sourceSpec,
+    source: "package",
+    packageName: null,
+    packageRoot: null,
+    manifestPath: null,
+    manifest: null,
+    checks: [],
+    errors: [],
+    smoke: null,
+    executesPackageCode: true
+  };
+  if (!sourceSpec || sourceSpec.startsWith("-")) {
+    payload.errors.push("Usage: topogram extractor check <path-or-package>");
+    payload.checks.push({ name: "source", ok: false, message: payload.errors[0] });
+    return payload;
+  }
+
+  const loaded = loadExtractorPackageAdapterForSpec(sourceSpec, options);
+  payload.source = loaded.source;
+  payload.packageName = loaded.packageName;
+  payload.packageRoot = loaded.packageRoot;
+  payload.manifestPath = loaded.manifestPath;
+  payload.manifest = loaded.manifest;
+  if (!loaded.manifest) {
+    payload.errors.push(...loaded.errors);
+    payload.checks.push({ name: "manifest-load", ok: false, message: loaded.errors.join(" ") || "Could not load extractor manifest." });
+    return payload;
+  }
+  payload.checks.push({ name: "manifest-load", ok: true, message: loaded.manifestPath || sourceSpec });
+  if (!loaded.adapter || loaded.errors.length > 0) {
+    payload.errors.push(...loaded.errors);
+    payload.checks.push({ name: "adapter-load", ok: false, message: loaded.errors.join(" ") || "Could not load extractor adapter." });
+    return payload;
+  }
+  payload.checks.push({ name: "adapter-load", ok: true, message: "Adapter export loaded." });
+
+  const adapterValidation = validateExtractorAdapter(loaded.adapter, loaded.manifest);
+  payload.checks.push({
+    name: "adapter-shape",
+    ok: adapterValidation.errors.length === 0,
+    message: adapterValidation.errors.length === 0 ? "Adapter shape is valid." : adapterValidation.errors.join(" ")
+  });
+  if (adapterValidation.errors.length > 0) {
+    payload.errors.push(...adapterValidation.errors);
+    return payload;
+  }
+
+  const context = createExtractorSmokeContext();
+  let totalFindings = 0;
+  let totalCandidateKeys = 0;
+  let totalDiagnostics = 0;
+  for (const extractor of adapterValidation.extractors) {
+    try {
+      const detection = extractor.detect(context) || { score: 0, reasons: [] };
+      if (!detection || typeof detection !== "object" || typeof detection.score !== "number") {
+        payload.errors.push(`Extractor '${extractor.id}' detect(context) must return { score, reasons }.`);
+        continue;
+      }
+      const result = extractor.extract(context) || { findings: [], candidates: {} };
+      const validation = validateExtractResult(result);
+      if (!validation.ok || !validation.smoke) {
+        payload.errors.push(`Extractor '${extractor.id}' ${validation.message}.`);
+        continue;
+      }
+      totalFindings += validation.smoke.findings;
+      totalCandidateKeys += validation.smoke.candidateKeys;
+      totalDiagnostics += validation.smoke.diagnostics;
+    } catch (error) {
+      payload.errors.push(`Extractor '${extractor.id}' smoke failed: ${error instanceof Error ? error.message : String(error)}`);
+    }
+  }
+  payload.checks.push({
+    name: "smoke-extract",
+    ok: payload.errors.length === 0,
+    message: payload.errors.length === 0 ? `Ran ${adapterValidation.extractors.length} extractor smoke check(s).` : payload.errors.join(" ")
+  });
+  payload.smoke = {
+    extractors: adapterValidation.extractors.length,
+    findings: totalFindings,
+    candidateKeys: totalCandidateKeys,
+    diagnostics: totalDiagnostics
+  };
+  payload.ok = payload.errors.length === 0;
+  return payload;
+}
+

package/src/extractor/packages.js

@@ -0,0 +1,295 @@
+// @ts-check
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  isPathSpec,
+  loadInstalledPackageAdapter,
+  loadLocalPackageAdapter,
+  packageNameFromSpec
+} from "../package-adapters/index.js";
+import {
+  EXTRACTOR_MANIFESTS,
+  getExtractorManifest,
+  loadPackageExtractorManifest,
+  validateExtractorManifest
+} from "./registry.js";
+import {
+  effectiveExtractorPolicy,
+  extractorPackageAllowed,
+  extractorPolicyDiagnosticsForPackages,
+  loadExtractorPolicy
+} from "../extractor-policy.js";
+import { createImportContext } from "../import/core/context.js";
+import { getBundledExtractorById, getBundledExtractorPack } from "../import/core/registry.js";
+
+/**
+ * @typedef {import("./registry.js").ExtractorManifest} ExtractorManifest
+ */
+
+/**
+ * @param {string} packageRoot
+ * @returns {string|null}
+ */
+function packageNameForRoot(packageRoot) {
+  const packageJsonPath = path.join(packageRoot, "package.json");
+  if (!fs.existsSync(packageJsonPath)) {
+    return null;
+  }
+  try {
+    const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
+    return typeof packageJson.name === "string" && packageJson.name.length > 0 ? packageJson.name : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * @param {any} adapter
+ * @param {ExtractorManifest} manifest
+ * @returns {{ extractors: any[], errors: string[] }}
+ */
+export function validateExtractorAdapter(adapter, manifest) {
+  const errors = [];
+  if (!adapter || typeof adapter !== "object" || Array.isArray(adapter)) {
+    return { extractors: [], errors: ["Extractor adapter must export an object."] };
+  }
+  const adapterManifest = adapter.manifest;
+  if (!adapterManifest || adapterManifest.id !== manifest.id || adapterManifest.version !== manifest.version) {
+    errors.push("Extractor adapter must export manifest matching topogram-extractor.json.");
+  }
+  if (!Array.isArray(adapter.extractors) || adapter.extractors.length === 0) {
+    errors.push("Extractor adapter must export a non-empty extractors array.");
+  }
+  const manifestExtractorIds = new Set(manifest.extractors || []);
+  const manifestTracks = new Set(manifest.tracks || []);
+  const extractors = Array.isArray(adapter.extractors) ? adapter.extractors : [];
+  for (const extractor of extractors) {
+    if (!extractor || typeof extractor !== "object") {
+      errors.push("Extractor entries must be objects.");
+      continue;
+    }
+    if (typeof extractor.id !== "string" || extractor.id.length === 0) {
+      errors.push("Extractor entries must have a non-empty id.");
+    } else if (!manifestExtractorIds.has(extractor.id)) {
+      errors.push(`Extractor '${extractor.id}' is not declared by manifest extractors.`);
+    }
+    if (typeof extractor.track !== "string" || !manifestTracks.has(extractor.track)) {
+      errors.push(`Extractor '${extractor.id || "unknown"}' has track '${extractor.track || "unknown"}' not declared by manifest tracks.`);
+    }
+    if (typeof extractor.detect !== "function") {
+      errors.push(`Extractor '${extractor.id || "unknown"}' must expose detect(context).`);
+    }
+    if (typeof extractor.extract !== "function") {
+      errors.push(`Extractor '${extractor.id || "unknown"}' must expose extract(context).`);
+    }
+  }
+  return { extractors, errors };
+}
+
+/**
+ * @param {string} sourceSpec
+ * @param {{ cwd?: string }} [options]
+ * @returns {{ source: "path"|"package", packageName: string|null, packageRoot: string|null, manifestPath: string|null, manifest: ExtractorManifest|null, errors: string[] }}
+ */
+export function loadExtractorPackageManifestForSpec(sourceSpec, options = {}) {
+  const cwd = path.resolve(options.cwd || process.cwd());
+  if (isPathSpec(sourceSpec, cwd)) {
+    const packageRoot = path.resolve(cwd, sourceSpec);
+    const manifestPath = path.join(packageRoot, "topogram-extractor.json");
+    if (!fs.existsSync(packageRoot) || !fs.statSync(packageRoot).isDirectory()) {
+      return { source: "path", packageName: null, packageRoot, manifestPath, manifest: null, errors: [`Extractor path '${packageRoot}' must be a directory.`] };
+    }
+    if (!fs.existsSync(manifestPath)) {
+      return { source: "path", packageName: packageNameForRoot(packageRoot), packageRoot, manifestPath, manifest: null, errors: [`Extractor path '${packageRoot}' is missing topogram-extractor.json.`] };
+    }
+    try {
+      const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+      const validation = validateExtractorManifest(manifest);
+      return {
+        source: "path",
+        packageName: typeof manifest.package === "string" ? manifest.package : packageNameForRoot(packageRoot),
+        packageRoot,
+        manifestPath,
+        manifest: validation.ok ? manifest : null,
+        errors: validation.errors
+      };
+    } catch (error) {
+      return {
+        source: "path",
+        packageName: packageNameForRoot(packageRoot),
+        packageRoot,
+        manifestPath,
+        manifest: null,
+        errors: [`Extractor manifest could not be read: ${error instanceof Error ? error.message : String(error)}`]
+      };
+    }
+  }
+
+  const packageName = packageNameFromSpec(sourceSpec);
+  const loaded = loadPackageExtractorManifest(packageName, cwd);
+  return {
+    source: "package",
+    packageName,
+    packageRoot: loaded.packageRoot,
+    manifestPath: loaded.manifestPath,
+    manifest: loaded.manifest,
+    errors: loaded.errors
+  };
+}
+
+/**
+ * @param {string} sourceSpec
+ * @param {{ cwd?: string }} [options]
+ * @returns {{ adapter: any|null, source: "path"|"package", packageName: string|null, packageRoot: string|null, manifestPath: string|null, manifest: ExtractorManifest|null, errors: string[] }}
+ */
+export function loadExtractorPackageAdapterForSpec(sourceSpec, options = {}) {
+  const loadedManifest = loadExtractorPackageManifestForSpec(sourceSpec, options);
+  if (!loadedManifest.manifest) {
+    return { ...loadedManifest, adapter: null };
+  }
+  const loadedAdapter = loadedManifest.source === "path"
+    ? loadLocalPackageAdapter({
+        packageRoot: loadedManifest.packageRoot || path.resolve(options.cwd || process.cwd(), sourceSpec),
+        exportName: loadedManifest.manifest.export,
+        packageLabel: "Extractor package"
+      })
+    : loadInstalledPackageAdapter({
+        packageName: loadedManifest.packageName || packageNameFromSpec(sourceSpec),
+        rootDir: path.resolve(options.cwd || process.cwd()),
+        exportName: loadedManifest.manifest.export,
+        packageLabel: "Extractor package"
+      });
+  return {
+    ...loadedManifest,
+    adapter: loadedAdapter.adapter,
+    errors: [...loadedManifest.errors, ...(loadedAdapter.error ? [loadedAdapter.error] : [])]
+  };
+}
+
+/**
+ * @param {string} packageName
+ * @param {string} version
+ * @returns {import("../extractor-policy.js").PackageExtractorBinding}
+ */
+function packageBinding(packageName, version) {
+  return {
+    packageName,
+    version
+  };
+}
+
+/**
+ * @param {any} context
+ * @returns {any}
+ */
+function packageExtractorContext(context) {
+  return {
+    paths: context.paths,
+    options: context.options || {},
+    priorResults: context.priorResults || {},
+    scanDocsSummary: context.scanDocsSummary || null,
+    helpers: {
+      path: context.helpers?.path,
+      readTextIfExists: context.helpers?.readTextIfExists,
+      readJsonIfExists: context.helpers?.readJsonIfExists
+    }
+  };
+}
+
+/**
+ * @param {any} context
+ * @returns {{ extractors: any[], diagnostics: any[], provenance: any[] }}
+ */
+export function packageExtractorsForContext(context) {
+  if (context.packageExtractorState) {
+    return context.packageExtractorState;
+  }
+  const cwd = path.resolve(context.options?.cwd || process.cwd());
+  const policyInfo = loadExtractorPolicy(cwd, context.options?.extractorPolicyPath || null);
+  const policy = effectiveExtractorPolicy(policyInfo);
+  const specs = [
+    ...(Array.isArray(policy.enabledPackages) ? policy.enabledPackages : []),
+    ...(Array.isArray(context.options?.extractorSpecs) ? context.options.extractorSpecs : [])
+  ].filter(Boolean);
+  /** @type {any[]} */
+  const extractors = [];
+  /** @type {any[]} */
+  const diagnostics = [...policyInfo.diagnostics];
+  /** @type {any[]} */
+  const provenance = [];
+  const seen = new Set();
+
+  for (const spec of specs) {
+    if (seen.has(spec)) continue;
+    seen.add(spec);
+    const bundledPack = getBundledExtractorPack(spec);
+    if (bundledPack) {
+      extractors.push(...bundledPack.extractors);
+      provenance.push({ source: "bundled", id: bundledPack.manifest.id, version: bundledPack.manifest.version, extractors: bundledPack.manifest.extractors });
+      continue;
+    }
+    const bundledExtractor = getBundledExtractorById(spec);
+    if (bundledExtractor) {
+      extractors.push(bundledExtractor);
+      provenance.push({ source: "bundled", id: bundledExtractor.id, version: "1", extractors: [bundledExtractor.id] });
+      continue;
+    }
+    const packageManifest = loadExtractorPackageManifestForSpec(spec, { cwd });
+    if (!packageManifest.manifest) {
+      diagnostics.push({ severity: "error", code: "extractor_package_manifest_failed", message: packageManifest.errors.join(" "), packageName: packageManifest.packageName, spec });
+      continue;
+    }
+    const packageName = packageManifest.packageName || packageManifest.manifest.package || spec;
+    const policyDiagnostics = extractorPolicyDiagnosticsForPackages(policyInfo, [packageBinding(packageName, packageManifest.manifest.version)], "extractor-policy");
+    const errors = policyDiagnostics.filter((diagnostic) => diagnostic.severity === "error");
+    diagnostics.push(...policyDiagnostics);
+    if (errors.length > 0 || !extractorPackageAllowed(policy, packageName)) {
+      continue;
+    }
+    const adapter = loadExtractorPackageAdapterForSpec(spec, { cwd });
+    if (!adapter.adapter || adapter.errors.length > 0) {
+      diagnostics.push(...adapter.errors.map((message) => ({ severity: "error", code: "extractor_package_load_failed", message, packageName, spec })));
+      continue;
+    }
+    const adapterValidation = validateExtractorAdapter(adapter.adapter, packageManifest.manifest);
+    if (adapterValidation.errors.length > 0) {
+      diagnostics.push(...adapterValidation.errors.map((message) => ({ severity: "error", code: "extractor_adapter_invalid", message, packageName, spec })));
+      continue;
+    }
+    for (const extractor of adapterValidation.extractors) {
+      extractors.push({
+        ...extractor,
+        source: "package",
+        manifestId: packageManifest.manifest.id,
+        packageName,
+        packageContext: packageExtractorContext
+      });
+    }
+    provenance.push({
+      source: "package",
+      id: packageManifest.manifest.id,
+      version: packageManifest.manifest.version,
+      packageName,
+      manifestPath: packageManifest.manifestPath,
+      packageRoot: packageManifest.packageRoot,
+      extractors: packageManifest.manifest.extractors
+    });
+  }
+
+  context.packageExtractorState = { extractors, diagnostics, provenance };
+  return context.packageExtractorState;
+}
+
+/**
+ * @returns {any}
+ */
+export function createExtractorSmokeContext() {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "topogram-extractor-check."));
+  fs.writeFileSync(path.join(root, "package.json"), "{\"scripts\":{\"check\":\"topogram check\"}}\n", "utf8");
+  return packageExtractorContext(createImportContext(root, { from: "cli" }));
+}
+
+export { EXTRACTOR_MANIFESTS, getExtractorManifest };