@topogram/cli 0.3.79 → 0.3.80

package/src/generator/registry/index.js

@@ -1,8 +1,11 @@
 // @ts-check
 
-import fs from "node:fs";
-import path from "node:path";
-import { createRequire } from "node:module";
+import {
+  loadPackageManifest,
+  packageInstallCommand,
+  packageInstallHint,
+  resolvePackageManifestPath
+} from "../../package-adapters/index.js";
 import { UI_GENERATOR_RENDERED_COMPONENT_PATTERNS } from "../../ui/taxonomy.js";
 
 /**
@@ -208,20 +211,12 @@ export function getGeneratorManifest(generatorId) {
   return GENERATOR_BY_ID.get(generatorId) || null;
 }
 
-/**
- * @param {string|null|undefined} rootDir
- * @returns {string}
- */
-function packageResolutionBase(rootDir) {
-  return path.join(rootDir || process.cwd(), "package.json");
-}
-
 /**
  * @param {string|null|undefined} packageName
  * @returns {string|null}
  */
 export function packageGeneratorInstallCommand(packageName) {
-  return packageName ? `npm install -D ${packageName}` : null;
+  return packageInstallCommand(packageName);
 }
 
 /**
@@ -229,8 +224,7 @@ export function packageGeneratorInstallCommand(packageName) {
  * @returns {string|null}
  */
 export function packageGeneratorInstallHint(packageName) {
-  const command = packageGeneratorInstallCommand(packageName);
-  return command ? `Install it from the project root with: ${command}` : null;
+  return packageInstallHint(packageName);
 }
 
 /**
@@ -239,41 +233,7 @@ export function packageGeneratorInstallHint(packageName) {
  * @returns {{ manifestPath: string|null, packageRoot: string|null, error: string|null }}
  */
 export function resolvePackageGeneratorManifestPath(packageName, rootDir = process.cwd()) {
-  const requireFromRoot = createRequire(packageResolutionBase(rootDir));
-  try {
-    const manifestPath = requireFromRoot.resolve(`${packageName}/topogram-generator.json`);
-    return {
-      manifestPath,
-      packageRoot: path.dirname(manifestPath),
-      error: null
-    };
-  } catch (manifestError) {
-    try {
-      const packageJsonPath = requireFromRoot.resolve(`${packageName}/package.json`);
-      const packageRoot = path.dirname(packageJsonPath);
-      const manifestPath = path.join(packageRoot, "topogram-generator.json");
-      if (!fs.existsSync(manifestPath)) {
-        return {
-          manifestPath: null,
-          packageRoot,
-          error: `Generator package '${packageName}' is missing topogram-generator.json`
-        };
-      }
-      return {
-        manifestPath,
-        packageRoot,
-        error: null
-      };
-    } catch {
-      const detail = manifestError instanceof Error ? manifestError.message : String(manifestError);
-      const installHint = packageGeneratorInstallHint(packageName);
-      return {
-        manifestPath: null,
-        packageRoot: null,
-        error: `Generator package '${packageName}' could not be resolved from '${rootDir || process.cwd()}': ${detail}${installHint ? `. ${installHint}` : ""}`
-      };
-    }
-  }
+  return resolvePackageManifestPath(packageName, "topogram-generator.json", rootDir, "Generator package");
 }
 
 /**
@@ -282,32 +242,13 @@ export function resolvePackageGeneratorManifestPath(packageName, rootDir = proce
  * @returns {{ manifest: GeneratorManifest|null, errors: string[], manifestPath: string|null, packageRoot: string|null }}
  */
 export function loadPackageGeneratorManifest(packageName, rootDir = process.cwd()) {
-  const resolved = resolvePackageGeneratorManifestPath(packageName, rootDir);
-  if (!resolved.manifestPath) {
-    return {
-      manifest: null,
-      errors: [resolved.error || `Generator package '${packageName}' could not be resolved`],
-      manifestPath: null,
-      packageRoot: resolved.packageRoot
-    };
-  }
-  try {
-    const manifest = JSON.parse(fs.readFileSync(resolved.manifestPath, "utf8"));
-    const validation = validateGeneratorManifest(manifest);
-    return {
-      manifest: validation.ok ? manifest : null,
-      errors: validation.errors,
-      manifestPath: resolved.manifestPath,
-      packageRoot: resolved.packageRoot
-    };
-  } catch (error) {
-    return {
-      manifest: null,
-      errors: [`Generator package '${packageName}' manifest could not be read: ${error instanceof Error ? error.message : String(error)}`],
-      manifestPath: resolved.manifestPath,
-      packageRoot: resolved.packageRoot
-    };
-  }
+  return loadPackageManifest({
+    packageName,
+    rootDir,
+    manifestFile: "topogram-generator.json",
+    packageLabel: "Generator package",
+    validateManifest: validateGeneratorManifest
+  });
 }
 
 /**

package/src/generator-policy.js

@@ -4,6 +4,12 @@ import fs from "node:fs";
 import path from "node:path";
 
 import { stableStringify } from "./format.js";
+import {
+  optionalStringArray,
+  optionalStringRecord,
+  packageAllowedByPolicy,
+  packageScopeFromName as sharedPackageScopeFromName
+} from "./package-adapters/index.js";
 
 export const GENERATOR_POLICY_FILE = "topogram.generator-policy.json";
 
@@ -66,47 +72,6 @@ function generatorPolicyDiagnostic(input) {
   };
 }
 
-/**
- * @param {unknown} value
- * @param {string} fieldName
- * @param {string} policyPath
- * @returns {string[]}
- */
-function optionalStringArray(value, fieldName, policyPath) {
-  if (value == null) {
-    return [];
-  }
-  if (!Array.isArray(value)) {
-    throw new Error(`${policyPath} ${fieldName} must be an array of strings.`);
-  }
-  return value.map((item) => {
-    if (typeof item !== "string" || item.length === 0) {
-      throw new Error(`${policyPath} ${fieldName} must contain only non-empty strings.`);
-    }
-    return item;
-  });
-}
-
-/**
- * @param {unknown} value
- * @param {string} policyPath
- * @returns {Record<string, string>}
- */
-function optionalStringRecord(value, policyPath) {
-  if (value == null) {
-    return {};
-  }
-  if (typeof value !== "object" || Array.isArray(value)) {
-    throw new Error(`${policyPath} pinnedVersions must be an object of package-or-generator ids to versions.`);
-  }
-  return Object.fromEntries(Object.entries(value).map(([key, item]) => {
-    if (typeof item !== "string" || item.length === 0) {
-      throw new Error(`${policyPath} pinnedVersions['${key}'] must be a non-empty string.`);
-    }
-    return [key, item];
-  }));
-}
-
 /**
  * @returns {GeneratorPolicy}
  */
@@ -136,7 +101,7 @@ export function validateGeneratorPolicy(value, policyPath) {
       ? defaults.allowedPackageScopes
       : optionalStringArray(raw.allowedPackageScopes, "allowedPackageScopes", policyPath),
     allowedPackages: optionalStringArray(raw.allowedPackages, "allowedPackages", policyPath),
-    pinnedVersions: optionalStringRecord(raw.pinnedVersions, policyPath)
+    pinnedVersions: optionalStringRecord(raw.pinnedVersions, policyPath, "package-or-generator ids")
   };
 }
 
@@ -145,16 +110,7 @@ export function validateGeneratorPolicy(value, policyPath) {
  * @returns {string|null}
  */
 export function packageScopeFromName(packageName) {
-  return packageName.startsWith("@") ? packageName.split("/")[0] || null : null;
-}
-
-/**
- * @param {string} allowed
- * @param {string|null} scope
- * @returns {boolean}
- */
-function packageScopeMatches(allowed, scope) {
-  return Boolean(scope && (allowed === scope || allowed === `${scope}/*`));
+  return sharedPackageScopeFromName(packageName);
 }
 
 /**
@@ -163,11 +119,7 @@ function packageScopeMatches(allowed, scope) {
  * @returns {boolean}
  */
 export function generatorPackageAllowed(policy, packageName) {
-  if (policy.allowedPackages.includes(packageName)) {
-    return true;
-  }
-  const scope = packageScopeFromName(packageName);
-  return policy.allowedPackageScopes.some((allowed) => packageScopeMatches(allowed, scope));
+  return packageAllowedByPolicy(policy, packageName);
 }
 
 /**

package/src/import/core/registry.d.ts

@@ -1,4 +1,7 @@
 export const extractorRegistry: Record<string, any[]>;
 export const enricherRegistry: Record<string, any[]>;
+export const BUILTIN_EXTRACTOR_PACKS: any[];
 export function getExtractorsForTrack(...args: any[]): any[];
 export function getEnrichersForTrack(...args: any[]): any[];
+export function getBundledExtractorPack(...args: any[]): any;
+export function getBundledExtractorById(...args: any[]): any;

package/src/import/core/registry.js

@@ -57,14 +57,80 @@ import { railsControllerEnricher } from "../enrichers/rails-controllers.js";
 import { workflowTargetStateEnricher } from "../enrichers/workflow-target-state.js";
 import { docLinkingEnricher } from "../enrichers/doc-linking.js";
 
-export const extractorRegistry = {
-  db: [prismaExtractor, djangoModelsExtractor, efCoreExtractor, roomExtractor, swiftDataExtractor, dotnetModelsExtractor, flutterEntitiesExtractor, reactNativeEntitiesExtractor, railsSchemaExtractor, liquibaseExtractor, myBatisXmlExtractor, jpaExtractor, drizzleExtractor, sqlExtractor, snapshotExtractor],
-  api: [openApiExtractor, openApiCodeExtractor, graphQlSdlExtractor, graphQlCodeFirstExtractor, trpcExtractor, aspNetCoreExtractor, retrofitExtractor, swiftWebApiExtractor, flutterDioExtractor, reactNativeRepositoryExtractor, fastifyExtractor, expressExtractor, djangoRoutesExtractor, railsRoutesExtractor, micronautExtractor, jaxRsExtractor, springWebExtractor, nextRouteExtractor, genericRouteFallbackExtractor, nextServerActionExtractor, nextAuthExtractor],
-  ui: [nextAppRouterUiExtractor, nextPagesRouterUiExtractor, androidComposeUiExtractor, blazorUiExtractor, razorPagesUiExtractor, swiftUiExtractor, uiKitExtractor, mauiXamlUiExtractor, flutterScreensUiExtractor, reactNativeScreensExtractor, reactRouterUiExtractor, svelteKitUiExtractor, backendOnlyUiExtractor],
-  cli: [genericCliExtractor],
-  workflows: [genericWorkflowExtractor],
-  verification: [genericVerificationExtractor]
-};
+function extractorPack(id, tracks, extractors, candidateKinds, stack = {}, capabilities = {}) {
+  return {
+    manifest: {
+      id,
+      version: "1",
+      tracks,
+      source: "bundled",
+      extractors: extractors.map((extractor) => extractor.id),
+      stack,
+      capabilities,
+      candidateKinds,
+      evidenceTypes: ["runtime_source", "parser_config", "docs", "tests", "fixtures"]
+    },
+    extractors
+  };
+}
+
+export const BUILTIN_EXTRACTOR_PACKS = [
+  extractorPack(
+    "topogram/db-extractors",
+    ["db"],
+    [prismaExtractor, djangoModelsExtractor, efCoreExtractor, roomExtractor, swiftDataExtractor, dotnetModelsExtractor, flutterEntitiesExtractor, reactNativeEntitiesExtractor, railsSchemaExtractor, liquibaseExtractor, myBatisXmlExtractor, jpaExtractor, drizzleExtractor, sqlExtractor, snapshotExtractor],
+    ["entity", "enum", "relation", "index", "maintained_db_migration_seam"],
+    { domain: "database" },
+    { schema: true, migrations: true, maintainedSeams: true }
+  ),
+  extractorPack(
+    "topogram/api-extractors",
+    ["api"],
+    [openApiExtractor, openApiCodeExtractor, graphQlSdlExtractor, graphQlCodeFirstExtractor, trpcExtractor, aspNetCoreExtractor, retrofitExtractor, swiftWebApiExtractor, flutterDioExtractor, reactNativeRepositoryExtractor, fastifyExtractor, expressExtractor, djangoRoutesExtractor, railsRoutesExtractor, micronautExtractor, jaxRsExtractor, springWebExtractor, nextRouteExtractor, genericRouteFallbackExtractor, nextServerActionExtractor, nextAuthExtractor],
+    ["capability", "route", "stack"],
+    { domain: "api" },
+    { routes: true, openapi: true, graphql: true }
+  ),
+  extractorPack(
+    "topogram/ui-extractors",
+    ["ui"],
+    [nextAppRouterUiExtractor, nextPagesRouterUiExtractor, androidComposeUiExtractor, blazorUiExtractor, razorPagesUiExtractor, swiftUiExtractor, uiKitExtractor, mauiXamlUiExtractor, flutterScreensUiExtractor, reactNativeScreensExtractor, reactRouterUiExtractor, svelteKitUiExtractor, backendOnlyUiExtractor],
+    ["screen", "route", "action", "flow", "widget", "shape", "stack"],
+    { domain: "ui" },
+    { screens: true, widgets: true, flows: true }
+  ),
+  extractorPack(
+    "topogram/cli-extractors",
+    ["cli"],
+    [genericCliExtractor],
+    ["command", "capability", "cli_surface"],
+    { domain: "cli" },
+    { commands: true, options: true, effects: true }
+  ),
+  extractorPack(
+    "topogram/workflow-extractors",
+    ["workflows"],
+    [genericWorkflowExtractor],
+    ["workflow", "workflow_state", "workflow_transition"],
+    { domain: "workflow" },
+    { workflows: true }
+  ),
+  extractorPack(
+    "topogram/verification-extractors",
+    ["verification"],
+    [genericVerificationExtractor],
+    ["verification", "scenario", "framework", "script"],
+    { domain: "verification" },
+    { verifications: true }
+  )
+];
+
+export const extractorRegistry = Object.fromEntries(
+  ["db", "api", "ui", "cli", "workflows", "verification"].map((track) => [
+    track,
+    BUILTIN_EXTRACTOR_PACKS.flatMap((pack) => pack.extractors).filter((extractor) => extractor.track === track)
+  ])
+);
 
 export const enricherRegistry = {
   db: [railsModelEnricher],
@@ -82,3 +148,11 @@ export function getExtractorsForTrack(track) {
 export function getEnrichersForTrack(track) {
   return enricherRegistry[track] || [];
 }
+
+export function getBundledExtractorPack(id) {
+  return BUILTIN_EXTRACTOR_PACKS.find((pack) => pack.manifest.id === id) || null;
+}
+
+export function getBundledExtractorById(id) {
+  return BUILTIN_EXTRACTOR_PACKS.flatMap((pack) => pack.extractors).find((extractor) => extractor.id === id) || null;
+}

package/src/import/core/runner/run.js

@@ -6,6 +6,7 @@ import { parseImportTracks } from "./options.js";
 import { appReportMarkdown, reportMarkdown } from "./reports.js";
 import { runTrack } from "./tracks.js";
 import { draftUiProjectionFiles } from "./ui-drafts.js";
+import { packageExtractorsForContext } from "../../../extractor/packages.js";
 
 /**
  * @param {string} inputPath
@@ -64,6 +65,7 @@ export function runImportApp(inputPath, options = {}) {
     tracks,
     findings_count: Object.values(findings).reduce((total, entries) => total + entries.length, 0),
     extractor_detections: Object.fromEntries(Object.entries(resultsByTrack).map(([track, result]) => [track, result.extractor_detections])),
+    package_extractors: packageExtractorsForContext(context).provenance,
     candidates
   };
 

package/src/import/core/runner/tracks.js

@@ -2,6 +2,7 @@
 
 import { getEnrichersForTrack, getExtractorsForTrack } from "../registry.js";
 import { normalizeCandidatesForTrack } from "./candidates.js";
+import { packageExtractorsForContext } from "../../../extractor/packages.js";
 
 /**
  * @param {any} context
@@ -10,7 +11,12 @@ import { normalizeCandidatesForTrack } from "./candidates.js";
  */
 function sortExtractors(context, extractors) {
   return extractors
-    .map((extractor) => ({ extractor, detection: extractor.detect(context) || { score: 0, reasons: [] } }))
+    .map((extractor) => {
+      const extractorContext = extractor.source === "package" && typeof extractor.packageContext === "function"
+        ? extractor.packageContext(context)
+        : context;
+      return { extractor, detection: extractor.detect(extractorContext) || { score: 0, reasons: [] } };
+    })
     .filter((entry) => entry.detection.score > 0)
     .sort((a, b) => b.detection.score - a.detection.score || a.extractor.id.localeCompare(b.extractor.id));
 }
@@ -20,7 +26,7 @@ function sortExtractors(context, extractors) {
  * @param {Array<{ extractor: any, detection: any }>} detections
  * @returns {Array<{ extractor: any, detection: any }>}
  */
-function selectDetectionsForTrack(track, detections) {
+function selectBundledDetectionsForTrack(track, detections) {
   if (track === "db") {
     const prisma = detections.find((entry) => entry.extractor.id === "db.prisma");
     if (prisma) return [prisma];
@@ -90,6 +96,20 @@ function selectDetectionsForTrack(track, detections) {
   return detections;
 }
 
+/**
+ * @param {string} track
+ * @param {Array<{ extractor: any, detection: any }>} detections
+ * @returns {Array<{ extractor: any, detection: any }>}
+ */
+function selectDetectionsForTrack(track, detections) {
+  const packageDetections = detections.filter((entry) => entry.extractor.source === "package");
+  const bundledDetections = selectBundledDetectionsForTrack(
+    track,
+    detections.filter((entry) => entry.extractor.source !== "package")
+  );
+  return [...bundledDetections, ...packageDetections];
+}
+
 /**
  * @param {string} track
  * @returns {any}
@@ -113,6 +133,35 @@ function initialCandidatesForTrack(track) {
   return { workflows: [], workflow_states: [], workflow_transitions: [] };
 }
 
+/**
+ * @param {any} extractor
+ * @param {any} result
+ * @returns {void}
+ */
+function assertExtractorResultShape(extractor, result) {
+  const label = extractor?.id || "unknown";
+  if (!result || typeof result !== "object" || Array.isArray(result)) {
+    throw new Error(`Extractor '${label}' extract(context) must return an object.`);
+  }
+  if (result.findings != null && !Array.isArray(result.findings)) {
+    throw new Error(`Extractor '${label}' extract(context) findings must be an array when present.`);
+  }
+  if (result.diagnostics != null && !Array.isArray(result.diagnostics)) {
+    throw new Error(`Extractor '${label}' extract(context) diagnostics must be an array when present.`);
+  }
+  if (result.candidates == null) {
+    result.candidates = {};
+  }
+  if (!result.candidates || typeof result.candidates !== "object" || Array.isArray(result.candidates)) {
+    throw new Error(`Extractor '${label}' extract(context) candidates must be an object.`);
+  }
+  for (const [key, value] of Object.entries(result.candidates)) {
+    if (!Array.isArray(value)) {
+      throw new Error(`Extractor '${label}' extract(context) candidates.${key} must be an array.`);
+    }
+  }
+}
+
 /**
  * @param {any} context
  * @param {string} track
@@ -121,9 +170,22 @@ function initialCandidatesForTrack(track) {
 export function runTrack(context, track) {
   const findings = [];
   const rawCandidates = initialCandidatesForTrack(track);
+  const packageState = packageExtractorsForContext(context);
+  const packageErrors = packageState.diagnostics.filter((diagnostic) => diagnostic.severity !== "warning");
+  if (packageErrors.length > 0) {
+    throw new Error(packageErrors.map((diagnostic) => diagnostic.message || String(diagnostic)).join("\n"));
+  }
+  const extractors = [
+    ...getExtractorsForTrack(track),
+    ...packageState.extractors.filter((extractor) => extractor.track === track)
+  ];
 
-  for (const { extractor, detection } of selectDetectionsForTrack(track, sortExtractors(context, getExtractorsForTrack(track)))) {
-    const result = extractor.extract(context) || { findings: [], candidates: {} };
+  for (const { extractor, detection } of selectDetectionsForTrack(track, sortExtractors(context, extractors))) {
+    const extractorContext = extractor.source === "package" && typeof extractor.packageContext === "function"
+      ? extractor.packageContext(context)
+      : context;
+    const result = extractor.extract(extractorContext) || { findings: [], candidates: {} };
+    assertExtractorResultShape(extractor, result);
     findings.push({
       extractor: extractor.id,
       detection,

package/src/import/provenance.js

@@ -53,7 +53,8 @@ export function writeTopogramImportRecord(projectRoot, input) {
     extract: {
       tracks: input.tracks || [],
       findingsCount: input.findingsCount || 0,
-      candidateCounts: input.candidateCounts || {}
+      candidateCounts: input.candidateCounts || {},
+      extractorPackages: input.extractorPackages || []
     },
     ownership: {
       extractedArtifacts: "project-owned",

package/src/package-adapters/adapter.js

@@ -0,0 +1,64 @@
+// @ts-check
+
+import path from "node:path";
+import { createRequire } from "node:module";
+
+/**
+ * @param {any} moduleValue
+ * @param {string|null|undefined} exportName
+ * @returns {any}
+ */
+export function selectPackageExport(moduleValue, exportName) {
+  if (exportName) {
+    return moduleValue?.[exportName] || moduleValue?.default?.[exportName] || null;
+  }
+  return moduleValue?.default || moduleValue;
+}
+
+/**
+ * @param {{
+ *   packageRoot: string,
+ *   exportName?: string|null,
+ *   packageLabel: string
+ * }} options
+ * @returns {{ adapter: any|null, error: string|null }}
+ */
+export function loadLocalPackageAdapter(options) {
+  try {
+    const packageJsonPath = path.join(options.packageRoot, "package.json");
+    const requireFromPackage = createRequire(packageJsonPath);
+    return {
+      adapter: selectPackageExport(requireFromPackage(options.packageRoot), options.exportName),
+      error: null
+    };
+  } catch (error) {
+    return {
+      adapter: null,
+      error: `${options.packageLabel} export could not be loaded from '${options.packageRoot}': ${error instanceof Error ? error.message : String(error)}`
+    };
+  }
+}
+
+/**
+ * @param {{
+ *   packageName: string,
+ *   rootDir: string,
+ *   exportName?: string|null,
+ *   packageLabel: string
+ * }} options
+ * @returns {{ adapter: any|null, error: string|null }}
+ */
+export function loadInstalledPackageAdapter(options) {
+  try {
+    const requireFromRoot = createRequire(path.join(options.rootDir, "package.json"));
+    return {
+      adapter: selectPackageExport(requireFromRoot(options.packageName), options.exportName),
+      error: null
+    };
+  } catch (error) {
+    return {
+      adapter: null,
+      error: `${options.packageLabel} '${options.packageName}' export could not be loaded from '${options.rootDir}': ${error instanceof Error ? error.message : String(error)}`
+    };
+  }
+}

package/src/package-adapters/file-map.js

@@ -0,0 +1,30 @@
+// @ts-check
+
+import path from "node:path";
+
+/**
+ * @param {any} files
+ * @param {{ filePathMessage: string, contentMessage: (filePath: string) => string }} messages
+ * @returns {{ ok: boolean, message: string }}
+ */
+export function validateRelativeStringFileMap(files, messages) {
+  if (!files || typeof files !== "object" || Array.isArray(files)) {
+    return { ok: false, message: "files must be an object" };
+  }
+  for (const [filePath, content] of Object.entries(files)) {
+    const normalizedPath = typeof filePath === "string" ? path.normalize(filePath) : "";
+    if (
+      typeof filePath !== "string" ||
+      filePath.length === 0 ||
+      path.isAbsolute(filePath) ||
+      normalizedPath === ".." ||
+      normalizedPath.startsWith(`..${path.sep}`)
+    ) {
+      return { ok: false, message: messages.filePathMessage };
+    }
+    if (typeof content !== "string") {
+      return { ok: false, message: messages.contentMessage(filePath) };
+    }
+  }
+  return { ok: true, message: "files are valid" };
+}

package/src/package-adapters/index.js

@@ -0,0 +1,27 @@
+// @ts-check
+
+export {
+  loadInstalledPackageAdapter,
+  loadLocalPackageAdapter,
+  selectPackageExport
+} from "./adapter.js";
+export {
+  loadPackageManifest,
+  resolvePackageManifestPath
+} from "./manifest.js";
+export {
+  isPathSpec,
+  packageInstallCommand,
+  packageInstallHint,
+  packageNameFromSpec,
+  packageResolutionBase
+} from "./spec.js";
+export {
+  optionalStringArray,
+  optionalStringRecord,
+  packageAllowedByPolicy,
+  packageScopeFromName
+} from "./policy.js";
+export {
+  validateRelativeStringFileMap
+} from "./file-map.js";