@cdxgen/cdxgen 0.0.1 → 13.0.0

package/lib/inventory/aiInventory.js

@@ -0,0 +1,414 @@
+import { getAllFiles } from "../core/fs.js";
+import { agentFormulationParser } from "./agentFormulationParser.js";
+import {
+  collectHuggingFaceRepoAiInventory,
+  collectJsAiInventory,
+  collectNotebookAiInventory,
+  collectPromptConfigAiInventory,
+  collectPythonAiInventory,
+} from "./aiCollector.js";
+import { communityAiConfigParser } from "./communityAiConfigParser.js";
+import { mergeServices, trimComponents } from "./depsUtils.js";
+import { classifyMcpReference } from "./mcp.js";
+import { mcpConfigParser } from "./mcpConfigParser.js";
+
+/** Project types treated as AI-related inventory for collection and parsing. */
+export const AI_INVENTORY_PROJECT_TYPES = ["mcp", "ai-skill", "ai"];
+/**
+ * Set of file-kind labels that identify AI instruction and agent-definition
+ * files (for example copilot instructions, crew agents/tasks, custom tools).
+ */
+export const AI_INSTRUCTION_FILE_KINDS = new Set([
+  "agent-config",
+  "agent-definition",
+  "agent-instructions",
+  "ai-agent-file",
+  "copilot-instructions",
+  "copilot-setup-workflow",
+  "crew-agent",
+  "crew-task",
+  "crew-tool",
+  "custom-command",
+  "custom-tool",
+  "graph-definition",
+]);
+/** Constant label used to classify AI skill definition files. */
+export const AI_SKILL_FILE_KIND = "skill-file";
+/** Constant label used to classify MCP (Model Context Protocol) config files. */
+export const MCP_CONFIG_FILE_KIND = "mcp-config";
+
+const AI_INVENTORY_FILE_KINDS = new Set([
+  "agent-config",
+  "agent-definition",
+  "agent-instructions",
+  "ai-agent-file",
+  "copilot-instructions",
+  "copilot-setup-workflow",
+  "crew-agent",
+  "crew-task",
+  "crew-tool",
+  "custom-command",
+  "custom-tool",
+  "graph-definition",
+  AI_SKILL_FILE_KIND,
+]);
+
+const AI_INVENTORY_PARSERS = [
+  {
+    id: agentFormulationParser.id,
+    parser: agentFormulationParser,
+    types: ["mcp", "ai-skill"],
+  },
+  {
+    id: mcpConfigParser.id,
+    parser: mcpConfigParser,
+    types: ["mcp"],
+  },
+  {
+    id: communityAiConfigParser.id,
+    parser: communityAiConfigParser,
+    types: ["ai-skill"],
+  },
+];
+
+const AI_LANGUAGE_COLLECTORS = [
+  {
+    id: "javascript-ai",
+    collect: collectJsAiInventory,
+  },
+  {
+    id: "python-ai",
+    collect: collectPythonAiInventory,
+  },
+  {
+    id: "notebook-ai",
+    collect: collectNotebookAiInventory,
+  },
+  {
+    id: "prompt-config-ai",
+    collect: collectPromptConfigAiInventory,
+  },
+  {
+    id: "huggingface-repo-ai",
+    collect: collectHuggingFaceRepoAiInventory,
+  },
+];
+
+/**
+ * Read a property value from an inventory subject.
+ *
+ * @param {Object} subject component or service
+ * @param {string} name property name
+ * @returns {string|undefined} matching property value
+ */
+export function inventoryPropertyValue(subject, name) {
+  return subject?.properties?.find((property) => property.name === name)?.value;
+}
+
+function hasPropertyPrefix(subject, prefix) {
+  return (subject?.properties || []).some((property) =>
+    property?.name?.startsWith(prefix),
+  );
+}
+
+function uniqueNonEmptyTypes(types) {
+  return [...new Set((types || []).filter(Boolean))];
+}
+
+/**
+ * Determine whether CLI project type options include an AI inventory selector.
+ *
+ * @param {string|string[]|undefined} optionValue raw CLI option value
+ * @param {string} type AI inventory type
+ * @returns {boolean} true when the requested type is enabled
+ */
+export function optionIncludesAiInventoryProjectType(optionValue, type) {
+  const values = Array.isArray(optionValue)
+    ? optionValue
+    : optionValue
+      ? [optionValue]
+      : [];
+  return values.some((value) => {
+    const normalizedValue = String(value).toLowerCase();
+    if (type === "ai-skill") {
+      return ["ai-skill", "skill", "skills"].includes(normalizedValue);
+    }
+    if (type === "ai") {
+      return ["ai", "aibom", "ai-bom"].includes(normalizedValue);
+    }
+    return normalizedValue === type;
+  });
+}
+
+/**
+ * Classify an inventory subject into AI inventory types.
+ *
+ * @param {Object} subject component or service
+ * @returns {string[]} matching inventory types
+ */
+export function inventoryTypesForSubject(subject) {
+  const types = new Set();
+  const fileKind = inventoryPropertyValue(subject, "cdx:file:kind");
+  if (
+    fileKind === MCP_CONFIG_FILE_KIND ||
+    subject?.group === "mcp" ||
+    classifyMcpReference(subject).isMcp ||
+    hasPropertyPrefix(subject, "cdx:mcp:") ||
+    (subject?.tags || []).some((tag) => String(tag || "").startsWith("mcp"))
+  ) {
+    types.add("mcp");
+  }
+  if (
+    AI_INVENTORY_FILE_KINDS.has(fileKind) ||
+    hasPropertyPrefix(subject, "cdx:agent:") ||
+    hasPropertyPrefix(subject, "cdx:skill:") ||
+    hasPropertyPrefix(subject, "cdx:tool:") ||
+    hasPropertyPrefix(subject, "cdx:langgraph:") ||
+    hasPropertyPrefix(subject, "cdx:crewai:")
+  ) {
+    types.add("ai-skill");
+  }
+  if (hasPropertyPrefix(subject, "cdx:ai:")) {
+    types.add("ai");
+  }
+  if (
+    inventoryPropertyValue(subject, "cdx:mcp:inventorySource") === "agent-file"
+  ) {
+    types.add("ai-skill");
+  }
+  return Array.from(types);
+}
+
+/**
+ * Check whether a subject belongs to a specific AI inventory type.
+ *
+ * @param {Object} subject component or service
+ * @param {string} type AI inventory type
+ * @returns {boolean} true when the subject matches
+ */
+export function matchesAiInventoryType(subject, type) {
+  return inventoryTypesForSubject(subject).includes(type);
+}
+
+/**
+ * Check whether a subject should be excluded by an AI inventory type filter.
+ *
+ * @param {Object} subject component or service
+ * @param {string} type AI inventory type
+ * @returns {boolean} true when the subject matches the exclusion
+ */
+export function matchesAiInventoryExcludeType(subject, type) {
+  if (type === "mcp") {
+    // Genuine MCP SDK dependency packages are real supply-chain components,
+    // so --exclude-type mcp drops only the inventory overlay, never packages.
+    // enrichComponentWithMcpMetadata() stamps cdx:mcp:package=true exclusively
+    // on such packages; primitives, services and config files never carry it.
+    if (inventoryPropertyValue(subject, "cdx:mcp:package") === "true") {
+      return false;
+    }
+    // Inclusion now classifies the whole overlay (services, primitives and
+    // mcp-config files) as "mcp", so exclusion mirrors it exactly and the two
+    // can no longer drift apart as new MCP signals are added.
+    return matchesAiInventoryType(subject, "mcp");
+  }
+  return matchesAiInventoryType(subject, type);
+}
+
+/**
+ * Filter components or services by requested AI inventory types.
+ *
+ * @param {Object[]} subjects inventory subjects
+ * @param {string[]} types allowed AI inventory types
+ * @returns {Object[]} filtered subjects
+ */
+export function filterInventorySubjectsByTypes(subjects, types) {
+  const allowedTypes = uniqueNonEmptyTypes(types);
+  if (!allowedTypes.length) {
+    return [];
+  }
+  return (subjects || []).filter((subject) =>
+    inventoryTypesForSubject(subject).some((type) =>
+      allowedTypes.includes(type),
+    ),
+  );
+}
+
+/**
+ * Filter dependency edges to include only retained AI inventory subjects.
+ *
+ * @param {Object[]} dependencies dependency edges
+ * @param {Object[]} components retained components
+ * @param {Object[]} services retained services
+ * @returns {Object[]} filtered dependency edges
+ */
+export function filterInventoryDependencies(
+  dependencies,
+  components,
+  services,
+) {
+  const allowedRefs = new Set(
+    []
+      .concat(components || [])
+      .concat(services || [])
+      .map((subject) => subject?.["bom-ref"])
+      .filter(Boolean),
+  );
+  return (dependencies || [])
+    .filter((dependency) => allowedRefs.has(dependency.ref))
+    .map((dependency) => {
+      const filteredDependency = {
+        ref: dependency.ref,
+      };
+      if (dependency.dependsOn?.length) {
+        filteredDependency.dependsOn = dependency.dependsOn.filter((ref) =>
+          allowedRefs.has(ref),
+        );
+      }
+      if (dependency.provides?.length) {
+        filteredDependency.provides = dependency.provides.filter((ref) =>
+          allowedRefs.has(ref),
+        );
+      }
+      return filteredDependency;
+    });
+}
+
+/**
+ * Collect AI inventory subjects and dependency edges for the requested types.
+ *
+ * @param {string} discoveryPath directory or file being analyzed
+ * @param {Object} options collection options
+ * @param {string[]} types requested inventory types
+ * @returns {{ components: Object[], dependencies: Object[], services: Object[] }} AI inventory
+ */
+export function collectAiInventory(discoveryPath, options, types) {
+  const requestedTypes = uniqueNonEmptyTypes(types);
+  if (!requestedTypes.length) {
+    return { components: [], dependencies: [], services: [] };
+  }
+  let components = [];
+  const dependencies = [];
+  let services = [];
+  for (const parserEntry of AI_INVENTORY_PARSERS) {
+    if (!parserEntry.types.some((type) => requestedTypes.includes(type))) {
+      continue;
+    }
+    const matchedFiles = [];
+    for (const pattern of parserEntry.parser.patterns) {
+      const found = getAllFiles(discoveryPath, pattern, options);
+      if (found?.length) {
+        matchedFiles.push(...found);
+      }
+    }
+    const uniqueMatchedFiles = [...new Set(matchedFiles)];
+    if (!uniqueMatchedFiles.length) {
+      continue;
+    }
+    let result;
+    try {
+      result = parserEntry.parser.parse(uniqueMatchedFiles, options);
+    } catch (err) {
+      console.warn(
+        `[aiInventory] Parser "${parserEntry.id}" threw an error:`,
+        err.message,
+      );
+      continue;
+    }
+    if (result?.components?.length) {
+      components = components.concat(result.components);
+    }
+    if (result?.services?.length) {
+      services = mergeServices(services, result.services);
+    }
+    if (result?.dependencies?.length) {
+      dependencies.push(...result.dependencies);
+    }
+  }
+  if (requestedTypes.includes("ai")) {
+    for (const collectorEntry of AI_LANGUAGE_COLLECTORS) {
+      let collectorInventory = {
+        components: [],
+        dependencies: [],
+        services: [],
+      };
+      try {
+        collectorInventory = collectorEntry.collect(discoveryPath, options);
+      } catch (err) {
+        console.warn(
+          `[aiInventory] ${collectorEntry.id} collector threw an error:`,
+          err.message,
+        );
+        continue;
+      }
+      if (collectorInventory.components?.length) {
+        components = components.concat(collectorInventory.components);
+      }
+      if (collectorInventory.services?.length) {
+        services = mergeServices(services, collectorInventory.services);
+      }
+      if (collectorInventory.dependencies?.length) {
+        dependencies.push(...collectorInventory.dependencies);
+      }
+    }
+  }
+
+  components = trimComponents(
+    filterInventorySubjectsByTypes(components, requestedTypes),
+  );
+  services = mergeServices(
+    [],
+    filterInventorySubjectsByTypes(services, requestedTypes),
+  );
+  return {
+    components,
+    dependencies: filterInventoryDependencies(
+      dependencies,
+      components,
+      services,
+    ),
+    services,
+  };
+}
+
+/**
+ * Summarize collected AI inventory counts for reporting.
+ *
+ * @param {{ components?: Object[], services?: Object[] }} inventory collected AI inventory
+ * @returns {{
+ *   aiComponentCount: number,
+ *   aiServiceCount: number,
+ *   instructionCount: number,
+ *   mcpConfigCount: number,
+ *   skillComponentCount: number,
+ * }} summary counts
+ */
+export function summarizeAiInventory(inventory) {
+  const components = inventory?.components || [];
+  const services = inventory?.services || [];
+  return {
+    aiComponentCount: components.filter((component) =>
+      inventoryTypesForSubject(component).includes("ai"),
+    ).length,
+    aiServiceCount: services.filter((service) =>
+      inventoryTypesForSubject(service).includes("ai"),
+    ).length,
+    instructionCount: components.filter((component) =>
+      AI_INSTRUCTION_FILE_KINDS.has(
+        inventoryPropertyValue(component, "cdx:file:kind"),
+      ),
+    ).length,
+    mcpConfigCount: components.filter(
+      (component) =>
+        inventoryPropertyValue(component, "cdx:file:kind") ===
+        MCP_CONFIG_FILE_KIND,
+    ).length,
+    mcpServiceCount: services.filter((service) =>
+      inventoryTypesForSubject(service).includes("mcp"),
+    ).length,
+    skillCount: components.filter(
+      (component) =>
+        inventoryPropertyValue(component, "cdx:file:kind") ===
+        AI_SKILL_FILE_KIND,
+    ).length,
+  };
+}

package/lib/inventory/aiModelVariants.js

@@ -0,0 +1,121 @@
+const AI_MODEL_VARIANT_PATTERNS = [
+  {
+    variant: "fine-tuned",
+    patterns: [/\bfine[-_\s]?tun(?:e|ed|ing)\b/iu, /\bfinetun(?:e|ed|ing)\b/iu],
+  },
+  {
+    variant: "distilled",
+    patterns: [/\bdistill(?:ed|ation)?\b/iu],
+  },
+  {
+    variant: "quantized",
+    patterns: [
+      /\bquantiz(?:e|ed|ation)\b/iu,
+      /\b(?:awq|gptq|gguf|bnb|int4|int8|4-bit|8-bit|fp8)\b/iu,
+      /\bq\d(?:_[a-z0-9]+)*\b/iu,
+    ],
+  },
+  {
+    variant: "adapter",
+    patterns: [/\badapter\b/iu, /\blora\b/iu, /\bqlora\b/iu, /\bpeft\b/iu],
+  },
+  {
+    variant: "merged",
+    patterns: [/\bmerge[dr]?\b/iu, /\bmergekit\b/iu],
+  },
+  {
+    variant: "abliterated",
+    patterns: [/\babliterat(?:e|ed|ion)\b/iu],
+  },
+  {
+    variant: "unlocked",
+    patterns: [
+      /\bunlocked\b/iu,
+      /\buncensored\b/iu,
+      /\bunfiltered\b/iu,
+      /\bde-?aligned\b/iu,
+      /\bjailbreak(?:ed)?\b/iu,
+    ],
+  },
+];
+
+const AI_MODEL_RELATION_VARIANTS = new Map([
+  ["adapter", "adapter"],
+  ["distillation", "distilled"],
+  ["distilled", "distilled"],
+  ["finetune", "fine-tuned"],
+  ["fine-tune", "fine-tuned"],
+  ["fine_tune", "fine-tuned"],
+  ["finetuned", "fine-tuned"],
+  ["merge", "merged"],
+  ["merged", "merged"],
+  ["quantized", "quantized"],
+]);
+
+const toSignalStrings = (value) =>
+  (Array.isArray(value) ? value : value === undefined ? [] : [value])
+    .flatMap((entry) =>
+      typeof entry === "string"
+        ? [entry]
+        : entry && typeof entry === "object"
+          ? Object.values(entry)
+          : [],
+    )
+    .filter(Boolean)
+    .map((entry) => String(entry));
+
+/**
+ * Normalize a list of detected AI model variant labels into a unique string array.
+ *
+ * @param {unknown[]} [variants=[]] detected variant candidates
+ * @returns {string[]} normalized variant labels
+ */
+export function normalizeDetectedVariants(variants = []) {
+  return [
+    ...new Set(
+      (variants || []).filter(Boolean).map((variant) => String(variant)),
+    ),
+  ];
+}
+
+/**
+ * Detect normalized AI model variant labels from names, metadata, and notes.
+ *
+ * @param {{
+ *   description?: string,
+ *   metadata?: unknown[],
+ *   modelName?: string,
+ *   notes?: unknown[],
+ *   quantization?: string,
+ *   relation?: string,
+ *   tags?: unknown[],
+ * }} [signals] variant detection signals
+ * @returns {string[]} normalized variant labels
+ */
+export function detectAiModelVariants(signals = {}) {
+  const detected = new Set();
+  const relation = String(signals?.relation || "")
+    .trim()
+    .toLowerCase();
+  if (relation && AI_MODEL_RELATION_VARIANTS.has(relation)) {
+    detected.add(AI_MODEL_RELATION_VARIANTS.get(relation));
+  }
+  if (signals?.quantization) {
+    detected.add("quantized");
+  }
+  const texts = [
+    ...toSignalStrings(signals?.modelName),
+    ...toSignalStrings(signals?.description),
+    ...toSignalStrings(signals?.tags),
+    ...toSignalStrings(signals?.notes),
+    ...toSignalStrings(signals?.metadata),
+  ];
+  for (const { variant, patterns } of AI_MODEL_VARIANT_PATTERNS) {
+    if (texts.some((text) => patterns.some((pattern) => pattern.test(text)))) {
+      detected.add(variant);
+    }
+  }
+  return AI_MODEL_VARIANT_PATTERNS.map(({ variant }) => variant).filter(
+    (variant) => detected.has(variant),
+  );
+}