@cdxgen/cdxgen 13.3.0 → 13.5.0

package/lib/inventory/agentSkillSpec.js

@@ -0,0 +1,17 @@
+/**
+ * Whether the frontmatter of a SKILL.md satisfies the open Agent Skills
+ * constraints (agentskills.io): a name of 1-64 lowercase letters, digits and
+ * hyphens, and a description of 1-1024 characters.
+ *
+ * @param {{name?: unknown, description?: unknown}|undefined} metadata Frontmatter values
+ * @returns {boolean} true when both constraints hold
+ */
+export function isAgentSkillSpecValid(metadata) {
+  return (
+    typeof metadata?.name === "string" &&
+    /^[a-z0-9-]{1,64}$/u.test(metadata.name) &&
+    typeof metadata?.description === "string" &&
+    metadata.description.length >= 1 &&
+    metadata.description.length <= 1024
+  );
+}

package/lib/inventory/aiCollector.js

@@ -90,8 +90,11 @@ const isVendoredArtifactPath = (filePath) => {
 };
 const GGUF_MIN_ALIGNMENT = 4;
 const GGUF_MAX_ALIGNMENT = 1024 * 1024;
+// Tested against full paths. The config exceptions follow IGNORE_FILE_PATTERN
+// in analyzer.js, with the app exception anchored to a path separator instead
+// of the start of a file name.
 const IGNORE_SOURCE_FILE_PATTERN =
-  /(^|\/|\\)(__tests__|fixtures?|examples?|samples?)($|\/|\\)|(^|\/|\\)(test|spec|mock|setup-jest|conftest|sitecustomize)\.(js|ts|tsx|py)$|(?<!vite\.|vue\.)(conf|config)\.(js|ts|tsx)$/iu;
+  /(^|\/|\\)(__tests__|fixtures?|examples?|samples?)($|\/|\\)|(^|\/|\\)(test|spec|mock|setup-jest|conftest|sitecustomize)\.([cm]?[jt]sx?|py)$|(?<!vite\.|vue\.|svelte\.|astro\.|(?:^|\/|\\)app\.)(conf|config)\.[cm]?[jt]sx?$/iu;
 const HOST_PROVIDER_PATTERNS = [
   ["openai", /(?:^|\.)openai\.com$/iu],
   ["anthropic", /(?:^|\.)anthropic\.com$/iu],

package/lib/inventory/analyzer.js

@@ -68,9 +68,19 @@ const IGNORE_DIRS = readEnvironmentVariable("ASTGEN_IGNORE_DIRS")
       "coverage",
     ];
 
+// The pattern is tested against file names. Config files are skipped because
+// their imports are mostly build and test tooling. The exceptions hold code the
+// application is built from: the Vite, Vue CLI, Svelte, and Astro configs name
+// the build plugins and integrations the app needs, and app.config.* holds
+// application code, such as the root providers of an Angular app. The app
+// exception is anchored to the start of the name, so names such as
+// webapp.config.ts are still skipped. [cm]?[jt]sx? covers every JavaScript and
+// TypeScript module extension, as the default test globs of Jest and Vitest do.
+// Keep the config exceptions in step with IGNORE_SOURCE_FILE_PATTERN in
+// aiCollector.js.
 const IGNORE_FILE_PATTERN = new RegExp(
   readEnvironmentVariable("ASTGEN_IGNORE_FILE_PATTERN") ||
-    "(test|spec|mock|setup-jest|\\.d)\\.(js|ts|tsx)$|(?<!vite\\.|vue\\.|svelte\\.)(conf|config)\\.(js|ts|tsx)$",
+    "(test|spec|mock|setup-jest|\\.d)\\.[cm]?[jt]sx?$|(?<!vite\\.|vue\\.|svelte\\.|astro\\.|^app\\.)(conf|config)\\.[cm]?[jt]sx?$",
   "i",
 );
 
@@ -210,6 +220,46 @@ const getAllFiles = (
   return result;
 };
 
+/**
+ * Walk a directory tree once and return the files whose names end with any of
+ * the given extensions.
+ *
+ * Files are grouped by extension in the order of `extensions`, and each group
+ * keeps the walk order. That is the order one walk per extension produced, and
+ * occurrences reach the BOM in it.
+ *
+ * @param {string} dir Directory to walk
+ * @param {string[]} extensions File extensions, each with its leading dot
+ * @param {boolean|Object} deepOrOptions Deep flag, or `{ deep, exclude, noIgnore }`
+ * @returns {string[]} Matching file paths
+ */
+const getAllFilesByExtensions = (dir, extensions, deepOrOptions) => {
+  const searchOptions = normalizeAnalyzerSearchOptions(deepOrOptions);
+  const files = getAllFiles(
+    searchOptions.deep,
+    dir,
+    undefined,
+    undefined,
+    undefined,
+    new RegExp(
+      `(?:${extensions.map((extension) => `\\${extension}`).join("|")})$`,
+    ),
+    dir,
+    searchOptions.exclude,
+    searchOptions.noIgnore,
+  );
+  const filesByExtension = extensions.map(() => []);
+  for (const file of files) {
+    const extensionIndex = extensions.findIndex((extension) =>
+      file.endsWith(extension),
+    );
+    if (extensionIndex !== -1) {
+      filesByExtension[extensionIndex].push(file);
+    }
+  }
+  return filesByExtension.flat();
+};
+
 const babelParserOptions = {
   sourceType: "unambiguous",
   allowImportExportEverywhere: true,
@@ -1752,37 +1802,29 @@ export const parseFileASTTreeCollected = (src, file) => {
   return { imports, exports };
 };
 
+const JS_TS_SOURCE_EXTENSIONS = [
+  ".js",
+  ".jsx",
+  ".cjs",
+  ".mjs",
+  ".ts",
+  ".tsx",
+  ".mts",
+  ".cts",
+  ".vue",
+  ".svelte",
+];
+
 /**
- * Return paths to all (j|tsx?) files.
+ * Return paths to all JavaScript, TypeScript, Vue, and Svelte source files,
+ * found in one walk of the tree.
+ *
+ * @param {string} src Directory to walk
+ * @param {boolean|Object} deep Deep flag, or `{ deep, exclude, noIgnore }`
+ * @returns {string[]} Source file paths, grouped by extension
  */
 const getAllSrcJSAndTSFiles = (src, deep) =>
-  Promise.all(
-    [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].map((extension) => {
-      const searchOptions = normalizeAnalyzerSearchOptions(deep);
-      return getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      );
-    }),
-  );
+  getAllFilesByExtensions(src, JS_TS_SOURCE_EXTENSIONS, deep);
 
 const getAngularWorkspaceManifestFiles = (
   dir,
@@ -2139,51 +2181,42 @@ const parseAngularStyleFiles = (
   excludePatterns,
   noIgnore = false,
 ) => {
-  for (const extension of ANGULAR_STYLE_FILE_EXTENSIONS) {
-    const styleFiles = getAllFiles(
-      deep,
-      src,
-      extension,
-      undefined,
-      undefined,
-      undefined,
-      src,
-      excludePatterns,
-      noIgnore,
-    );
-    for (const styleFile of styleFiles) {
-      let content;
-      try {
-        content = readFileSync(styleFile, "utf-8");
-      } catch {
-        continue;
-      }
-      for (const referencePattern of STYLE_PACKAGE_REFERENCE_PATTERNS) {
-        for (const match of content.matchAll(referencePattern)) {
-          const reference = match[1];
-          if (!reference) {
-            continue;
-          }
-          if (
-            match[0]?.trim().startsWith("url") &&
-            !isPackageStyleUrlReference(reference)
-          ) {
-            continue;
-          }
-          const prefix = content.slice(0, match.index || 0);
-          const lineNumber = prefix.split(/\r?\n/).length;
-          const lineStart = prefix.lastIndexOf("\n");
-          const columnNumber =
-            lineStart >= 0 ? prefix.length - lineStart - 1 : 0;
-          setAngularPackageRef(
-            allImports,
-            allExports,
-            src,
-            styleFile,
-            reference,
-            { column: columnNumber, line: lineNumber },
-          );
+  const styleFiles = getAllFilesByExtensions(
+    src,
+    ANGULAR_STYLE_FILE_EXTENSIONS,
+    { deep, exclude: excludePatterns, noIgnore },
+  );
+  for (const styleFile of styleFiles) {
+    let content;
+    try {
+      content = readFileSync(styleFile, "utf-8");
+    } catch {
+      continue;
+    }
+    for (const referencePattern of STYLE_PACKAGE_REFERENCE_PATTERNS) {
+      for (const match of content.matchAll(referencePattern)) {
+        const reference = match[1];
+        if (!reference) {
+          continue;
         }
+        if (
+          match[0]?.trim().startsWith("url") &&
+          !isPackageStyleUrlReference(reference)
+        ) {
+          continue;
+        }
+        const prefix = content.slice(0, match.index || 0);
+        const lineNumber = prefix.split(/\r?\n/).length;
+        const lineStart = prefix.lastIndexOf("\n");
+        const columnNumber = lineStart >= 0 ? prefix.length - lineStart - 1 : 0;
+        setAngularPackageRef(
+          allImports,
+          allExports,
+          src,
+          styleFile,
+          reference,
+          { column: columnNumber, line: lineNumber },
+        );
       }
     }
   }
@@ -3478,8 +3511,7 @@ export const findJSImportsExports = async (src, deep) => {
     // scope of astWorker.js. Absence is silent — tier 1 covers all scope
     // evidence.
     await primeSvelteParser();
-    const promiseMap = await getAllSrcJSAndTSFiles(src, deep);
-    const srcFiles = promiseMap.flat();
+    const srcFiles = getAllSrcJSAndTSFiles(src, deep);
     const tasks = srcFiles.map((file) => ({ src, file }));
     const parallelWorthwhile =
       totalSourceBytes(srcFiles) >= MIN_PARALLEL_PARSE_BYTES;
@@ -4580,8 +4612,7 @@ export const analyzeJsCryptoFile = (filePath) => {
 export const detectJsCryptoInventory = async (src, deep = false) => {
   let srcFiles = [];
   try {
-    const promiseMap = await getAllSrcJSAndTSFiles(src, deep);
-    srcFiles = promiseMap.flat();
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch {
     return { algorithms: [], libraries: [] };
   }
@@ -4626,31 +4657,7 @@ export const detectExtensionCapabilities = (src, deep = false) => {
   }
   let srcFiles = [];
   try {
-    const searchOptions = normalizeAnalyzerSearchOptions(deep);
-    srcFiles = [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].flatMap((extension) =>
-      getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      ),
-    );
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch (_err) {
     return { capabilities: [], indicators: {} };
   }
@@ -5857,31 +5864,7 @@ export const detectMcpInventory = (src, deep = false) => {
   const servicesByKey = new Map();
   let srcFiles = [];
   try {
-    const searchOptions = normalizeAnalyzerSearchOptions(deep);
-    srcFiles = [
-      ".js",
-      ".jsx",
-      ".cjs",
-      ".mjs",
-      ".ts",
-      ".tsx",
-      ".mts",
-      ".cts",
-      ".vue",
-      ".svelte",
-    ].flatMap((extension) =>
-      getAllFiles(
-        searchOptions.deep,
-        src,
-        extension,
-        undefined,
-        undefined,
-        undefined,
-        src,
-        searchOptions.exclude,
-        searchOptions.noIgnore,
-      ),
-    );
+    srcFiles = getAllSrcJSAndTSFiles(src, deep);
   } catch {
     return { components: [], dependencies: [], services: [] };
   }

package/lib/inventory/apkInstalledDb.js

@@ -0,0 +1,243 @@
+// Bounded, read-only reader of the apk installed database.
+//
+// osquery has no apk table, so an Alpine, Wolfi or Chainguard host would have no
+// OS package inventory in an OBOM, and the PHP host probe needs the same
+// database to tell which PHP files a package owns. Both read it through here, so
+// what the OBOM lists and what the probe treats as owned follow one parser: the
+// same file, location order, caps and record rules.
+//
+// The database is `/lib/apk/db/installed`. A usr-merged system keeps it at
+// `/usr/lib/apk/db/installed` (and `/lib` is a link to `/usr/lib`, so the two
+// names are one file), and Alpaquita keeps it under `/var/lib/apk/db`. The first
+// of those that exists is the database. One that exists but cannot be read
+// leaves the packages unknown: it is reported as unreadable, never read in part
+// and never skipped for the next name.
+//
+// A record is a run of `<letter>:<value>` lines ended by a blank line. P: names
+// the package, V: its version, A: its architecture, o: its origin (the source
+// package), m: its maintainer, L: its licence, U: its home page and T: its
+// description. F: names a directory and each R: after it a file in that
+// directory. A record may start with C: (Alpine) or P: (Wolfi).
+
+import { statSync } from "node:fs";
+import { join } from "node:path";
+import process from "node:process";
+
+import { isDryRun, recordActivity } from "../core/activity.js";
+import { safeExistsSync, safeReadFileSync } from "../core/fs.js";
+import { thoughtLog } from "../core/logger.js";
+
+/** The database, or the packages it lists, past these is unreadable rather than read in part. */
+export const MAX_APK_DATABASE_BYTES = 64 * 1024 * 1024;
+export const MAX_APK_PACKAGES = 100000;
+
+/** Where apk keeps the installed database, relative to the root, in the order they are tried. */
+export const APK_INSTALLED_DB_PATHS = [
+  "lib/apk/db/installed",
+  "usr/lib/apk/db/installed",
+  "var/lib/apk/db/installed",
+];
+
+// No field of a package record is longer than this once read.
+const MAX_FIELD_LENGTH = 4096;
+
+const FIELD_NAMES = {
+  P: "name",
+  V: "version",
+  A: "arch",
+  o: "origin",
+  m: "maintainer",
+  L: "license",
+  U: "url",
+  T: "description",
+};
+
+function readBounded(file, maxBytes) {
+  try {
+    const stats = statSync(file);
+    if (!stats.isFile() || stats.size > maxBytes) {
+      return undefined;
+    }
+  } catch {
+    return undefined;
+  }
+  const text = safeReadFileSync(file, "utf-8");
+  // A file that grew between the stat and the read is over the cap too.
+  return typeof text === "string" && text.length <= maxBytes ? text : undefined;
+}
+
+function newRecord() {
+  return { fields: {}, fileCount: 0, files: [], touched: false };
+}
+
+/**
+ * Parse the text of an installed database.
+ *
+ * @param {string} text Database text
+ * @param {Set<string>|undefined} wantedFiles Absolute paths whose owner is wanted
+ * @param {number} maxPackages Most packages to accept
+ * @returns {{packages: Object[], malformed: number, overflow: boolean}}
+ */
+function parseDatabase(text, wantedFiles, maxPackages) {
+  const packages = [];
+  let malformed = 0;
+  let overflow = false;
+  let record = newRecord();
+  let directory = "";
+  const finishRecord = () => {
+    const { fields, fileCount, files, touched } = record;
+    record = newRecord();
+    directory = "";
+    if (!touched) {
+      return;
+    }
+    // A package with no name or no version cannot be named by a purl.
+    if (!fields.name || !fields.version) {
+      malformed++;
+      return;
+    }
+    packages.push({ ...fields, fileCount, files });
+    if (packages.length > maxPackages) {
+      overflow = true;
+    }
+  };
+  let position = 0;
+  while (position <= text.length && !overflow) {
+    let end = text.indexOf("\n", position);
+    if (end === -1) {
+      end = text.length;
+    }
+    let line = text.slice(position, end);
+    position = end + 1;
+    if (line.endsWith("\r")) {
+      line = line.slice(0, -1);
+    }
+    if (line === "") {
+      finishRecord();
+      continue;
+    }
+    if (line.length < 2 || line[1] !== ":") {
+      continue;
+    }
+    const letter = line[0];
+    const value = line.slice(2);
+    if (Object.hasOwn(FIELD_NAMES, letter)) {
+      record.touched = true;
+      if (value) {
+        record.fields[FIELD_NAMES[letter]] =
+          value.length > MAX_FIELD_LENGTH
+            ? value.slice(0, MAX_FIELD_LENGTH)
+            : value;
+      }
+    } else if (letter === "F") {
+      record.touched = true;
+      directory = value;
+    } else if (letter === "R") {
+      record.touched = true;
+      record.fileCount++;
+      if (wantedFiles) {
+        const file = `/${directory ? `${directory}/` : ""}${value}`;
+        if (wantedFiles.has(file)) {
+          record.files.push(file);
+        }
+      }
+    }
+  }
+  if (!overflow) {
+    finishRecord();
+  }
+  return { packages, malformed, overflow };
+}
+
+/**
+ * Read the packages of the apk installed database.
+ *
+ * @param {Object} [options] Read options
+ * @param {string} [options.rootPath="/"] Directory the database paths hang off;
+ *   tests point it at a built tree
+ * @param {string} [options.databaseFile] Read this file instead of looking
+ *   under `rootPath`
+ * @param {Set<string>} [options.wantedFiles] Absolute paths whose owner is
+ *   wanted. A package comes back with the ones it owns in `files`; the other
+ *   files of a package are only counted, so a large database stays small.
+ * @param {number} [options.maxBytes] Largest database to read, 64 MiB by default
+ * @param {number} [options.maxPackages] Most packages to accept, 100000 by default
+ * @param {string} [options.platform] `linux` by default for the process; an apk
+ *   database exists on Linux only
+ * @returns {{state: string, path?: string, packages: Object[], malformed: number}}
+ *   `state` is `ok`, `absent` (no database, or not Linux), `unreadable` (a
+ *   database that could not be read in full) or `skipped` (a dry run).
+ *   `malformed` counts the records left out for having no name or version.
+ *   Each package has `name`, `version` and `fileCount`, `files`, and, when the
+ *   record has them, `arch`, `origin`, `maintainer`, `license`, `url` and
+ *   `description`.
+ */
+export function readApkInstalledDb(options = {}) {
+  const result = (state, extra = {}) => ({
+    state,
+    packages: [],
+    malformed: 0,
+    ...extra,
+  });
+  const platform = options.platform || process.platform;
+  if (platform !== "linux") {
+    return result("absent");
+  }
+  const rootPath = options.rootPath || "/";
+  if (isDryRun) {
+    recordActivity({
+      capability: "apk-installed-db",
+      kind: "read",
+      reason:
+        "Dry run mode blocks reading the apk installed database, which lives outside the project.",
+      status: "blocked",
+      target: options.databaseFile || rootPath,
+    });
+    return result("skipped");
+  }
+  const candidates = options.databaseFile
+    ? [options.databaseFile]
+    : APK_INSTALLED_DB_PATHS.map((location) => join(rootPath, location));
+  const file = candidates.find((candidate) => safeExistsSync(candidate));
+  if (!file) {
+    return result("absent");
+  }
+  const unreadable = (reason) => {
+    thoughtLog(`The apk installed database ${file} was not read: ${reason}.`);
+    recordActivity({
+      capability: "apk-installed-db",
+      kind: "read",
+      reason: `The apk installed database ${file} could not be read: ${reason}.`,
+      status: "failed",
+      target: file,
+    });
+    return result("unreadable", { path: file });
+  };
+  const maxBytes = options.maxBytes ?? MAX_APK_DATABASE_BYTES;
+  const text = readBounded(file, maxBytes);
+  if (text === undefined) {
+    return unreadable(`it is not a file of at most ${maxBytes} bytes`);
+  }
+  const maxPackages = options.maxPackages ?? MAX_APK_PACKAGES;
+  const { packages, malformed, overflow } = parseDatabase(
+    text,
+    options.wantedFiles,
+    maxPackages,
+  );
+  if (overflow) {
+    return unreadable(`it lists more than ${maxPackages} packages`);
+  }
+  if (malformed) {
+    thoughtLog(
+      `${malformed} record(s) of the apk installed database ${file} have no name or no version and were left out.`,
+    );
+  }
+  recordActivity({
+    capability: "apk-installed-db",
+    kind: "read",
+    reason: `Read ${packages.length} package(s) from the apk installed database ${file}.`,
+    status: "completed",
+    target: file,
+  });
+  return result("ok", { path: file, packages, malformed });
+}