@cdxgen/cdxgen 0.0.1 → 13.0.0

package/lib/managers/piptree.js

@@ -0,0 +1,267 @@
+/**
+ * The idea behind this plugin came from the excellent pipdeptree package
+ * https://github.com/tox-dev/pipdeptree
+ *
+ * We use the internal pip api to construct the dependency tree for modern python + pip environments
+ */
+import { readFileSync } from "node:fs";
+import { delimiter, join } from "node:path";
+
+import {
+  getTmpDir,
+  safeExistsSync,
+  safeMkdtempSync,
+  safeRmSync,
+  safeSpawnSync,
+  safeWriteSync,
+} from "../core/fs.js";
+
+const PIP_TREE_PLUGIN_CONTENT = `
+import importlib.metadata as importlib_metadata
+import json
+import sys
+
+from pip._internal.metadata import pkg_resources
+
+REQUIREMENT_MODULE_FOUND = False
+try:
+    from packaging.requirements import Requirement
+    REQUIREMENT_MODULE_FOUND = True
+except ImportError:
+    try:
+        from pip._vendor.packaging.requirements import Requirement
+        REQUIREMENT_MODULE_FOUND = True
+    except ImportError:
+        pass
+
+def frozen_req_from_dist(dist):
+    try:
+        from pip._internal.operations.freeze import FrozenRequirement
+    except ImportError:
+        from pip import FrozenRequirement
+    try:
+        from pip._internal import metadata
+
+        dist = metadata.pkg_resources.Distribution(dist)
+        try:
+            fr = FrozenRequirement.from_dist(dist)
+        except TypeError:
+            fr = FrozenRequirement.from_dist(dist, [])
+        return str(fr).strip()
+    except ImportError:
+        pass
+
+
+def get_installed_distributions(python_path=None):
+    dists = pkg_resources.Environment.from_paths(python_path).iter_installed_distributions(
+        local_only=False,
+        skip=(),
+        user_only=False,
+    )
+    return [d._dist for d in dists]
+
+
+def _get_extra_deps_from_dist(dist):
+    extra_deps = {}
+    if not dist:
+        return extra_deps
+    # all requirements, some of which may be extra-only:
+    reqs = dist.metadata.get_all('Requires-Dist') or []
+    # extras this package defines:
+    extras = dist.metadata.get_all('Provides-Extra') or []
+    for req_str in reqs:
+        req = Requirement(req_str)
+        if req.marker and 'extra' in str(req.marker):
+            # evaluate marker for each declared extra
+            for extra in extras:
+                if req.marker.evaluate({'extra': extra}):
+                    extra_deps.setdefault(extra, []).append({"name": str(req.name), "versionSpecifiers": str(req.specifier), "url": str(req.url) if req.url else None})
+    return extra_deps
+
+
+def _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps, visited=None):
+    dependencies = []
+    if not extra_deps:
+        return dependencies
+    if visited is None:
+        visited = set()
+
+    # Treat an extra with the name 'all' as dependencies
+    all_deps = extra_deps.get("all", [])
+    for dep in all_deps:
+        dep_name = dep["name"]
+        if dep_name in visited:
+            continue  # Avoid cycles
+        visited.add(dep_name)
+
+        dversion = name_version_cache.get(dep_name)
+        if not dversion:
+            continue
+        dversionSpecifiers = dep.get("versionSpecifiers")
+        dpurl = f"pkg:pypi/{dep_name.lower()}@{dversion}"
+        dextra_deps = _get_extra_deps_from_dist(name_dist_cache.get(dep_name))
+        ddependencies = _get_deps_from_extras(
+            name_version_cache, name_dist_cache, dextra_deps, visited.copy()
+        )
+        dependencies.append({
+            "name": dep_name,
+            "version": dversion,
+            "versionSpecifiers": dversionSpecifiers,
+            "purl": dpurl,
+            "extra_deps": dextra_deps,
+            "dependencies": ddependencies
+        })
+    return dependencies
+
+
+def get_installed_with_extras():
+    result = {}
+    if not REQUIREMENT_MODULE_FOUND:
+        return result
+    name_version_cache = {}
+    name_dist_cache = {}
+    for dist in importlib_metadata.distributions():
+        name = dist.metadata['Name']
+        version = dist.version or ""
+        name_version_cache[name] = version
+        name_dist_cache[name] = dist
+    for dist in importlib_metadata.distributions():
+        name = dist.metadata['Name'] or ""
+        version = dist.version or ""
+        # extras this package defines:
+        extras = dist.metadata.get_all('Provides-Extra') or []
+        # map each extra → its extra-only dependencies
+        extra_deps = _get_extra_deps_from_dist(dist)
+        purl = f"pkg:pypi/{name.lower()}@{version}"
+        dependencies = _get_deps_from_extras(name_version_cache, name_dist_cache, extra_deps, set())
+        result[purl] = {
+            'name': name,
+            'version': version,
+            'extras': extras,
+            'purl': purl,
+            'extra_deps': extra_deps,
+            "dependencies": dependencies
+        }
+    return result
+
+
+def find_deps(idx, path, purl, reqs, global_installed, traverse_count):
+    freqs = []
+    for r in reqs:
+        d = idx.get(r.key)
+        if not d:
+            continue
+        r.project_name = d.project_name if d is not None else r.project_name
+        if r.key in path:
+            print(f"Cycle detected: {' -> '.join(path)} -> {r.key}")
+            continue
+        current_path = path + [r.key]
+        specs = sorted(r.specs, reverse=True)
+        specs_str = ",".join(["".join(sp) for sp in specs]) if specs else ""
+        dreqs = d.requires()
+        name = r.project_name
+        version = importlib_metadata.version(r.key)
+        purl = f"pkg:pypi/{name.lower()}@{version}"
+        extra_deps = global_installed.get(purl, {}).get("extra_deps", {})
+        dependencies = find_deps(idx, current_path, purl, dreqs, global_installed, traverse_count + 1) if dreqs and traverse_count < 200 else []
+        all_dependencies = global_installed.get(purl, {}).get("dependencies", [])
+        freqs.append(
+            {
+                "name": name,
+                "version": version,
+                "versionSpecifiers": specs_str,
+                'purl': purl,
+                "extra_deps": extra_deps,
+                "dependencies": dependencies + all_dependencies,
+            }
+        )
+    return freqs
+
+
+def main(argv):
+    out_file = "piptree.json" if len(argv) < 2 else argv[-1]
+    tree = []
+    global_installed = get_installed_with_extras()
+    pkgs = get_installed_distributions(python_path=None)
+    idx = {p.key: p for p in pkgs}
+    traverse_count = 0
+    for p in pkgs:
+        fr = frozen_req_from_dist(p)
+        if not fr.startswith('# Editable'):
+            tmpA = fr.split("==")
+        else:
+            fr = p.key
+            tmpA = [fr,p.version]
+        name = tmpA[0]
+        if name.startswith("-e"):
+            name = name.split("#egg=")[-1].split(" ")[0].split("&")[0]
+        version = "latest"
+        if len(tmpA) == 2:
+            version = tmpA[1]
+        pkgName = name.split(" ")[0]
+        purl = f"pkg:pypi/{pkgName.lower()}@{version}"
+        extra_deps = global_installed.get(purl, {}).get("extra_deps", "")
+        all_dependencies = global_installed.get(purl, {}).get("dependencies", [])
+        dependencies = find_deps(idx, [p.key], purl, p.requires(), global_installed, traverse_count + 1)
+        tree.append(
+            {
+                "name": pkgName,
+                "version": version,
+                "purl": purl,
+                "extra_deps": extra_deps,
+                "dependencies": dependencies + all_dependencies,
+            }
+        )
+    with open(out_file, mode="w", encoding="utf-8") as fp:
+        json.dump(tree, fp)
+
+
+if __name__ == "__main__":
+    main(sys.argv)
+`;
+
+/**
+ * Execute the piptree plugin and return the generated tree as json object.
+ * The resulting tree would also include dependencies belonging to pip.
+ * Usage analysis is performed at a later stage to mark many of these packages as optional.
+ *
+ * @param {Object} env Environment variables to use
+ * @param {String} python_cmd Python command to use
+ * @param {String} basePath Current working directory
+ *
+ * @returns {Object} Dependency tree
+ */
+export const getTreeWithPlugin = (env, python_cmd, basePath) => {
+  let tree = [];
+  const tempDir = safeMkdtempSync(join(getTmpDir(), "cdxgen-piptree-"));
+  const pipPlugin = join(tempDir, "piptree.py");
+  const pipTreeJson = join(tempDir, "piptree.json");
+  const pipPluginArgs = [pipPlugin, pipTreeJson];
+  safeWriteSync(pipPlugin, PIP_TREE_PLUGIN_CONTENT);
+  if (env.PIP_TARGET) {
+    if (!env.PYTHONPATH) {
+      env.PYTHONPATH = "";
+    }
+    if (!env.PYTHONPATH.includes(env.PIP_TARGET)) {
+      env.PYTHONPATH = `${env.PYTHONPATH}${delimiter}${env.PIP_TARGET}`;
+    }
+  }
+  const result = safeSpawnSync(python_cmd, pipPluginArgs, {
+    cwd: basePath,
+    env,
+  });
+  if (result.status !== 0 || result.error) {
+    if (result.stdout || result.stderr) {
+      console.log(result.stdout, result.stderr);
+    }
+  }
+  if (safeExistsSync(pipTreeJson)) {
+    tree = JSON.parse(
+      readFileSync(pipTreeJson, {
+        encoding: "utf-8",
+      }),
+    );
+  }
+  safeRmSync(tempDir, { recursive: true, force: true });
+  return tree;
+};

package/lib/parsers/cmakeCache.js

@@ -0,0 +1,199 @@
+/**
+ * Pure parser for CMakeCache.txt.
+ *
+ * CMakeCache is produced by `cmake` during configuration. Each non-comment line
+ * is `KEY:TYPE=VALUE`, where TYPE is one of BOOL, PATH, FILEPATH, STRING,
+ * INTERNAL, STATIC, UNINITIALIZED. Comment lines start with `//`; a leading
+ * `# This is the CMakeCache file.` header and blank lines are ignored.
+ *
+ * This module is layer 1 (pure text in, data out): it has no filesystem or
+ * subprocess access. The git/build-directory orchestration that consumes these
+ * facts lives in `lib/ecosystems/cmakeResolver.js`.
+ */
+
+const FETCH_SOURCE_DIR_PREFIX = "FETCHCONTENT_SOURCE_DIR_";
+
+const CACHE_TYPE_PATTERN =
+  /^(?<key>[A-Za-z0-9_.+-]+):(?<type>BOOL|PATH|FILEPATH|STRING|INTERNAL|STATIC|UNINITIALIZED)=(?<value>.*)$/;
+
+/**
+ * Parse the raw text of a CMakeCache.txt file into an ordered key→entry map.
+ *
+ * Only well-formed `KEY:TYPE=VALUE` lines are captured. Comment (`//`), blank,
+ * and the `# This is the CMakeCache file.` header lines are ignored, as are
+ * lines that do not carry a recognised CMake cache type.
+ *
+ * @param {string} text Raw CMakeCache.txt contents
+ * @returns {Map<string, {type: string, value: string}>} Ordered map of cache entries
+ */
+export function parseCmakeCache(text) {
+  const map = new Map();
+  if (!text || typeof text !== "string") {
+    return map;
+  }
+  for (const rawLine of text.split(/\r\n|\r|\n/)) {
+    const line = rawLine.trimEnd();
+    if (!line) {
+      continue;
+    }
+    if (line.startsWith("//") || line.startsWith("#")) {
+      continue;
+    }
+    const match = line.match(CACHE_TYPE_PATTERN);
+    if (!match) {
+      continue;
+    }
+    const { key, type, value } = match.groups;
+    if (map.has(key)) {
+      continue;
+    }
+    map.set(key, { type, value });
+  }
+  return map;
+}
+
+/**
+ * Extract the version from a `FIND_PACKAGE_MESSAGE_DETAILS_<Pkg>:INTERNAL`
+ * value.
+ *
+ * The value is a sequence of `[...]` groups whose last group is
+ * `[v<VERSION>()]`. The version may be empty (`[v()]`). Only the final
+ * `[v...]` group is matched so a `[v...]` fragment inside an earlier bracket
+ * (a library path) cannot poison the result.
+ *
+ * @param {string} value The INTERNAL value after `=`
+ * @returns {string|null} The resolved version, empty string when the group is
+ *   present but empty, or `null` when no `[v...]` group is present.
+ */
+export function parseFindPackageVersion(value) {
+  if (!value || typeof value !== "string") {
+    return null;
+  }
+  let lastVersion = null;
+  let i = 0;
+  while (i < value.length) {
+    const open = value.indexOf("[v", i);
+    if (open === -1) {
+      break;
+    }
+    const close = value.indexOf("]", open);
+    if (close === -1) {
+      break;
+    }
+    const group = value.slice(open + 1, close);
+    if (group.startsWith("v") && group.includes("(")) {
+      const openParen = group.indexOf("(");
+      lastVersion = group.slice(1, openParen);
+    }
+    i = close + 1;
+  }
+  return lastVersion;
+}
+
+/**
+ * Resolve the high-level facts a CMakeCache exposes about a configured build.
+ *
+ * Harvests the root project, every project directory, FetchContent sources, and
+ * the resolved versions of `find_package` lookups. `<name>_VERSION` is **not**
+ * present in the cache for subprojects, so fetched/submodule versions must be
+ * resolved from git or the gitclone script instead.
+ *
+ * @param {Map<string, {type: string, value: string}>} map Output of {@link parseCmakeCache}
+ * @returns {{ rootProject: {name?: string, version?: string} | null, projects: Map<string, {version?: string, sourceDir?: string, binaryDir?: string, isTopLevel?: boolean, kind: string}>, findPackages: Map<string, string>, fetchContentBase?: string }}
+ */
+export function resolveCmakeCacheFacts(map) {
+  const rootProject = { name: undefined, version: undefined };
+  const rootName = map.get("CMAKE_PROJECT_NAME")?.value;
+  const rootVersion = map.get("CMAKE_PROJECT_VERSION")?.value;
+  if (rootName) {
+    rootProject.name = rootName;
+  }
+  if (rootVersion) {
+    rootProject.version = rootVersion;
+  }
+
+  const projects = new Map();
+  const findPackages = new Map();
+  let fetchContentBase;
+
+  for (const [key, entry] of map) {
+    if (key.startsWith("FETCHCONTENT_BASE_DIR") && entry.type === "PATH") {
+      fetchContentBase = entry.value;
+      continue;
+    }
+    if (key.startsWith(FETCH_SOURCE_DIR_PREFIX) && entry.type === "PATH") {
+      const depName = key.slice(FETCH_SOURCE_DIR_PREFIX.length).toLowerCase();
+      const project = projects.get(depName) || {
+        kind: "fetch",
+      };
+      project.kind = "fetch";
+      project.sourceDir = entry.value;
+      projects.set(depName, project);
+      continue;
+    }
+    const sourceDirMatch = key.match(/^(.+)_SOURCE_DIR$/);
+    if (sourceDirMatch && entry.type === "STATIC") {
+      const name = sourceDirMatch[1].toLowerCase();
+      if (name === "cmake") {
+        continue;
+      }
+      const project = projects.get(name) || { kind: "internal" };
+      project.sourceDir = entry.value;
+      if (!project.kind || project.kind === "internal") {
+        project.kind =
+          rootName && name === rootName.toLowerCase() ? "internal" : "internal";
+      }
+      projects.set(name, project);
+      continue;
+    }
+    const binaryDirMatch = key.match(/^(.+)_BINARY_DIR$/);
+    if (binaryDirMatch && entry.type === "STATIC") {
+      const name = binaryDirMatch[1].toLowerCase();
+      if (name === "cmake") {
+        continue;
+      }
+      const project = projects.get(name) || { kind: "internal" };
+      project.binaryDir = entry.value;
+      projects.set(name, project);
+      continue;
+    }
+    const isTopLevelMatch = key.match(/^(.+)_IS_TOP_LEVEL$/);
+    if (isTopLevelMatch && entry.type === "STATIC") {
+      const name = isTopLevelMatch[1].toLowerCase();
+      const project = projects.get(name) || { kind: "internal" };
+      project.isTopLevel = entry.value === "ON";
+      projects.set(name, project);
+      continue;
+    }
+    const findPkgMatch = key.match(/^FIND_PACKAGE_MESSAGE_DETAILS_(.+)$/);
+    if (findPkgMatch && entry.type === "INTERNAL") {
+      const pkg = findPkgMatch[1];
+      const version = parseFindPackageVersion(entry.value);
+      if (version !== null) {
+        findPackages.set(pkg, version);
+      }
+    }
+  }
+
+  for (const [name, project] of projects) {
+    if (project.kind === "fetch") {
+      continue;
+    }
+    if (project.isTopLevel === true) {
+      project.kind = "internal";
+    } else if (project.isTopLevel === false) {
+      project.kind = "submodule";
+    }
+    if (rootVersion && rootName && name === rootName.toLowerCase()) {
+      project.version = rootVersion;
+    }
+    projects.set(name, project);
+  }
+
+  return {
+    rootProject: rootName || rootVersion ? rootProject : null,
+    projects,
+    findPackages,
+    fetchContentBase,
+  };
+}