@cdxgen/cdxgen 13.0.1 → 13.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -16
- package/bin/audit.js +6 -0
- package/bin/cdxgen.js +141 -7
- package/bin/evinse.js +3 -1
- package/data/component-tags.json +8 -1
- package/data/crypto-oid.json +88 -12
- package/data/frameworks-list.json +86 -1
- package/data/remediations.json +1167 -0
- package/data/rules/ai-bom.yaml +78 -0
- package/data/rules/asar-archives.yaml +1 -1
- package/data/rules/build-fidelity.yaml +782 -0
- package/data/rules/cargo-rust.yaml +132 -0
- package/data/rules/mcp-servers.yaml +122 -0
- package/data/rules/package-integrity.yaml +13 -13
- package/lib/audit/index.js +48 -4
- package/lib/audit/scoring.js +2 -8
- package/lib/audit/targets.js +6 -1
- package/lib/cli/bomAssembly.js +32 -1
- package/lib/cli/cliOptions.js +94 -7
- package/lib/cli/index.js +268 -49
- package/lib/cli/jsBom.js +91 -5
- package/lib/cli/jvmBom.js +1515 -1314
- package/lib/cli/managedBom.js +249 -28
- package/lib/cli/nativeBom.js +986 -10
- package/lib/core/activity.js +120 -2
- package/lib/core/buildLedger.js +799 -0
- package/lib/core/deferredExit.js +141 -0
- package/lib/core/env.js +108 -2
- package/lib/core/fs.js +60 -0
- package/lib/core/logger.js +12 -99
- package/lib/core/severity.js +16 -0
- package/lib/core/syncFileWriter.js +106 -0
- package/lib/core/userAgent.js +151 -0
- package/lib/ecosystems/cargoCache.js +364 -0
- package/lib/ecosystems/caxa.js +13 -5
- package/lib/ecosystems/cppEvidence.js +40 -2
- package/lib/ecosystems/ecosystems.js +250 -91
- package/lib/ecosystems/golem.js +11 -1
- package/lib/ecosystems/gradleutils.js +498 -17
- package/lib/ecosystems/kosi.js +1256 -0
- package/lib/ecosystems/parsers-crystal.js +179 -0
- package/lib/ecosystems/parsers-dotnet.js +69 -39
- package/lib/ecosystems/parsers-elm.js +816 -0
- package/lib/ecosystems/parsers-gleam.js +1 -1
- package/lib/ecosystems/parsers-js.js +26 -0
- package/lib/ecosystems/parsers-julia.js +268 -0
- package/lib/ecosystems/parsers-jvm.js +9 -0
- package/lib/ecosystems/parsers-lua.js +218 -0
- package/lib/ecosystems/parsers-misc.js +188 -2
- package/lib/ecosystems/parsers-nim.js +221 -0
- package/lib/ecosystems/parsers-opam.js +180 -0
- package/lib/ecosystems/parsers-perl.js +277 -0
- package/lib/ecosystems/parsers-python.js +172 -36
- package/lib/ecosystems/parsers-r.js +148 -0
- package/lib/ecosystems/parsers-rebar.js +411 -0
- package/lib/ecosystems/parsers-rust.js +944 -19
- package/lib/ecosystems/parsers-spack.js +204 -0
- package/lib/ecosystems/parsers-stack.js +264 -0
- package/lib/ecosystems/parsers-terraform.js +194 -0
- package/lib/ecosystems/pypiClassifiers.js +326 -0
- package/lib/ecosystems/pythonutils.js +49 -1
- package/lib/ecosystems/rubyutils.js +13 -0
- package/lib/ecosystems/rusi.js +126 -6
- package/lib/evinser/evinser.js +141 -13
- package/lib/helpers/agenticToolutils.js +925 -0
- package/lib/helpers/remote/dependency-track.js +81 -14
- package/lib/inventory/agentFormulationParser.js +57 -4
- package/lib/inventory/aiCollector.js +504 -10
- package/lib/inventory/aiInventory.js +12 -0
- package/lib/inventory/analyzer.js +488 -30
- package/lib/inventory/astWorker.js +6 -0
- package/lib/inventory/auditCategories.js +17 -0
- package/lib/inventory/cbomutils.js +30 -0
- package/lib/inventory/cdxrs.js +7 -2
- package/lib/inventory/communityAiConfigParser.js +215 -2
- package/lib/inventory/depsUtils.js +72 -0
- package/lib/inventory/dosai.js +508 -22
- package/lib/inventory/envcontext.js +215 -24
- package/lib/inventory/evidenceUtils.js +13 -0
- package/lib/inventory/fetchBatch.js +131 -2
- package/lib/inventory/fetchRate.js +74 -11
- package/lib/inventory/forgeEnricher.js +3 -2
- package/lib/inventory/jvmToolEnv.js +594 -0
- package/lib/inventory/mcpConfigParser.js +166 -2
- package/lib/inventory/mcpDiscovery.js +11 -0
- package/lib/inventory/mcpServerManifest.js +389 -0
- package/lib/inventory/mcpbBundle.js +419 -0
- package/lib/inventory/osinfo.js +1 -0
- package/lib/inventory/plugins.js +3 -0
- package/lib/inventory/protobom.js +54 -88
- package/lib/inventory/protobomLoader.js +4 -0
- package/lib/inventory/rustFormulationParser.js +1 -1
- package/lib/inventory/source.js +10 -4
- package/lib/inventory/stylePackageRefs.js +41 -0
- package/lib/inventory/svelteUtils.js +524 -0
- package/lib/inventory/toolRequirements.js +903 -0
- package/lib/managers/binary.js +24 -0
- package/lib/managers/docker.js +26 -5
- package/lib/parsers/huggingfaceUtils.js +15 -7
- package/lib/parsers/onnx.js +294 -0
- package/lib/parsers/safetensors.js +114 -0
- package/lib/server/server.js +1 -1
- package/lib/stages/postgen/annotationBuilder.js +97 -0
- package/lib/stages/postgen/auditBom.js +76 -79
- package/lib/stages/postgen/introspection/annotate.js +320 -0
- package/lib/stages/postgen/introspection/commandFacts.js +293 -0
- package/lib/stages/postgen/introspection/emit.js +192 -0
- package/lib/stages/postgen/introspection/formulationEvidence.js +427 -0
- package/lib/stages/postgen/introspection/markers.js +250 -0
- package/lib/stages/postgen/introspection/reflect.js +1214 -0
- package/lib/stages/postgen/introspection/report.js +1208 -0
- package/lib/stages/postgen/introspection/score.js +1067 -0
- package/lib/stages/postgen/introspection/shapeCommand.js +304 -0
- package/lib/stages/postgen/parentEdge.js +156 -0
- package/lib/stages/postgen/postgen.js +284 -3
- package/lib/stages/postgen/ruleEngine.js +17 -0
- package/lib/stages/pregen/envAudit.js +42 -2
- package/lib/stages/pregen/pregen.js +1171 -6
- package/lib/third-party/arborist/snapshots/calc-dep-flags.test.cjs +462 -0
- package/lib/validator/bomValidator.js +11 -2
- package/package.json +26 -26
- package/types/lib/audit/index.d.ts.map +1 -1
- package/types/lib/audit/scoring.d.ts +2 -8
- package/types/lib/audit/scoring.d.ts.map +1 -1
- package/types/lib/audit/targets.d.ts.map +1 -1
- package/types/lib/cli/bomAssembly.d.ts.map +1 -1
- package/types/lib/cli/cliOptions.d.ts +2 -1
- package/types/lib/cli/cliOptions.d.ts.map +1 -1
- package/types/lib/cli/index.d.ts +10 -2
- package/types/lib/cli/index.d.ts.map +1 -1
- package/types/lib/cli/jsBom.d.ts +23 -0
- package/types/lib/cli/jsBom.d.ts.map +1 -1
- package/types/lib/cli/jvmBom.d.ts.map +1 -1
- package/types/lib/cli/managedBom.d.ts +33 -0
- package/types/lib/cli/managedBom.d.ts.map +1 -1
- package/types/lib/cli/nativeBom.d.ts +88 -8
- package/types/lib/cli/nativeBom.d.ts.map +1 -1
- package/types/lib/core/activity.d.ts.map +1 -1
- package/types/lib/core/buildLedger.d.ts +272 -0
- package/types/lib/core/buildLedger.d.ts.map +1 -0
- package/types/lib/core/deferredExit.d.ts +84 -0
- package/types/lib/core/deferredExit.d.ts.map +1 -0
- package/types/lib/core/env.d.ts +43 -1
- package/types/lib/core/env.d.ts.map +1 -1
- package/types/lib/core/fs.d.ts +21 -0
- package/types/lib/core/fs.d.ts.map +1 -1
- package/types/lib/core/logger.d.ts +2 -11
- package/types/lib/core/logger.d.ts.map +1 -1
- package/types/lib/core/severity.d.ts +11 -0
- package/types/lib/core/severity.d.ts.map +1 -0
- package/types/lib/core/syncFileWriter.d.ts +33 -0
- package/types/lib/core/syncFileWriter.d.ts.map +1 -0
- package/types/lib/core/userAgent.d.ts +56 -0
- package/types/lib/core/userAgent.d.ts.map +1 -0
- package/types/lib/ecosystems/cargoCache.d.ts +75 -0
- package/types/lib/ecosystems/cargoCache.d.ts.map +1 -0
- package/types/lib/ecosystems/caxa.d.ts.map +1 -1
- package/types/lib/ecosystems/cppEvidence.d.ts +1 -1
- package/types/lib/ecosystems/cppEvidence.d.ts.map +1 -1
- package/types/lib/ecosystems/ecosystems.d.ts +15 -1
- package/types/lib/ecosystems/ecosystems.d.ts.map +1 -1
- package/types/lib/ecosystems/gradleutils.d.ts +53 -0
- package/types/lib/ecosystems/gradleutils.d.ts.map +1 -1
- package/types/lib/ecosystems/kosi.d.ts +73 -0
- package/types/lib/ecosystems/kosi.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-crystal.d.ts +27 -0
- package/types/lib/ecosystems/parsers-crystal.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-dotnet.d.ts +19 -0
- package/types/lib/ecosystems/parsers-dotnet.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-elm.d.ts +79 -0
- package/types/lib/ecosystems/parsers-elm.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-js.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-julia.d.ts +57 -0
- package/types/lib/ecosystems/parsers-julia.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-jvm.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-lua.d.ts +50 -0
- package/types/lib/ecosystems/parsers-lua.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-misc.d.ts +23 -0
- package/types/lib/ecosystems/parsers-misc.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-nim.d.ts +43 -0
- package/types/lib/ecosystems/parsers-nim.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-opam.d.ts +31 -0
- package/types/lib/ecosystems/parsers-opam.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-perl.d.ts +36 -0
- package/types/lib/ecosystems/parsers-perl.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-python.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-r.d.ts +35 -0
- package/types/lib/ecosystems/parsers-r.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-rebar.d.ts +42 -0
- package/types/lib/ecosystems/parsers-rebar.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-rust.d.ts +199 -0
- package/types/lib/ecosystems/parsers-rust.d.ts.map +1 -1
- package/types/lib/ecosystems/parsers-spack.d.ts +33 -0
- package/types/lib/ecosystems/parsers-spack.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-stack.d.ts +35 -0
- package/types/lib/ecosystems/parsers-stack.d.ts.map +1 -0
- package/types/lib/ecosystems/parsers-terraform.d.ts +34 -0
- package/types/lib/ecosystems/parsers-terraform.d.ts.map +1 -0
- package/types/lib/ecosystems/pypiClassifiers.d.ts +80 -0
- package/types/lib/ecosystems/pypiClassifiers.d.ts.map +1 -0
- package/types/lib/ecosystems/pythonutils.d.ts +21 -1
- package/types/lib/ecosystems/pythonutils.d.ts.map +1 -1
- package/types/lib/ecosystems/rubyutils.d.ts.map +1 -1
- package/types/lib/ecosystems/rusi.d.ts.map +1 -1
- package/types/lib/evinser/evinser.d.ts +34 -1
- package/types/lib/evinser/evinser.d.ts.map +1 -1
- package/types/lib/helpers/agenticToolutils.d.ts +87 -0
- package/types/lib/helpers/agenticToolutils.d.ts.map +1 -0
- package/types/lib/helpers/remote/dependency-track.d.ts +45 -4
- package/types/lib/helpers/remote/dependency-track.d.ts.map +1 -1
- package/types/lib/inventory/agentFormulationParser.d.ts.map +1 -1
- package/types/lib/inventory/aiCollector.d.ts.map +1 -1
- package/types/lib/inventory/aiInventory.d.ts.map +1 -1
- package/types/lib/inventory/analyzer.d.ts.map +1 -1
- package/types/lib/inventory/auditCategories.d.ts +12 -0
- package/types/lib/inventory/auditCategories.d.ts.map +1 -1
- package/types/lib/inventory/cbomutils.d.ts.map +1 -1
- package/types/lib/inventory/communityAiConfigParser.d.ts.map +1 -1
- package/types/lib/inventory/depsUtils.d.ts.map +1 -1
- package/types/lib/inventory/dosai.d.ts +43 -0
- package/types/lib/inventory/dosai.d.ts.map +1 -1
- package/types/lib/inventory/envcontext.d.ts.map +1 -1
- package/types/lib/inventory/evidenceUtils.d.ts.map +1 -1
- package/types/lib/inventory/fetchBatch.d.ts +43 -0
- package/types/lib/inventory/fetchBatch.d.ts.map +1 -1
- package/types/lib/inventory/fetchRate.d.ts +9 -6
- package/types/lib/inventory/fetchRate.d.ts.map +1 -1
- package/types/lib/inventory/forgeEnricher.d.ts.map +1 -1
- package/types/lib/inventory/jvmToolEnv.d.ts +190 -0
- package/types/lib/inventory/jvmToolEnv.d.ts.map +1 -0
- package/types/lib/inventory/mcpConfigParser.d.ts.map +1 -1
- package/types/lib/inventory/mcpDiscovery.d.ts.map +1 -1
- package/types/lib/inventory/mcpServerManifest.d.ts +19 -0
- package/types/lib/inventory/mcpServerManifest.d.ts.map +1 -0
- package/types/lib/inventory/mcpbBundle.d.ts +19 -0
- package/types/lib/inventory/mcpbBundle.d.ts.map +1 -0
- package/types/lib/inventory/osinfo.d.ts.map +1 -1
- package/types/lib/inventory/plugins.d.ts.map +1 -1
- package/types/lib/inventory/protobom.d.ts +18 -13
- package/types/lib/inventory/protobom.d.ts.map +1 -1
- package/types/lib/inventory/protobomLoader.d.ts.map +1 -1
- package/types/lib/inventory/source.d.ts.map +1 -1
- package/types/lib/inventory/stylePackageRefs.d.ts +17 -0
- package/types/lib/inventory/stylePackageRefs.d.ts.map +1 -0
- package/types/lib/inventory/svelteUtils.d.ts +124 -0
- package/types/lib/inventory/svelteUtils.d.ts.map +1 -0
- package/types/lib/inventory/toolRequirements.d.ts +216 -0
- package/types/lib/inventory/toolRequirements.d.ts.map +1 -0
- package/types/lib/managers/binary.d.ts.map +1 -1
- package/types/lib/managers/docker.d.ts.map +1 -1
- package/types/lib/parsers/huggingfaceUtils.d.ts +8 -1
- package/types/lib/parsers/huggingfaceUtils.d.ts.map +1 -1
- package/types/lib/parsers/onnx.d.ts +28 -0
- package/types/lib/parsers/onnx.d.ts.map +1 -0
- package/types/lib/parsers/safetensors.d.ts +22 -0
- package/types/lib/parsers/safetensors.d.ts.map +1 -0
- package/types/lib/stages/postgen/annotationBuilder.d.ts +53 -0
- package/types/lib/stages/postgen/annotationBuilder.d.ts.map +1 -0
- package/types/lib/stages/postgen/auditBom.d.ts.map +1 -1
- package/types/lib/stages/postgen/introspection/annotate.d.ts +81 -0
- package/types/lib/stages/postgen/introspection/annotate.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts +32 -0
- package/types/lib/stages/postgen/introspection/commandFacts.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts +71 -0
- package/types/lib/stages/postgen/introspection/emit.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts +154 -0
- package/types/lib/stages/postgen/introspection/formulationEvidence.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts +58 -0
- package/types/lib/stages/postgen/introspection/markers.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts +77 -0
- package/types/lib/stages/postgen/introspection/reflect.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/report.d.ts +85 -0
- package/types/lib/stages/postgen/introspection/report.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/score.d.ts +149 -0
- package/types/lib/stages/postgen/introspection/score.d.ts.map +1 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts +79 -0
- package/types/lib/stages/postgen/introspection/shapeCommand.d.ts.map +1 -0
- package/types/lib/stages/postgen/parentEdge.d.ts +35 -0
- package/types/lib/stages/postgen/parentEdge.d.ts.map +1 -0
- package/types/lib/stages/postgen/postgen.d.ts +35 -2
- package/types/lib/stages/postgen/postgen.d.ts.map +1 -1
- package/types/lib/stages/postgen/ruleEngine.d.ts.map +1 -1
- package/types/lib/stages/pregen/envAudit.d.ts.map +1 -1
- package/types/lib/stages/pregen/pregen.d.ts +131 -0
- package/types/lib/stages/pregen/pregen.d.ts.map +1 -1
- package/types/lib/validator/bomValidator.d.ts.map +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { readFileSync, statSync } from "node:fs";
|
|
2
|
-
import { basename, dirname, extname, relative } from "node:path";
|
|
2
|
+
import { basename, dirname, extname, join, relative } from "node:path";
|
|
3
3
|
|
|
4
4
|
import { getAllFiles } from "../core/fs.js";
|
|
5
5
|
import {
|
|
@@ -31,6 +31,8 @@ import {
|
|
|
31
31
|
toHuggingFacePurl,
|
|
32
32
|
} from "../parsers/huggingfaceUtils.js";
|
|
33
33
|
import { parseOllamaModelfile } from "../parsers/ollama.js";
|
|
34
|
+
import { readOnnxHeader } from "../parsers/onnx.js";
|
|
35
|
+
import { readSafetensorsHeader } from "../parsers/safetensors.js";
|
|
34
36
|
import {
|
|
35
37
|
detectAiModelVariants,
|
|
36
38
|
normalizeDetectedVariants,
|
|
@@ -52,6 +54,40 @@ const PROMPT_CONFIG_PATTERNS = [
|
|
|
52
54
|
];
|
|
53
55
|
const MODEFILE_PATTERNS = ["Modelfile", "**/Modelfile", "**/Modelfile.*"];
|
|
54
56
|
const GGUF_PATTERNS = ["**/*.gguf"];
|
|
57
|
+
const SAFETENSORS_PATTERNS = ["**/*.safetensors"];
|
|
58
|
+
const ONNX_PATTERNS = ["**/*.onnx"];
|
|
59
|
+
// PyTorch pickle-based checkpoints. Loading these executes arbitrary code, so
|
|
60
|
+
// they are inventoried and flagged rather than parsed. The `.pth` extension is
|
|
61
|
+
// intentionally excluded: in the wild it is overwhelmingly the Python
|
|
62
|
+
// path-configuration file (site-packages shims), not a PyTorch checkpoint.
|
|
63
|
+
const PICKLE_MODEL_PATTERNS = ["**/*.{bin,pt,ckpt}"];
|
|
64
|
+
// Directories whose contents are never real model artifacts. Filtering these
|
|
65
|
+
// keeps the expensive `**/*.bin` walk from surfacing dependency payloads.
|
|
66
|
+
const MODEL_ARTIFACT_IGNORE_DIRS = [
|
|
67
|
+
"site-packages",
|
|
68
|
+
"node_modules",
|
|
69
|
+
"vendor",
|
|
70
|
+
"dist-packages",
|
|
71
|
+
".venv",
|
|
72
|
+
"venv",
|
|
73
|
+
];
|
|
74
|
+
// A real checkpoint is not a few bytes. Anything smaller is a shim or stub.
|
|
75
|
+
const MIN_PICKLE_MODEL_BYTES = 4096;
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Whether a discovered artifact path sits inside a dependency payload
|
|
79
|
+
* directory. Weights vendored under a dependency tree belong to that
|
|
80
|
+
* dependency, not to the project being scanned.
|
|
81
|
+
*
|
|
82
|
+
* @param {string} filePath Discovered file path
|
|
83
|
+
* @returns {boolean} true when the path should be skipped
|
|
84
|
+
*/
|
|
85
|
+
const isVendoredArtifactPath = (filePath) => {
|
|
86
|
+
const normalized = String(filePath).replaceAll("\\", "/");
|
|
87
|
+
return MODEL_ARTIFACT_IGNORE_DIRS.some((segment) =>
|
|
88
|
+
normalized.includes(`/${segment}/`),
|
|
89
|
+
);
|
|
90
|
+
};
|
|
55
91
|
const GGUF_MIN_ALIGNMENT = 4;
|
|
56
92
|
const GGUF_MAX_ALIGNMENT = 1024 * 1024;
|
|
57
93
|
const IGNORE_SOURCE_FILE_PATTERN =
|
|
@@ -72,10 +108,35 @@ const HOST_PROVIDER_PATTERNS = [
|
|
|
72
108
|
["replicate", /(?:^|\.)replicate\.com$/iu],
|
|
73
109
|
["perplexity", /(?:^|\.)perplexity\.ai$/iu],
|
|
74
110
|
["vertex-ai", /(?:^|\.)aiplatform\.googleapis\.com$/iu],
|
|
111
|
+
["xai", /(?:^|\.)x\.ai$/iu],
|
|
112
|
+
[
|
|
113
|
+
"aws-bedrock",
|
|
114
|
+
/(?:^|\.)bedrock(?:-runtime)?(?:\.[a-z0-9-]+)?\.amazonaws\.com$/iu,
|
|
115
|
+
],
|
|
116
|
+
["openrouter", /(?:^|\.)openrouter\.ai$/iu],
|
|
117
|
+
["cerebras", /(?:^|\.)cerebras\.ai$/iu],
|
|
118
|
+
["sambanova", /(?:^|\.)sambanova\.ai$/iu],
|
|
119
|
+
["moonshot", /(?:^|\.)moonshot\.(?:cn|ai)$/iu],
|
|
120
|
+
["zhipu", /(?:^|\.)bigmodel\.cn$/iu],
|
|
121
|
+
["dashscope", /(?:^|\.)dashscope(?:-intl)?\.aliyuncs\.com$/iu],
|
|
122
|
+
["nvidia-nim", /(?:^|\.)api\.nvidia\.com$/iu],
|
|
123
|
+
["baseten", /(?:^|\.)baseten\.co$/iu],
|
|
124
|
+
["novita", /(?:^|\.)novita\.ai$/iu],
|
|
125
|
+
["deepinfra", /(?:^|\.)deepinfra\.com$/iu],
|
|
75
126
|
["ollama", /(?:^|\.)ollama\.com$/iu],
|
|
76
|
-
["ollama", /(?:^|\.)localhost$/iu],
|
|
77
|
-
["ollama", /(?:^|\.)127\.0\.0\.1$/iu],
|
|
78
127
|
];
|
|
128
|
+
// Local inference servers are distinguished by their conventional loopback
|
|
129
|
+
// port rather than by hostname, since they all bind to localhost. When the
|
|
130
|
+
// port is unknown, the endpoint is labeled `local-inference` and marked for
|
|
131
|
+
// review instead of being guessed as any single tool.
|
|
132
|
+
const LOCAL_INFERENCE_PORTS = new Map([
|
|
133
|
+
["1234", "lm-studio"],
|
|
134
|
+
["8000", "vllm"],
|
|
135
|
+
["8080", "llamafile"],
|
|
136
|
+
["1337", "jan"],
|
|
137
|
+
["4891", "gpt4all"],
|
|
138
|
+
["11434", "ollama"],
|
|
139
|
+
]);
|
|
79
140
|
const AI_PACKAGE_REGISTRY = [
|
|
80
141
|
{ pattern: /^openai$/u, provider: "openai", serviceName: "OpenAI API" },
|
|
81
142
|
{
|
|
@@ -126,7 +187,7 @@ const AI_PACKAGE_REGISTRY = [
|
|
|
126
187
|
{ pattern: /^@mlc-ai\//u, runtime: "mlc" },
|
|
127
188
|
{ pattern: /^groq-sdk$|^groq$/u, provider: "groq", serviceName: "Groq API" },
|
|
128
189
|
{
|
|
129
|
-
pattern: /^cohere-ai$/u,
|
|
190
|
+
pattern: /^cohere-ai$|^cohere$/u,
|
|
130
191
|
provider: "cohere",
|
|
131
192
|
serviceName: "Cohere API",
|
|
132
193
|
},
|
|
@@ -179,6 +240,85 @@ const AI_PACKAGE_REGISTRY = [
|
|
|
179
240
|
provider: "perplexity",
|
|
180
241
|
serviceName: "Perplexity API",
|
|
181
242
|
},
|
|
243
|
+
// Agent frameworks and SDKs from the 2025-2026 wave. These identify the
|
|
244
|
+
// orchestration layer a project builds on, which is often more meaningful for
|
|
245
|
+
// an AI-BOM than the underlying provider SDK alone. Patterns accept both the
|
|
246
|
+
// registry package name (hyphenated) and the imported module name
|
|
247
|
+
// (underscored) because Python import statements normalize hyphens to
|
|
248
|
+
// underscores.
|
|
249
|
+
{
|
|
250
|
+
pattern:
|
|
251
|
+
/^claude[-_]agent[-_]sdk$|^@anthropic-ai\/claude-agent-sdk$|^claude[-_]code[-_]sdk$/u,
|
|
252
|
+
framework: "claude-agent-sdk",
|
|
253
|
+
provider: "anthropic",
|
|
254
|
+
serviceName: "Anthropic API",
|
|
255
|
+
},
|
|
256
|
+
{ pattern: /^google[-_]adk$|^@google\/adk$/u, framework: "google-adk" },
|
|
257
|
+
{
|
|
258
|
+
pattern: /^pydantic[-_]ai(?:[-_]slim)?$|^pydantic[-_]graph$/u,
|
|
259
|
+
framework: "pydantic-ai",
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
pattern: /^strands[-_]agents(?:[-_]tools|[-_]builder)?$/u,
|
|
263
|
+
framework: "strands-agents",
|
|
264
|
+
},
|
|
265
|
+
{ pattern: /^agno$|^phidata$/u, framework: "agno" },
|
|
266
|
+
{
|
|
267
|
+
pattern:
|
|
268
|
+
/^agent[-_]framework(?:[-_]core|[-_]azure[-_]ai|[-_]copilotstudio)?$/u,
|
|
269
|
+
framework: "microsoft-agent-framework",
|
|
270
|
+
},
|
|
271
|
+
{ pattern: /^semantic[-_]kernel$/u, framework: "semantic-kernel" },
|
|
272
|
+
{ pattern: /^deepagents$/u, framework: "deepagents" },
|
|
273
|
+
{
|
|
274
|
+
pattern: /^smolagents$/u,
|
|
275
|
+
framework: "smolagents",
|
|
276
|
+
provider: "huggingface",
|
|
277
|
+
},
|
|
278
|
+
{ pattern: /^dspy(?:[-_]ai)?$/u, framework: "dspy" },
|
|
279
|
+
{ pattern: /^haystack[-_]ai$|^farm[-_]haystack$/u, framework: "haystack" },
|
|
280
|
+
{ pattern: /^instructor$/u, framework: "instructor" },
|
|
281
|
+
{ pattern: /^browser[-_]use$/u, framework: "browser-use" },
|
|
282
|
+
{
|
|
283
|
+
pattern: /^openai[-_]agents$/u,
|
|
284
|
+
framework: "openai-agents",
|
|
285
|
+
provider: "openai",
|
|
286
|
+
serviceName: "OpenAI API",
|
|
287
|
+
},
|
|
288
|
+
{ pattern: /^crewai(?:[-_]tools)?$/u, framework: "crewai" },
|
|
289
|
+
// Note: OpenAI Swarm is intentionally not matched by a bare `swarm` name.
|
|
290
|
+
// On npm/PyPI `swarm` refers to unrelated projects (Ethereum Swarm, Docker
|
|
291
|
+
// Swarm tooling); OpenAI Swarm is installed from Git, not under that name.
|
|
292
|
+
// Provider SDKs and OpenAI-compatible gateway clients for the wider set of
|
|
293
|
+
// inference providers now in common use.
|
|
294
|
+
{
|
|
295
|
+
pattern: /^@ai-sdk\/xai$|^langchain[-_]xai$|^xai[-_]sdk$/u,
|
|
296
|
+
provider: "xai",
|
|
297
|
+
serviceName: "xAI Grok API",
|
|
298
|
+
},
|
|
299
|
+
{
|
|
300
|
+
pattern:
|
|
301
|
+
/^@aws-sdk\/client-bedrock-runtime$|^langchain[-_]aws$|^@langchain\/aws$/u,
|
|
302
|
+
provider: "aws-bedrock",
|
|
303
|
+
serviceName: "AWS Bedrock",
|
|
304
|
+
},
|
|
305
|
+
{
|
|
306
|
+
pattern: /^openrouter$|^@openrouter\/ai-sdk-provider$/u,
|
|
307
|
+
provider: "openrouter",
|
|
308
|
+
serviceName: "OpenRouter API",
|
|
309
|
+
},
|
|
310
|
+
{
|
|
311
|
+
pattern:
|
|
312
|
+
/^@cerebras\/cerebras[_-]cloud[_-]sdk$|^cerebras[-_]cloud[-_]sdk$/u,
|
|
313
|
+
provider: "cerebras",
|
|
314
|
+
serviceName: "Cerebras API",
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
pattern: /^dashscope$/u,
|
|
318
|
+
provider: "dashscope",
|
|
319
|
+
serviceName: "Alibaba DashScope API",
|
|
320
|
+
},
|
|
321
|
+
{ pattern: /^zhipuai$/u, provider: "zhipu", serviceName: "Zhipu GLM API" },
|
|
182
322
|
];
|
|
183
323
|
const serializeAiService = (service) => ({
|
|
184
324
|
"bom-ref": service["bom-ref"],
|
|
@@ -252,7 +392,7 @@ const extractImportNames = (raw) => {
|
|
|
252
392
|
imports.push({ index: match.index, names: [match[1]] });
|
|
253
393
|
}
|
|
254
394
|
for (const match of raw.matchAll(
|
|
255
|
-
/from\s+([A-Za-z0-9_.-]+)\s+import\b|import\
|
|
395
|
+
/from\s+([A-Za-z0-9_.-]+)\s+import\b|import[ \t]+([A-Za-z0-9_.,\- \t]+)/gu,
|
|
256
396
|
)) {
|
|
257
397
|
if (linePrefixForIndex(raw, match.index) !== "") {
|
|
258
398
|
continue;
|
|
@@ -326,11 +466,19 @@ const isLocalAiHostname = (hostname) => {
|
|
|
326
466
|
const hostProviderFromValue = (urlValue) => {
|
|
327
467
|
try {
|
|
328
468
|
const parsed = new URL(urlValue);
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
469
|
+
const hostMatch = HOST_PROVIDER_PATTERNS.find(([, pattern]) =>
|
|
470
|
+
pattern.test(parsed.hostname),
|
|
471
|
+
)?.[0];
|
|
472
|
+
if (hostMatch) {
|
|
473
|
+
return hostMatch;
|
|
474
|
+
}
|
|
475
|
+
// Local inference servers all bind to loopback, so disambiguate by the
|
|
476
|
+
// conventional port. An unknown local port is labeled generically rather
|
|
477
|
+
// than guessed as Ollama.
|
|
478
|
+
if (isLocalAiHostname(parsed.hostname)) {
|
|
479
|
+
return LOCAL_INFERENCE_PORTS.get(parsed.port) || "local-inference";
|
|
480
|
+
}
|
|
481
|
+
return undefined;
|
|
334
482
|
} catch {
|
|
335
483
|
return undefined;
|
|
336
484
|
}
|
|
@@ -426,6 +574,82 @@ const quantizationValueFromFilename = (fileName) => {
|
|
|
426
574
|
return legacyQuantizationValueFromFilename(fileName);
|
|
427
575
|
};
|
|
428
576
|
|
|
577
|
+
// Quantization methods that are distinct from the container format. These often
|
|
578
|
+
// appear in a model or file name and are useful for hardware-fit review.
|
|
579
|
+
const QUANTIZATION_METHOD_PATTERNS = [
|
|
580
|
+
["gptq", /\bgptq\b/iu],
|
|
581
|
+
["awq", /\bawq\b/iu],
|
|
582
|
+
["exl2", /\bexl2\b/iu],
|
|
583
|
+
["exl3", /\bexl3\b/iu],
|
|
584
|
+
["aqlm", /\baqlm\b/iu],
|
|
585
|
+
// Marlin is a GPTQ/AWQ kernel that appears as a suffix (gptq-marlin) or a
|
|
586
|
+
// quant_method value, so require that shape rather than the bare word, which
|
|
587
|
+
// is also an ordinary name.
|
|
588
|
+
["marlin", /(?:gptq|awq)[-_]?marlin|[-_]marlin\b|^marlin$/iu],
|
|
589
|
+
["bitsandbytes", /\bbnb\b|bitsandbytes/iu],
|
|
590
|
+
];
|
|
591
|
+
|
|
592
|
+
/**
|
|
593
|
+
* Detect a non-GGUF quantization method from a model name, file name, or an
|
|
594
|
+
* explicit `quantization_config.quant_method` value.
|
|
595
|
+
*
|
|
596
|
+
* @param {string} candidate Model name, file name, or quant method string
|
|
597
|
+
* @returns {string|undefined} Normalized quantization method label
|
|
598
|
+
*/
|
|
599
|
+
const quantizationMethodFromText = (candidate) => {
|
|
600
|
+
const normalized = String(candidate || "");
|
|
601
|
+
if (!normalized) {
|
|
602
|
+
return undefined;
|
|
603
|
+
}
|
|
604
|
+
for (const [method, pattern] of QUANTIZATION_METHOD_PATTERNS) {
|
|
605
|
+
if (pattern.test(normalized)) {
|
|
606
|
+
return method;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
return undefined;
|
|
610
|
+
};
|
|
611
|
+
|
|
612
|
+
// Matches the Hugging Face sharded-weights suffix, e.g.
|
|
613
|
+
// `model-00001-of-00003.safetensors`. Grouping by the de-sharded stem lets a
|
|
614
|
+
// multi-file checkpoint be modeled as one component instead of one per shard.
|
|
615
|
+
const SAFETENSORS_SHARD_PATTERN = /-\d{5}-of-\d{5}$/u;
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Compute the de-sharded model stem for a safetensors file name.
|
|
619
|
+
*
|
|
620
|
+
* @param {string} fileName safetensors base name (without extension)
|
|
621
|
+
* @returns {{ stem: string, isShard: boolean }} de-sharded stem and shard flag
|
|
622
|
+
*/
|
|
623
|
+
const deshardedStem = (fileName) => {
|
|
624
|
+
const isShard = SAFETENSORS_SHARD_PATTERN.test(fileName);
|
|
625
|
+
return {
|
|
626
|
+
stem: fileName.replace(SAFETENSORS_SHARD_PATTERN, ""),
|
|
627
|
+
isShard,
|
|
628
|
+
};
|
|
629
|
+
};
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Read the model identity a sibling `config.json` declares, when present, so a
|
|
633
|
+
* safetensors artifact attaches to the repository model rather than a filename.
|
|
634
|
+
*
|
|
635
|
+
* @param {string} directoryPath Directory containing the safetensors file
|
|
636
|
+
* @returns {string|undefined} Model identity from `_name_or_path`
|
|
637
|
+
*/
|
|
638
|
+
const modelIdentityFromSiblingConfig = (directoryPath) => {
|
|
639
|
+
try {
|
|
640
|
+
const config = JSON.parse(
|
|
641
|
+
readFileSync(join(directoryPath, "config.json"), "utf-8"),
|
|
642
|
+
);
|
|
643
|
+
const nameOrPath = config?._name_or_path;
|
|
644
|
+
if (typeof nameOrPath === "string" && nameOrPath.trim()) {
|
|
645
|
+
return nameOrPath.trim();
|
|
646
|
+
}
|
|
647
|
+
} catch {
|
|
648
|
+
// no readable sibling config
|
|
649
|
+
}
|
|
650
|
+
return undefined;
|
|
651
|
+
};
|
|
652
|
+
|
|
429
653
|
const createExternalReference = (type, url, comment) => {
|
|
430
654
|
const sanitizedUrl = sanitizeBomUrl(url);
|
|
431
655
|
if (!sanitizedUrl) {
|
|
@@ -1237,6 +1461,20 @@ const syncComponentProperties = (component, data = {}) => {
|
|
|
1237
1461
|
data.quantization,
|
|
1238
1462
|
);
|
|
1239
1463
|
}
|
|
1464
|
+
if (data.quantizationMethod) {
|
|
1465
|
+
addUniqueProperty(
|
|
1466
|
+
component.properties,
|
|
1467
|
+
"cdx:ai:quantizationMethod",
|
|
1468
|
+
data.quantizationMethod,
|
|
1469
|
+
);
|
|
1470
|
+
}
|
|
1471
|
+
if (data.unsafeDeserialization) {
|
|
1472
|
+
addUniqueProperty(
|
|
1473
|
+
component.properties,
|
|
1474
|
+
"cdx:ai:unsafeDeserialization",
|
|
1475
|
+
"true",
|
|
1476
|
+
);
|
|
1477
|
+
}
|
|
1240
1478
|
for (const variant of data.variants || []) {
|
|
1241
1479
|
addUniqueProperty(component.properties, "cdx:ai:variant", variant);
|
|
1242
1480
|
}
|
|
@@ -1540,9 +1778,11 @@ const createModelComponent = (componentsByKey, seed, occurrenceLocation) => {
|
|
|
1540
1778
|
parameterCount: seed.parameterCount,
|
|
1541
1779
|
provider: seed.provider,
|
|
1542
1780
|
quantization: seed.quantization,
|
|
1781
|
+
quantizationMethod: seed.quantizationMethod,
|
|
1543
1782
|
reviewNeeded: seed.reviewNeeded,
|
|
1544
1783
|
runtime: seed.runtime,
|
|
1545
1784
|
source: seed.source,
|
|
1785
|
+
unsafeDeserialization: seed.unsafeDeserialization,
|
|
1546
1786
|
});
|
|
1547
1787
|
if (occurrenceLocation) {
|
|
1548
1788
|
appendOccurrence(component.evidence, occurrenceLocation);
|
|
@@ -1891,6 +2131,260 @@ export function collectJsAiInventory(discoveryPath, options = {}) {
|
|
|
1891
2131
|
);
|
|
1892
2132
|
}
|
|
1893
2133
|
|
|
2134
|
+
const safetensorsGroups = new Map();
|
|
2135
|
+
for (const filePath of getMatchingFiles(
|
|
2136
|
+
discoveryPath,
|
|
2137
|
+
SAFETENSORS_PATTERNS,
|
|
2138
|
+
options,
|
|
2139
|
+
)) {
|
|
2140
|
+
if (isVendoredArtifactPath(filePath)) {
|
|
2141
|
+
continue;
|
|
2142
|
+
}
|
|
2143
|
+
const header = readSafetensorsHeader(filePath);
|
|
2144
|
+
if (!header) {
|
|
2145
|
+
continue;
|
|
2146
|
+
}
|
|
2147
|
+
const fileComponent = createAiFileComponent(
|
|
2148
|
+
componentsByKey,
|
|
2149
|
+
discoveryPath,
|
|
2150
|
+
filePath,
|
|
2151
|
+
"model-artifact-file",
|
|
2152
|
+
["safetensors"],
|
|
2153
|
+
);
|
|
2154
|
+
addUniqueProperty(
|
|
2155
|
+
fileComponent.properties,
|
|
2156
|
+
"cdx:safetensors:tensorCount",
|
|
2157
|
+
header.tensorCount || undefined,
|
|
2158
|
+
);
|
|
2159
|
+
if (header.dtypes.length) {
|
|
2160
|
+
addUniqueProperty(
|
|
2161
|
+
fileComponent.properties,
|
|
2162
|
+
"cdx:safetensors:dtypes",
|
|
2163
|
+
header.dtypes.join(","),
|
|
2164
|
+
);
|
|
2165
|
+
}
|
|
2166
|
+
const directoryPath = dirname(filePath);
|
|
2167
|
+
const { stem, isShard } = deshardedStem(basename(filePath, ".safetensors"));
|
|
2168
|
+
// Group shards of the same checkpoint (and co-located single files) so a
|
|
2169
|
+
// multi-file model is one component. The sibling config identity, when
|
|
2170
|
+
// present, keeps the group aligned with the model the HF parser found.
|
|
2171
|
+
const groupKey = `${directoryPath}::${stem}`;
|
|
2172
|
+
let group = safetensorsGroups.get(groupKey);
|
|
2173
|
+
if (!group) {
|
|
2174
|
+
group = {
|
|
2175
|
+
directoryPath,
|
|
2176
|
+
stem,
|
|
2177
|
+
isShard,
|
|
2178
|
+
// The first matching file on the sorted discovery list. Occurrence
|
|
2179
|
+
// evidence must point at a path that exists, so a sharded group cites
|
|
2180
|
+
// its first shard rather than a synthesized `<stem>.safetensors`.
|
|
2181
|
+
firstFilePath: filePath,
|
|
2182
|
+
fileRefs: [],
|
|
2183
|
+
totalParameters: 0,
|
|
2184
|
+
dtypes: new Set(),
|
|
2185
|
+
quantMethod: undefined,
|
|
2186
|
+
};
|
|
2187
|
+
safetensorsGroups.set(groupKey, group);
|
|
2188
|
+
}
|
|
2189
|
+
group.isShard = group.isShard || isShard;
|
|
2190
|
+
group.fileRefs.push(fileComponent["bom-ref"]);
|
|
2191
|
+
group.totalParameters += header.totalParameters || 0;
|
|
2192
|
+
for (const dtype of header.dtypes) {
|
|
2193
|
+
group.dtypes.add(dtype);
|
|
2194
|
+
}
|
|
2195
|
+
group.quantMethod =
|
|
2196
|
+
group.quantMethod ||
|
|
2197
|
+
quantizationMethodFromText(header.metadata?.quant_method) ||
|
|
2198
|
+
quantizationMethodFromText(basename(filePath));
|
|
2199
|
+
}
|
|
2200
|
+
// A directory can hold several distinct checkpoints (for example
|
|
2201
|
+
// `model.safetensors` alongside `adapter_model.safetensors`). They share one
|
|
2202
|
+
// sibling config.json, so resolve the config identity only for the group that
|
|
2203
|
+
// carries the primary weights and let the others keep their own stem.
|
|
2204
|
+
const groupsByDirectory = new Map();
|
|
2205
|
+
for (const group of safetensorsGroups.values()) {
|
|
2206
|
+
const peers = groupsByDirectory.get(group.directoryPath) || [];
|
|
2207
|
+
peers.push(group);
|
|
2208
|
+
groupsByDirectory.set(group.directoryPath, peers);
|
|
2209
|
+
}
|
|
2210
|
+
for (const group of safetensorsGroups.values()) {
|
|
2211
|
+
const peers = groupsByDirectory.get(group.directoryPath) || [];
|
|
2212
|
+
const isPrimaryGroup =
|
|
2213
|
+
peers.length === 1 ||
|
|
2214
|
+
group.stem === "model" ||
|
|
2215
|
+
group.stem === "pytorch_model" ||
|
|
2216
|
+
group.isShard;
|
|
2217
|
+
const siblingIdentity = isPrimaryGroup
|
|
2218
|
+
? modelIdentityFromSiblingConfig(group.directoryPath)
|
|
2219
|
+
: undefined;
|
|
2220
|
+
const modelId = siblingIdentity || group.stem;
|
|
2221
|
+
const component = createModelComponent(
|
|
2222
|
+
componentsByKey,
|
|
2223
|
+
{
|
|
2224
|
+
artifactFormat: "safetensors",
|
|
2225
|
+
confidence: siblingIdentity ? "high" : "medium",
|
|
2226
|
+
modelFamily: modelFamilyFromName(modelId),
|
|
2227
|
+
modelId,
|
|
2228
|
+
parameterCount: group.totalParameters || undefined,
|
|
2229
|
+
provider: siblingIdentity?.includes("/") ? "huggingface" : "local",
|
|
2230
|
+
quantizationMethod: group.quantMethod,
|
|
2231
|
+
reviewNeeded: true,
|
|
2232
|
+
source: "local-artifact",
|
|
2233
|
+
tags: ["safetensors", "local-model"],
|
|
2234
|
+
variants: group.isShard ? ["sharded"] : undefined,
|
|
2235
|
+
},
|
|
2236
|
+
relativeOccurrenceLocation(discoveryPath, group.firstFilePath),
|
|
2237
|
+
);
|
|
2238
|
+
if (group.dtypes.size) {
|
|
2239
|
+
addUniqueProperty(
|
|
2240
|
+
component.properties,
|
|
2241
|
+
"cdx:safetensors:dtypes",
|
|
2242
|
+
Array.from(group.dtypes).sort().join(","),
|
|
2243
|
+
);
|
|
2244
|
+
}
|
|
2245
|
+
// Two groups can still resolve to the same model component (for example a
|
|
2246
|
+
// repository whose config identity matches its stem), so merge the artifact
|
|
2247
|
+
// edges instead of replacing them.
|
|
2248
|
+
const existingRefs = dependencyMap.get(component["bom-ref"]) || new Set();
|
|
2249
|
+
for (const fileRef of group.fileRefs) {
|
|
2250
|
+
existingRefs.add(fileRef);
|
|
2251
|
+
}
|
|
2252
|
+
dependencyMap.set(component["bom-ref"], existingRefs);
|
|
2253
|
+
}
|
|
2254
|
+
|
|
2255
|
+
for (const filePath of getMatchingFiles(
|
|
2256
|
+
discoveryPath,
|
|
2257
|
+
ONNX_PATTERNS,
|
|
2258
|
+
options,
|
|
2259
|
+
)) {
|
|
2260
|
+
if (isVendoredArtifactPath(filePath)) {
|
|
2261
|
+
continue;
|
|
2262
|
+
}
|
|
2263
|
+
const fileComponent = createAiFileComponent(
|
|
2264
|
+
componentsByKey,
|
|
2265
|
+
discoveryPath,
|
|
2266
|
+
filePath,
|
|
2267
|
+
"model-artifact-file",
|
|
2268
|
+
["onnx"],
|
|
2269
|
+
);
|
|
2270
|
+
const modelId = basename(filePath, ".onnx");
|
|
2271
|
+
const component = createModelComponent(
|
|
2272
|
+
componentsByKey,
|
|
2273
|
+
{
|
|
2274
|
+
artifactFormat: "onnx",
|
|
2275
|
+
confidence: "high",
|
|
2276
|
+
modelFamily: modelFamilyFromName(modelId),
|
|
2277
|
+
modelId,
|
|
2278
|
+
provider: "local",
|
|
2279
|
+
reviewNeeded: true,
|
|
2280
|
+
source: "local-artifact",
|
|
2281
|
+
tags: ["onnx", "local-model"],
|
|
2282
|
+
},
|
|
2283
|
+
relativeOccurrenceLocation(discoveryPath, filePath),
|
|
2284
|
+
);
|
|
2285
|
+
// The protobuf header is read by offset without loading the graph, so
|
|
2286
|
+
// producer and opset provenance is cheap even for large models.
|
|
2287
|
+
const onnxHeader = readOnnxHeader(filePath);
|
|
2288
|
+
if (onnxHeader) {
|
|
2289
|
+
if (onnxHeader.irVersion) {
|
|
2290
|
+
addUniqueProperty(
|
|
2291
|
+
component.properties,
|
|
2292
|
+
"cdx:onnx:irVersion",
|
|
2293
|
+
String(onnxHeader.irVersion),
|
|
2294
|
+
);
|
|
2295
|
+
}
|
|
2296
|
+
// Producer names are tool identifiers such as "pytorch"; they name the
|
|
2297
|
+
// exporter rather than any user or environment detail.
|
|
2298
|
+
if (onnxHeader.producerName) {
|
|
2299
|
+
addUniqueProperty(
|
|
2300
|
+
component.properties,
|
|
2301
|
+
"cdx:onnx:producer",
|
|
2302
|
+
onnxHeader.producerName,
|
|
2303
|
+
);
|
|
2304
|
+
}
|
|
2305
|
+
if (onnxHeader.producerVersion) {
|
|
2306
|
+
addUniqueProperty(
|
|
2307
|
+
component.properties,
|
|
2308
|
+
"cdx:onnx:producerVersion",
|
|
2309
|
+
onnxHeader.producerVersion,
|
|
2310
|
+
);
|
|
2311
|
+
}
|
|
2312
|
+
if (onnxHeader.opsets.length) {
|
|
2313
|
+
addUniqueProperty(
|
|
2314
|
+
component.properties,
|
|
2315
|
+
"cdx:onnx:opsets",
|
|
2316
|
+
onnxHeader.opsets
|
|
2317
|
+
.map((opset) => `${opset.domain || "ai.onnx"}:${opset.version}`)
|
|
2318
|
+
.join(","),
|
|
2319
|
+
);
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
dependencyMap.set(
|
|
2323
|
+
fileComponent["bom-ref"],
|
|
2324
|
+
new Set([component["bom-ref"]]),
|
|
2325
|
+
);
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
for (const filePath of getMatchingFiles(
|
|
2329
|
+
discoveryPath,
|
|
2330
|
+
PICKLE_MODEL_PATTERNS,
|
|
2331
|
+
options,
|
|
2332
|
+
)) {
|
|
2333
|
+
// Only treat these as model artifacts when the name looks model-like and
|
|
2334
|
+
// the file is large enough to be a real checkpoint. Dependency payload
|
|
2335
|
+
// directories are skipped entirely.
|
|
2336
|
+
if (isVendoredArtifactPath(filePath)) {
|
|
2337
|
+
continue;
|
|
2338
|
+
}
|
|
2339
|
+
const lowerName = basename(filePath).toLowerCase();
|
|
2340
|
+
const looksLikeModel =
|
|
2341
|
+
/(model|weights|pytorch|checkpoint|ckpt|adapter|lora|diffusion|embed)/u.test(
|
|
2342
|
+
lowerName,
|
|
2343
|
+
);
|
|
2344
|
+
if (!looksLikeModel) {
|
|
2345
|
+
continue;
|
|
2346
|
+
}
|
|
2347
|
+
let artifactBytes = 0;
|
|
2348
|
+
try {
|
|
2349
|
+
artifactBytes = statSync(filePath).size;
|
|
2350
|
+
} catch {
|
|
2351
|
+
continue;
|
|
2352
|
+
}
|
|
2353
|
+
if (artifactBytes < MIN_PICKLE_MODEL_BYTES) {
|
|
2354
|
+
continue;
|
|
2355
|
+
}
|
|
2356
|
+
const fileComponent = createAiFileComponent(
|
|
2357
|
+
componentsByKey,
|
|
2358
|
+
discoveryPath,
|
|
2359
|
+
filePath,
|
|
2360
|
+
"model-artifact-file",
|
|
2361
|
+
["pytorch-pickle"],
|
|
2362
|
+
);
|
|
2363
|
+
const modelId = basename(filePath).replace(/\.[^.]+$/u, "");
|
|
2364
|
+
const component = createModelComponent(
|
|
2365
|
+
componentsByKey,
|
|
2366
|
+
{
|
|
2367
|
+
artifactFormat: "pytorch-pickle",
|
|
2368
|
+
confidence: "medium",
|
|
2369
|
+
modelFamily: modelFamilyFromName(modelId),
|
|
2370
|
+
modelId,
|
|
2371
|
+
provider: "local",
|
|
2372
|
+
quantizationMethod: quantizationMethodFromText(lowerName),
|
|
2373
|
+
reviewNeeded: true,
|
|
2374
|
+
source: "local-artifact",
|
|
2375
|
+
tags: ["pytorch", "pickle", "local-model"],
|
|
2376
|
+
// Loading a pickle checkpoint can execute arbitrary code; surface this
|
|
2377
|
+
// so security audit rules can flag untrusted model artifacts.
|
|
2378
|
+
unsafeDeserialization: true,
|
|
2379
|
+
},
|
|
2380
|
+
relativeOccurrenceLocation(discoveryPath, filePath),
|
|
2381
|
+
);
|
|
2382
|
+
dependencyMap.set(
|
|
2383
|
+
fileComponent["bom-ref"],
|
|
2384
|
+
new Set([component["bom-ref"]]),
|
|
2385
|
+
);
|
|
2386
|
+
}
|
|
2387
|
+
|
|
1894
2388
|
return buildAiInventoryResult(componentsByKey, servicesByKey, dependencyMap);
|
|
1895
2389
|
}
|
|
1896
2390
|
|
|
@@ -10,7 +10,9 @@ import {
|
|
|
10
10
|
import { communityAiConfigParser } from "./communityAiConfigParser.js";
|
|
11
11
|
import { mergeServices, trimComponents } from "./depsUtils.js";
|
|
12
12
|
import { classifyMcpReference } from "./mcp.js";
|
|
13
|
+
import { mcpbBundleParser } from "./mcpbBundle.js";
|
|
13
14
|
import { mcpConfigParser } from "./mcpConfigParser.js";
|
|
15
|
+
import { mcpServerManifestParser } from "./mcpServerManifest.js";
|
|
14
16
|
|
|
15
17
|
/** Project types treated as AI-related inventory for collection and parsing. */
|
|
16
18
|
export const AI_INVENTORY_PROJECT_TYPES = ["mcp", "ai-skill", "ai"];
|
|
@@ -64,6 +66,16 @@ const AI_INVENTORY_PARSERS = [
|
|
|
64
66
|
parser: mcpConfigParser,
|
|
65
67
|
types: ["mcp"],
|
|
66
68
|
},
|
|
69
|
+
{
|
|
70
|
+
id: mcpServerManifestParser.id,
|
|
71
|
+
parser: mcpServerManifestParser,
|
|
72
|
+
types: ["mcp"],
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
id: mcpbBundleParser.id,
|
|
76
|
+
parser: mcpbBundleParser,
|
|
77
|
+
types: ["mcp"],
|
|
78
|
+
},
|
|
67
79
|
{
|
|
68
80
|
id: communityAiConfigParser.id,
|
|
69
81
|
parser: communityAiConfigParser,
|