knodin 0.7.6 → 0.8.2

package/dist/src/relationship-adapters.js

@@ -0,0 +1,377 @@
+import childProcess from "node:child_process";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { compareBytes } from "./compare.js";
+import { gitExecutable } from "./git-executable.js";
+const MAX_FILES = 5_000;
+const MAX_CONTRACT_BYTES = 2 * 1024 * 1024;
+export function trackedRepositoryFiles(repository) {
+    try {
+        return (childProcess
+            .execFileSync(gitExecutable(), ["ls-files", "-z"], {
+            cwd: repository,
+            encoding: "utf-8",
+            stdio: ["ignore", "pipe", "ignore"],
+        })
+            .split("\0")
+            .filter(Boolean)
+            // Byte order: this sort decides which files survive the MAX_FILES cap,
+            // so collation would change which files are scanned at all.
+            .sort(compareBytes)
+            .slice(0, MAX_FILES));
+    }
+    catch {
+        return [];
+    }
+}
+function lineNumber(content, offset) {
+    return content.slice(0, offset).split("\n").length;
+}
+function matchAll(content, pattern, build) {
+    const findings = [];
+    for (const match of content.matchAll(pattern)) {
+        findings.push({ ...build(match), line: lineNumber(content, match.index ?? 0) });
+    }
+    return findings;
+}
+function packageFindings(file, content) {
+    if (file === "package.json") {
+        try {
+            const manifest = JSON.parse(content);
+            const findings = [];
+            for (const section of ["dependencies", "devDependencies", "peerDependencies"]) {
+                const dependencies = manifest[section];
+                if (typeof dependencies !== "object" || dependencies === null)
+                    continue;
+                for (const name of Object.keys(dependencies).sort(compareBytes)) {
+                    findings.push({
+                        type: "depends_on",
+                        target: `pkg:npm/${encodeURIComponent(name)}`,
+                        line: lineNumber(content, Math.max(content.indexOf(`"${name}"`), 0)),
+                        adapter: "package-manifest",
+                        confidence: 1,
+                    });
+                }
+            }
+            return findings;
+        }
+        catch {
+            return [];
+        }
+    }
+    if (file === "go.mod") {
+        return content.split(/\r?\n/).flatMap((line, index) => {
+            const [moduleName, version] = line.trim().split(/\s+/, 2);
+            if (!moduleName?.includes(".") || !version?.startsWith("v"))
+                return [];
+            return [
+                {
+                    type: "depends_on",
+                    target: `pkg:golang/${moduleName}`,
+                    line: index + 1,
+                    adapter: "package-manifest",
+                    confidence: 1,
+                },
+            ];
+        });
+    }
+    if (file === "Cargo.toml" || file === "pyproject.toml") {
+        return matchAll(content, /^([A-Za-z0-9_.-]+)[ \t]*=[ \t]*(?:["'{])/gm, (match) => ({
+            type: "depends_on",
+            target: `pkg:${file === "Cargo.toml" ? "cargo" : "pypi"}/${match[1]}`,
+            adapter: "package-manifest",
+            confidence: 0.9,
+        }));
+    }
+    return [];
+}
+function contractFindings(file, content) {
+    const lower = file.toLowerCase();
+    if (/openapi|swagger/.test(lower) && /\.(json|ya?ml)$/.test(lower)) {
+        return matchAll(content, /^[ \t]{0,8}(\/[^:\s]+)[ \t]*:/gm, (match) => ({
+            type: "provides_api",
+            target: `openapi:${match[1]}`,
+            adapter: "openapi",
+            confidence: 1,
+        }));
+    }
+    if (/asyncapi/.test(lower)) {
+        return matchAll(content, /^[ \t]{2,8}([^:\s]+)[ \t]*:/gm, (match) => ({
+            type: "publishes_event",
+            target: `asyncapi:${match[1]}`,
+            adapter: "asyncapi",
+            confidence: 0.9,
+        }));
+    }
+    if (lower.endsWith(".graphql") || lower.endsWith(".gql")) {
+        return matchAll(content, /^[ \t]*(?:type|interface|input)[ \t]+([A-Za-z_]\w*)/gm, (match) => ({
+            type: "provides_api",
+            target: `graphql:${match[1]}`,
+            adapter: "graphql",
+            confidence: 1,
+        }));
+    }
+    if (lower.endsWith(".proto")) {
+        return matchAll(content, /^[ \t]*service[ \t]+([A-Za-z_]\w*)/gm, (match) => ({
+            type: "provides_api",
+            target: `protobuf:${match[1]}`,
+            adapter: "protobuf",
+            confidence: 1,
+        }));
+    }
+    return [];
+}
+function infrastructureFindings(file, content) {
+    const lower = file.toLowerCase();
+    const findings = [];
+    if (lower.endsWith(".tf")) {
+        findings.push(...matchAll(content, /^[ \t]*source[ \t]*=[ \t]*"([^"\r\n]+)"/gm, (match) => ({
+            type: "depends_on",
+            target: `terraform:${match[1]}`,
+            adapter: "terraform",
+            confidence: 1,
+        })), ...matchAll(content, /^[ \t]*backend[ \t]+"([^"\r\n]+)"/gm, (match) => ({
+            type: "reads_output",
+            target: `terraform-state:${match[1]}`,
+            adapter: "terraform",
+            confidence: 0.9,
+        })), ...matchAll(content, /^[ \t]*output[ \t]+"([^"\r\n]+)"/gm, (match) => ({
+            type: "provisions",
+            target: `terraform-output:${match[1]}`,
+            adapter: "terraform",
+            confidence: 1,
+        })));
+    }
+    if (/\.(ya?ml|tpl)$/.test(lower)) {
+        findings.push(...matchAll(content, /^[ \t]*image:[ \t]*["']?([^"'\s]+)["']?/gm, (match) => ({
+            type: "uses_image",
+            target: `oci:${match[1]}`,
+            adapter: lower.includes("helm") || lower.includes("chart") ? "helm" : "kubernetes",
+            confidence: 1,
+        })));
+    }
+    return findings;
+}
+function provenanceFindings(file, content) {
+    const lower = file.toLowerCase();
+    if (lower.endsWith(".map")) {
+        try {
+            const sourceMap = JSON.parse(content);
+            if (!Array.isArray(sourceMap.sources))
+                return [];
+            return sourceMap.sources.flatMap((source) => typeof source === "string"
+                ? [
+                    {
+                        type: "generated_from",
+                        target: `file:${source}`,
+                        line: 1,
+                        adapter: "source-map",
+                        confidence: 1,
+                    },
+                ]
+                : []);
+        }
+        catch {
+            return [];
+        }
+    }
+    if (/(?:^|\/)(?:bom|sbom)(?:\.[^/]*)?\.json$/.test(lower)) {
+        try {
+            const sbom = JSON.parse(content);
+            if (!Array.isArray(sbom.components))
+                return [];
+            return sbom.components.flatMap((component) => {
+                if (!component || typeof component !== "object")
+                    return [];
+                const purl = component.purl;
+                return typeof purl === "string"
+                    ? [
+                        {
+                            type: "depends_on",
+                            target: purl,
+                            line: lineNumber(content, Math.max(content.indexOf(purl), 0)),
+                            adapter: "cyclonedx-sbom",
+                            confidence: 1,
+                        },
+                    ]
+                    : [];
+            });
+        }
+        catch {
+            return [];
+        }
+    }
+    if (lower.endsWith(".intoto.jsonl") || lower.includes("provenance")) {
+        return matchAll(content, /"uri"[ \t]*:[ \t]*"([^"\r\n]+)"/g, (match) => ({
+            type: "generated_from",
+            target: match[1] ?? "unknown",
+            adapter: "slsa-provenance",
+            confidence: 1,
+        }));
+    }
+    return [];
+}
+function sourceFindings(content, repository, knownPaths) {
+    const findings = [];
+    for (const [knownPath, identity] of knownPaths) {
+        for (const match of content.matchAll(/(["'`])([^"'`\n]+)\1/g)) {
+            const literal = match[2];
+            if (!literal || path.resolve(repository, literal) !== path.resolve(knownPath))
+                continue;
+            findings.push({
+                type: "references_path",
+                target: identity,
+                line: lineNumber(content, match.index ?? 0),
+                adapter: "exact-path-literal",
+                confidence: 1,
+            });
+        }
+    }
+    findings.push(...matchAll(content, /\b(?:process\.env\.|os\.environ\[["'])([A-Z][A-Z0-9_]{2,})/g, (match) => ({
+        type: "reads_config",
+        target: `config:${match[1]}`,
+        adapter: "configuration-key",
+        confidence: 0.85,
+    })));
+    return findings;
+}
+/** Extract bounded, source-evidenced non-symbol relationships from one checkout. */
+export function extractRepositoryRelationships(repositoryId, repository, knownPaths = new Map()) {
+    const relationships = [];
+    for (const file of trackedRepositoryFiles(repository)) {
+        const absolute = path.join(repository, file);
+        let stat;
+        try {
+            stat = fs.statSync(absolute);
+        }
+        catch {
+            continue;
+        }
+        if (!stat.isFile() || stat.size > MAX_CONTRACT_BYTES)
+            continue;
+        let content;
+        try {
+            content = fs.readFileSync(absolute, "utf-8");
+        }
+        catch {
+            continue;
+        }
+        const findings = [
+            ...packageFindings(file, content),
+            ...contractFindings(file, content),
+            ...infrastructureFindings(file, content),
+            ...provenanceFindings(file, content),
+            ...sourceFindings(content, repository, knownPaths),
+        ];
+        for (const finding of findings) {
+            relationships.push({
+                type: finding.type,
+                from: repositoryId,
+                to: finding.target,
+                targetIdentity: finding.target,
+                evidence: {
+                    location: `${file}:${finding.line}`,
+                    line: finding.line,
+                    adapter: finding.adapter,
+                    kind: "extracted",
+                    confidence: finding.confidence,
+                    freshness: "current",
+                    indexGeneration: null,
+                },
+            });
+        }
+    }
+    // Byte order over a NUL-joined composite key: collation may ignore the NUL
+    // separators entirely, which blurs the field boundaries this key depends on.
+    return relationships.sort((left, right) => compareBytes(`${left.from}\0${left.type}\0${left.to}\0${left.evidence.location}`, `${right.from}\0${right.type}\0${right.to}\0${right.evidence.location}`));
+}
+/** Stream a file hash so duplicate detection never buffers an entire artifact. */
+export async function hashFileStream(file) {
+    const digest = crypto.createHash("sha256");
+    await new Promise((resolve, reject) => {
+        const stream = fs.createReadStream(file);
+        stream.on("data", (chunk) => digest.update(chunk));
+        stream.on("error", reject);
+        stream.on("end", resolve);
+    });
+    return digest.digest("hex");
+}
+function filesGroupedBySize(repositories) {
+    const bySize = new Map();
+    const orderedRepositories = [...repositories].sort((left, right) => compareBytes(left.id, right.id));
+    for (const repository of orderedRepositories) {
+        for (const file of trackedRepositoryFiles(repository.path)) {
+            const absolute = path.join(repository.path, file);
+            try {
+                const stat = fs.statSync(absolute);
+                if (!stat.isFile())
+                    continue;
+                const candidates = bySize.get(stat.size) ?? [];
+                candidates.push({ repositoryId: repository.id, file, absolute });
+                bySize.set(stat.size, candidates);
+            }
+            catch {
+                // A concurrently removed file is absent from this evidence snapshot.
+            }
+        }
+    }
+    return bySize;
+}
+async function filesGroupedByHash(candidates) {
+    const byHash = new Map();
+    for (const candidate of candidates) {
+        const hash = await hashFileStream(candidate.absolute);
+        const matches = byHash.get(hash) ?? [];
+        matches.push(candidate);
+        byHash.set(hash, matches);
+    }
+    return byHash;
+}
+function duplicateRelationships(matches) {
+    const repositoryIds = new Set(matches.map(({ repositoryId }) => repositoryId));
+    if (repositoryIds.size < 2)
+        return [];
+    // Byte order: `ordered[0]` becomes the canonical side of every emitted
+    // duplicate_candidate, so collation would change the relationship direction.
+    const ordered = [...matches].sort((left, right) => compareBytes(`${left.repositoryId}\0${left.file}`, `${right.repositoryId}\0${right.file}`));
+    const canonical = ordered[0];
+    if (!canonical)
+        return [];
+    return ordered.slice(1).flatMap((duplicate) => {
+        if (duplicate.repositoryId === canonical.repositoryId)
+            return [];
+        return [
+            {
+                type: "duplicate_candidate",
+                from: duplicate.repositoryId,
+                to: canonical.repositoryId,
+                targetIdentity: canonical.repositoryId,
+                evidence: {
+                    location: duplicate.file,
+                    adapter: "streaming-sha256",
+                    kind: "inferred",
+                    confidence: 1,
+                    freshness: "current",
+                    indexGeneration: null,
+                },
+            },
+        ];
+    });
+}
+/**
+ * Detect exact tracked-file matches across repositories. Equality is reported
+ * only as a duplicate candidate; it never asserts generated/vendor lineage.
+ */
+export async function detectDuplicateCandidates(repositories) {
+    const relationships = [];
+    for (const candidates of filesGroupedBySize(repositories).values()) {
+        if (candidates.length < 2)
+            continue;
+        const byHash = await filesGroupedByHash(candidates);
+        for (const matches of byHash.values()) {
+            relationships.push(...duplicateRelationships(matches));
+        }
+    }
+    return relationships.sort((left, right) => compareBytes(`${left.from}\0${left.to}\0${left.evidence.location}`, `${right.from}\0${right.to}\0${right.evidence.location}`));
+}