dsh-code-security 0.3.8 → 0.4.0

package/lib/deps.js

@@ -1,179 +1,206 @@
-/**
- * SBOM-lite：解析常见依赖清单，输出依赖名称、版本约束与风险标记。
- * 只做确定性解析，不联网、不执行包管理器。
- *
- * @module dsh-code-security/deps
- */
+/** Offline manifest inventory. Parsing/coverage failures are explicit, never a clean audit. */
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { parse as parseToml } from 'smol-toml';
 import { DEFAULT_EXCLUDE_DIRS } from './config.js';
+import { redactText } from './privacy.js';
+import { policyExcludesDir } from './policy.js';
+const LOCKFILES = ['package-lock.json', 'npm-shrinkwrap.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lock', 'uv.lock', 'poetry.lock', 'Pipfile.lock', 'pdm.lock', 'pylock.toml', 'Cargo.lock', 'composer.lock', 'Gemfile.lock', 'go.mod', 'requirements.txt', 'pom.xml'];
 const MANIFESTS = ['package.json', 'requirements.txt', 'pyproject.toml', 'go.mod', 'Cargo.toml', 'composer.json', 'Gemfile'];
-function asRecord(value) {
-    return typeof value === 'object' && value !== null ? value : {};
-}
+const record = (value) => value && typeof value === 'object' && !Array.isArray(value) ? value : {};
 function riskOfSpec(spec) {
     const s = spec.trim();
-    if (s === '')
+    if (!s)
         return { risk: 'unpinned', note: '未声明版本约束' };
-    if (s === '*' || /^latest$/i.test(s) || /^x$/i.test(s))
-        return { risk: 'wildcard', note: '版本约束为通配' };
-    if (/^(?:https?|git|git+|github:)/i.test(s))
-        return { risk: /#([0-9a-f]{7,40})\b/i.test(s) ? 'exact' : 'git', note: /#/.test(s) ? 'git 源已带提交锁定' : 'git 源未锁定到提交' };
-    if (/^(?:file|link):/i.test(s))
-        return { risk: 'local', note: '本地/链接依赖，发布时会失配' };
-    if (/^[0-9]+(?:\.[A-Za-z0-9.*-]+)+$/.test(s) || /^==[0-9]/.test(s))
+    if (/^(?:file|link|workspace|path):/i.test(s))
+        return { risk: 'local', note: '本地或工作区依赖' };
+    if (/^(?:https?|git|github:)/i.test(s))
+        return /#[a-f0-9]{7,40}$/i.test(s) ? { risk: 'exact', note: 'Git 提交已固定' } : { risk: 'git', note: '外部源未固定 Git 提交' };
+    if (s === '*' || /^latest$/i.test(s) || /(?:^|\.)[x*](?:\.|$)/i.test(s))
+        return { risk: 'wildcard', note: '浮动版本' };
+    if (/^(?:==)?v?\d+(?:\.\d+)+(?:[-+][\w.-]+)?$/.test(s))
         return { risk: 'exact', note: '' };
     if (/[<>=~^]/.test(s) || s.includes('||'))
         return { risk: 'range', note: '' };
-    return { risk: 'unpinned', note: '无法识别版本约束' };
-}
-function pushDeps(list, entries, manager, file, kind) {
-    for (const [name, rawSpec] of Object.entries(entries)) {
-        const spec = typeof rawSpec === 'string' ? rawSpec : '';
-        const { risk, note } = riskOfSpec(spec);
-        list.push({ name, spec, manager, file, kind, risk, note });
-    }
+    return { risk: 'unpinned', note: '未能识别固定版本' };
 }
-function parsePackageJson(text, file) {
-    try {
-        const pkg = JSON.parse(text);
-        const list = [];
-        pushDeps(list, asRecord(pkg.dependencies), 'npm', file, 'runtime');
-        pushDeps(list, asRecord(pkg.devDependencies), 'npm', file, 'dev');
-        pushDeps(list, asRecord(pkg.peerDependencies), 'npm', file, 'peer');
-        pushDeps(list, asRecord(pkg.optionalDependencies), 'npm', file, 'optional');
-        return list;
+export async function scanDeps(root, options = {}) {
+    const dependencies = [], warnings = [];
+    const lockfiles = [];
+    let filesScanned = 0, filesSkipped = 0, visited = 0, complete = true;
+    const warn = (text) => { complete = false; if (warnings.length < 100)
+        warnings.push(text); };
+    function add(name, rawSpec, manager, file, kind) {
+        dependencies.push({ name: redactText(name), spec: redactText(rawSpec), manager, file, kind, ...riskOfSpec(rawSpec) });
     }
-    catch {
-        return [];
+    function table(value, manager, file, kind) {
+        if (value === undefined)
+            return;
+        if (!value || typeof value !== 'object' || Array.isArray(value)) {
+            warn(file + ' 的依赖表格式无效');
+            return;
+        }
+        for (const [name, valueSpec] of Object.entries(record(value))) {
+            if (typeof valueSpec !== 'string' && (!valueSpec || typeof valueSpec !== 'object' || Array.isArray(valueSpec))) {
+                warn(file + ' 含无效依赖声明');
+                continue;
+            }
+            const obj = record(valueSpec);
+            const spec = typeof valueSpec === 'string' ? valueSpec : typeof obj.git === 'string' ? obj.git + (typeof obj.rev === 'string' ? '#' + obj.rev : '') : typeof obj.path === 'string' ? 'path:' + obj.path : typeof obj.version === 'string' ? obj.version : '';
+            add(name, spec, manager, file, kind);
+        }
     }
-}
-function parseRequirements(text, file) {
-    const list = [];
-    for (const raw of text.split(/\r?\n/)) {
-        const line = raw.trim();
-        if (line === '' || line.startsWith('#') || line.startsWith('-'))
-            continue;
-        const m = /^([A-Za-z0-9_.\-]+)(.*)$/.exec(line);
-        if (m === null)
-            continue;
-        const spec = m[2]?.trim() ?? '';
-        const { risk, note } = riskOfSpec(spec);
-        list.push({ name: m[1], spec, manager: 'pypi', file, kind: 'runtime', risk, note });
+    function requirement(value, file, kind) {
+        if (typeof value !== 'string') {
+            warn(file + ' 含尚未展开的依赖组引用');
+            return;
+        }
+        const match = /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]+\])?\s*(.*)$/.exec(value);
+        if (!match) {
+            warn(file + ' 有未识别的 Python 依赖');
+            return;
+        }
+        add(match[1], match[2].split(';', 1)[0].trim().replace(/^@\s*/, ''), 'pypi', file, kind);
     }
-    return list;
-}
-function parsePyproject(text, file) {
-    const list = [];
-    const re = /^\s*([A-Za-z0-9_.\-]+)\s*(?:[<>=!~^]+[^\s,]+)?/gm;
-    for (const section of text.matchAll(/\[project\.(?:optional-)?dependencies\][\s\S]*?(?=\n\[|$)/g)) {
-        for (const line of section[0].split(/\r?\n/)) {
-            const trimmed = line.trim();
-            if (trimmed === '' || trimmed.startsWith('['))
+    function pyproject(text, file) {
+        const doc = parseToml(text), project = record(doc.project);
+        if (project.dependencies !== undefined && !Array.isArray(project.dependencies))
+            throw new Error('invalid dependencies');
+        for (const item of project.dependencies ?? [])
+            requirement(item, file, 'runtime');
+        for (const items of Object.values(record(project['optional-dependencies']))) {
+            if (!Array.isArray(items)) {
+                warn(file + ' 可选依赖不是数组');
                 continue;
-            const m = /^([A-Za-z0-9_.\-]+)\s*(.*)$/.exec(trimmed.replace(/^"|"$/g, ''));
-            if (m === null)
+            }
+            items.forEach(item => requirement(item, file, 'optional'));
+        }
+        for (const items of Object.values(record(doc['dependency-groups']))) {
+            if (!Array.isArray(items)) {
+                warn(file + ' 依赖组不是数组');
                 continue;
-            const spec = m[2]?.trim() ?? '';
-            const { risk, note } = riskOfSpec(spec);
-            list.push({ name: m[1], spec, manager: 'pypi', file, kind: 'runtime', risk, note });
+            }
+            items.forEach(item => requirement(item, file, 'dev'));
         }
+        const poetry = record(record(doc.tool).poetry);
+        const poetryTable = (value, kind) => table(Object.fromEntries(Object.entries(record(value)).filter(([name]) => name !== 'python')), 'pypi', file, kind);
+        poetryTable(poetry.dependencies, 'runtime');
+        poetryTable(poetry['dev-dependencies'], 'dev');
+        for (const group of Object.values(record(poetry.group)))
+            poetryTable(record(group).dependencies, 'dev');
+        if (Array.isArray(project.dynamic) && project.dynamic.includes('dependencies'))
+            warn(file + ' 的 dependencies 为动态声明，未执行项目代码读取');
     }
-    return list;
-}
-function parseGoMod(text, file) {
-    const list = [];
-    const re = /^\s*([A-Za-z0-9_.\/\-]+)\s+(v[0-9][^\s]*)/gm;
-    for (const m of text.matchAll(re)) {
-        list.push({ name: m[1], spec: m[2], manager: 'go', file, kind: 'runtime', risk: 'exact', note: '' });
-    }
-    return list;
-}
-function parseCargo(text, file) {
-    const list = [];
-    for (const line of text.split(/\r?\n/)) {
-        const m = /^\s*([A-Za-z0-9_.\-]+)\s*=\s*(.+)$/.exec(line);
-        if (m === null)
-            continue;
-        const spec = m[2].trim().replace(/^["']|["']$/g, '');
-        const isGit = /git\s*=|branch\s*=|rev\s*=/.test(spec);
-        const risk = isGit ? (spec.includes('rev') ? 'exact' : 'git') : 'range';
-        list.push({ name: m[1], spec, manager: 'cargo', file, kind: 'runtime', risk, note: isGit && !spec.includes('rev') ? 'git 源未锁定 rev' : '' });
-    }
-    return list;
-}
-function parseComposer(text, file) {
-    const list = [];
-    try {
-        const pkg = JSON.parse(text);
-        for (const [key, entries] of [['require', 'runtime'], ['require-dev', 'dev']]) {
-            pushDeps(list, asRecord(pkg[key]), 'composer', file, entries);
+    function parse(text, base, file) {
+        if (base === 'package.json' || base === 'composer.json') {
+            const pkg = JSON.parse(text);
+            if (!pkg || typeof pkg !== 'object' || Array.isArray(pkg))
+                throw new Error('invalid manifest');
+            if (base === 'package.json')
+                for (const [field, kind] of [['dependencies', 'runtime'], ['devDependencies', 'dev'], ['peerDependencies', 'peer'], ['optionalDependencies', 'optional']])
+                    table(pkg[field], 'npm', file, kind);
+            else {
+                table(pkg.require, 'composer', file, 'runtime');
+                table(pkg['require-dev'], 'composer', file, 'dev');
+            }
         }
+        else if (base === 'pyproject.toml')
+            pyproject(text, file);
+        else if (base === 'requirements.txt')
+            for (const line of text.split(/\r?\n/)) {
+                const item = line.trim();
+                if (!item || item.startsWith('#'))
+                    continue;
+                if (item.startsWith('-')) {
+                    warn(file + ' 包含外部文件/安装选项，未执行解析');
+                    continue;
+                }
+                requirement(item.split(/\s+#/, 1)[0], file, 'runtime');
+            }
+        else if (base === 'Cargo.toml') {
+            const doc = parseToml(text);
+            const cargo = (value) => { const obj = record(value); for (const [field, kind] of [['dependencies', 'runtime'], ['dev-dependencies', 'dev'], ['build-dependencies', 'dev']])
+                table(obj[field], 'cargo', file, kind); };
+            cargo(doc);
+            cargo(record(doc.workspace));
+            Object.values(record(doc.target)).forEach(cargo);
+        }
+        else if (base === 'go.mod') {
+            for (const m of text.matchAll(/^\s*(?:require\s+)?([A-Za-z0-9_.\/-]+)\s+(v[0-9][^\s]*)/gm))
+                add(m[1], m[2], 'go', file, 'runtime');
+            if (/^\s*replace\s/m.test(text))
+                warn(file + ' 含 replace，清单不代表最终解析图');
+        }
+        else
+            for (const m of text.matchAll(/^\s*gem\s+['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"]\s*)?/gm))
+                add(m[1], m[2] ?? '', 'ruby', file, 'runtime');
     }
-    catch {
-        // ignore malformed
-    }
-    return list;
-}
-function parseGemfile(text, file) {
-    const list = [];
-    for (const m of text.matchAll(/^\s*gem\s+['"]([^'"]+)['"]\s*(?:,\s*['"]([^'"]*)['"]\s*)?/gm)) {
-        const spec = m[2] ?? '';
-        const { risk, note } = riskOfSpec(spec);
-        list.push({ name: m[1], spec, manager: 'ruby', file, kind: 'runtime', risk, note });
-    }
-    return list;
-}
-/** 扫描依赖清单，返回 SBOM-lite 结果。 */
-export async function scanDeps(root) {
-    const dependencies = [];
-    const warnings = [];
-    let filesScanned = 0;
-    const visit = async (absolute, rel) => {
+    async function visit(absolute, rel) {
+        options.signal?.throwIfAborted();
+        if (++visited > 100000 || filesScanned >= (options.maxFiles ?? 5000)) {
+            warn('已达到扫描数量上限');
+            return;
+        }
+        if (rel.split('/').some(p => DEFAULT_EXCLUDE_DIRS.includes(p)) || options.policy && policyExcludesDir(options.policy, rel)) {
+            filesSkipped++;
+            return;
+        }
         let stat;
         try {
-            stat = await fs.stat(absolute);
+            stat = await fs.lstat(absolute);
         }
         catch {
+            filesSkipped++;
+            warn('无法读取 ' + rel);
+            return;
+        }
+        if (stat.isSymbolicLink()) {
+            filesSkipped++;
+            if (absolute === path.resolve(root))
+                warn('目标为链接，未读取链接内容');
             return;
         }
         if (stat.isDirectory()) {
-            if (DEFAULT_EXCLUDE_DIRS.includes(path.basename(absolute)))
+            let entries;
+            try {
+                entries = await fs.readdir(absolute, { withFileTypes: true });
+            }
+            catch {
+                warn('无法遍历 ' + rel);
                 return;
-            const entries = await fs.readdir(absolute, { withFileTypes: true });
-            for (const entry of entries)
-                await visit(path.join(absolute, entry.name), rel === '' ? entry.name : rel + '/' + entry.name);
+            }
+            for (const entry of entries) {
+                if (visited > 100000 || filesScanned >= (options.maxFiles ?? 5000)) {
+                    warn('已达到扫描数量上限');
+                    break;
+                }
+                ;
+                await visit(path.join(absolute, entry.name), rel ? rel + '/' + entry.name : entry.name);
+            }
             return;
         }
-        const base = path.basename(absolute);
-        if (!MANIFESTS.includes(base))
+        if (!stat.isFile() || !MANIFESTS.includes(path.basename(absolute)) && !LOCKFILES.includes(path.basename(absolute)))
+            return;
+        if (stat.size > (options.maxFileBytes ?? 2 * 1024 * 1024)) {
+            filesSkipped++;
+            warn(rel + ' 超过文件大小上限');
             return;
-        filesScanned += 1;
-        let text = '';
+        }
+        filesScanned++;
         try {
-            text = await fs.readFile(absolute, 'utf8');
+            const text = await fs.readFile(absolute, { encoding: 'utf8', signal: options.signal }), file = rel || path.basename(absolute);
+            if (LOCKFILES.includes(path.basename(absolute))) {
+                lockfiles.push(file);
+                options.onDocument?.({ file, text });
+            }
+            if (MANIFESTS.includes(path.basename(absolute)))
+                parse(text, path.basename(absolute), file);
         }
         catch {
-            warnings.push('无法读取 ' + rel);
-            return;
+            options.signal?.throwIfAborted();
+            warn('无法解析 ' + (rel || path.basename(absolute)) + '：清单格式无效，本文件不计作已确认安全');
         }
-        if (base === 'package.json')
-            dependencies.push(...parsePackageJson(text, rel));
-        else if (base === 'requirements.txt')
-            dependencies.push(...parseRequirements(text, rel));
-        else if (base === 'pyproject.toml')
-            dependencies.push(...parsePyproject(text, rel));
-        else if (base === 'go.mod')
-            dependencies.push(...parseGoMod(text, rel));
-        else if (base === 'Cargo.toml')
-            dependencies.push(...parseCargo(text, rel));
-        else if (base === 'composer.json')
-            dependencies.push(...parseComposer(text, rel));
-        else if (base === 'Gemfile')
-            dependencies.push(...parseGemfile(text, rel));
-    };
-    await visit(root, '');
-    dependencies.sort((a, b) => a.file.localeCompare(b.file) || a.name.localeCompare(b.name));
-    const risky = dependencies.filter((item) => item.risk !== 'exact' && item.risk !== 'range');
-    return { root, filesScanned, dependencies, riskyCount: risky.length, warnings };
+    }
+    await visit(path.resolve(root), '');
+    return { root, filesScanned, filesSkipped, dependencies, riskyCount: dependencies.filter(item => !['exact', 'range'].includes(item.risk)).length, warnings, complete, lockfiles };
 }

package/lib/engines.d.ts

@@ -0,0 +1,50 @@
+import type { ResolvedSecureConfig, SourceEngine } from './config.js';
+import type { ProcessRunner } from './runner.js';
+import type { Finding } from './scanner.js';
+export type EngineId = SourceEngine | 'osv';
+export interface EngineCoverage {
+    engine: EngineId;
+    status: 'completed' | 'partial' | 'not-installed' | 'failed' | 'timed-out' | 'not-run';
+    version?: string;
+    reason?: string;
+    findingsCount?: number;
+    durationMs?: number;
+}
+export interface ScanDocument {
+    file: string;
+    text: string;
+    selectedLines?: ReadonlySet<number>;
+}
+/** File modification dates describe the cache, not the age of every advisory inside it. */
+export declare function databaseHealth(cfg: ResolvedSecureConfig): Promise<{
+    directory: string;
+    available: boolean;
+    databases: {
+        ecosystem: string;
+        fileModifiedAt: string;
+    }[];
+}>;
+export declare function probeEngine(engine: Exclude<EngineId, 'builtin'>, cfg: ResolvedSecureConfig, runner: ProcessRunner, signal?: AbortSignal): Promise<EngineCoverage>;
+export declare function scanExternal(engine: 'betterleaks' | 'semgrep', documents: ScanDocument[], cfg: ResolvedSecureConfig, runner: ProcessRunner, signal?: AbortSignal): Promise<{
+    findings: Finding[];
+    coverage: EngineCoverage;
+}>;
+export declare function mergeFindings(findings: Finding[]): Finding[];
+export declare function auditDependencies(documents: ScanDocument[], mode: 'offline' | 'online', cfg: ResolvedSecureConfig, runner: ProcessRunner, signal?: AbortSignal): Promise<{
+    mode: "offline" | "online";
+    coverage: EngineCoverage;
+    packagesScanned: number;
+    vulnerabilities: never[];
+} | {
+    mode: "offline" | "online";
+    coverage: {
+        findingsCount: number;
+        reason?: string;
+        status: "completed" | "partial" | "not-run";
+        engine: EngineId;
+        version?: string;
+        durationMs?: number;
+    };
+    packagesScanned: number;
+    vulnerabilities: Record<string, unknown>[];
+}>;