arceus-s 1.6.4

package/dist/core/group/extractors/node-workspace-extractor.js

@@ -0,0 +1,207 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
+import { logger } from '../../logger.js';
+async function parsePackageManifest(repoPath) {
+    const pkgPath = path.join(repoPath, 'package.json');
+    let content;
+    try {
+        content = await fs.readFile(pkgPath, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+    let pkg;
+    try {
+        pkg = JSON.parse(content);
+    }
+    catch {
+        return null;
+    }
+    const name = typeof pkg.name === 'string' ? pkg.name : '';
+    if (!name)
+        return null;
+    const deps = [];
+    for (const field of ['dependencies', 'devDependencies', 'peerDependencies']) {
+        const section = pkg[field];
+        if (section && typeof section === 'object') {
+            deps.push(...Object.keys(section));
+        }
+    }
+    return { name, workspaceDeps: [...new Set(deps)] };
+}
+async function scanImports(repoPath, knownPackages) {
+    const results = [];
+    const sourceFiles = await findSourceFiles(repoPath);
+    for (const relFile of sourceFiles) {
+        const absPath = path.join(repoPath, relFile);
+        let content;
+        try {
+            content = await fs.readFile(absPath, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        // ES import: import { Foo, Bar } from '<pkg>'
+        // Also: import { Foo as Baz } from '<pkg>'
+        const esImportRegex = /^import\s+\{([^}]+)\}\s+from\s+['"]([^'"]+)['"]/gm;
+        let match;
+        while ((match = esImportRegex.exec(content)) !== null) {
+            const importClause = match[1];
+            const modulePath = match[2];
+            const pkgName = resolvePackageName(modulePath);
+            if (!pkgName || !knownPackages.has(pkgName))
+                continue;
+            const symbols = parseImportClause(importClause);
+            for (const sym of symbols) {
+                if (isExportedName(sym)) {
+                    results.push({ packageName: pkgName, symbolName: sym, filePath: relFile });
+                }
+            }
+        }
+        // ES import default: import Foo from '<pkg>'
+        const defaultImportRegex = /^import\s+([A-Z][A-Za-z0-9]*)\s+from\s+['"]([^'"]+)['"]/gm;
+        while ((match = defaultImportRegex.exec(content)) !== null) {
+            const symbolName = match[1];
+            const modulePath = match[2];
+            const pkgName = resolvePackageName(modulePath);
+            if (!pkgName || !knownPackages.has(pkgName))
+                continue;
+            if (isExportedName(symbolName)) {
+                results.push({ packageName: pkgName, symbolName, filePath: relFile });
+            }
+        }
+        // CommonJS: const { Foo, Bar } = require('<pkg>')
+        const cjsRegex = /(?:const|let|var)\s+\{([^}]+)\}\s*=\s*require\s*\(\s*['"]([^'"]+)['"]\s*\)/gm;
+        while ((match = cjsRegex.exec(content)) !== null) {
+            const importClause = match[1];
+            const modulePath = match[2];
+            const pkgName = resolvePackageName(modulePath);
+            if (!pkgName || !knownPackages.has(pkgName))
+                continue;
+            const symbols = parseImportClause(importClause);
+            for (const sym of symbols) {
+                if (isExportedName(sym)) {
+                    results.push({ packageName: pkgName, symbolName: sym, filePath: relFile });
+                }
+            }
+        }
+    }
+    return results;
+}
+function resolvePackageName(modulePath) {
+    if (modulePath.startsWith('.') || modulePath.startsWith('/'))
+        return null;
+    // Scoped: @scope/pkg or @scope/pkg/sub
+    if (modulePath.startsWith('@')) {
+        const parts = modulePath.split('/');
+        if (parts.length >= 2)
+            return `${parts[0]}/${parts[1]}`;
+        return null;
+    }
+    // Unscoped: pkg or pkg/sub
+    return modulePath.split('/')[0];
+}
+function parseImportClause(clause) {
+    return clause
+        .split(',')
+        .map((s) => {
+        const trimmed = s.trim();
+        // Handle `Foo as Bar` — use the original export name
+        const asMatch = trimmed.match(/^(\S+)\s+as\s+/);
+        return asMatch ? asMatch[1] : trimmed;
+    })
+        .filter(Boolean);
+}
+function isExportedName(name) {
+    return /^[A-Z][A-Za-z0-9]*$/.test(name);
+}
+async function findSourceFiles(repoPath) {
+    const results = [];
+    const EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.mts', '.cts']);
+    const ig = await loadIgnoreRules(repoPath);
+    async function walk(dir, rel) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel + '/'))
+                    continue;
+                await walk(path.join(dir, entry.name), childRel);
+            }
+            else {
+                const ext = path.extname(entry.name);
+                if (EXTENSIONS.has(ext)) {
+                    if (shouldIgnorePath(childRel))
+                        continue;
+                    if (ig && ig.ignores(childRel))
+                        continue;
+                    results.push(childRel);
+                }
+            }
+        }
+    }
+    await walk(repoPath, '');
+    return results;
+}
+export async function extractNodeWorkspaceLinks(repos, repoPaths, _dbExecutors) {
+    const packagesByName = new Map();
+    const packagesByGroupPath = new Map();
+    for (const [groupPath] of Object.entries(repos)) {
+        const repoPath = repoPaths.get(groupPath);
+        if (!repoPath)
+            continue;
+        const manifest = await parsePackageManifest(repoPath);
+        if (!manifest)
+            continue;
+        const meta = {
+            name: manifest.name,
+            groupPath,
+            repoPath,
+            workspaceDeps: manifest.workspaceDeps,
+        };
+        const existing = packagesByName.get(manifest.name);
+        if (existing) {
+            logger.warn(`[node-workspace-extractor] duplicate package name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`);
+            continue;
+        }
+        packagesByName.set(manifest.name, meta);
+        packagesByGroupPath.set(groupPath, meta);
+    }
+    const links = [];
+    const seen = new Set();
+    for (const [, pkg] of packagesByGroupPath) {
+        const groupPkgDeps = pkg.workspaceDeps.filter((d) => packagesByName.has(d));
+        if (groupPkgDeps.length === 0)
+            continue;
+        const knownPackages = new Set(groupPkgDeps);
+        const imports = await scanImports(pkg.repoPath, knownPackages);
+        for (const imp of imports) {
+            const providerPkg = packagesByName.get(imp.packageName);
+            if (!providerPkg)
+                continue;
+            const qualifiedContract = `${imp.packageName}::${imp.symbolName}`;
+            const key = `${pkg.groupPath}→${providerPkg.groupPath}::${qualifiedContract}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            const link = {
+                from: providerPkg.groupPath,
+                to: pkg.groupPath,
+                type: 'custom',
+                contract: qualifiedContract,
+                role: 'provider',
+            };
+            links.push(link);
+        }
+    }
+    return { links, discoveredPackages: packagesByGroupPath };
+}

package/dist/core/group/extractors/python-workspace-extractor.d.ts

@@ -0,0 +1,15 @@
+import type { CypherExecutor } from '../contract-extractor.js';
+import type { GroupManifestLink } from '../types.js';
+interface PythonPackageMeta {
+    name: string;
+    importName: string;
+    groupPath: string;
+    repoPath: string;
+    workspaceDeps: string[];
+}
+export interface PythonWorkspaceResult {
+    links: GroupManifestLink[];
+    discoveredPackages: Map<string, PythonPackageMeta>;
+}
+export declare function extractPythonWorkspaceLinks(repos: Record<string, string>, repoPaths: Map<string, string>, _dbExecutors?: Map<string, CypherExecutor>): Promise<PythonWorkspaceResult>;
+export {};

package/dist/core/group/extractors/python-workspace-extractor.js

@@ -0,0 +1,205 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { shouldIgnorePath, loadIgnoreRules } from '../../../config/ignore-service.js';
+import { logger } from '../../logger.js';
+async function parsePythonManifest(repoPath) {
+    const pyprojectPath = path.join(repoPath, 'pyproject.toml');
+    let content = null;
+    try {
+        content = await fs.readFile(pyprojectPath, 'utf-8');
+    }
+    catch {
+        // fall through to setup.py
+    }
+    if (content)
+        return parsePyproject(content);
+    const setupPyPath = path.join(repoPath, 'setup.py');
+    try {
+        content = await fs.readFile(setupPyPath, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+    return parseSetupPy(content);
+}
+function parsePyproject(content) {
+    const nameMatch = content.match(/^\[project\]\s*\n(?:[^\n\[]*\n)*?name\s*=\s*"([^"]+)"/m);
+    if (!nameMatch)
+        return null;
+    const name = nameMatch[1];
+    const importName = name.replace(/-/g, '_');
+    const deps = [];
+    const depsMatch = content.match(/^\[project\]\s*\n[\s\S]*?dependencies\s*=\s*\[([\s\S]*?)\]/m);
+    if (depsMatch) {
+        const depLines = depsMatch[1].matchAll(/"([^"]+)"/g);
+        for (const m of depLines) {
+            deps.push(extractPepName(m[1]));
+        }
+    }
+    const optMatch = content.match(/\[project\.optional-dependencies\]\s*\n([\s\S]*?)(?=\n\[|$)/);
+    if (optMatch) {
+        const optDeps = optMatch[1].matchAll(/"([^"]+)"/g);
+        for (const m of optDeps) {
+            deps.push(extractPepName(m[1]));
+        }
+    }
+    return { name, importName, deps: [...new Set(deps)] };
+}
+function parseSetupPy(content) {
+    const nameMatch = content.match(/name\s*=\s*['"]([^'"]+)['"]/);
+    if (!nameMatch)
+        return null;
+    const name = nameMatch[1];
+    const importName = name.replace(/-/g, '_');
+    const deps = [];
+    const installMatch = content.match(/install_requires\s*=\s*\[([\s\S]*?)\]/);
+    if (installMatch) {
+        const depLines = installMatch[1].matchAll(/['"]([^'"]+)['"]/g);
+        for (const m of depLines) {
+            deps.push(extractPepName(m[1]));
+        }
+    }
+    return { name, importName, deps: [...new Set(deps)] };
+}
+function extractPepName(spec) {
+    return spec.split(/[><=!~;\[]/)[0].trim();
+}
+async function scanPythonImports(repoPath, knownPackages) {
+    const results = [];
+    const sourceFiles = await findPythonFiles(repoPath);
+    for (const relFile of sourceFiles) {
+        const absPath = path.join(repoPath, relFile);
+        let content;
+        try {
+            content = await fs.readFile(absPath, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        // from <pkg> import Foo, Bar
+        // from <pkg>.module import Foo
+        const fromImportRegex = /^from\s+(\w[\w.]*)\s+import\s+(.+)/gm;
+        let match;
+        while ((match = fromImportRegex.exec(content)) !== null) {
+            const modulePath = match[1];
+            const importClause = match[2];
+            const rootModule = modulePath.split('.')[0];
+            const originalName = knownPackages.get(rootModule);
+            if (!originalName)
+                continue;
+            if (importClause.trim() === '(')
+                continue;
+            const symbols = importClause
+                .replace(/\(|\)/g, '')
+                .split(',')
+                .map((s) => {
+                const trimmed = s.trim();
+                const asMatch = trimmed.match(/^(\S+)\s+as\s+/);
+                return asMatch ? asMatch[1] : trimmed;
+            })
+                .filter(Boolean);
+            for (const sym of symbols) {
+                if (isPascalCase(sym)) {
+                    results.push({ packageName: originalName, symbolName: sym, filePath: relFile });
+                }
+            }
+        }
+    }
+    return results;
+}
+function isPascalCase(name) {
+    return /^[A-Z][A-Za-z0-9]*$/.test(name);
+}
+async function findPythonFiles(repoPath) {
+    const results = [];
+    const ig = await loadIgnoreRules(repoPath);
+    async function walk(dir, rel) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel + '/'))
+                    continue;
+                await walk(path.join(dir, entry.name), childRel);
+            }
+            else if (entry.name.endsWith('.py')) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel))
+                    continue;
+                results.push(childRel);
+            }
+        }
+    }
+    await walk(repoPath, '');
+    return results;
+}
+export async function extractPythonWorkspaceLinks(repos, repoPaths, _dbExecutors) {
+    const packagesByImportName = new Map();
+    const packagesByGroupPath = new Map();
+    for (const [groupPath] of Object.entries(repos)) {
+        const repoPath = repoPaths.get(groupPath);
+        if (!repoPath)
+            continue;
+        const manifest = await parsePythonManifest(repoPath);
+        if (!manifest)
+            continue;
+        const meta = {
+            name: manifest.name,
+            importName: manifest.importName,
+            groupPath,
+            repoPath,
+            workspaceDeps: manifest.deps,
+        };
+        const existing = packagesByImportName.get(manifest.importName);
+        if (existing) {
+            logger.warn(`[python-workspace-extractor] duplicate package "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`);
+            continue;
+        }
+        packagesByImportName.set(manifest.importName, meta);
+        packagesByGroupPath.set(groupPath, meta);
+    }
+    const links = [];
+    const seen = new Set();
+    for (const [, pkg] of packagesByGroupPath) {
+        const normalizedDeps = pkg.workspaceDeps.map((d) => d.replace(/-/g, '_'));
+        const groupPkgDeps = normalizedDeps.filter((d) => packagesByImportName.has(d));
+        if (groupPkgDeps.length === 0)
+            continue;
+        const knownPackages = new Map();
+        for (const dep of groupPkgDeps) {
+            const meta = packagesByImportName.get(dep);
+            if (meta)
+                knownPackages.set(dep, meta.name);
+        }
+        const imports = await scanPythonImports(pkg.repoPath, knownPackages);
+        for (const imp of imports) {
+            const providerImportName = imp.packageName.replace(/-/g, '_');
+            const providerPkg = packagesByImportName.get(providerImportName);
+            if (!providerPkg)
+                continue;
+            const qualifiedContract = `${providerPkg.name}::${imp.symbolName}`;
+            const key = `${pkg.groupPath}→${providerPkg.groupPath}::${qualifiedContract}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            const link = {
+                from: providerPkg.groupPath,
+                to: pkg.groupPath,
+                type: 'custom',
+                contract: qualifiedContract,
+                role: 'provider',
+            };
+            links.push(link);
+        }
+    }
+    return { links, discoveredPackages: packagesByGroupPath };
+}

package/dist/core/group/extractors/rust-workspace-extractor.d.ts

@@ -0,0 +1,44 @@
+import type { CypherExecutor } from '../contract-extractor.js';
+import type { GroupManifestLink } from '../types.js';
+/**
+ * Discover cross-crate contracts in a Rust workspace by reading each
+ * member's `Cargo.toml` dependencies and scanning source files for
+ * `use <workspace_dep>::<Type>` imports.
+ *
+ * Emits `GroupManifestLink[]` with `type: 'custom'` that feed into the
+ * existing ManifestExtractor pipeline — no new matching logic needed.
+ *
+ * Designed for the group-level sync pipeline: it receives all repos in
+ * a group and produces cross-repo links between them.
+ */
+interface CrateMeta {
+    name: string;
+    groupPath: string;
+    repoPath: string;
+    workspaceDeps: string[];
+}
+/**
+ * Linear-time `[package].name = "..."` lookup. The previous regex
+ * `^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"` had a nested
+ * lazy quantifier on `\n` that CodeQL js/redos flagged as exponential
+ * on inputs like `[package]\n` + many bare `\n`. We walk lines
+ * explicitly: scan from the first `[package]` header until we hit the
+ * next `[...]` section header, looking for the `name = "..."` line.
+ * O(n) with the line count.
+ *
+ * Exported so the U8 ReDoS regression test can drive the production
+ * line-walk directly with adversarial fixtures (multi-line strings,
+ * trailing sections, etc.) instead of duplicating it inline.
+ */
+export declare function parseCargoPackageName(content: string): string | null;
+export interface RustWorkspaceResult {
+    links: GroupManifestLink[];
+    discoveredCrates: Map<string, CrateMeta>;
+}
+/**
+ * Discover cross-crate contracts across all Rust repos in a group.
+ *
+ * Returns `GroupManifestLink[]` ready to feed into `ManifestExtractor`.
+ */
+export declare function extractRustWorkspaceLinks(repos: Record<string, string>, repoPaths: Map<string, string>, _dbExecutors?: Map<string, CypherExecutor>): Promise<RustWorkspaceResult>;
+export {};