arceus-s 1.6.4

package/dist/core/group/extractors/rust-workspace-extractor.js

@@ -0,0 +1,240 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { shouldIgnorePath } from '../../../config/ignore-service.js';
+import { loadIgnoreRules } from '../../../config/ignore-service.js';
+import { logger } from '../../logger.js';
+/**
+ * Linear-time `[package].name = "..."` lookup. The previous regex
+ * `^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"` had a nested
+ * lazy quantifier on `\n` that CodeQL js/redos flagged as exponential
+ * on inputs like `[package]\n` + many bare `\n`. We walk lines
+ * explicitly: scan from the first `[package]` header until we hit the
+ * next `[...]` section header, looking for the `name = "..."` line.
+ * O(n) with the line count.
+ *
+ * Exported so the U8 ReDoS regression test can drive the production
+ * line-walk directly with adversarial fixtures (multi-line strings,
+ * trailing sections, etc.) instead of duplicating it inline.
+ */
+export function parseCargoPackageName(content) {
+    const lines = content.split('\n');
+    const packageStart = lines.findIndex((l) => l.trim() === '[package]');
+    if (packageStart < 0)
+        return null;
+    for (let i = packageStart + 1; i < lines.length; i++) {
+        const line = lines[i].trimStart();
+        if (line.startsWith('['))
+            break; // hit the next section header
+        const m = /^name\s*=\s*"([^"]+)"/.exec(line);
+        if (m)
+            return m[1];
+    }
+    return null;
+}
+/**
+ * Parse a Cargo.toml to extract the crate name and workspace dependency
+ * names. Uses simple line-based parsing — no TOML library needed for
+ * the subset we care about.
+ */
+async function parseCrateManifest(repoPath) {
+    const cargoPath = path.join(repoPath, 'Cargo.toml');
+    let content;
+    try {
+        content = await fs.readFile(cargoPath, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+    const name = parseCargoPackageName(content) ?? '';
+    const workspaceDeps = [];
+    // Match dependencies that use workspace = true, which indicates they
+    // are workspace-internal deps:
+    //   dep_name = { workspace = true }
+    //   dep_name.workspace = true
+    //
+    // Also match plain path dependencies:
+    //   dep_name = { path = "../other" }
+    const depSections = content.matchAll(/\[(dependencies|dev-dependencies|build-dependencies)\]\s*\n([\s\S]*?)(?=\n\[|$)/g);
+    for (const section of depSections) {
+        const sectionBody = section[2];
+        // workspace = true style
+        const wsMatches = sectionBody.matchAll(/^(\w[\w-]*)\s*=\s*\{[^}]*workspace\s*=\s*true[^}]*\}/gm);
+        for (const m of wsMatches)
+            workspaceDeps.push(m[1]);
+        // dotted workspace style: dep_name.workspace = true
+        const dottedMatches = sectionBody.matchAll(/^(\w[\w-]*)\.workspace\s*=\s*true/gm);
+        for (const m of dottedMatches)
+            workspaceDeps.push(m[1]);
+        // path = "../other" style (local path deps within workspace)
+        const pathMatches = sectionBody.matchAll(/^(\w[\w-]*)\s*=\s*\{[^}]*path\s*=\s*"[^"]*"[^}]*\}/gm);
+        for (const m of pathMatches)
+            workspaceDeps.push(m[1]);
+    }
+    if (!name)
+        return null;
+    return { name, workspaceDeps: [...new Set(workspaceDeps)] };
+}
+/**
+ * Scan Rust source files for `use <crate>::<path>::<Symbol>` patterns
+ * where <crate> is a known workspace dependency.
+ */
+async function scanImports(repoPath, knownCrates) {
+    const results = [];
+    const normalizedCrates = new Map();
+    for (const c of knownCrates) {
+        normalizedCrates.set(c.replace(/-/g, '_'), c);
+    }
+    const sourceFiles = await findRustFiles(repoPath);
+    for (const relFile of sourceFiles) {
+        const absPath = path.join(repoPath, relFile);
+        let content;
+        try {
+            content = await fs.readFile(absPath, 'utf-8');
+        }
+        catch {
+            continue;
+        }
+        // Match patterns:
+        //   use crate_name::Type;
+        //   use crate_name::module::Type;
+        //   use crate_name::{Type1, Type2};
+        //   use crate_name::module::{Type1, Type2};
+        const useRegex = /^use\s+(\w+)::(.+);/gm;
+        let match;
+        while ((match = useRegex.exec(content)) !== null) {
+            const crateName = match[1];
+            const originalCrateName = normalizedCrates.get(crateName);
+            if (!originalCrateName)
+                continue;
+            const importPath = match[2].trim();
+            // Handle grouped imports: {Type1, Type2, module::Type3}
+            const braceMatch = importPath.match(/\{([^}]+)\}/);
+            if (braceMatch) {
+                const items = braceMatch[1].split(',').map((s) => s.trim());
+                for (const item of items) {
+                    const symbolName = extractSymbolName(item);
+                    if (symbolName && isTypeName(symbolName)) {
+                        results.push({ crateName: originalCrateName, symbolName, filePath: relFile });
+                    }
+                }
+            }
+            else {
+                const symbolName = extractSymbolName(importPath);
+                if (symbolName && isTypeName(symbolName)) {
+                    results.push({ crateName: originalCrateName, symbolName, filePath: relFile });
+                }
+            }
+        }
+    }
+    return results;
+}
+/** Extract the final symbol name from a path like `module::submod::TypeName`. */
+function extractSymbolName(importPath) {
+    const trimmed = importPath.trim();
+    if (!trimmed || trimmed === '*' || trimmed === 'self')
+        return null;
+    const parts = trimmed.split('::');
+    return parts[parts.length - 1].trim() || null;
+}
+/**
+ * Heuristic: in Rust, types (structs, enums, traits) are PascalCase.
+ * Functions and modules are snake_case. We only want types as cross-crate
+ * contracts — functions are too granular and modules too broad.
+ */
+function isTypeName(name) {
+    return /^[A-Z][A-Za-z0-9]*$/.test(name);
+}
+async function findRustFiles(repoPath) {
+    const results = [];
+    const ig = await loadIgnoreRules(repoPath);
+    async function walk(dir, rel) {
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+            if (entry.isDirectory()) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel + '/'))
+                    continue;
+                await walk(path.join(dir, entry.name), childRel);
+            }
+            else if (entry.name.endsWith('.rs')) {
+                if (shouldIgnorePath(childRel))
+                    continue;
+                if (ig && ig.ignores(childRel))
+                    continue;
+                results.push(childRel);
+            }
+        }
+    }
+    await walk(repoPath, '');
+    return results;
+}
+/**
+ * Discover cross-crate contracts across all Rust repos in a group.
+ *
+ * Returns `GroupManifestLink[]` ready to feed into `ManifestExtractor`.
+ */
+export async function extractRustWorkspaceLinks(repos, repoPaths, _dbExecutors) {
+    // Phase 1: Parse all Cargo.toml files to build crate registry
+    const cratesByName = new Map();
+    const cratesByGroupPath = new Map();
+    for (const [groupPath] of Object.entries(repos)) {
+        const repoPath = repoPaths.get(groupPath);
+        if (!repoPath)
+            continue;
+        const manifest = await parseCrateManifest(repoPath);
+        if (!manifest)
+            continue;
+        const meta = {
+            name: manifest.name,
+            groupPath,
+            repoPath,
+            workspaceDeps: manifest.workspaceDeps,
+        };
+        const existing = cratesByName.get(manifest.name);
+        if (existing) {
+            logger.warn(`[rust-workspace-extractor] duplicate crate name "${manifest.name}" in "${groupPath}" and "${existing.groupPath}" — skipping "${groupPath}"`);
+            continue;
+        }
+        cratesByName.set(manifest.name, meta);
+        cratesByGroupPath.set(groupPath, meta);
+    }
+    // Phase 2: For each crate, identify which of its workspace deps are
+    // also in this group (i.e., repos we can link to)
+    const links = [];
+    const seen = new Set();
+    for (const [, crate] of cratesByGroupPath) {
+        const groupCrateDeps = crate.workspaceDeps.filter((d) => cratesByName.has(d));
+        if (groupCrateDeps.length === 0)
+            continue;
+        // Phase 3: Scan source files for imports from workspace deps
+        const knownCrates = new Set(groupCrateDeps);
+        const imports = await scanImports(crate.repoPath, knownCrates);
+        for (const imp of imports) {
+            const providerCrate = cratesByName.get(imp.crateName);
+            if (!providerCrate)
+                continue;
+            const qualifiedContract = `${imp.crateName}::${imp.symbolName}`;
+            const key = `${crate.groupPath}→${providerCrate.groupPath}::${qualifiedContract}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            const link = {
+                from: providerCrate.groupPath,
+                to: crate.groupPath,
+                type: 'custom',
+                contract: qualifiedContract,
+                role: 'provider',
+            };
+            links.push(link);
+        }
+    }
+    return { links, discoveredCrates: cratesByGroupPath };
+}

package/dist/core/group/extractors/thrift-extractor.d.ts

@@ -0,0 +1,22 @@
+import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
+import type { ExtractedContract, RepoHandle } from '../types.js';
+export interface ThriftServiceInfo {
+    namespace: string;
+    serviceName: string;
+    methods: string[];
+    thriftPath: string;
+}
+export interface ThriftContext {
+    namespacesByThrift: Map<string, string>;
+    servicesByName: Map<string, ThriftServiceInfo[]>;
+}
+export declare function thriftMethodContractId(namespace: string, serviceName: string, methodName: string): string;
+export declare function thriftServiceContractId(namespace: string, serviceName: string): string;
+export declare function buildThriftContext(repoPath: string): Promise<ThriftContext>;
+export declare class ThriftExtractor implements ContractExtractor {
+    type: "thrift";
+    canExtract(_repo: RepoHandle): Promise<boolean>;
+    extract(_dbExecutor: CypherExecutor | null, repoPath: string, _repo: RepoHandle): Promise<ExtractedContract[]>;
+    private detectionToContract;
+    private dedupe;
+}

package/dist/core/group/extractors/thrift-extractor.js

@@ -0,0 +1,283 @@
+import { glob } from 'glob';
+import Parser from 'tree-sitter';
+import { readSafe } from './fs-utils.js';
+import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
+import { getPluginForFile, THRIFT_SCAN_GLOB, } from './thrift-patterns/index.js';
+function normalizeThriftPath(rel) {
+    return rel.replace(/\\/g, '/');
+}
+export function thriftMethodContractId(namespace, serviceName, methodName) {
+    const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
+    return `thrift::${prefix}/${methodName}`;
+}
+export function thriftServiceContractId(namespace, serviceName) {
+    const prefix = namespace ? `${namespace}.${serviceName}` : serviceName;
+    return `thrift::${prefix}/*`;
+}
+/**
+ * Replace Thrift comments and string literals with spaces while preserving
+ * newlines and character offsets. Service block scanning can then count braces
+ * without being confused by examples or comments inside the IDL.
+ */
+function stripThriftCommentsAndStrings(content) {
+    const out = new Array(content.length);
+    let i = 0;
+    while (i < content.length) {
+        const ch = content[i];
+        const next = content[i + 1];
+        if (ch === '/' && next === '/') {
+            out[i] = ' ';
+            out[i + 1] = ' ';
+            i += 2;
+            while (i < content.length && content[i] !== '\n') {
+                out[i] = content[i] === '\r' ? '\r' : ' ';
+                i++;
+            }
+            continue;
+        }
+        if (ch === '#') {
+            out[i] = ' ';
+            i++;
+            while (i < content.length && content[i] !== '\n') {
+                out[i] = content[i] === '\r' ? '\r' : ' ';
+                i++;
+            }
+            continue;
+        }
+        if (ch === '/' && next === '*') {
+            out[i] = ' ';
+            out[i + 1] = ' ';
+            i += 2;
+            while (i < content.length) {
+                if (content[i] === '*' && content[i + 1] === '/') {
+                    out[i] = ' ';
+                    out[i + 1] = ' ';
+                    i += 2;
+                    break;
+                }
+                out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
+                i++;
+            }
+            continue;
+        }
+        if (ch === '"' || ch === "'") {
+            const quote = ch;
+            out[i] = ' ';
+            i++;
+            while (i < content.length) {
+                const c = content[i];
+                if (c === '\\' && i + 1 < content.length) {
+                    out[i] = ' ';
+                    out[i + 1] = ' ';
+                    i += 2;
+                    continue;
+                }
+                if (c === quote) {
+                    out[i] = ' ';
+                    i++;
+                    break;
+                }
+                out[i] = c === '\n' || c === '\r' ? c : ' ';
+                i++;
+            }
+            continue;
+        }
+        out[i] = ch;
+        i++;
+    }
+    return out.join('');
+}
+function extractNamespace(sanitizedContent) {
+    const namespaces = [];
+    const namespaceRe = /^\s*namespace\s+([A-Za-z_*][\w.*-]*)\s+([A-Za-z_][\w.]*)\s*$/gm;
+    let match;
+    while ((match = namespaceRe.exec(sanitizedContent)) !== null) {
+        namespaces.push({ language: match[1], namespace: match[2] });
+    }
+    return (namespaces.find((entry) => entry.language === 'java')?.namespace ??
+        namespaces[0]?.namespace ??
+        '');
+}
+function extractServiceBlocks(sanitizedContent) {
+    const results = [];
+    const headerRe = /service\s+([A-Za-z_]\w*)\s*(?:extends\s+[A-Za-z_][\w.]*)?\s*\{/g;
+    let headerMatch;
+    while ((headerMatch = headerRe.exec(sanitizedContent)) !== null) {
+        const serviceName = headerMatch[1];
+        const bodyStart = headerMatch.index + headerMatch[0].length;
+        let depth = 1;
+        let pos = bodyStart;
+        while (pos < sanitizedContent.length && depth > 0) {
+            const ch = sanitizedContent[pos];
+            if (ch === '{')
+                depth++;
+            else if (ch === '}')
+                depth--;
+            pos++;
+        }
+        if (depth !== 0)
+            continue;
+        results.push({
+            name: serviceName,
+            body: sanitizedContent.slice(bodyStart, pos - 1),
+        });
+    }
+    return results;
+}
+function extractMethods(sanitizedServiceBody) {
+    const methods = [];
+    const methodRe = /(?:^|[;,\n\r])\s*(?:oneway\s+)?[A-Za-z_][\w.]*(?:\s*<[^(){};]*>)?\s+([A-Za-z_]\w*)\s*\(/g;
+    let match;
+    while ((match = methodRe.exec(sanitizedServiceBody)) !== null) {
+        methods.push(match[1]);
+    }
+    return methods;
+}
+function thriftSourceScanSymbolUid(contractId, role, filePath, symbolName) {
+    const contractKey = contractId.startsWith('thrift::')
+        ? contractId.slice('thrift::'.length)
+        : contractId;
+    return ['source-scan::thrift', role, contractKey, normalizeThriftPath(filePath), symbolName].join('::');
+}
+function makeContract(cid, role, filePath, symbolName, confidence, meta) {
+    return {
+        contractId: cid,
+        type: 'thrift',
+        role,
+        symbolUid: thriftSourceScanSymbolUid(cid, role, filePath, symbolName),
+        symbolRef: { filePath: normalizeThriftPath(filePath), name: symbolName },
+        symbolName,
+        confidence,
+        meta: { ...meta, extractionStrategy: 'source_scan' },
+    };
+}
+export async function buildThriftContext(repoPath) {
+    const thriftFiles = await glob('**/*.thrift', {
+        cwd: repoPath,
+        absolute: false,
+        nodir: true,
+        // TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
+        // (the canonical ingestion ignore filter, like include-extractor.ts now uses).
+        // New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
+        // third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
+        ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
+    });
+    const namespacesByThrift = new Map();
+    const servicesByName = new Map();
+    for (const rel of thriftFiles) {
+        const thriftPath = normalizeThriftPath(rel);
+        const content = readSafe(repoPath, rel);
+        if (!content)
+            continue;
+        const sanitized = stripThriftCommentsAndStrings(content);
+        const namespace = extractNamespace(sanitized);
+        namespacesByThrift.set(thriftPath, namespace);
+        for (const block of extractServiceBlocks(sanitized)) {
+            const methods = extractMethods(block.body);
+            const info = {
+                namespace,
+                serviceName: block.name,
+                methods,
+                thriftPath,
+            };
+            const existing = servicesByName.get(block.name) ?? [];
+            existing.push(info);
+            servicesByName.set(block.name, existing);
+        }
+    }
+    return { namespacesByThrift, servicesByName };
+}
+export class ThriftExtractor {
+    type = 'thrift';
+    async canExtract(_repo) {
+        return true;
+    }
+    async extract(_dbExecutor, repoPath, _repo) {
+        const out = [];
+        const context = await buildThriftContext(repoPath);
+        for (const infos of context.servicesByName.values()) {
+            for (const info of infos) {
+                for (const methodName of info.methods) {
+                    const symbolName = `${info.serviceName}.${methodName}`;
+                    out.push(makeContract(thriftMethodContractId(info.namespace, info.serviceName, methodName), 'provider', info.thriftPath, symbolName, 0.85, {
+                        namespace: info.namespace,
+                        service: info.serviceName,
+                        method: methodName,
+                        source: 'thrift_idl',
+                    }));
+                }
+            }
+        }
+        const sourceFiles = await glob(THRIFT_SCAN_GLOB, {
+            cwd: repoPath,
+            absolute: false,
+            nodir: true,
+            // TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
+            // (the canonical ingestion ignore filter, like include-extractor.ts now uses).
+            // New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
+            // third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
+            ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
+        });
+        const parser = new Parser();
+        for (const rel of sourceFiles) {
+            const plugin = getPluginForFile(rel);
+            if (!plugin)
+                continue;
+            const content = readSafe(repoPath, rel);
+            if (!content)
+                continue;
+            let detections = [];
+            try {
+                parser.setLanguage(plugin.language);
+                const tree = parseSourceSafe(parser, content);
+                detections = plugin.scan(tree);
+            }
+            catch {
+                continue;
+            }
+            for (const detection of detections) {
+                const contract = this.detectionToContract(detection, rel, context);
+                if (contract)
+                    out.push(contract);
+            }
+        }
+        return this.dedupe(out);
+    }
+    detectionToContract(detection, filePath, context) {
+        const candidates = context.servicesByName.get(detection.serviceName) ?? [];
+        if (candidates.length > 1)
+            return null;
+        const info = candidates[0];
+        if (info) {
+            if (!info.methods.includes(detection.methodName))
+                return null;
+            return makeContract(thriftMethodContractId(info.namespace, info.serviceName, detection.methodName), detection.role, filePath, detection.symbolName, detection.confidenceWithIdl, {
+                namespace: info.namespace,
+                service: info.serviceName,
+                method: detection.methodName,
+                source: detection.source,
+            });
+        }
+        if (detection.role !== 'consumer' ||
+            !detection.methodName ||
+            !detection.usesGeneratedServiceMember) {
+            return null;
+        }
+        return makeContract(thriftMethodContractId('', detection.serviceName, detection.methodName), detection.role, filePath, detection.symbolName, detection.confidenceWithoutIdl, {
+            service: detection.serviceName,
+            method: detection.methodName,
+            source: 'java_thrift_consumer_weak',
+        });
+    }
+    dedupe(items) {
+        const byKey = new Map();
+        for (const c of items) {
+            const key = `${c.contractId}|${c.role}|${c.symbolRef.filePath}|${c.symbolName}`;
+            const existing = byKey.get(key);
+            if (!existing || c.confidence > existing.confidence) {
+                byKey.set(key, c);
+            }
+        }
+        return Array.from(byKey.values());
+    }
+}

package/dist/core/group/extractors/thrift-patterns/index.d.ts

@@ -0,0 +1,4 @@
+import type { ThriftLanguagePlugin } from './types.js';
+export type { ThriftDetection, ThriftLanguagePlugin, ThriftRole } from './types.js';
+export declare const THRIFT_SCAN_GLOB = "**/*.java";
+export declare function getPluginForFile(rel: string): ThriftLanguagePlugin | undefined;

package/dist/core/group/extractors/thrift-patterns/index.js

@@ -0,0 +1,10 @@
+import * as path from 'node:path';
+import { JAVA_THRIFT_PLUGIN } from './java.js';
+const REGISTRY = {
+    '.java': JAVA_THRIFT_PLUGIN,
+};
+export const THRIFT_SCAN_GLOB = '**/*.java';
+export function getPluginForFile(rel) {
+    const ext = path.extname(rel).toLowerCase();
+    return REGISTRY[ext];
+}

package/dist/core/group/extractors/thrift-patterns/java.d.ts

@@ -0,0 +1,2 @@
+import type { ThriftLanguagePlugin } from './types.js';
+export declare const JAVA_THRIFT_PLUGIN: ThriftLanguagePlugin;