arceus-s 1.6.4

package/dist/core/group/extractors/grpc-extractor.js

@@ -0,0 +1,414 @@
+import * as path from 'node:path';
+import { glob } from 'glob';
+import Parser from 'tree-sitter';
+import { createIgnoreFilter } from '../../../config/ignore-service.js';
+import { readSafe } from './fs-utils.js';
+import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
+import { logger } from '../../logger.js';
+import { GRPC_SCAN_GLOB, getPluginForFile, hasProtoPlugin, } from './grpc-patterns/index.js';
+/**
+ * Language-agnostic orchestrator for gRPC (provider + consumer) contract
+ * extraction.
+ *
+ * Two parts:
+ *
+ * 1. **`.proto` parsing** — tree-sitter when `tree-sitter-proto` is
+ *    installed (optionalDependency vendored in `vendor/tree-sitter-proto/`),
+ *    via the `.proto` entry in `grpc-patterns/` and `hasProtoPlugin`.
+ *    When the grammar isn't available (platform incompatibility, native
+ *    build failure) the orchestrator falls back to the in-process
+ *    string-sanitizing parser defined below (`stripProtoCommentsAndStrings`
+ *    + `extractServiceBlocks`). The fallback preserves offsets so any
+ *    downstream regex scans run against a sanitized copy without
+ *    affecting line numbers of the original.
+ *
+ * 2. **Source-scan providers / consumers** — delegated to per-language
+ *    plugins in `./grpc-patterns/`. The orchestrator imports NO
+ *    tree-sitter grammars or query strings — each plugin owns its own.
+ */
+// ─── .proto fallback parser (used only when tree-sitter-proto is absent) ───
+function contractId(pkg, service, method) {
+    const prefix = pkg ? `${pkg}.${service}` : service;
+    return `grpc::${prefix}/${method}`;
+}
+function serviceOnlyContractId(serviceName) {
+    return `grpc::${serviceName}/*`;
+}
+/**
+ * Replace all .proto comments and string literals with spaces, preserving the
+ * original length and character offsets of the input. This lets downstream
+ * regex / brace-depth parsers run on a "sanitized" copy without having to
+ * understand proto syntax, while any RegExp.exec/index-based lookups that
+ * were already positional against `content` continue to work against the
+ * original string.
+ *
+ * Supported comment forms: `// line comment`, `/* block comment * /`.
+ * Supported strings: double-quoted ("…") and single-quoted ('…') with `\`
+ * escape handling. Raw/unterminated strings are not supported — we stop
+ * on a line break for line-style comments and on EOF for unterminated
+ * strings/blocks, which matches how most real proto files parse.
+ */
+function stripProtoCommentsAndStrings(content) {
+    const out = new Array(content.length);
+    let i = 0;
+    while (i < content.length) {
+        const ch = content[i];
+        const next = content[i + 1];
+        // Line comment: // ... \n
+        if (ch === '/' && next === '/') {
+            out[i] = ' ';
+            out[i + 1] = ' ';
+            i += 2;
+            while (i < content.length && content[i] !== '\n') {
+                out[i] = content[i] === '\r' ? '\r' : ' ';
+                i++;
+            }
+            continue;
+        }
+        // Block comment: /* ... */
+        if (ch === '/' && next === '*') {
+            out[i] = ' ';
+            out[i + 1] = ' ';
+            i += 2;
+            while (i < content.length) {
+                if (content[i] === '*' && content[i + 1] === '/') {
+                    out[i] = ' ';
+                    out[i + 1] = ' ';
+                    i += 2;
+                    break;
+                }
+                // Preserve newlines so line numbers stay stable for downstream code.
+                out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
+                i++;
+            }
+            continue;
+        }
+        // String literal: "..." or '...'
+        if (ch === '"' || ch === "'") {
+            const quote = ch;
+            out[i] = ' '; // replace opening quote
+            i++;
+            while (i < content.length) {
+                const c = content[i];
+                if (c === '\\' && i + 1 < content.length) {
+                    // Skip escaped pair (e.g. \" \n \\)
+                    out[i] = ' ';
+                    out[i + 1] = ' ';
+                    i += 2;
+                    continue;
+                }
+                if (c === quote) {
+                    out[i] = ' ';
+                    i++;
+                    break;
+                }
+                // Preserve newlines; proto technically disallows unescaped newlines
+                // inside strings, but real files occasionally have them.
+                out[i] = c === '\n' || c === '\r' ? c : ' ';
+                i++;
+            }
+            continue;
+        }
+        out[i] = ch;
+        i++;
+    }
+    return out.join('');
+}
+function extractServiceBlocks(content) {
+    const results = [];
+    // Sanitize comments and string literals so braces inside them don't
+    // throw off the depth counter. The sanitized copy has the same length
+    // and offsets as the original, so we use it ONLY to scan for service
+    // headers and braces; the service body we return is sliced from the
+    // ORIGINAL content to preserve exact source text for downstream use.
+    const sanitized = stripProtoCommentsAndStrings(content);
+    const headerRe = /service\s+(\w+)\s*\{/g;
+    let headerMatch;
+    while ((headerMatch = headerRe.exec(sanitized)) !== null) {
+        const serviceName = headerMatch[1];
+        const bodyStart = headerMatch.index + headerMatch[0].length;
+        let depth = 1;
+        let pos = bodyStart;
+        while (pos < sanitized.length && depth > 0) {
+            const ch = sanitized[pos];
+            if (ch === '{')
+                depth++;
+            else if (ch === '}')
+                depth--;
+            pos++;
+        }
+        // If EOF before depth returns to 0, skip incomplete service
+        if (depth !== 0)
+            continue;
+        // body is between opening { (consumed by regex) and closing } (pos is one past it)
+        const body = content.slice(bodyStart, pos - 1);
+        results.push({ name: serviceName, body });
+    }
+    return results;
+}
+function makeContract(cid, role, filePath, symbolName, confidence, meta) {
+    return {
+        contractId: cid,
+        type: 'grpc',
+        role,
+        symbolUid: '',
+        symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: symbolName },
+        symbolName,
+        confidence,
+        meta: { ...meta, extractionStrategy: 'source_scan' },
+    };
+}
+function normalizeProtoPath(rel) {
+    return rel.replace(/\\/g, '/');
+}
+function extractProtoImports(content) {
+    const imports = [];
+    const re = /^\s*import\s+"([^"]+)"\s*;/gm;
+    let match;
+    while ((match = re.exec(content)) !== null) {
+        imports.push(match[1]);
+    }
+    return imports;
+}
+function longestSharedSegmentRun(aPath, bPath) {
+    const a = aPath.split('/').filter(Boolean);
+    const b = bPath.split('/').filter(Boolean);
+    let best = 0;
+    for (let i = 0; i < a.length; i++) {
+        for (let j = 0; j < b.length; j++) {
+            let run = 0;
+            while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) {
+                run++;
+            }
+            if (run > best)
+                best = run;
+        }
+    }
+    return best;
+}
+async function buildProtoContext(repoPath) {
+    const servicesByName = new Map();
+    // `.arcignore` / `.gitignore` honoured via the shared IgnoreService —
+    // see `filesystem-walker.ts` for the canonical pattern. Replaces a
+    // hardcoded `[node_modules, .git, vendor]` array; those names plus the
+    // rest of `DEFAULT_IGNORE_LIST` are still excluded by default (#1185).
+    const protoIgnoreFilter = await createIgnoreFilter(repoPath);
+    const protoFiles = await glob('**/*.proto', {
+        cwd: repoPath,
+        absolute: false,
+        nodir: true,
+        ignore: protoIgnoreFilter,
+    });
+    const contents = new Map();
+    for (const rel of protoFiles) {
+        const content = readSafe(repoPath, rel);
+        if (!content)
+            continue;
+        contents.set(normalizeProtoPath(rel), content);
+    }
+    const packagesByProto = new Map();
+    const resolvePackage = (protoPath, seen = new Set()) => {
+        if (packagesByProto.has(protoPath))
+            return packagesByProto.get(protoPath) ?? '';
+        if (seen.has(protoPath))
+            return '';
+        const content = contents.get(protoPath);
+        if (!content)
+            return '';
+        seen.add(protoPath);
+        const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m);
+        if (pkgMatch?.[1]) {
+            packagesByProto.set(protoPath, pkgMatch[1]);
+            return pkgMatch[1];
+        }
+        for (const importPath of extractProtoImports(content)) {
+            const normalizedImport = normalizeProtoPath(importPath);
+            const candidates = [
+                normalizeProtoPath(path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport))),
+                normalizedImport,
+            ];
+            for (const candidate of candidates) {
+                if (!contents.has(candidate))
+                    continue;
+                const inheritedPackage = resolvePackage(candidate, seen);
+                if (inheritedPackage) {
+                    packagesByProto.set(protoPath, inheritedPackage);
+                    return inheritedPackage;
+                }
+            }
+        }
+        packagesByProto.set(protoPath, '');
+        return '';
+    };
+    for (const rel of protoFiles) {
+        const normalizedRel = normalizeProtoPath(rel);
+        const content = contents.get(normalizedRel);
+        if (!content)
+            continue;
+        const pkg = resolvePackage(normalizedRel);
+        const serviceBlocks = extractServiceBlocks(content);
+        for (const block of serviceBlocks) {
+            const rpcRe = /rpc\s+(\w+)\s*\(/g;
+            const methods = [];
+            let m;
+            while ((m = rpcRe.exec(block.body)) !== null) {
+                methods.push(m[1]);
+            }
+            const info = {
+                package: pkg,
+                serviceName: block.name,
+                methods,
+                protoPath: normalizedRel,
+            };
+            const existing = servicesByName.get(block.name) ?? [];
+            existing.push(info);
+            servicesByName.set(block.name, existing);
+        }
+    }
+    return { packagesByProto, servicesByName };
+}
+export async function buildProtoMap(repoPath) {
+    const { servicesByName } = await buildProtoContext(repoPath);
+    return servicesByName;
+}
+export function resolveProtoConflict(serviceName, sourceFilePath, candidates) {
+    if (candidates.length === 0)
+        return null;
+    if (candidates.length === 1)
+        return candidates[0];
+    const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath));
+    const scored = candidates.map((c) => {
+        const protoDir = normalizeProtoPath(path.dirname(c.protoPath));
+        return { candidate: c, score: longestSharedSegmentRun(sourceDir, protoDir) };
+    });
+    let maxScore = -1;
+    for (const s of scored) {
+        if (s.score > maxScore)
+            maxScore = s.score;
+    }
+    const winners = scored.filter((s) => s.score === maxScore);
+    // Path heuristic cannot uniquely identify a winner — refuse to guess.
+    // Ties (including all-zero ties) would otherwise silently merge unrelated
+    // services under a fabricated package-qualified contract id.
+    if (winners.length !== 1) {
+        const paths = candidates.map((c) => c.protoPath).join(', ');
+        logger.warn(`[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`);
+        return null;
+    }
+    return winners[0].candidate;
+}
+export function serviceContractId(pkg, serviceName) {
+    const prefix = pkg ? `${pkg}.${serviceName}` : serviceName;
+    return `grpc::${prefix}/*`;
+}
+// ─── Orchestrator ────────────────────────────────────────────────────
+export class GrpcExtractor {
+    type = 'grpc';
+    async canExtract(_repo) {
+        return true;
+    }
+    async extract(_dbExecutor, repoPath, _repo) {
+        const out = [];
+        const protoContext = await buildProtoContext(repoPath);
+        const protoMap = protoContext.servicesByName;
+        // ─── Proto files — definitive provider source ─────────────────
+        // When tree-sitter-proto is available, .proto files are handled by
+        // the plugin loop below (they're in GRPC_SCAN_GLOB). Otherwise
+        // emit provider contracts directly from the proto map that
+        // `buildProtoContext` already built — no second glob / parse pass.
+        if (!hasProtoPlugin) {
+            for (const infos of protoMap.values()) {
+                for (const info of infos) {
+                    for (const methodName of info.methods) {
+                        const cid = contractId(info.package, info.serviceName, methodName);
+                        out.push(makeContract(cid, 'provider', info.protoPath, `${info.serviceName}.${methodName}`, 0.85, {
+                            package: info.package,
+                            service: info.serviceName,
+                            method: methodName,
+                            source: 'proto',
+                        }));
+                    }
+                }
+            }
+        }
+        // ─── Source files (+ .proto when plugin available) ────────────
+        // Honour `.arcignore` / `.gitignore` via the shared IgnoreService —
+        // mirrors `filesystem-walker.ts`. Replaces a hardcoded
+        // `[node_modules, .git, vendor, dist, build]` array; those names are all
+        // in `DEFAULT_IGNORE_LIST`, so default behaviour is preserved (#1185).
+        const sourceIgnoreFilter = await createIgnoreFilter(repoPath);
+        const sourceFiles = await glob(GRPC_SCAN_GLOB, {
+            cwd: repoPath,
+            ignore: sourceIgnoreFilter,
+            nodir: true,
+        });
+        const parser = new Parser();
+        for (const rel of sourceFiles) {
+            const plugin = getPluginForFile(rel);
+            if (!plugin)
+                continue;
+            const content = readSafe(repoPath, rel);
+            if (!content)
+                continue;
+            let detections = [];
+            try {
+                parser.setLanguage(plugin.language);
+                const tree = parseSourceSafe(parser, content);
+                detections = plugin.scan(tree);
+            }
+            catch {
+                continue;
+            }
+            for (const d of detections) {
+                const contract = this.detectionToContract(d, rel, protoMap);
+                if (contract)
+                    out.push(contract);
+            }
+        }
+        return this.dedupe(out);
+    }
+    /**
+     * Convert a plugin `GrpcDetection` into a concrete `ExtractedContract`
+     * by resolving the short service name against the proto map, building
+     * either a service-level (`grpc::pkg.Svc/*`) or method-level
+     * (`grpc::pkg.Svc/Method`) contract id, and selecting confidence
+     * based on whether the proto map had an entry.
+     */
+    detectionToContract(d, filePath, protoMap) {
+        const candidates = protoMap.get(d.serviceName) ?? [];
+        const proto = resolveProtoConflict(d.serviceName, filePath, candidates);
+        // If there were proto candidates but resolution was ambiguous, skip
+        // contract emission rather than fabricating a package-qualified id from
+        // an arbitrary candidate. resolveProtoConflict already warned.
+        if (candidates.length > 0 && proto === null)
+            return null;
+        const pkg = proto?.package ?? '';
+        const cid = d.methodName
+            ? contractId(pkg, d.serviceName, d.methodName)
+            : proto
+                ? serviceContractId(pkg, d.serviceName)
+                : serviceOnlyContractId(d.serviceName);
+        const confidence = proto ? d.confidenceWithProto : d.confidenceWithoutProto;
+        const meta = {
+            service: d.serviceName,
+            source: d.source,
+        };
+        if (d.methodName)
+            meta.method = d.methodName;
+        return makeContract(cid, d.role, filePath, d.symbolName, confidence, meta);
+    }
+    dedupe(items) {
+        const byKey = new Map();
+        for (const c of items) {
+            const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`;
+            const existing = byKey.get(k);
+            if (!existing ||
+                c.confidence > existing.confidence ||
+                (c.confidence === existing.confidence &&
+                    String(c.meta.source) < String(existing.meta.source))) {
+                byKey.set(k, c);
+            }
+        }
+        return Array.from(byKey.values());
+    }
+}

package/dist/core/group/extractors/grpc-patterns/go.d.ts

@@ -0,0 +1,2 @@
+import type { GrpcLanguagePlugin } from './types.js';
+export declare const GO_GRPC_PLUGIN: GrpcLanguagePlugin;

package/dist/core/group/extractors/grpc-patterns/go.js

@@ -0,0 +1,97 @@
+import Go from 'tree-sitter-go';
+import { compilePatterns, runCompiledPatterns, } from '../tree-sitter-scanner.js';
+/**
+ * Go gRPC plugin. Detects:
+ *   - Provider: `pb.RegisterXxxServer(...)` calls
+ *   - Provider: `pb.UnimplementedXxxServer` embedded in a struct
+ *   - Consumer: `pb.NewXxxClient(conn)` calls
+ */
+const REGISTER_RE = /^Register(\w+)Server$/;
+const UNIMPLEMENTED_RE = /^Unimplemented(\w+)Server$/;
+const NEW_CLIENT_RE = /^New(\w+)Client$/;
+// Any `xxx.<fn>(...)` call — plugin filters the field identifier text.
+const SELECTOR_CALL_PATTERNS = compilePatterns({
+    name: 'go-grpc-selector-call',
+    language: Go,
+    patterns: [
+        {
+            meta: {},
+            query: `
+        (call_expression
+          function: (selector_expression
+            field: (field_identifier) @fn))
+      `,
+        },
+    ],
+});
+// Any `qualified_type` used as a struct field — for `pb.UnimplementedXxxServer`.
+const STRUCT_EMBEDDING_PATTERNS = compilePatterns({
+    name: 'go-grpc-struct-embedding',
+    language: Go,
+    patterns: [
+        {
+            meta: {},
+            query: `
+        (struct_type
+          (field_declaration_list
+            (field_declaration
+              type: (qualified_type
+                name: (type_identifier) @field_type))))
+      `,
+        },
+    ],
+});
+export const GO_GRPC_PLUGIN = {
+    name: 'go-grpc',
+    language: Go,
+    scan(tree) {
+        const out = [];
+        for (const match of runCompiledPatterns(SELECTOR_CALL_PATTERNS, tree)) {
+            const fnNode = match.captures.fn;
+            if (!fnNode)
+                continue;
+            const fnText = fnNode.text;
+            const registerMatch = REGISTER_RE.exec(fnText);
+            if (registerMatch) {
+                out.push({
+                    role: 'provider',
+                    serviceName: registerMatch[1],
+                    symbolName: fnText,
+                    source: 'go_register',
+                    confidenceWithProto: 0.8,
+                    confidenceWithoutProto: 0.65,
+                });
+                continue;
+            }
+            const newClientMatch = NEW_CLIENT_RE.exec(fnText);
+            if (newClientMatch) {
+                out.push({
+                    role: 'consumer',
+                    serviceName: newClientMatch[1],
+                    symbolName: fnText,
+                    source: 'go_client',
+                    confidenceWithProto: 0.75,
+                    confidenceWithoutProto: 0.55,
+                });
+                continue;
+            }
+        }
+        for (const match of runCompiledPatterns(STRUCT_EMBEDDING_PATTERNS, tree)) {
+            const fieldNode = match.captures.field_type;
+            if (!fieldNode)
+                continue;
+            const unimpl = UNIMPLEMENTED_RE.exec(fieldNode.text);
+            if (!unimpl)
+                continue;
+            out.push({
+                role: 'provider',
+                serviceName: unimpl[1],
+                symbolName: fieldNode.text,
+                source: 'go_unimplemented',
+                confidenceWithProto: 0.8,
+                confidenceWithoutProto: 0.65,
+            });
+        }
+        return out;
+    },
+};

package/dist/core/group/extractors/grpc-patterns/index.d.ts

@@ -0,0 +1,19 @@
+import type { GrpcLanguagePlugin } from './types.js';
+export type { GrpcDetection, GrpcLanguagePlugin, GrpcRole } from './types.js';
+export { PROTO_GRPC_PLUGIN, extractPackageFromTree } from './proto.js';
+/**
+ * Glob for source files worth scanning for gRPC server/client patterns.
+ * Includes `.proto` when the grammar is available.
+ */
+export declare const GRPC_SCAN_GLOB: string;
+/**
+ * Whether the tree-sitter proto plugin is available. The orchestrator
+ * uses this to decide between the tree-sitter path and the fallback
+ * manual parser for `.proto` files.
+ */
+export declare const hasProtoPlugin: boolean;
+/**
+ * Return the gRPC plugin registered for the given file's extension,
+ * or `undefined` if the extension is not registered.
+ */
+export declare function getPluginForFile(rel: string): GrpcLanguagePlugin | undefined;

package/dist/core/group/extractors/grpc-patterns/index.js

@@ -0,0 +1,46 @@
+import * as path from 'node:path';
+import { GO_GRPC_PLUGIN } from './go.js';
+import { JAVA_GRPC_PLUGIN } from './java.js';
+import { PYTHON_GRPC_PLUGIN } from './python.js';
+import { JAVASCRIPT_GRPC_PLUGIN, TYPESCRIPT_GRPC_PLUGIN, TSX_GRPC_PLUGIN } from './node.js';
+import { PROTO_GRPC_PLUGIN } from './proto.js';
+export { PROTO_GRPC_PLUGIN, extractPackageFromTree } from './proto.js';
+/**
+ * File-extension → gRPC language plugin registry. Mirrors the shape
+ * of `http-patterns/index.ts` and `topic-patterns/index.ts`.
+ *
+ * `.proto` files are registered only when `tree-sitter-proto` is
+ * available (it's an optionalDependency). When absent, the orchestrator
+ * falls back to the built-in manual proto parser.
+ */
+const REGISTRY = {
+    '.go': GO_GRPC_PLUGIN,
+    '.java': JAVA_GRPC_PLUGIN,
+    '.py': PYTHON_GRPC_PLUGIN,
+    '.js': JAVASCRIPT_GRPC_PLUGIN,
+    '.jsx': JAVASCRIPT_GRPC_PLUGIN,
+    '.ts': TYPESCRIPT_GRPC_PLUGIN,
+    '.tsx': TSX_GRPC_PLUGIN,
+    ...(PROTO_GRPC_PLUGIN ? { '.proto': PROTO_GRPC_PLUGIN } : {}),
+};
+/**
+ * Glob for source files worth scanning for gRPC server/client patterns.
+ * Includes `.proto` when the grammar is available.
+ */
+export const GRPC_SCAN_GLOB = PROTO_GRPC_PLUGIN
+    ? '**/*.{go,java,py,ts,tsx,js,jsx,proto}'
+    : '**/*.{go,java,py,ts,tsx,js,jsx}';
+/**
+ * Whether the tree-sitter proto plugin is available. The orchestrator
+ * uses this to decide between the tree-sitter path and the fallback
+ * manual parser for `.proto` files.
+ */
+export const hasProtoPlugin = PROTO_GRPC_PLUGIN !== null;
+/**
+ * Return the gRPC plugin registered for the given file's extension,
+ * or `undefined` if the extension is not registered.
+ */
+export function getPluginForFile(rel) {
+    const ext = path.extname(rel).toLowerCase();
+    return REGISTRY[ext];
+}

package/dist/core/group/extractors/grpc-patterns/java.d.ts

@@ -0,0 +1,2 @@
+import type { GrpcLanguagePlugin } from './types.js';
+export declare const JAVA_GRPC_PLUGIN: GrpcLanguagePlugin;