@voybio/ace-swarm 0.1.0

package/dist/astgrep-index.js

@@ -0,0 +1,476 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { isAbsolute, relative, resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+import { appendStatusEvent } from "./status-events.js";
+import { safeRead, safeWrite, WORKSPACE_ROOT, wsPath } from "./helpers.js";
+const CODE_EXTENSIONS = new Set(["ts", "tsx", "js", "mjs", "cjs", "py", "rs", "go"]);
+const EXCLUDE_DIRS = new Set([
+    ".git",
+    "node_modules",
+    "dist",
+    "target",
+    ".venv",
+    "__pycache__",
+    ".DS_Store",
+]);
+function normalizeRelPath(path) {
+    return path.replace(/\\/g, "/").replace(/^\.?\//, "");
+}
+function isInside(base, target) {
+    const rel = relative(base, target);
+    return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel));
+}
+function resolveScope(scope) {
+    const rel = normalizeRelPath(scope ?? ".");
+    const abs = resolve(WORKSPACE_ROOT, rel || ".");
+    if (!isInside(WORKSPACE_ROOT, abs)) {
+        throw new Error(`Scope escapes workspace root: ${scope}`);
+    }
+    return { abs, rel: rel || "." };
+}
+function detectAstgrepCommand() {
+    for (const cmd of ["ast-grep", "sg"]) {
+        const probe = spawnSync(cmd, ["--version"], { encoding: "utf8" });
+        if (probe.status === 0)
+            return cmd;
+    }
+    return null;
+}
+function safeReadText(path) {
+    try {
+        return readFileSync(path, "utf8");
+    }
+    catch {
+        return "";
+    }
+}
+function collectFiles(rootAbs) {
+    const output = [];
+    function walk(currentAbs) {
+        const entries = readdirSync(currentAbs, { withFileTypes: true });
+        entries.sort((a, b) => a.name.localeCompare(b.name));
+        for (const entry of entries) {
+            if (EXCLUDE_DIRS.has(entry.name))
+                continue;
+            const abs = resolve(currentAbs, entry.name);
+            if (entry.isDirectory()) {
+                walk(abs);
+                continue;
+            }
+            if (!entry.isFile())
+                continue;
+            const ext = entry.name.includes(".")
+                ? entry.name.slice(entry.name.lastIndexOf(".") + 1).toLowerCase()
+                : "";
+            if (!CODE_EXTENSIONS.has(ext))
+                continue;
+            output.push(abs);
+        }
+    }
+    walk(rootAbs);
+    return output;
+}
+function normalizeHitPath(path) {
+    return normalizeRelPath(path);
+}
+function grepPatternInFiles(files, pattern) {
+    const lines = [];
+    for (const abs of files) {
+        const text = safeReadText(abs);
+        if (!text)
+            continue;
+        const rel = normalizeHitPath(relative(WORKSPACE_ROOT, abs));
+        const rows = text.split(/\r?\n/);
+        rows.forEach((row, idx) => {
+            if (pattern.test(row)) {
+                lines.push(`${rel}:${idx + 1}:${row.trim()}`);
+            }
+        });
+    }
+    return lines;
+}
+function parseAstgrepOutput(stdout) {
+    const trimmed = stdout.trim();
+    if (!trimmed)
+        return [];
+    try {
+        const parsed = JSON.parse(trimmed);
+        if (Array.isArray(parsed))
+            return parsed;
+        if (parsed &&
+            typeof parsed === "object" &&
+            "matches" in parsed &&
+            Array.isArray(parsed.matches)) {
+            return parsed.matches;
+        }
+    }
+    catch {
+        // Fall through to line-based parse.
+    }
+    const matches = [];
+    for (const line of trimmed.split(/\r?\n/)) {
+        const candidate = line.trim();
+        if (!candidate.startsWith("{") || !candidate.endsWith("}"))
+            continue;
+        try {
+            matches.push(JSON.parse(candidate));
+        }
+        catch {
+            // Ignore malformed lines.
+        }
+    }
+    return matches;
+}
+function runAstgrep(astgrepCmd, pattern, lang, paths) {
+    if (!astgrepCmd || paths.length === 0)
+        return [];
+    const args = ["--pattern", pattern, "--lang", lang, "--json", "--color", "never", ...paths];
+    const res = spawnSync(astgrepCmd, args, {
+        cwd: WORKSPACE_ROOT,
+        encoding: "utf8",
+        maxBuffer: 16 * 1024 * 1024,
+    });
+    if (typeof res.stdout !== "string" || res.stdout.trim().length === 0)
+        return [];
+    return parseAstgrepOutput(res.stdout);
+}
+function extractMetaVarText(match, key) {
+    const raw = match.metaVariables?.[key];
+    if (!raw)
+        return undefined;
+    if (typeof raw === "string")
+        return raw;
+    if (raw && typeof raw === "object" && "text" in raw && typeof raw.text === "string") {
+        return raw.text;
+    }
+    if (Array.isArray(raw)) {
+        const first = raw[0];
+        if (typeof first === "string")
+            return first;
+        if (first && typeof first === "object" && "text" in first) {
+            const text = first.text;
+            if (typeof text === "string")
+                return text;
+        }
+    }
+    return undefined;
+}
+function uniqueSorted(values, limit) {
+    return [...new Set(values.filter(Boolean))].sort((a, b) => a.localeCompare(b)).slice(0, limit);
+}
+function extensionOf(path) {
+    const idx = path.lastIndexOf(".");
+    if (idx <= 0 || idx >= path.length - 1)
+        return undefined;
+    return path.slice(idx + 1).toLowerCase();
+}
+function appendEvidenceLine(timestamp, astgrepCmd, stats) {
+    const existing = safeRead("agent-state/EVIDENCE_LOG.md");
+    const seed = existing.startsWith("[FILE NOT FOUND]") || existing.startsWith("[ACCESS DENIED]")
+        ? "# EVIDENCE LOG\n\nAppend-only validation evidence.\n\n## Entries\n"
+        : existing.trimEnd();
+    const entry = [
+        "",
+        `[${timestamp}] AST discovery refresh via agent-astgrep`,
+        `- ast-grep command: ${astgrepCmd ?? "not found"}`,
+        `- language profile entries: ${stats.language_profile_entries}`,
+        `- hotspot files: ${stats.hotspot_files}`,
+        `- rep_astgrep source paths: ${stats.rep_sources}`,
+        "- outputs: agent-state/AST_GREP_INDEX.md, agent-state/AST_GREP_INDEX.json",
+    ].join("\n");
+    safeWrite("agent-state/EVIDENCE_LOG.md", `${seed}${entry}\n`);
+}
+export function refreshAstgrepIndex(input = {}) {
+    const generatedAt = new Date().toISOString();
+    const scope = resolveScope(input.scope);
+    const appendEvidence = input.append_evidence ?? true;
+    const includeRepCorpus = input.include_rep_corpus ?? true;
+    const emitEvent = input.emit_event ?? true;
+    const astgrepCmd = detectAstgrepCommand();
+    const preferredRoots = [
+        "ace-mcp-server/src",
+        "ace-mcp-server/test",
+        "engineering-state/src",
+        "src",
+        "test",
+        "tests",
+        "__tests__",
+    ]
+        .map((rel) => resolve(WORKSPACE_ROOT, rel))
+        .filter((abs) => isInside(scope.abs, abs) && existsSync(abs) && statSync(abs).isDirectory());
+    const roots = preferredRoots.length > 0 ? preferredRoots : [scope.abs];
+    const codeFiles = roots.flatMap((root) => collectFiles(root));
+    const languageProfile = {};
+    for (const abs of codeFiles) {
+        const ext = extensionOf(abs);
+        if (!ext)
+            continue;
+        languageProfile[ext] = (languageProfile[ext] ?? 0) + 1;
+    }
+    const entryPointNames = new Set(["index.ts", "main.py", "main.rs", "server.ts", "app.py"]);
+    const entryPoints = uniqueSorted(codeFiles
+        .filter((abs) => entryPointNames.has(abs.split(/[\\/]/).pop() ?? ""))
+        .map((abs) => normalizeHitPath(relative(WORKSPACE_ROOT, abs))), 30);
+    const tsRoots = roots.filter((root) => codeFiles.some((file) => file.startsWith(root) && /\.(ts|tsx|js)$/i.test(file)));
+    const pyRoots = roots.filter((root) => codeFiles.some((file) => file.startsWith(root) && /\.py$/i.test(file)));
+    const exportedFnMatches = runAstgrep(astgrepCmd, "export function $NAME($$$ARGS)", "ts", tsRoots);
+    const exportedClassMatches = runAstgrep(astgrepCmd, "export class $NAME", "ts", tsRoots);
+    const asyncFnMatches = runAstgrep(astgrepCmd, "async function $NAME($$$ARGS)", "ts", tsRoots);
+    const tsItMatches = runAstgrep(astgrepCmd, "it($$$ARGS)", "ts", tsRoots);
+    const tsTestMatches = runAstgrep(astgrepCmd, "test($$$ARGS)", "ts", tsRoots);
+    const pyTestMatches = runAstgrep(astgrepCmd, "def test_$NAME($$$ARGS):", "python", pyRoots);
+    const handoffRefMatches = runAstgrep(astgrepCmd, '$X(\"SWARM_HANDOFF.$Y\")', "ts", tsRoots);
+    const exportedFunctions = uniqueSorted(exportedFnMatches
+        .map((m) => extractMetaVarText(m, "NAME"))
+        .filter((v) => Boolean(v)), 40);
+    const exportedClasses = uniqueSorted(exportedClassMatches
+        .map((m) => extractMetaVarText(m, "NAME"))
+        .filter((v) => Boolean(v)), 30);
+    const asyncFunctions = uniqueSorted(asyncFnMatches
+        .map((m) => extractMetaVarText(m, "NAME"))
+        .filter((v) => Boolean(v)), 40);
+    // Fallback symbol extraction when AST match shapes differ across language versions.
+    const tsFiles = codeFiles.filter((file) => /\.(ts|tsx|js)$/i.test(file));
+    if (exportedFunctions.length === 0 || exportedClasses.length === 0 || asyncFunctions.length === 0) {
+        const regexExportFn = /^\s*export\s+function\s+([A-Za-z0-9_]+)/gm;
+        const regexExportClass = /^\s*export\s+class\s+([A-Za-z0-9_]+)/gm;
+        const regexAsyncFn = /^\s*async\s+function\s+([A-Za-z0-9_]+)/gm;
+        const fallbackFns = [];
+        const fallbackClasses = [];
+        const fallbackAsync = [];
+        for (const file of tsFiles) {
+            const text = safeReadText(file);
+            if (!text)
+                continue;
+            for (const match of text.matchAll(regexExportFn)) {
+                if (match[1])
+                    fallbackFns.push(match[1]);
+            }
+            for (const match of text.matchAll(regexExportClass)) {
+                if (match[1])
+                    fallbackClasses.push(match[1]);
+            }
+            for (const match of text.matchAll(regexAsyncFn)) {
+                if (match[1])
+                    fallbackAsync.push(match[1]);
+            }
+        }
+        if (exportedFunctions.length === 0) {
+            exportedFunctions.push(...uniqueSorted(fallbackFns, 40));
+        }
+        if (exportedClasses.length === 0) {
+            exportedClasses.push(...uniqueSorted(fallbackClasses, 30));
+        }
+        if (asyncFunctions.length === 0) {
+            asyncFunctions.push(...uniqueSorted(fallbackAsync, 40));
+        }
+    }
+    const routeHits = grepPatternInFiles(codeFiles.filter((abs) => normalizeHitPath(relative(WORKSPACE_ROOT, abs)).startsWith("ace-mcp-server/src/")), /(route_task|routingMap|ace-orchestrator)/i);
+    const indexHits = grepPatternInFiles(codeFiles.filter((abs) => normalizeHitPath(relative(WORKSPACE_ROOT, abs)).startsWith("ace-mcp-server/src/")), /(agent-state\/index\.json|scanWorkspaceDelta)/i);
+    // Use comment-anchored patterns to avoid counting identifiers like TodoNodeStatus.
+    // Match lines that contain a comment marker followed by TODO/FIXME/HACK/XXX.
+    const todoHits = grepPatternInFiles(codeFiles, /(\/\/|\/\*|#|<!--|--|%)\s*.*(TODO|FIXME|HACK|XXX)\b/i);
+    const hotspotMap = new Map();
+    for (const row of [...routeHits, ...indexHits, ...todoHits]) {
+        const file = row.split(":")[0] ?? "";
+        if (!file)
+            continue;
+        hotspotMap.set(file, (hotspotMap.get(file) ?? 0) + 1);
+    }
+    const hotspots = [...hotspotMap.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 15)
+        .map(([file, hits]) => ({ file, hits }));
+    let repSources = [];
+    let repTopDirectories = [];
+    let repExtensions = [];
+    let repKeywordHits = 0;
+    const repPath = wsPath("rep_astgrep.cxml");
+    if (includeRepCorpus && existsSync(repPath) && statSync(repPath).isFile()) {
+        const repText = safeReadText(repPath);
+        repSources = [...repText.matchAll(/<source>([^<]+)<\/source>/g)].map((m) => m[1].trim());
+        repSources = uniqueSorted(repSources, 5000);
+        const directoryCounts = new Map();
+        const extensionCounts = new Map();
+        for (const source of repSources) {
+            const top = source.split("/")[0] ?? source;
+            directoryCounts.set(top, (directoryCounts.get(top) ?? 0) + 1);
+            const ext = extensionOf(source);
+            if (ext)
+                extensionCounts.set(ext, (extensionCounts.get(ext) ?? 0) + 1);
+        }
+        repTopDirectories = [...directoryCounts.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 12)
+            .map(([directory, files]) => ({ directory, files }));
+        repExtensions = [...extensionCounts.entries()]
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 12)
+            .map(([extension, files]) => ({ extension, files }));
+        const keywordPattern = /(ast-grep|sg |playground|pattern|rewrite|rule|yaml|napi|pyo3|Cargo\.toml)/i;
+        const matchingLines = repText
+            .split(/\r?\n/)
+            .filter((line) => keywordPattern.test(line));
+        // Keep this bounded for token efficiency and stable trend tracking.
+        repKeywordHits = Math.min(matchingLines.length, 120);
+    }
+    const commandsUsed = [
+        "filesystem profile scan (language mix + entry points)",
+        `${astgrepCmd ?? "ast-grep-unavailable"} structural scans (exports/classes/async/tests/handoff refs)`,
+        "targeted source grep scans (routing/index/todo signals)",
+        "rep_astgrep.cxml corpus mining",
+    ];
+    const qualitySignals = {
+        astgrep_available: Boolean(astgrepCmd),
+        astgrep_command: astgrepCmd,
+        ts_test_blocks: tsItMatches.length + tsTestMatches.length,
+        python_test_defs: pyTestMatches.length,
+        todo_fixme_hack_xxx: todoHits.length,
+        handoff_schema_refs: handoffRefMatches.length,
+        route_logic_refs: routeHits.length,
+        index_usage_refs: indexHits.length,
+    };
+    const notes = [];
+    if (!astgrepCmd) {
+        notes.push("ast-grep binary not found; AST-level metrics are unavailable.");
+    }
+    if (repSources.length === 0) {
+        notes.push("rep_astgrep.cxml source inventory unavailable or empty.");
+    }
+    notes.push("index.json remains summary-only; use AST_GREP_INDEX artifacts for structural discovery.");
+    const routingHints = [];
+    if (hotspots.length > 0) {
+        routingHints.push(`Highest-signal hotspot: ${hotspots[0].file}`);
+    }
+    if (routeHits.length > 0) {
+        routingHints.push("Routing logic references are concentrated in ace-mcp-server/src.");
+    }
+    if (indexHits.length > 0) {
+        routingHints.push("Index usage hooks are present; refresh AST index when scope changes.");
+    }
+    if (repKeywordHits > 0) {
+        routingHints.push("rep_astgrep.cxml provides relevant AST-grep reference coverage for discovery tuning.");
+    }
+    const payload = {
+        generated_at: generatedAt,
+        scope: scope.rel,
+        commands_used: commandsUsed,
+        language_profile: languageProfile,
+        top_symbols: {
+            ts_exported_functions: exportedFunctions,
+            ts_exported_classes: exportedClasses,
+            ts_async_functions: asyncFunctions,
+            entry_points: entryPoints,
+        },
+        hotspots,
+        quality_signals: qualitySignals,
+        rep_astgrep_reference: {
+            source_paths: repSources.length,
+            top_directories: repTopDirectories,
+            extensions: repExtensions,
+            keyword_hits: repKeywordHits,
+        },
+        routing_hints: routingHints,
+        notes,
+    };
+    const markdown = [
+        "# AST_GREP_INDEX",
+        "",
+        `Generated: ${generatedAt}`,
+        `Scope: ${scope.rel}`,
+        "",
+        "## Commands Used",
+        ...commandsUsed.map((row) => `- ${row}`),
+        "",
+        "## Language Profile",
+        ...(Object.entries(languageProfile).length > 0
+            ? Object.entries(languageProfile)
+                .sort((a, b) => b[1] - a[1])
+                .map(([ext, count]) => `- ${ext}: ${count}`)
+            : ["- No source files detected in scope."]),
+        "",
+        "## Top Symbols",
+        `- TS exported functions (${exportedFunctions.length} sampled): ${exportedFunctions.join(", ") || "none"}`,
+        `- TS exported classes (${exportedClasses.length} sampled): ${exportedClasses.join(", ") || "none"}`,
+        `- TS async functions (${asyncFunctions.length} sampled): ${asyncFunctions.join(", ") || "none"}`,
+        `- Entry points (${entryPoints.length}): ${entryPoints.join(", ") || "none"}`,
+        "",
+        "## Hotspot Files",
+        ...(hotspots.length > 0
+            ? hotspots.map((row) => `- ${row.file} (${row.hits} hits)`)
+            : ["- No hotspot signals detected."]),
+        "",
+        "## Quality Signals",
+        `- ast-grep available: ${qualitySignals.astgrep_available}`,
+        `- TS test blocks: ${qualitySignals.ts_test_blocks}`,
+        `- Python test defs: ${qualitySignals.python_test_defs}`,
+        `- TODO/FIXME/HACK/XXX matches: ${qualitySignals.todo_fixme_hack_xxx}`,
+        `- Handoff schema refs: ${qualitySignals.handoff_schema_refs}`,
+        `- Route logic refs: ${qualitySignals.route_logic_refs}`,
+        `- Index usage refs: ${qualitySignals.index_usage_refs}`,
+        "",
+        "## rep_astgrep.cxml Signals",
+        `- Source paths discovered: ${repSources.length}`,
+        ...(repTopDirectories.length > 0
+            ? repTopDirectories.map((row) => `- Top dir ${row.directory}: ${row.files}`)
+            : ["- No top directory data."]),
+        ...(repExtensions.length > 0
+            ? repExtensions.map((row) => `- Extension ${row.extension}: ${row.files}`)
+            : ["- No extension distribution data."]),
+        `- Keyword hits: ${repKeywordHits}`,
+        "",
+        "## Routing Hints",
+        ...(routingHints.length > 0 ? routingHints.map((row) => `- ${row}`) : ["- No routing hints derived."]),
+        "",
+        "## Notes",
+        ...notes.map((row) => `- ${row}`),
+        "",
+    ].join("\n");
+    const outputJsonPath = safeWrite("agent-state/AST_GREP_INDEX.json", JSON.stringify(payload, null, 2));
+    const outputMdPath = safeWrite("agent-state/AST_GREP_INDEX.md", markdown);
+    if (appendEvidence) {
+        appendEvidenceLine(generatedAt, astgrepCmd, {
+            language_profile_entries: Object.keys(languageProfile).length,
+            hotspot_files: hotspots.length,
+            rep_sources: repSources.length,
+        });
+    }
+    let eventPath;
+    let eventId;
+    if (emitEvent) {
+        const status = astgrepCmd ? "pass" : "blocked";
+        const eventType = astgrepCmd ? "DISCOVERY_INDEX_READY" : "DISCOVERY_SCOPE_BLOCKED";
+        const event = appendStatusEvent({
+            source_module: "capability-astgrep",
+            event_type: eventType,
+            status,
+            summary: `AST discovery index refreshed for scope ${scope.rel}`,
+            objective_id: "discovery-index",
+            payload: {
+                astgrep_available: Boolean(astgrepCmd),
+                hotspot_files: hotspots.length,
+                rep_sources: repSources.length,
+                evidence_ref: "agent-state/AST_GREP_INDEX.json",
+            },
+        });
+        eventPath = event.path;
+        eventId = event.event.event_id;
+    }
+    return {
+        generated_at: generatedAt,
+        scope: scope.rel,
+        astgrep_command: astgrepCmd,
+        output_md_path: outputMdPath,
+        output_json_path: outputJsonPath,
+        evidence_appended: appendEvidence,
+        event_path: eventPath,
+        event_id: eventId,
+        stats: {
+            language_profile_entries: Object.keys(languageProfile).length,
+            hotspot_files: hotspots.length,
+            rep_sources: repSources.length,
+            todo_signals: todoHits.length,
+        },
+    };
+}
+//# sourceMappingURL=astgrep-index.js.map

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map