@deftai/directive-core 0.55.0

package/dist/codebase/default-extractor.js

@@ -0,0 +1,713 @@
+import { createHash } from "node:crypto";
+import { existsSync, globSync, readdirSync, readFileSync, realpathSync, statSync, } from "node:fs";
+import { isAbsolute, join, relative, resolve } from "node:path";
+import { extractCodeStructure, loadJsonFile } from "../verify-source/code-structure-validate.js";
+import { sortedStringifyPretty } from "./json.js";
+import { CODEBASE_MAP_FORMAT_VERSION, CODEBASE_MAP_KIND, CODEBASE_PROVIDER_CONTRACT_VERSION, } from "./projection-registry.js";
+export const DEFAULT_PROVIDER_NAME = "directive-default-extractor";
+export const DEFAULT_PROVIDER_VERSION = "0.1";
+export const MAX_IMPORT_SCAN_BYTES = 262_144;
+export const MAX_FILES_PER_MODULE = 100;
+export const MAX_EVIDENCE_PER_EDGE = 5;
+export const SKIP_DIRS = new Set([
+    ".git",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".venv",
+    "__pycache__",
+    "build",
+    "dist",
+    "node_modules",
+    "swarm-worktrees",
+]);
+export const LANGUAGE_BY_SUFFIX = {
+    ".go": "Go",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".json": "JSON",
+    ".md": "Markdown",
+    ".py": "Python",
+    ".sh": "Shell",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".yaml": "YAML",
+    ".yml": "YAML",
+};
+export const ENTRYPOINT_NAMES = new Set([
+    "__main__.py",
+    "cli.py",
+    "cmd.py",
+    "index.js",
+    "index.ts",
+    "main.go",
+    "main.py",
+    "run",
+    "run.py",
+]);
+export class CodeStructureConfigError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "CodeStructureConfigError";
+    }
+}
+function posixPath(path) {
+    return path.replace(/\\/g, "/");
+}
+function relativeFile(path, projectRoot) {
+    return posixPath(relative(projectRoot, path));
+}
+export function fileSha256(path) {
+    return createHash("sha256").update(readFileSync(path)).digest("hex");
+}
+function sourceContentHash(projectRoot, relPaths) {
+    const digest = createHash("sha256");
+    let fileCount = 0;
+    for (const relPath of [...new Set(relPaths)].sort()) {
+        const path = join(projectRoot, relPath);
+        if (!existsSync(path)) {
+            continue;
+        }
+        let st;
+        try {
+            st = statSync(path);
+        }
+        catch {
+            continue;
+        }
+        if (!st.isFile()) {
+            continue;
+        }
+        fileCount += 1;
+        digest.update(Buffer.from(relPath, "utf8"));
+        digest.update(Buffer.from([0]));
+        digest.update(Buffer.from(fileSha256(path), "ascii"));
+        digest.update(Buffer.from([0]));
+    }
+    return {
+        algorithm: "sha256",
+        scope: "codeStructure-and-module-files",
+        value: digest.digest("hex"),
+        fileCount,
+    };
+}
+function sourcePathRel(projectRoot, sourcePath) {
+    const rel = posixPath(relative(projectRoot, resolve(sourcePath)));
+    if (rel === "" || rel.startsWith("../") || rel === ".." || isAbsolute(rel)) {
+        return null;
+    }
+    return rel;
+}
+/** Return the authored codeStructure source path used by the default extractor. */
+export function defaultCodeStructurePath(projectRoot, codeStructurePath) {
+    return codeStructurePath ?? join(projectRoot, "vbrief", "PROJECT-DEFINITION.vbrief.json");
+}
+function stableId(value) {
+    let slug = "";
+    for (let i = 0; i < value.length; i += 1) {
+        const c = value.charCodeAt(i);
+        if ((c >= 97 && c <= 122) || (c >= 48 && c <= 57)) {
+            slug += value.charAt(i).toLowerCase();
+        }
+        else if (slug.length > 0 && slug.charAt(slug.length - 1) !== "-") {
+            slug += "-";
+        }
+    }
+    while (slug.endsWith("-")) {
+        slug = slug.slice(0, -1);
+    }
+    return slug.length > 0 ? slug : "root";
+}
+function hasGlobMagic(value) {
+    return value.includes("*") || value.includes("?") || value.includes("[");
+}
+function repoFiles(projectRoot) {
+    const files = [];
+    function walk(dir) {
+        let entries;
+        try {
+            entries = readdirSync(dir);
+        }
+        catch {
+            return;
+        }
+        for (const name of entries.sort()) {
+            const full = join(dir, name);
+            let st;
+            try {
+                st = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (st.isDirectory()) {
+                if (!SKIP_DIRS.has(name)) {
+                    walk(full);
+                }
+            }
+            else if (st.isFile()) {
+                files.push(full);
+            }
+        }
+    }
+    walk(projectRoot);
+    return files.sort((a, b) => relativeFile(a, projectRoot).localeCompare(relativeFile(b, projectRoot)));
+}
+function globFiles(projectRoot, globs) {
+    const files = new Map();
+    for (const globValue of globs) {
+        let matches;
+        try {
+            matches = globSync(globValue, { cwd: projectRoot });
+        }
+        catch {
+            continue;
+        }
+        for (const match of matches) {
+            const full = join(projectRoot, match);
+            let st;
+            try {
+                st = statSync(full);
+            }
+            catch {
+                continue;
+            }
+            if (!st.isFile()) {
+                continue;
+            }
+            const relParts = relativeFile(full, projectRoot).split("/");
+            if (relParts.some((part) => SKIP_DIRS.has(part))) {
+                continue;
+            }
+            files.set(relativeFile(full, projectRoot), full);
+        }
+    }
+    return [...files.keys()].sort().map((key) => files.get(key));
+}
+function loadAuthoredCodeStructure(projectRoot, codeStructurePath) {
+    const path = defaultCodeStructurePath(projectRoot, codeStructurePath);
+    if (!existsSync(path)) {
+        return [null, null];
+    }
+    let data;
+    try {
+        data = loadJsonFile(path);
+    }
+    catch (err) {
+        throw new CodeStructureConfigError(String(err));
+    }
+    const extracted = extractCodeStructure(data);
+    if (extracted === null) {
+        return [null, null];
+    }
+    return [{ record: extracted.record, home: extracted.home }, extracted.home];
+}
+function modulePrefixes(module, files, projectRoot) {
+    const prefixes = new Set();
+    const globs = module.pathGlobs;
+    if (Array.isArray(globs)) {
+        for (const globValue of globs) {
+            if (typeof globValue !== "string") {
+                continue;
+            }
+            const first = globValue.split("/", 1)[0] ?? "";
+            if (first && !hasGlobMagic(first) && first !== "." && first !== "**") {
+                prefixes.add(first.endsWith(".py") ? first.slice(0, -3) : first);
+            }
+        }
+    }
+    for (const filePath of files) {
+        const relParts = relativeFile(filePath, projectRoot).split("/");
+        if (relParts.length > 1) {
+            prefixes.add(relParts[0] ?? "");
+        }
+        else if (filePath.endsWith(".py")) {
+            const base = relativeFile(filePath, projectRoot).split("/").pop() ?? "";
+            prefixes.add(base.endsWith(".py") ? base.slice(0, -3) : base);
+        }
+    }
+    prefixes.delete("");
+    return prefixes;
+}
+function curatedModules(projectRoot, codeStructure) {
+    const artifacts = [];
+    const fileToModule = new Map();
+    const prefixesByModule = new Map();
+    const degraded = [];
+    const modules = codeStructure.modules;
+    if (!Array.isArray(modules)) {
+        return [artifacts, fileToModule, prefixesByModule, degraded];
+    }
+    for (const rawModule of modules) {
+        if (typeof rawModule !== "object" || rawModule === null || Array.isArray(rawModule)) {
+            continue;
+        }
+        const mod = rawModule;
+        const moduleId = String(mod.id ?? "unknown");
+        const globs = Array.isArray(mod.pathGlobs)
+            ? mod.pathGlobs.filter((v) => typeof v === "string")
+            : [];
+        const files = globFiles(projectRoot, globs);
+        const relFiles = files.map((p) => relativeFile(p, projectRoot));
+        for (const relPath of relFiles) {
+            if (!fileToModule.has(relPath)) {
+                fileToModule.set(relPath, moduleId);
+            }
+        }
+        if (relFiles.length > MAX_FILES_PER_MODULE) {
+            degraded.push({
+                code: "MODULE-FILES-TRUNCATED",
+                module: moduleId,
+                message: `Module file list was truncated to ${MAX_FILES_PER_MODULE} deterministic entries.`,
+            });
+        }
+        artifacts.push({
+            id: moduleId,
+            name: mod.name,
+            purpose: mod.purpose,
+            pathGlobs: globs,
+            fileCount: relFiles.length,
+            files: relFiles.slice(0, MAX_FILES_PER_MODULE),
+            derivedFrom: {
+                intent: "codeStructure.modules[]",
+                files: "repository-glob-walk",
+            },
+        });
+        prefixesByModule.set(moduleId, modulePrefixes(mod, files, projectRoot));
+    }
+    return [artifacts, fileToModule, prefixesByModule, degraded];
+}
+function directoryModules(projectRoot) {
+    const grouped = new Map();
+    for (const filePath of repoFiles(projectRoot)) {
+        const parts = relativeFile(filePath, projectRoot).split("/");
+        if (parts.length === 0) {
+            continue;
+        }
+        const top = parts.length > 1 ? (parts[0] ?? "root-files") : "root-files";
+        const list = grouped.get(top) ?? [];
+        list.push(filePath);
+        grouped.set(top, list);
+    }
+    const modules = [];
+    const fileToModule = new Map();
+    const prefixesByModule = new Map();
+    const degradedMarkers = [
+        {
+            code: "NO-CODESTRUCTURE",
+            message: "No authored codeStructure metadata was found; modules were derived from " +
+                "top-level repository paths.",
+        },
+    ];
+    for (const top of [...grouped.keys()].sort()) {
+        const moduleId = stableId(top);
+        const paths = (grouped.get(top) ?? []).sort((a, b) => relativeFile(a, projectRoot).localeCompare(relativeFile(b, projectRoot)));
+        const relFiles = paths.map((p) => relativeFile(p, projectRoot));
+        for (const relPath of relFiles) {
+            if (!fileToModule.has(relPath)) {
+                fileToModule.set(relPath, moduleId);
+            }
+        }
+        if (relFiles.length > MAX_FILES_PER_MODULE) {
+            degradedMarkers.push({
+                code: "MODULE-FILES-TRUNCATED",
+                module: moduleId,
+                message: `Module file list was truncated to ${MAX_FILES_PER_MODULE} deterministic entries.`,
+            });
+        }
+        modules.push({
+            id: moduleId,
+            name: top,
+            purpose: null,
+            pathGlobs: [top !== "root-files" ? `${top}/**/*` : "*"],
+            fileCount: relFiles.length,
+            files: relFiles.slice(0, MAX_FILES_PER_MODULE),
+            derivedFrom: {
+                intent: "directory-derived-fallback",
+                files: "repository-tree-walk",
+            },
+        });
+        prefixesByModule.set(moduleId, top !== "root-files" ? new Set([top]) : new Set());
+    }
+    return [modules, fileToModule, prefixesByModule, degradedMarkers];
+}
+function matchPyImport(line) {
+    let i = 0;
+    while (i < line.length && (line.charCodeAt(i) === 32 || line.charCodeAt(i) === 9)) {
+        i += 1;
+    }
+    if (line.slice(i, i + 6) === "import" && (i + 6 >= line.length || line.charCodeAt(i + 6) <= 32)) {
+        i += 6;
+        while (i < line.length && line.charCodeAt(i) <= 32) {
+            i += 1;
+        }
+        let name = "";
+        const c0 = line.charCodeAt(i);
+        if (!((c0 >= 65 && c0 <= 90) || (c0 >= 97 && c0 <= 122) || c0 === 95)) {
+            return null;
+        }
+        while (i < line.length) {
+            const c = line.charCodeAt(i);
+            if ((c >= 65 && c <= 90) ||
+                (c >= 97 && c <= 122) ||
+                (c >= 48 && c <= 57) ||
+                c === 95 ||
+                c === 46) {
+                name += line.charAt(i);
+                i += 1;
+            }
+            else {
+                break;
+            }
+        }
+        return name.length > 0 ? name : null;
+    }
+    return null;
+}
+function matchPyFromImport(line) {
+    let i = 0;
+    while (i < line.length && (line.charCodeAt(i) === 32 || line.charCodeAt(i) === 9)) {
+        i += 1;
+    }
+    if (line.slice(i, i + 4) !== "from") {
+        return null;
+    }
+    i += 4;
+    while (i < line.length && line.charCodeAt(i) <= 32) {
+        i += 1;
+    }
+    let name = "";
+    while (i < line.length) {
+        const c = line.charCodeAt(i);
+        if ((c >= 65 && c <= 90) ||
+            (c >= 97 && c <= 122) ||
+            (c >= 48 && c <= 57) ||
+            c === 95 ||
+            c === 46) {
+            name += line.charAt(i);
+            i += 1;
+        }
+        else {
+            break;
+        }
+    }
+    while (i < line.length && line.charCodeAt(i) <= 32) {
+        i += 1;
+    }
+    if (line.slice(i, i + 6) !== "import") {
+        return null;
+    }
+    return name.length > 0 ? name : null;
+}
+function extractQuotedImport(line, prefix) {
+    const idx = line.indexOf(prefix);
+    if (idx < 0) {
+        return null;
+    }
+    const q1 = line.indexOf('"', idx);
+    const q2 = line.indexOf("'", idx);
+    let q = -1;
+    let quote = '"';
+    if (q1 >= 0 && (q2 < 0 || q1 < q2)) {
+        q = q1;
+        quote = '"';
+    }
+    else if (q2 >= 0) {
+        q = q2;
+        quote = "'";
+    }
+    if (q < 0) {
+        return null;
+    }
+    const end = line.indexOf(quote, q + 1);
+    if (end < 0) {
+        return null;
+    }
+    return line.slice(q + 1, end);
+}
+function readImports(path) {
+    const ext = path.slice(path.lastIndexOf("."));
+    if (![".go", ".js", ".jsx", ".py", ".ts", ".tsx"].includes(ext)) {
+        return [];
+    }
+    let st;
+    try {
+        st = statSync(path);
+    }
+    catch {
+        return [];
+    }
+    if (st.size > MAX_IMPORT_SCAN_BYTES) {
+        return [];
+    }
+    let text;
+    try {
+        text = readFileSync(path, { encoding: "utf8" });
+    }
+    catch {
+        return [];
+    }
+    const lines = text.split("\n");
+    const imports = [];
+    let inGoImportBlock = false;
+    for (let lineNumber = 1; lineNumber <= lines.length; lineNumber += 1) {
+        const line = lines[lineNumber - 1] ?? "";
+        const stripped = line.trim();
+        if (ext === ".go") {
+            if (stripped === "import (") {
+                inGoImportBlock = true;
+                continue;
+            }
+            if (inGoImportBlock && stripped === ")") {
+                inGoImportBlock = false;
+                continue;
+            }
+            if (inGoImportBlock) {
+                const q = stripped.indexOf('"');
+                if (q >= 0) {
+                    const end = stripped.indexOf('"', q + 1);
+                    if (end > q) {
+                        imports.push([lineNumber, stripped.slice(q + 1, end)]);
+                    }
+                }
+                continue;
+            }
+            const goMatch = stripped.match(/^\s*import\s+"([^"]+)"/);
+            if (goMatch?.[1] !== undefined) {
+                imports.push([lineNumber, goMatch[1]]);
+            }
+            continue;
+        }
+        const pyImport = matchPyImport(line);
+        if (pyImport !== null) {
+            imports.push([lineNumber, pyImport]);
+            continue;
+        }
+        const pyFrom = matchPyFromImport(line);
+        if (pyFrom !== null) {
+            imports.push([lineNumber, pyFrom]);
+            continue;
+        }
+        const esFrom = extractQuotedImport(line, "from");
+        if (esFrom !== null && line.includes("import")) {
+            imports.push([lineNumber, esFrom]);
+            continue;
+        }
+        const requireVal = extractQuotedImport(line, "require(");
+        if (requireVal !== null) {
+            imports.push([lineNumber, requireVal]);
+            continue;
+        }
+        const bareImport = extractQuotedImport(line, "import");
+        if (bareImport !== null) {
+            imports.push([lineNumber, bareImport]);
+        }
+    }
+    return imports;
+}
+function importTargets(ref, prefixesByModule) {
+    if (ref.startsWith(".")) {
+        return new Set();
+    }
+    const normalized = ref.startsWith("@") ? ref.slice(1) : ref;
+    let firstSegment = normalized;
+    const slashIdx = normalized.search(/[/.]/);
+    if (slashIdx >= 0) {
+        firstSegment = normalized.slice(0, slashIdx);
+    }
+    const targets = new Set();
+    for (const [moduleId, prefixes] of prefixesByModule) {
+        if (prefixes.has(firstSegment) || prefixes.has(ref)) {
+            targets.add(moduleId);
+        }
+    }
+    return targets;
+}
+function couplingEdges(projectRoot, fileToModule, prefixesByModule) {
+    const edges = new Map();
+    const sortedFiles = [...fileToModule.entries()].sort((a, b) => a[0].localeCompare(b[0]));
+    for (const [relPath, sourceModule] of sortedFiles) {
+        const path = join(projectRoot, relPath);
+        for (const [lineNumber, importRef] of readImports(path)) {
+            for (const targetModule of importTargets(importRef, prefixesByModule)) {
+                if (targetModule === sourceModule) {
+                    continue;
+                }
+                const key = `${sourceModule}\0${targetModule}`;
+                let evidence = edges.get(key);
+                if (evidence === undefined) {
+                    evidence = [];
+                    edges.set(key, evidence);
+                }
+                if (evidence.length < MAX_EVIDENCE_PER_EDGE) {
+                    evidence.push({ path: relPath, line: lineNumber, import: importRef });
+                }
+            }
+        }
+    }
+    const out = [];
+    for (const key of [...edges.keys()].sort()) {
+        const [source, target] = key.split("\0");
+        out.push({
+            from: source,
+            to: target,
+            derivedFrom: "import-line-heuristic",
+            confidence: "heuristic",
+            evidence: edges.get(key),
+        });
+    }
+    return out;
+}
+function entryPoints(fileToModule) {
+    const entries = [];
+    for (const [relPath, moduleId] of [...fileToModule.entries()].sort((a, b) => a[0].localeCompare(b[0]))) {
+        const parts = relPath.split("/");
+        const name = parts[parts.length - 1] ?? relPath;
+        if (ENTRYPOINT_NAMES.has(name) || parts[0] === "cmd") {
+            entries.push({
+                path: relPath,
+                module: moduleId,
+                derivedFrom: "filename-heuristic",
+                confidence: "heuristic",
+            });
+        }
+    }
+    return entries;
+}
+function languageDistribution(fileToModule) {
+    const counts = new Map();
+    for (const relPath of fileToModule.keys()) {
+        const dot = relPath.lastIndexOf(".");
+        const suffix = dot >= 0 ? relPath.slice(dot) : "";
+        const language = LANGUAGE_BY_SUFFIX[suffix] ?? "Other";
+        counts.set(language, (counts.get(language) ?? 0) + 1);
+    }
+    return [...counts.entries()]
+        .sort((a, b) => a[0].localeCompare(b[0]))
+        .map(([language, files]) => ({
+        language,
+        files,
+        derivedFrom: "extension-heuristic",
+    }));
+}
+/** Build a deterministic tier-1 codebase-map artifact. */
+export function buildCodebaseMap(projectRoot, options = {}) {
+    const root = realpathSync(resolve(projectRoot));
+    const [codeStructure] = loadAuthoredCodeStructure(root, options.codeStructurePath);
+    const sourcePath = defaultCodeStructurePath(root, options.codeStructurePath);
+    let modules;
+    let fileToModule;
+    let prefixesByModule;
+    let degraded;
+    if (codeStructure !== null) {
+        [modules, fileToModule, prefixesByModule, degraded] = curatedModules(root, codeStructure.record);
+    }
+    else {
+        [modules, fileToModule, prefixesByModule, degraded] = directoryModules(root);
+    }
+    const contentHashPaths = [...fileToModule.keys()];
+    const sourceRelPath = sourcePathRel(root, sourcePath);
+    if (sourceRelPath !== null) {
+        contentHashPaths.push(sourceRelPath);
+    }
+    degraded = [
+        ...degraded,
+        {
+            code: "AST-FREE-HEURISTICS",
+            message: "Default extractor uses repository walking and import-line heuristics only; " +
+                "no AST or language parser provider was configured.",
+        },
+    ];
+    if (options.fallbackReason) {
+        degraded.push({ code: "PROVIDER-FALLBACK", message: options.fallbackReason });
+    }
+    const provider = {
+        name: DEFAULT_PROVIDER_NAME,
+        version: DEFAULT_PROVIDER_VERSION,
+        mode: "default",
+        degraded: true,
+    };
+    if (options.fallbackReason) {
+        provider.fallbackReason = options.fallbackReason;
+    }
+    return {
+        formatVersion: CODEBASE_MAP_FORMAT_VERSION,
+        contractVersion: CODEBASE_PROVIDER_CONTRACT_VERSION,
+        kind: CODEBASE_MAP_KIND,
+        provider,
+        source: {
+            projectRoot: root,
+            codeStructurePath: sourcePath,
+            codeStructureHome: codeStructure?.home ?? null,
+            contentHash: sourceContentHash(root, contentHashPaths),
+        },
+        modules,
+        coupling: couplingEdges(root, fileToModule, prefixesByModule),
+        entryPoints: entryPoints(fileToModule),
+        languageDistribution: languageDistribution(fileToModule),
+        degraded,
+    };
+}
+export function configErrorToDict(path, error) {
+    return {
+        path,
+        ok: false,
+        errors: [{ code: "CS-CONFIG", message: String(error), location: path }],
+        warnings: [],
+    };
+}
+/** CLI entry point. */
+export function runDefaultExtractorCli(argv) {
+    let projectRoot = ".";
+    let codeStructurePath;
+    for (let i = 0; i < argv.length; i += 1) {
+        const arg = argv[i];
+        if (arg === "--project-root") {
+            const value = argv[i + 1];
+            if (value === undefined) {
+                return {
+                    exitCode: 2,
+                    stdout: "",
+                    stderr: "argument --project-root: expected one argument\n",
+                };
+            }
+            projectRoot = value;
+            i += 1;
+        }
+        else if (arg?.startsWith("--project-root=")) {
+            projectRoot = arg.slice("--project-root=".length);
+        }
+        else if (arg === "--path") {
+            const value = argv[i + 1];
+            if (value === undefined) {
+                return { exitCode: 2, stdout: "", stderr: "argument --path: expected one argument\n" };
+            }
+            codeStructurePath = value;
+            i += 1;
+        }
+        else if (arg?.startsWith("--path=")) {
+            codeStructurePath = arg.slice("--path=".length);
+        }
+    }
+    const root = resolve(projectRoot);
+    try {
+        const artifact = buildCodebaseMap(root, { codeStructurePath });
+        return {
+            exitCode: 0,
+            stdout: sortedStringifyPretty(artifact),
+            stderr: "",
+        };
+    }
+    catch (err) {
+        if (err instanceof CodeStructureConfigError || err instanceof Error) {
+            const path = defaultCodeStructurePath(root, codeStructurePath);
+            return {
+                exitCode: 2,
+                stdout: "",
+                stderr: sortedStringifyPretty(configErrorToDict(path, err)),
+            };
+        }
+        throw err;
+    }
+}
+//# sourceMappingURL=default-extractor.js.map