@toolbaux/guardian 0.1.0

package/dist/extract/duplicates.js

@@ -0,0 +1,435 @@
+import crypto from "node:crypto";
+import path from "node:path";
+import ts from "typescript";
+import Parser from "tree-sitter";
+import Python from "tree-sitter-python";
+export async function findDuplicateFunctions(params) {
+    const { files, baseRoot, fileContents } = params;
+    const fingerprints = [];
+    for (const file of files) {
+        const ext = path.extname(file).toLowerCase();
+        const language = ext === ".py" ? "py" : "ts";
+        const content = fileContents.get(file);
+        if (typeof content !== "string" || content.length === 0) {
+            continue;
+        }
+        if (language === "py") {
+            fingerprints.push(...extractPythonFunctions(file, content));
+        }
+        else if (isTsLike(ext)) {
+            const lang = ext === ".js" || ext === ".jsx" || ext === ".mjs" || ext === ".cjs"
+                ? "js"
+                : "ts";
+            fingerprints.push(...extractTsFunctions(file, content, lang));
+        }
+    }
+    const duplicates = groupByHash(fingerprints);
+    const similar = [
+        ...findSimilarByCallPattern(fingerprints),
+        ...findSimilarByAstStructure(fingerprints)
+    ]
+        .sort((a, b) => b.similarity - a.similarity)
+        .slice(0, 50);
+    return {
+        duplicateFunctions: duplicates,
+        similarFunctions: similar
+    };
+}
+function groupByHash(fingerprints) {
+    const grouped = new Map();
+    for (const fingerprint of fingerprints) {
+        const entry = grouped.get(fingerprint.hash) ?? [];
+        entry.push(fingerprint);
+        grouped.set(fingerprint.hash, entry);
+    }
+    const groups = [];
+    for (const [hash, entries] of grouped.entries()) {
+        if (entries.length < 2) {
+            continue;
+        }
+        const size = Math.max(...entries.map((entry) => entry.size));
+        groups.push({
+            hash,
+            size,
+            functions: entries.map((entry) => ({
+                id: entry.id,
+                name: entry.name,
+                file: entry.file,
+                language: entry.language,
+                size: entry.size
+            }))
+        });
+    }
+    groups.sort((a, b) => b.size - a.size || b.functions.length - a.functions.length);
+    return groups;
+}
+function findSimilarByCallPattern(fingerprints) {
+    const bySize = new Map();
+    for (const fp of fingerprints) {
+        const callSize = fp.calls.length;
+        if (callSize === 0) {
+            continue;
+        }
+        const entry = bySize.get(callSize) ?? [];
+        entry.push(fp);
+        bySize.set(callSize, entry);
+    }
+    const pairs = [];
+    const sizes = Array.from(bySize.keys()).sort((a, b) => a - b);
+    const threshold = 0.8;
+    const maxPairs = 50;
+    for (const size of sizes) {
+        const bucket = bySize.get(size) ?? [];
+        for (let i = 0; i < bucket.length; i += 1) {
+            for (let j = i + 1; j < bucket.length; j += 1) {
+                const sim = jaccard(bucket[i].calls, bucket[j].calls);
+                if (sim >= threshold) {
+                    pairs.push({
+                        similarity: round(sim, 3),
+                        basis: "call_pattern",
+                        functions: [
+                            pickSummary(bucket[i]),
+                            pickSummary(bucket[j])
+                        ]
+                    });
+                    if (pairs.length >= maxPairs) {
+                        return pairs;
+                    }
+                }
+            }
+        }
+    }
+    return pairs;
+}
+function findSimilarByAstStructure(fingerprints) {
+    const minSize = 12;
+    const bucketSize = 10;
+    const buckets = new Map();
+    for (const fp of fingerprints) {
+        if (fp.size < minSize) {
+            continue;
+        }
+        const key = Math.floor(fp.size / bucketSize);
+        const entry = buckets.get(key) ?? [];
+        entry.push(fp);
+        buckets.set(key, entry);
+    }
+    const pairs = [];
+    const seen = new Set();
+    const threshold = 0.8;
+    const maxPairs = 50;
+    const keys = Array.from(buckets.keys()).sort((a, b) => a - b);
+    for (const key of keys) {
+        const group = buckets.get(key) ?? [];
+        const next = buckets.get(key + 1) ?? [];
+        for (let i = 0; i < group.length; i += 1) {
+            const base = group[i];
+            const candidates = [...group.slice(i + 1), ...next];
+            for (const candidate of candidates) {
+                if (base.hash === candidate.hash) {
+                    continue;
+                }
+                const pairKey = base.id < candidate.id ? `${base.id}::${candidate.id}` : `${candidate.id}::${base.id}`;
+                if (seen.has(pairKey)) {
+                    continue;
+                }
+                seen.add(pairKey);
+                const sim = jaccard(base.kindSet, candidate.kindSet);
+                if (sim >= threshold) {
+                    pairs.push({
+                        similarity: round(sim, 3),
+                        basis: "ast_structure",
+                        functions: [
+                            pickSummary(base),
+                            pickSummary(candidate)
+                        ]
+                    });
+                    if (pairs.length >= maxPairs) {
+                        return pairs;
+                    }
+                }
+            }
+        }
+    }
+    return pairs;
+}
+function pickSummary(fp) {
+    return {
+        id: fp.id,
+        name: fp.name,
+        file: fp.file,
+        language: fp.language
+    };
+}
+function jaccard(a, b) {
+    const setA = new Set(a);
+    const setB = new Set(b);
+    if (setA.size === 0 || setB.size === 0) {
+        return 0;
+    }
+    let intersection = 0;
+    for (const item of setA) {
+        if (setB.has(item)) {
+            intersection += 1;
+        }
+    }
+    const union = setA.size + setB.size - intersection;
+    if (union === 0) {
+        return 0;
+    }
+    return intersection / union;
+}
+function extractTsFunctions(file, source, language) {
+    const kind = inferScriptKind(file);
+    const sourceFile = ts.createSourceFile(file, source, ts.ScriptTarget.ES2022, true, kind);
+    const functions = [];
+    const visit = (node) => {
+        if (ts.isFunctionDeclaration(node) && node.name && node.body) {
+            functions.push(buildTsFingerprint(file, node.name.text, node.body, language));
+        }
+        if (ts.isMethodDeclaration(node) && node.name && node.body) {
+            const name = ts.isIdentifier(node.name) || ts.isStringLiteral(node.name)
+                ? node.name.text
+                : "method";
+            const className = ts.isClassDeclaration(node.parent) && node.parent.name
+                ? node.parent.name.text
+                : "Class";
+            functions.push(buildTsFingerprint(file, `${className}.${name}`, node.body, language));
+        }
+        if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) {
+            const initializer = node.initializer;
+            if (initializer && (ts.isArrowFunction(initializer) || ts.isFunctionExpression(initializer))) {
+                if (initializer.body) {
+                    functions.push(buildTsFingerprint(file, node.name.text, initializer.body, language));
+                }
+            }
+        }
+        ts.forEachChild(node, visit);
+    };
+    visit(sourceFile);
+    return functions;
+}
+function buildTsFingerprint(file, name, body, language) {
+    const tokens = [];
+    const calls = [];
+    const walk = (node) => {
+        tokens.push(normalizeTsKind(node));
+        if (ts.isCallExpression(node)) {
+            const callee = resolveTsCallee(node.expression);
+            if (callee) {
+                calls.push(callee);
+            }
+        }
+        ts.forEachChild(node, walk);
+    };
+    walk(body);
+    const payload = tokens.join("|");
+    const hash = crypto.createHash("sha1").update(payload).digest("hex");
+    const kindSet = Array.from(new Set(tokens));
+    const relative = file;
+    return {
+        id: `${relative}#${name}`,
+        name,
+        file: relative,
+        language,
+        size: tokens.length,
+        hash,
+        calls: Array.from(new Set(calls)),
+        kindSet
+    };
+}
+function normalizeTsKind(node) {
+    switch (node.kind) {
+        case ts.SyntaxKind.Identifier:
+            return "Identifier";
+        case ts.SyntaxKind.StringLiteral:
+        case ts.SyntaxKind.NoSubstitutionTemplateLiteral:
+        case ts.SyntaxKind.NumericLiteral:
+        case ts.SyntaxKind.BigIntLiteral:
+        case ts.SyntaxKind.RegularExpressionLiteral:
+            return "Literal";
+        default:
+            return ts.SyntaxKind[node.kind] || "Node";
+    }
+}
+function resolveTsCallee(expr) {
+    if (ts.isIdentifier(expr)) {
+        return expr.text;
+    }
+    if (ts.isPropertyAccessExpression(expr)) {
+        const base = resolveTsCallee(expr.expression);
+        if (base) {
+            return `${base}.${expr.name.text}`;
+        }
+        return expr.name.text;
+    }
+    if (ts.isElementAccessExpression(expr)) {
+        const base = resolveTsCallee(expr.expression);
+        if (base) {
+            return `${base}[]`;
+        }
+    }
+    if (expr.kind === ts.SyntaxKind.ThisKeyword) {
+        return "this";
+    }
+    return null;
+}
+function extractPythonFunctions(file, source) {
+    const parser = new Parser();
+    parser.setLanguage(Python);
+    let tree;
+    try {
+        tree = parser.parse(source);
+    }
+    catch {
+        return [];
+    }
+    const root = tree.rootNode;
+    const functions = [];
+    walk(root, (node) => {
+        if (node.type === "function_definition") {
+            if (!isTopLevel(node)) {
+                return;
+            }
+            const nameNode = node.childForFieldName("name");
+            const bodyNode = node.childForFieldName("body");
+            if (!nameNode || !bodyNode) {
+                return;
+            }
+            const name = nodeText(nameNode, source);
+            functions.push(buildPythonFingerprint(file, name, bodyNode, source));
+        }
+        if (node.type === "class_definition") {
+            if (!isTopLevel(node)) {
+                return;
+            }
+            const classNameNode = node.childForFieldName("name");
+            const bodyNode = node.childForFieldName("body");
+            if (!classNameNode || !bodyNode) {
+                return;
+            }
+            const className = nodeText(classNameNode, source);
+            for (const child of bodyNode.namedChildren) {
+                if (child.type !== "function_definition") {
+                    continue;
+                }
+                const methodNameNode = child.childForFieldName("name");
+                const methodBody = child.childForFieldName("body");
+                if (!methodNameNode || !methodBody) {
+                    continue;
+                }
+                const methodName = nodeText(methodNameNode, source);
+                functions.push(buildPythonFingerprint(file, `${className}.${methodName}`, methodBody, source));
+            }
+        }
+    });
+    return functions;
+}
+function buildPythonFingerprint(file, name, body, source) {
+    const tokens = [];
+    const calls = [];
+    walk(body, (node) => {
+        tokens.push(normalizePythonNode(node, source));
+        if (node.type === "call") {
+            const callee = node.childForFieldName("function");
+            if (callee) {
+                const callName = exprName(callee, source);
+                if (callName) {
+                    calls.push(callName);
+                }
+            }
+        }
+    });
+    const hash = crypto.createHash("sha1").update(tokens.join("|")).digest("hex");
+    const kindSet = Array.from(new Set(tokens));
+    return {
+        id: `${file}#${name}`,
+        name,
+        file,
+        language: "py",
+        size: tokens.length,
+        hash,
+        calls: Array.from(new Set(calls)),
+        kindSet
+    };
+}
+function normalizePythonNode(node, source) {
+    if (node.type === "identifier") {
+        return "id";
+    }
+    if (node.type === "string" ||
+        node.type === "integer" ||
+        node.type === "float" ||
+        node.type === "true" ||
+        node.type === "false" ||
+        node.type === "none") {
+        return "lit";
+    }
+    if (node.type === "attribute") {
+        return "attr";
+    }
+    return node.type;
+}
+function isTopLevel(node) {
+    return node.parent?.type === "module";
+}
+function walk(node, visit) {
+    visit(node);
+    for (const child of node.namedChildren) {
+        walk(child, visit);
+    }
+}
+function nodeText(node, source) {
+    return source.slice(node.startIndex, node.endIndex);
+}
+function exprName(node, source) {
+    if (node.type === "identifier") {
+        return nodeText(node, source);
+    }
+    if (node.type === "attribute") {
+        const objectNode = node.childForFieldName("object");
+        const attrNode = node.childForFieldName("attribute");
+        const base = objectNode ? exprName(objectNode, source) : null;
+        const attr = attrNode ? nodeText(attrNode, source) : null;
+        if (base && attr) {
+            return `${base}.${attr}`;
+        }
+        return attr ?? base;
+    }
+    return null;
+}
+function isTsLike(ext) {
+    return [
+        ".ts",
+        ".tsx",
+        ".js",
+        ".jsx",
+        ".mjs",
+        ".cjs",
+        ".mts",
+        ".cts"
+    ].includes(ext);
+}
+function inferScriptKind(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === ".tsx") {
+        return ts.ScriptKind.TSX;
+    }
+    if (ext === ".jsx") {
+        return ts.ScriptKind.JSX;
+    }
+    if (ext === ".js" || ext === ".mjs" || ext === ".cjs") {
+        return ts.ScriptKind.JS;
+    }
+    if (ext === ".mts") {
+        return ts.ScriptKind.TS;
+    }
+    if (ext === ".cts") {
+        return ts.ScriptKind.TS;
+    }
+    return ts.ScriptKind.TS;
+}
+function round(value, precision) {
+    const factor = 10 ** precision;
+    return Math.round(value * factor) / factor;
+}

package/dist/extract/feature-arcs.js

@@ -0,0 +1,138 @@
+/**
+ * Feature Arcs — builds a timeline of how feature areas evolve across sprints.
+ *
+ * Analogous to thread evolution in the book workflow (T8 evolving across 7 books):
+ * tracks which endpoints, models, and service calls each feature area added per sprint.
+ *
+ * Input:  directory of feature spec YAML files
+ * Output: feature-arcs.json
+ *
+ * Structure:
+ *   {
+ *     "auth": {
+ *       "sprint_1": { endpoints: [...], models: [...], maps_to: [...] },
+ *       "sprint_4": { ... },
+ *     },
+ *     "billing": { ... }
+ *   }
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import yaml from "js-yaml";
+import { parseFeatureSpec } from "../schema/feature-spec.js";
+/**
+ * Load all feature spec YAML files from a directory and build the arc timeline.
+ */
+export async function buildFeatureArcs(featureSpecsDir) {
+    const specs = await loadAllFeatureSpecs(featureSpecsDir);
+    const arcMap = new Map();
+    const untaggedFeatures = [];
+    const untaggedEndpoints = new Set();
+    const untaggedModels = new Set();
+    for (const spec of specs) {
+        const sprintKey = spec.sprint != null ? `sprint_${spec.sprint}` : "unsprinted";
+        const tags = spec.tags.length > 0 ? spec.tags : null;
+        if (!tags) {
+            untaggedFeatures.push(spec.feature);
+            spec.affected_endpoints.forEach((e) => untaggedEndpoints.add(e));
+            spec.affected_models.forEach((m) => untaggedModels.add(m));
+            continue;
+        }
+        for (const tag of tags) {
+            if (!arcMap.has(tag)) {
+                arcMap.set(tag, { tag, sprints: {}, total_endpoints: 0, total_models: 0 });
+            }
+            const arc = arcMap.get(tag);
+            if (!arc.sprints[sprintKey]) {
+                arc.sprints[sprintKey] = { endpoints: [], models: [], maps_to: [], features: [] };
+            }
+            const sprint = arc.sprints[sprintKey];
+            sprint.features.push(spec.feature);
+            spec.affected_endpoints.forEach((e) => {
+                if (!sprint.endpoints.includes(e))
+                    sprint.endpoints.push(e);
+            });
+            spec.affected_models.forEach((m) => {
+                if (!sprint.models.includes(m))
+                    sprint.models.push(m);
+            });
+            if (spec.maps_to && !sprint.maps_to.includes(spec.maps_to)) {
+                sprint.maps_to.push(spec.maps_to);
+            }
+        }
+    }
+    // Compute totals per arc (deduplicated across sprints)
+    for (const arc of arcMap.values()) {
+        const allEndpoints = new Set();
+        const allModels = new Set();
+        for (const sprint of Object.values(arc.sprints)) {
+            sprint.endpoints.forEach((e) => allEndpoints.add(e));
+            sprint.models.forEach((m) => allModels.add(m));
+        }
+        arc.total_endpoints = allEndpoints.size;
+        arc.total_models = allModels.size;
+    }
+    // Sort arcs alphabetically; sort sprint keys numerically
+    const arcs = {};
+    for (const [tag, arc] of Array.from(arcMap.entries()).sort((a, b) => a[0].localeCompare(b[0]))) {
+        arcs[tag] = {
+            ...arc,
+            sprints: sortSprintKeys(arc.sprints),
+        };
+    }
+    return {
+        generated_at: new Date().toISOString(),
+        arcs,
+        untagged: {
+            features: untaggedFeatures,
+            endpoints: Array.from(untaggedEndpoints),
+            models: Array.from(untaggedModels),
+        },
+    };
+}
+function sortSprintKeys(sprints) {
+    const sorted = {};
+    const keys = Object.keys(sprints).sort((a, b) => {
+        const na = parseInt(a.replace(/\D/g, ""), 10);
+        const nb = parseInt(b.replace(/\D/g, ""), 10);
+        if (!isNaN(na) && !isNaN(nb))
+            return na - nb;
+        return a.localeCompare(b);
+    });
+    for (const k of keys)
+        sorted[k] = sprints[k];
+    return sorted;
+}
+async function loadAllFeatureSpecs(dir) {
+    let entries;
+    try {
+        const dirEntries = await fs.readdir(dir);
+        entries = dirEntries
+            .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml"))
+            .map((f) => path.join(dir, f));
+    }
+    catch {
+        return [];
+    }
+    const specs = [];
+    for (const entry of entries) {
+        try {
+            const raw = await fs.readFile(entry, "utf8");
+            const parsed = yaml.load(raw);
+            const spec = parseFeatureSpec(parsed);
+            specs.push(spec);
+        }
+        catch {
+            // Skip malformed specs silently
+        }
+    }
+    return specs;
+}
+/**
+ * Write feature arcs to disk.
+ */
+export async function writeFeatureArcs(featureSpecsDir, outputPath) {
+    const arcs = await buildFeatureArcs(featureSpecsDir);
+    await fs.mkdir(path.dirname(outputPath), { recursive: true });
+    await fs.writeFile(outputPath, JSON.stringify(arcs, null, 2), "utf8");
+}

package/dist/extract/graph.js

@@ -0,0 +1,76 @@
+export function ensureNode(graph, node) {
+    if (!graph.has(node)) {
+        graph.set(node, new Set());
+    }
+}
+export function addEdge(graph, from, to) {
+    ensureNode(graph, from);
+    ensureNode(graph, to);
+    graph.get(from)?.add(to);
+}
+export function inboundCounts(graph, nodes) {
+    const counts = new Map();
+    for (const node of nodes) {
+        counts.set(node, 0);
+    }
+    for (const [from, neighbors] of graph) {
+        if (!counts.has(from)) {
+            counts.set(from, 0);
+        }
+        for (const neighbor of neighbors) {
+            counts.set(neighbor, (counts.get(neighbor) ?? 0) + 1);
+        }
+    }
+    return counts;
+}
+function normalizeCycle(cycle) {
+    if (cycle.length === 0) {
+        return cycle;
+    }
+    let minIndex = 0;
+    for (let i = 1; i < cycle.length; i += 1) {
+        if (cycle[i] < cycle[minIndex]) {
+            minIndex = i;
+        }
+    }
+    return [...cycle.slice(minIndex), ...cycle.slice(0, minIndex)];
+}
+function cycleKey(cycle) {
+    return normalizeCycle(cycle).join("->");
+}
+export function findCycles(graph) {
+    const nodes = Array.from(graph.keys()).sort((a, b) => a.localeCompare(b));
+    const visited = new Set();
+    const onStack = new Set();
+    const stack = [];
+    const cycles = new Map();
+    function dfs(node) {
+        visited.add(node);
+        stack.push(node);
+        onStack.add(node);
+        const neighbors = Array.from(graph.get(node) ?? []).sort((a, b) => a.localeCompare(b));
+        for (const neighbor of neighbors) {
+            if (!visited.has(neighbor)) {
+                dfs(neighbor);
+            }
+            else if (onStack.has(neighbor)) {
+                const startIndex = stack.indexOf(neighbor);
+                if (startIndex >= 0) {
+                    const cycle = stack.slice(startIndex);
+                    const key = cycleKey(cycle);
+                    if (!cycles.has(key)) {
+                        cycles.set(key, normalizeCycle(cycle));
+                    }
+                }
+            }
+        }
+        stack.pop();
+        onStack.delete(node);
+    }
+    for (const node of nodes) {
+        if (!visited.has(node)) {
+            dfs(node);
+        }
+    }
+    return Array.from(cycles.values()).sort((a, b) => a.join("->").localeCompare(b.join("->")));
+}