devlensio 0.2.0

package/dist/parser/index.js

@@ -0,0 +1,168 @@
+import { Project } from "ts-morph";
+import path from "path";
+import fs from "fs";
+import { extractComponents } from "./extractors/components.js";
+import { extractHooks } from "./extractors/hooks.js";
+import { extractFunctions } from "./extractors/functions.js";
+import { extractStores } from "./extractors/stores.js";
+import { detectFileDirective } from "./directives.js";
+import { createHash } from "crypto";
+// Directories to skip entirely while walking
+const IGNORE_DIRS = [
+    "node_modules",
+    "dist",
+    "build",
+    ".next",
+    "coverage",
+    "migrations",
+    ".git",
+];
+// File patterns to skip
+function shouldIgnoreFile(fileName) {
+    // In V1.0 I am now detecting test and story files as well because we have added an edge TESTS and 2 node types TEST and STORY
+    // if (/\.(test|spec)\.(ts|tsx|js|jsx)$/.test(fileName)) return true;
+    // if (/\.stories\.(ts|tsx|js|jsx)$/.test(fileName)) return true;
+    if (/\.d\.ts$/.test(fileName))
+        return true;
+    if (/\.config\.(ts|js)$/.test(fileName))
+        return true;
+    return false;
+}
+function getFileNodeType(fileName) {
+    if (/\.(test|spec)\.(ts|tsx|js|jsx)$/.test(fileName))
+        return "TEST";
+    if (/\.stories\.(ts|tsx|js|jsx)$/.test(fileName))
+        return "STORY";
+    return "FILE";
+}
+// Recursively walks directory and adds valid source files to the project
+function addFilesRecursively(dir, project) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        // If we can't read a directory just skip it
+        return;
+    }
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            // Skip ignored directories immediately
+            if (IGNORE_DIRS.includes(entry.name))
+                continue;
+            addFilesRecursively(fullPath, project);
+        }
+        else if (entry.isFile()) {
+            // Only process source files
+            if (!/\.(ts|tsx|js|jsx)$/.test(entry.name))
+                continue;
+            // Skip ignored file patterns
+            if (shouldIgnoreFile(entry.name))
+                continue;
+            project.addSourceFileAtPath(fullPath);
+        }
+    }
+}
+export function parseRepo(repoPath) {
+    // Set up ts-morph project
+    const project = new Project({
+        compilerOptions: {
+            allowJs: true, // support plain JS files
+            checkJs: false, // don't type check JS, just parse
+            jsx: 4, // support JSX (4 = React)
+            strict: false, // don't enforce strict mode on user's code
+        },
+        skipAddingFilesFromTsConfig: true,
+    });
+    // Walk directory and add files manually
+    // This approach works reliably on all platforms including Windows
+    addFilesRecursively(repoPath, project);
+    const sourceFiles = project.getSourceFiles();
+    const allNodes = [];
+    let skippedFiles = 0;
+    for (const file of sourceFiles) {
+        try {
+            const absFilePath = file.getFilePath();
+            const relativePath = path.relative(repoPath, absFilePath).replace(/\\/g, "/");
+            const fileType = getFileNodeType(path.basename(relativePath)); // returns either TEST / STORY / FILE type
+            // One FILE node per source file — represents the file itself in the graph
+            const fileNode = {
+                id: `file::${relativePath}`,
+                name: path.basename(relativePath),
+                type: fileType,
+                filePath: relativePath,
+                startLine: 1,
+                endLine: file.getEndLineNumber(),
+                parentFile: `file::${relativePath}`, //there is no parent file for file type node
+                metadata: {
+                    nodeCount: 0,
+                    childNodeIds: [],
+                    language: absFilePath.endsWith('.ts') || absFilePath.endsWith('.tsx') ? 'typescript' :
+                        absFilePath.endsWith('.js') || absFilePath.endsWith('.jsx') ? 'javascript' : 'unknown',
+                },
+            };
+            const fileDirective = detectFileDirective(file);
+            const components = extractComponents(file, fileDirective);
+            const hooks = extractHooks(file, fileDirective);
+            const functions = extractFunctions(file, fileDirective);
+            const stores = extractStores(file);
+            const extracted = [...components, ...hooks, ...functions, ...stores];
+            for (const node of extracted) {
+                // Normalize all extracted nodes to relative paths so every node in the
+                // graph uses the same coordinate system as the FILE nodes.
+                // Extractors store absolute ts-morph paths; we rewrite them here.
+                node.filePath = relativePath;
+                node.id = `${relativePath}::${node.name}`;
+                node.parentFile = `file::${relativePath}`;
+                if (node.rawCode) {
+                    node.codeHash = createHash("sha256").update(node.rawCode).digest("hex").slice(0, 16);
+                }
+            }
+            if (fileType === "TEST" || fileType === "STORY") {
+                // Do not add child nodes for test/story files — they are test helpers,
+                fileNode.metadata.testCases = extracted.map(n => n.name);
+                fileNode.metadata.nodeCount = 0;
+                fileNode.metadata.childNodeIds = [];
+                // File hash based on all child code combined
+                const fileRawCode = extracted.map(n => n.rawCode ?? "").join("\n");
+                if (fileRawCode.trim()) {
+                    fileNode.codeHash = createHash("sha256")
+                        .update(fileRawCode).digest("hex").slice(0, 16);
+                }
+                allNodes.push(fileNode); // only the file node, no children
+            }
+            else {
+                fileNode.metadata.nodeCount = extracted.length;
+                fileNode.metadata.childNodeIds = extracted.map(n => n.id);
+                // File node hash — based on all child code combined
+                const fileRawCode = extracted.map(n => n.rawCode ?? "").join("\n");
+                if (fileRawCode.trim()) {
+                    fileNode.codeHash = createHash("sha256").update(fileRawCode).digest("hex").slice(0, 16);
+                }
+                allNodes.push(fileNode, ...extracted);
+            }
+        }
+        catch (error) {
+            // Never let one bad file break the entire analysis
+            console.warn(`Skipped file due to error: ${file.getFilePath()}`);
+            skippedFiles++;
+        }
+    }
+    const componentCount = allNodes.filter((n) => n.type === "COMPONENT").length;
+    const hookCount = allNodes.filter((n) => n.type === "HOOK").length;
+    const functionCount = allNodes.filter((n) => n.type === "FUNCTION").length;
+    const storeCount = allNodes.filter((n) => n.type === "STATE_STORE").length;
+    return {
+        nodes: allNodes,
+        stats: {
+            totalFiles: sourceFiles.length,
+            totalNodes: allNodes.length,
+            componentCount,
+            hookCount,
+            functionCount,
+            storeCount,
+            skippedFiles,
+        },
+    };
+}

package/dist/parser/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/parser/index.test.js

@@ -0,0 +1,319 @@
+import fs from "fs";
+import path from "path";
+import os from "os";
+import { parseRepo } from "./index.js";
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+function createFakeRepo(files) {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "devlens-parser-test-"));
+    for (const [filePath, content] of Object.entries(files)) {
+        const fullPath = path.join(tmpDir, filePath);
+        fs.mkdirSync(path.dirname(fullPath), { recursive: true });
+        fs.writeFileSync(fullPath, content);
+    }
+    return tmpDir;
+}
+function deleteFakeRepo(repoPath) {
+    fs.rmSync(repoPath, { recursive: true, force: true });
+}
+// ─── Tests ────────────────────────────────────────────────────────────────────
+describe("parseRepo", () => {
+    // ─── Component Detection ───────────────────────────────────────────────────
+    it("should detect a function declaration component", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `
+        export function Button() {
+          return <button>Click me</button>;
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const component = result.nodes.find((n) => n.name === "Button");
+        expect(component).toBeDefined();
+        expect(component?.type).toBe("COMPONENT");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect an arrow function component", () => {
+        const repoPath = createFakeRepo({
+            "src/Card.tsx": `
+        export const Card = () => {
+          return <div>Card</div>;
+        };
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const component = result.nodes.find((n) => n.name === "Card");
+        expect(component).toBeDefined();
+        expect(component?.type).toBe("COMPONENT");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect hooks in component metadata", () => {
+        const repoPath = createFakeRepo({
+            "src/Counter.tsx": `
+        import { useState } from "react";
+        export function Counter() {
+          const [count, setCount] = useState(0);
+          return <div>{count}</div>;
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const component = result.nodes.find((n) => n.name === "Counter");
+        expect(component?.metadata.hasState).toBe(true);
+        expect(component?.metadata.hooks).toContain("useState");
+        deleteFakeRepo(repoPath);
+    });
+    it("should not detect lowercase functions as components", () => {
+        const repoPath = createFakeRepo({
+            "src/utils.tsx": `
+        export function formatDate() {
+          return <div>not a component</div>;
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const node = result.nodes.find((n) => n.name === "formatDate");
+        expect(node?.type).not.toBe("COMPONENT");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect React.memo wrapped component", () => {
+        const repoPath = createFakeRepo({
+            "src/MemoCard.tsx": `
+        export const MemoCard = React.memo(() => {
+          return <div>Memo</div>;
+        });
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const component = result.nodes.find((n) => n.name === "MemoCard");
+        expect(component).toBeDefined();
+        expect(component?.metadata.isMemoized).toBe(true);
+        deleteFakeRepo(repoPath);
+    });
+    // ─── Hook Detection ────────────────────────────────────────────────────────
+    it("should detect a custom hook", () => {
+        const repoPath = createFakeRepo({
+            "src/useAuth.ts": `
+        export function useAuth() {
+          const [user, setUser] = useState(null);
+          return { user };
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const hook = result.nodes.find((n) => n.name === "useAuth");
+        expect(hook).toBeDefined();
+        expect(hook?.type).toBe("HOOK");
+        deleteFakeRepo(repoPath);
+    });
+    it("should not create a HOOK node for built-in hooks", () => {
+        const repoPath = createFakeRepo({
+            "src/Component.tsx": `
+        export function MyComponent() {
+          const [x] = useState(0);
+          return <div>{x}</div>;
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const hook = result.nodes.find((n) => n.name === "useState" && n.type === "HOOK");
+        expect(hook).toBeUndefined();
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect async custom hook", () => {
+        const repoPath = createFakeRepo({
+            "src/useData.ts": `
+        export async function useData() {
+          const data = await fetch('/api/data');
+          return data.json();
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const hook = result.nodes.find((n) => n.name === "useData");
+        expect(hook?.metadata.isAsync).toBe(true);
+        deleteFakeRepo(repoPath);
+    });
+    // ─── Function Detection ────────────────────────────────────────────────────
+    it("should detect a regular function", () => {
+        const repoPath = createFakeRepo({
+            "src/utils.ts": `
+        export function calculateTotal(items: any[]) {
+          return items.reduce((sum, item) => sum + item.price, 0);
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const fn = result.nodes.find((n) => n.name === "calculateTotal");
+        expect(fn).toBeDefined();
+        expect(fn?.type).toBe("FUNCTION");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect async functions", () => {
+        const repoPath = createFakeRepo({
+            "src/api.ts": `
+        export async function fetchUser(id: string) {
+          const res = await fetch('/api/users/' + id);
+          return res.json();
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const fn = result.nodes.find((n) => n.name === "fetchUser");
+        expect(fn?.metadata.isAsync).toBe(true);
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect fetch api calls inside functions", () => {
+        const repoPath = createFakeRepo({
+            "src/api.ts": `
+        export async function getProducts() {
+          const res = await fetch('/api/products');
+          return res.json();
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const fn = result.nodes.find((n) => n.name === "getProducts");
+        expect((fn?.metadata.apiCalls).length).toBeGreaterThan(0);
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect error handling in functions", () => {
+        const repoPath = createFakeRepo({
+            "src/api.ts": `
+        export async function safeRequest() {
+          try {
+            const res = await fetch('/api/data');
+            return res.json();
+          } catch (error) {
+            console.error(error);
+          }
+        }
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const fn = result.nodes.find((n) => n.name === "safeRequest");
+        expect(fn?.metadata.hasErrorHandling).toBe(true);
+        deleteFakeRepo(repoPath);
+    });
+    // ─── Store Detection ───────────────────────────────────────────────────────
+    it("should detect a zustand store", () => {
+        const repoPath = createFakeRepo({
+            "src/store.ts": `
+        const useCartStore = create((set) => ({
+          items: [],
+          addItem: (item) => set((state) => ({ items: [...state.items, item] })),
+        }));
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const store = result.nodes.find((n) => n.name === "useCartStore");
+        expect(store).toBeDefined();
+        expect(store?.type).toBe("STATE_STORE");
+        expect(store?.metadata.storeType).toBe("zustand");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect zustand state shape and actions", () => {
+        const repoPath = createFakeRepo({
+            "src/store.ts": `
+        const useCartStore = create((set) => ({
+          items: [],
+          total: 0,
+          addItem: (item) => set((state) => ({ items: [...state.items, item] })),
+          clearCart: () => set({ items: [], total: 0 }),
+        }));
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const store = result.nodes.find((n) => n.name === "useCartStore");
+        expect(store?.metadata.stateShape).toContain("items");
+        expect(store?.metadata.stateShape).toContain("total");
+        expect(store?.metadata.actions).toContain("addItem");
+        expect(store?.metadata.actions).toContain("clearCart");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect a redux slice", () => {
+        const repoPath = createFakeRepo({
+            "src/cartSlice.ts": `
+        const cartSlice = createSlice({
+          name: 'cart',
+          initialState: { items: [], total: 0 },
+          reducers: {
+            addItem: (state, action) => { state.items.push(action.payload); },
+            clearCart: (state) => { state.items = []; },
+          }
+        });
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const store = result.nodes.find((n) => n.name === "cartSlice");
+        expect(store).toBeDefined();
+        expect(store?.type).toBe("STATE_STORE");
+        expect(store?.metadata.storeType).toBe("redux");
+        expect(store?.metadata.actions).toContain("addItem");
+        expect(store?.metadata.actions).toContain("clearCart");
+        deleteFakeRepo(repoPath);
+    });
+    it("should detect a react context store", () => {
+        const repoPath = createFakeRepo({
+            "src/AuthContext.ts": `
+        const AuthContext = createContext(null);
+      `,
+        });
+        const result = parseRepo(repoPath);
+        const store = result.nodes.find((n) => n.name === "AuthContext");
+        expect(store).toBeDefined();
+        expect(store?.type).toBe("STATE_STORE");
+        expect(store?.metadata.storeType).toBe("context");
+        deleteFakeRepo(repoPath);
+    });
+    // ─── Stats ─────────────────────────────────────────────────────────────────
+    it("should return correct file count in stats", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `export function Button() { return <button />; }`,
+            "src/useAuth.ts": `export function useAuth() { return {}; }`,
+            "src/utils.ts": `export function formatDate() { return ''; }`,
+        });
+        const result = parseRepo(repoPath);
+        expect(result.stats.totalFiles).toBe(3);
+        deleteFakeRepo(repoPath);
+    });
+    it("should count components correctly in stats", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `export function Button() { return <button />; }`,
+            "src/Card.tsx": `export function Card() { return <div />; }`,
+        });
+        const result = parseRepo(repoPath);
+        expect(result.stats.componentCount).toBe(2);
+        deleteFakeRepo(repoPath);
+    });
+    // ─── Noise Filtering ───────────────────────────────────────────────────────
+    it("should ignore node_modules", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `export function Button() { return <button />; }`,
+            "node_modules/react/index.js": `export function useState() {}`,
+        });
+        const result = parseRepo(repoPath);
+        const nodeModuleNode = result.nodes.find((n) => n.filePath.includes("node_modules"));
+        expect(nodeModuleNode).toBeUndefined();
+        deleteFakeRepo(repoPath);
+    });
+    it("should ignore test files", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `export function Button() { return <button />; }`,
+            "src/Button.test.tsx": `export function FakeButton() { return <button />; }`,
+        });
+        const result = parseRepo(repoPath);
+        const testNode = result.nodes.find((n) => n.filePath.includes(".test."));
+        expect(testNode).toBeUndefined();
+        deleteFakeRepo(repoPath);
+    });
+    it("should ignore config files", () => {
+        const repoPath = createFakeRepo({
+            "src/Button.tsx": `export function Button() { return <button />; }`,
+            "next.config.ts": `export default {};`,
+        });
+        const result = parseRepo(repoPath);
+        const configNode = result.nodes.find((n) => n.filePath.includes(".config."));
+        expect(configNode).toBeUndefined();
+        deleteFakeRepo(repoPath);
+    });
+});

package/dist/parser/typeUtils.d.ts

@@ -0,0 +1,9 @@
+import { SourceFile } from "ts-morph";
+export interface ParamInfo {
+    name: string;
+    type?: string;
+}
+export declare function extractParams(node: any): ParamInfo[];
+export declare function extractReturnTypeAnnotation(node: any): string | undefined;
+export declare function extractBareTypeNames(typeStrings: (string | undefined)[]): string[];
+export declare function extractReferencedInterfaces(sourceFile: SourceFile, typeNames: string[]): Record<string, Record<string, string>>;

package/dist/parser/typeUtils.js

@@ -0,0 +1,46 @@
+export function extractParams(node) {
+    const params = node.getParameters ? node.getParameters() : [];
+    return params.map((p) => ({
+        name: p.getName(),
+        type: p.getTypeNode()?.getText() ?? undefined,
+    }));
+}
+export function extractReturnTypeAnnotation(node) {
+    return node.getReturnTypeNode?.()?.getText() ?? undefined;
+}
+// Extract bare PascalCase type names from a set of type annotation strings.
+// e.g. ["FetchOptions", "Promise<User>"] → ["FetchOptions", "User"]
+export function extractBareTypeNames(typeStrings) {
+    const names = [];
+    for (const t of typeStrings) {
+        if (!t)
+            continue;
+        for (const m of t.matchAll(/\b([A-Z][A-Za-z0-9_]+)\b/g)) {
+            names.push(m[1]);
+        }
+    }
+    return [...new Set(names)];
+}
+// Looks up interface and type alias declarations in the CURRENT file only (O(1) depth).
+// Returns a map of typeName → { propName: propType }.
+export function extractReferencedInterfaces(sourceFile, typeNames) {
+    const result = {};
+    for (const typeName of typeNames) {
+        // Try interface declaration first
+        const iface = sourceFile.getInterface(typeName);
+        if (iface) {
+            const props = {};
+            for (const prop of iface.getProperties()) {
+                props[prop.getName()] = prop.getTypeNode()?.getText() ?? "unknown";
+            }
+            result[typeName] = props;
+            continue;
+        }
+        // Try type alias
+        const alias = sourceFile.getTypeAlias(typeName);
+        if (alias) {
+            result[typeName] = { _type: alias.getTypeNode()?.getText() ?? "unknown" };
+        }
+    }
+    return result;
+}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,50 @@
+import type { FilterThresholds } from "../scoring/noiseFilter.js";
+import type { CodeNode, CodeEdge, ProjectFingerprint, RouteNode, BackendRouteNode } from "../types.js";
+export type { FilterThresholds };
+export interface GitInfo {
+    commitHash: string;
+    branch: string;
+    message: string;
+    hasGit: boolean;
+}
+export interface PipelineOptions {
+    thresholds?: FilterThresholds;
+    onStep?: (step: "fingerprint" | "filesystem" | "parse" | "edges" | "scoring") => void;
+    includedThirdPartyLibs?: string[];
+}
+export interface PipelineStats {
+    totalNodesBeforeFilter: number;
+    totalEdgesBeforeFilter: number;
+    totalNodesAfterFilter: number;
+    totalEdgesAfterFilter: number;
+    removedNodeCount: number;
+    removedEdgeCount: number;
+    averageNodeScore: number;
+    topScoringNodes: {
+        name: string;
+        score: number;
+        type: string;
+    }[];
+    topScoringFiles: {
+        name: string;
+        score: number;
+        filePath: string;
+    }[];
+}
+export interface PipelineResult {
+    graphId: string;
+    repoPath: string;
+    analyzedAt: string;
+    fingerprint: ProjectFingerprint;
+    routes: RouteNode[] | BackendRouteNode[];
+    nodes: CodeNode[];
+    edges: CodeEdge[];
+    allNodes: CodeNode[];
+    allEdges: CodeEdge[];
+    nodeScores: Record<string, number>;
+    stats: PipelineStats;
+    isGithubRepo: boolean;
+    gitInfo: GitInfo;
+}
+export declare function analyzePipeline(repoPath: string, isGithubRepo: boolean, options?: PipelineOptions): Promise<PipelineResult>;
+export declare function refilterPipeline(stored: PipelineResult, thresholds: FilterThresholds): Pick<PipelineResult, "nodes" | "edges" | "stats">;