devlensio 0.2.0

package/dist/jobs/runner.js

@@ -0,0 +1,136 @@
+import { analyzePipeline } from "../pipeline/index.js";
+import { storage } from "../storage/index.js";
+// ─── runJob ───────────────────────────────────────────────────────────────────
+//
+// Entry point — called by InMemoryQueue.startJob().
+// Orchestrates Phase 1 (analysis) and Phase 2 (summarization) for one job.
+//
+// The runner is stateless — all state lives on the Job object in the queue.
+// This is what makes resume work: runner re-reads job state on every call.
+export async function runJob(job, queue) {
+    const q = queue; // safe — only InMemoryQueue used locally
+    queue.updateJob(job.jobId, {
+        status: "running",
+        startedAt: new Date().toISOString(),
+    });
+    // ── Phase 1 — Analysis ────────────────────────────────────────────────────
+    let graphId;
+    let commitHash;
+    try {
+        queue.updateJob(job.jobId, { phase: "analysis" });
+        queue.emitEvent(job.jobId, { event: "analysis_started", jobId: job.jobId });
+        const result = await analyzePipeline(job.repoPath, job.isGithubRepo, {
+            thresholds: job.thresholds,
+            includedThirdPartyLibs: job.includedThirdPartyLibs ?? [],
+            onStep: (step) => {
+                queue.emitEvent(job.jobId, {
+                    event: "analysis_progress",
+                    jobId: job.jobId,
+                    step,
+                });
+            },
+        });
+        storage.saveGraph(result, { force: job.forceSummarize });
+        graphId = result.graphId;
+        commitHash = result.gitInfo.commitHash;
+        queue.updateJob(job.jobId, { graphId });
+        queue.emitEvent(job.jobId, {
+            event: "analysis_complete",
+            jobId: job.jobId,
+            graphId,
+            nodeCount: result.nodes.length,
+            edgeCount: result.edges.length,
+        });
+        console.log(`\n📊 Phase 1 complete for job ${job.jobId} — graph ${graphId}`);
+        console.log(`   Nodes: ${result.nodes.length} | Edges: ${result.edges.length}`);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : "Analysis failed";
+        q._markFailed(job.jobId, message);
+        return;
+    }
+    // ── Cancel check after Phase 1 ────────────────────────────────────────────
+    const jobAfterPhase1 = queue.getJob(job.jobId);
+    if (jobAfterPhase1.cancelRequested) {
+        q._markCancelled(job.jobId, true);
+        return;
+    }
+    // ── skipSummarization — stop here, mark completed ─────────────────────────
+    // User chose analysis-only mode. Summaries can be triggered later via
+    // POST /api/graph/:graphId/:commitHash/summarize
+    if (job.skipSummarization) {
+        console.log(`⏭️  Job ${job.jobId} — skipSummarization=true, stopping after Phase 1`);
+        q._markCompleted(job.jobId, graphId);
+        return;
+    }
+    //if force summarization then remove the commit entirely from the meta 
+    console.log("Force Summarized ? : ", job.forceSummarize ?? "false");
+    if (job.forceSummarize) {
+        console.log("Removing Summarized Commits!");
+        storage.removeFromSummarizedCommits(graphId, commitHash);
+    }
+    // ── Phase 2 — Summarization ───────────────────────────────────────────────
+    try {
+        queue.updateJob(job.jobId, { phase: "summarization" });
+        const { runSummarization } = await import("../summarizer/index.js");
+        // Find previous summarized commit for smart reuse
+        const previousCommitHash = await storage.findLastSummarizedAncestor(graphId, commitHash, job.repoPath);
+        // Fetch routes and fingerprint from saved graph for summarizer context
+        const savedResult = storage.getGraph(graphId, commitHash);
+        await runSummarization({
+            job,
+            queue,
+            graphId,
+            commitHash,
+            repoPath: job.repoPath,
+            previousCommitHash,
+            routes: savedResult.routes,
+            callbacks: {
+                onStarted: (totalNodes) => {
+                    queue.updateJob(job.jobId, {
+                        summarizationTotal: totalNodes,
+                        summarizationCompleted: 0,
+                    });
+                    queue.emitEvent(job.jobId, {
+                        event: "summarization_started",
+                        jobId: job.jobId,
+                        totalNodes,
+                    });
+                },
+                onProgress: (completed, total, nodeName) => {
+                    queue.updateJob(job.jobId, {
+                        summarizationCompleted: completed,
+                        summarizationTotal: total,
+                    });
+                    queue.emitEvent(job.jobId, {
+                        event: "summarization_progress",
+                        jobId: job.jobId,
+                        completed,
+                        total,
+                        nodeName,
+                    });
+                },
+                onPause: () => {
+                    q._markPaused(job.jobId);
+                },
+                onCancel: (cleanedUp) => {
+                    q._markCancelled(job.jobId, cleanedUp);
+                },
+                onComplete: () => {
+                    queue.emitEvent(job.jobId, {
+                        event: "summarization_complete",
+                        jobId: job.jobId,
+                    });
+                    q._markCompleted(job.jobId, graphId);
+                },
+                onError: (error) => {
+                    q._markFailed(job.jobId, error);
+                },
+            },
+        });
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : "Summarization failed";
+        q._markFailed(job.jobId, message);
+    }
+}

package/dist/jobs/types.d.ts

@@ -0,0 +1,112 @@
+import type { DevLensConfig } from "../config/index.js";
+import type { FilterThresholds } from "../pipeline/index.js";
+export type JobStatus = "queued" | "running" | "paused" | "completed" | "cancelled" | "failed";
+export type JobPhase = "analysis" | "summarization";
+export type AnalysisStep = "fingerprint" | "filesystem" | "parse" | "edges" | "scoring";
+export type ProgressEvent = {
+    event: "queued";
+    jobId: string;
+    position: number;
+} | {
+    event: "analysis_started";
+    jobId: string;
+} | {
+    event: "analysis_progress";
+    jobId: string;
+    step: AnalysisStep;
+} | {
+    event: "analysis_complete";
+    jobId: string;
+    graphId: string;
+    nodeCount: number;
+    edgeCount: number;
+} | {
+    event: "summarization_started";
+    jobId: string;
+    totalNodes: number;
+} | {
+    event: "summarization_progress";
+    jobId: string;
+    completed: number;
+    total: number;
+    nodeName: string;
+} | {
+    event: "summarization_complete";
+    jobId: string;
+} | {
+    event: "paused";
+    jobId: string;
+    completedNodes: number;
+    totalNodes: number;
+} | {
+    event: "resumed";
+    jobId: string;
+    completedNodes: number;
+    totalNodes: number;
+} | {
+    event: "cancelled";
+    jobId: string;
+    cleanedUp: boolean;
+} | {
+    event: "completed";
+    jobId: string;
+    graphId: string;
+} | {
+    event: "failed";
+    jobId: string;
+    error: string;
+};
+export interface Job {
+    jobId: string;
+    status: JobStatus;
+    phase: JobPhase | null;
+    repoPath: string;
+    isGithubRepo: boolean;
+    thresholds?: FilterThresholds;
+    config: DevLensConfig;
+    skipSummarization?: boolean;
+    graphId?: string;
+    events: ProgressEvent[];
+    pauseRequested: boolean;
+    cancelRequested: boolean;
+    summarizationTotal?: number;
+    summarizationCompleted?: number;
+    createdAt: string;
+    startedAt?: string;
+    pausedAt?: string;
+    cancelledAt?: string;
+    completedAt?: string;
+    failedAt?: string;
+    error?: string;
+    forceSummarize?: boolean;
+    includedThirdPartyLibs?: string[];
+}
+export interface JobInput {
+    repoPath: string;
+    isGithubRepo?: boolean;
+    skipSummarization: boolean;
+    thresholds?: FilterThresholds;
+    config: DevLensConfig;
+    forceSummarize?: boolean;
+    includedThirdPartyLibs?: string[];
+}
+export interface JobSummary {
+    jobId: string;
+    status: JobStatus;
+    phase: JobPhase | null;
+    repoPath: string;
+    graphId?: string;
+    summarizationTotal?: number;
+    summarizationCompleted?: number;
+    createdAt: string;
+    startedAt?: string;
+    pausedAt?: string;
+    cancelledAt?: string;
+    completedAt?: string;
+    failedAt?: string;
+    error?: string;
+}
+export declare const TERMINAL_STATUSES: Set<JobStatus>;
+export declare function isTerminal(status: JobStatus): boolean;
+export declare function isResumable(status: JobStatus): boolean;
+export declare function toJobSummary(job: Job): JobSummary;

package/dist/jobs/types.js

@@ -0,0 +1,33 @@
+// Jobs in these statuses are done — they will never change state again.
+// Used by the queue to decide when to clean up SSE subscribers.
+export const TERMINAL_STATUSES = new Set([
+    "completed",
+    "failed",
+    "cancelled",
+]);
+export function isTerminal(status) {
+    return TERMINAL_STATUSES.has(status);
+}
+// Only paused jobs can be resumed.
+// Cancelled jobs cannot — their checkpoints are deleted.
+export function isResumable(status) {
+    return status === "paused";
+}
+export function toJobSummary(job) {
+    return {
+        jobId: job.jobId,
+        status: job.status,
+        phase: job.phase,
+        repoPath: job.repoPath,
+        graphId: job.graphId,
+        summarizationTotal: job.summarizationTotal,
+        summarizationCompleted: job.summarizationCompleted,
+        createdAt: job.createdAt,
+        startedAt: job.startedAt,
+        pausedAt: job.pausedAt,
+        cancelledAt: job.cancelledAt,
+        completedAt: job.completedAt,
+        failedAt: job.failedAt,
+        error: job.error,
+    };
+}

package/dist/parser/directives.d.ts

@@ -0,0 +1,4 @@
+import { SourceFile, Node } from "ts-morph";
+export type RenderingBoundary = "client" | "server" | null;
+export declare function detectFileDirective(sourceFile: SourceFile): RenderingBoundary;
+export declare function detectFunctionDirective(bodyNode: Node | undefined): RenderingBoundary;

package/dist/parser/directives.js

@@ -0,0 +1,31 @@
+import { SyntaxKind } from "ts-morph";
+function readDirectiveFromStatements(statements) {
+    const first = statements[0];
+    if (!first)
+        return null;
+    if (first.getKind() !== SyntaxKind.ExpressionStatement)
+        return null;
+    const expr = first.getExpression();
+    if (!expr || expr.getKind() !== SyntaxKind.StringLiteral)
+        return null;
+    const value = expr.getLiteralText();
+    if (value === "use client")
+        return "client";
+    if (value === "use server")
+        return "server";
+    return null;
+}
+export function detectFileDirective(sourceFile) {
+    return readDirectiveFromStatements(sourceFile.getStatements());
+}
+// Only "use server" is valid inside a function body (Next.js Server Actions).
+// Returns null for anything else.
+export function detectFunctionDirective(bodyNode) {
+    if (!bodyNode)
+        return null;
+    if (bodyNode.getKind() !== SyntaxKind.Block)
+        return null;
+    const result = readDirectiveFromStatements(bodyNode.getStatements());
+    // "use client" inside a function body is not valid per Next.js spec — ignore it.
+    return result === "server" ? "server" : null;
+}

package/dist/parser/extractors/components.d.ts

@@ -0,0 +1,5 @@
+import { SourceFile, Node } from "ts-morph";
+import type { CodeNode } from "../../types.js";
+import { type RenderingBoundary } from "../directives.js";
+export declare function returnsJSX(node: Node): boolean;
+export declare function extractComponents(file: SourceFile, fileDirective?: RenderingBoundary): CodeNode[];

package/dist/parser/extractors/components.js

@@ -0,0 +1,240 @@
+//This file will extract the React Components from the files
+import { SyntaxKind } from "ts-morph";
+import { detectFunctionDirective } from "../directives.js";
+import { extractReturnTypeAnnotation, extractReferencedInterfaces, } from "../typeUtils.js";
+// Generates a unique id for a node
+function makeId(filePath, name) {
+    return `${filePath}::${name}`;
+}
+// Checks if a node returns JSX by looking for JSX elements in its body
+export function returnsJSX(node) {
+    const jsxElements = node.getDescendantsOfKind(SyntaxKind.JsxElement);
+    const jsxSelfClosing = node.getDescendantsOfKind(SyntaxKind.JsxSelfClosingElement);
+    const jsxFragments = node.getDescendantsOfKind(SyntaxKind.JsxFragment);
+    return jsxElements.length > 0 || jsxSelfClosing.length > 0 || jsxFragments.length > 0;
+}
+// Extracts hooks used inside a node (store every expression starting with use, e.g. useState, useEffect, useCustomHook)
+function extractHooks(node) {
+    const calls = node.getDescendantsOfKind(SyntaxKind.CallExpression);
+    const hooks = [];
+    for (const call of calls) {
+        const name = call.getExpression().getText();
+        if (name.startsWith("use")) {
+            hooks.push(name);
+        }
+    }
+    return [...new Set(hooks)]; // deduplicate
+}
+// Extracts the context variable names passed to useContext() calls.
+// e.g. useContext(AuthContext) → ["AuthContext"]
+// This is stored separately from hooks so stateEdges.ts can do a direct
+// name lookup instead of relying on a fragile heuristic.
+function extractContextRefs(node) {
+    const refs = [];
+    for (const call of node.getDescendantsOfKind(SyntaxKind.CallExpression)) {
+        if (call.getExpression().getText() === "useContext") {
+            const arg = call.getArguments()[0];
+            if (arg)
+                refs.push(arg.getText());
+        }
+    }
+    return [...new Set(refs)];
+}
+// This will return all the external calls made inside the component (meaning all the calls except the calls to the inner functions and the hooks)
+function extractAllCalls(node) {
+    const innerFunctionNames = new Set();
+    // variable declearations like const fn = () => {} or const fn = function() {}
+    for (const varDecl of node.getDescendantsOfKind(SyntaxKind.VariableDeclaration)) {
+        const name = varDecl.getName();
+        const init = varDecl.getInitializer();
+        if (!init)
+            continue;
+        if (init.getKind() === SyntaxKind.ArrowFunction || init.getKind() === SyntaxKind.FunctionExpression) {
+            innerFunctionNames.add(name);
+        }
+    }
+    // Function declarations like function fn() {}
+    for (const fn of node.getDescendantsOfKind(SyntaxKind.FunctionDeclaration)) {
+        const name = fn.getName();
+        if (name)
+            innerFunctionNames.add(name);
+    }
+    const calls = node.getDescendantsOfKind(SyntaxKind.CallExpression);
+    const externalCalls = new Set();
+    for (const call of calls) {
+        const expr = call.getExpression();
+        const fullExpr = expr.getText();
+        const rootName = fullExpr.split(".")[0];
+        if (rootName.startsWith("use"))
+            continue; // skip hooks
+        if (rootName.startsWith("React"))
+            continue;
+        if (innerFunctionNames.has(rootName))
+            continue; // skip calls to inner functions
+        // Capture the full expression (e.g. "axios.get" not just "axios") so
+        // callEdges.ts can create a per-method third-party node when needed.
+        externalCalls.add(fullExpr);
+    }
+    return [...externalCalls];
+}
+// Checks if a component has any state (useState or useReducer)
+function hasState(hooks) {
+    return hooks.includes("useState") || hooks.includes("useReducer");
+}
+// Extracts prop types from the first parameter of a component function.
+// Handles three patterns:
+//   1. function Button({ color }: ButtonProps) — named ref → look up interface
+//   2. function Button({ color }: { color: string }) — inline object type literal
+//   3. const Button: React.FC<ButtonProps> = ... — generic type argument (regex)
+function extractPropTypes(fn, sourceFile) {
+    const params = fn.getParameters ? fn.getParameters() : [];
+    if (!params.length)
+        return undefined;
+    const firstParam = params[0];
+    const typeNode = firstParam.getTypeNode();
+    if (!typeNode)
+        return undefined;
+    const typeText = typeNode.getText();
+    // Pattern 2: inline object literal `{ color: string; onClick: () => void }`
+    if (typeText.startsWith("{")) {
+        try {
+            const typeLiteral = typeNode.asKind(SyntaxKind.TypeLiteral);
+            if (typeLiteral) {
+                const props = {};
+                for (const member of typeLiteral.getProperties()) {
+                    props[member.getName()] = member.getTypeNode()?.getText() ?? "unknown";
+                }
+                return Object.keys(props).length ? props : undefined;
+            }
+        }
+        catch {
+            return undefined;
+        }
+    }
+    // Pattern 1: named type reference like `ButtonProps`
+    const trimmed = typeText.trim();
+    if (/^[A-Z][A-Za-z0-9_]*$/.test(trimmed)) {
+        const result = extractReferencedInterfaces(sourceFile, [trimmed]);
+        return result[trimmed] ?? undefined;
+    }
+    return undefined;
+}
+export function extractComponents(file, fileDirective = null) {
+    const nodes = [];
+    const filePath = file.getFilePath();
+    // ─── Function Declarations ─────────────────────────────────────────────────
+    // e.g. function MyComponent() { return <div /> }
+    for (const fn of file.getFunctions()) {
+        const name = fn.getName();
+        if (!name)
+            continue;
+        // React components start with uppercase
+        if (!/^[A-Z]/.test(name))
+            continue;
+        // Must return JSX
+        if (!returnsJSX(fn))
+            continue;
+        const hooks = extractHooks(fn);
+        const externalCalls = extractAllCalls(fn);
+        const contextRefs = extractContextRefs(fn);
+        const renderingBoundary = detectFunctionDirective(fn.getBody()) ?? fileDirective;
+        const propTypes = extractPropTypes(fn, file);
+        const returnType = extractReturnTypeAnnotation(fn) ?? "JSX.Element";
+        nodes.push({
+            id: makeId(filePath, name),
+            name,
+            type: "COMPONENT",
+            filePath,
+            startLine: fn.getStartLineNumber(),
+            endLine: fn.getEndLineNumber(),
+            rawCode: fn.getText(),
+            metadata: {
+                hooks,
+                contextRefs,
+                uses: externalCalls,
+                hasState: hasState(hooks),
+                exportType: fn.isDefaultExport() ? "default" : "named",
+                propTypes,
+                returnType,
+                ...(renderingBoundary !== null && { renderingBoundary }),
+            },
+        });
+    }
+    // ─── Arrow Function Components ─────────────────────────────────────────────
+    // e.g. const MyComponent = () => <div />
+    // e.g. export const MyComponent = () => { return <div /> }
+    for (const variable of file.getVariableDeclarations()) {
+        const name = variable.getName();
+        // React components start with uppercase
+        if (!/^[A-Z]/.test(name))
+            continue;
+        const initializer = variable.getInitializer();
+        if (!initializer)
+            continue;
+        const isArrow = initializer.getKind() === SyntaxKind.ArrowFunction;
+        // Check for React.memo and React.forwardRef wrappers
+        const isMemoOrForwardRef = initializer.getKind() === SyntaxKind.CallExpression &&
+            (initializer.getText().startsWith("React.memo") ||
+                initializer.getText().startsWith("React.forwardRef") ||
+                initializer.getText().startsWith("memo(") ||
+                initializer.getText().startsWith("forwardRef("));
+        // Must be either a direct arrow function or a wrapped component
+        if (!isArrow && !isMemoOrForwardRef)
+            continue;
+        // For wrapped components we need to look inside the wrapper
+        // to find the actual arrow function to check for JSX
+        let nodeToAnalyze = initializer;
+        if (isMemoOrForwardRef) {
+            const callExpr = initializer.asKind(SyntaxKind.CallExpression);
+            const firstArg = callExpr?.getArguments()[0];
+            if (!firstArg)
+                continue;
+            // Inner component can be arrow function or regular function expression
+            const asArrow = firstArg.asKind(SyntaxKind.ArrowFunction);
+            const asFunctionExpr = firstArg.asKind(SyntaxKind.FunctionExpression);
+            const inner = asArrow ?? asFunctionExpr;
+            if (!inner)
+                continue;
+            nodeToAnalyze = inner;
+        }
+        // Must return JSX
+        if (!returnsJSX(nodeToAnalyze))
+            continue;
+        const hooks = extractHooks(nodeToAnalyze);
+        const externalCalls = extractAllCalls(nodeToAnalyze);
+        const contextRefs = extractContextRefs(nodeToAnalyze);
+        const renderingBoundary = detectFunctionDirective(nodeToAnalyze.getKind() === SyntaxKind.ArrowFunction
+            ? nodeToAnalyze.getBody()
+            : undefined) ?? fileDirective;
+        const propTypes = extractPropTypes(nodeToAnalyze, file);
+        const returnType = extractReturnTypeAnnotation(nodeToAnalyze) ?? "JSX.Element";
+        const variableStatement = variable.getVariableStatement();
+        const isExported = variableStatement
+            ? variableStatement.isExported()
+            : false;
+        const isDefault = variableStatement
+            ? variableStatement.isDefaultExport()
+            : false;
+        nodes.push({
+            id: makeId(filePath, name),
+            name,
+            type: "COMPONENT",
+            filePath,
+            startLine: variable.getStartLineNumber(),
+            endLine: variable.getEndLineNumber(),
+            rawCode: variable.getText(),
+            metadata: {
+                hooks,
+                contextRefs,
+                uses: externalCalls,
+                hasState: hasState(hooks),
+                exportType: isDefault ? "default" : isExported ? "named" : "none",
+                isMemoized: isMemoOrForwardRef,
+                propTypes,
+                returnType,
+                ...(renderingBoundary !== null && { renderingBoundary }),
+            },
+        });
+    }
+    return nodes;
+}

package/dist/parser/extractors/functions.d.ts

@@ -0,0 +1,4 @@
+import { SourceFile } from "ts-morph";
+import type { CodeNode } from "../../types.js";
+import { type RenderingBoundary } from "../directives.js";
+export declare function extractFunctions(file: SourceFile, fileDirective?: RenderingBoundary): CodeNode[];