devlensio 0.2.0

package/dist/summarizer/retry.js

@@ -0,0 +1,51 @@
+// Exponential backoff retry for LLM summarization calls.
+//
+// Retryable: rate limits (429), server errors (5xx), network errors (ECONNRESET etc.)
+// Not retryable: auth (401/403), not-found (404), bad-request (400) — retrying won't help.
+export const DEFAULT_RETRY_OPTIONS = {
+    maxAttempts: 4, // 1 initial + 3 retries
+    baseDelayMs: 1000,
+    maxDelayMs: 15000,
+};
+function isRetryable(err) {
+    const status = err?.status ?? err?.statusCode;
+    if (typeof status === "number") {
+        if (status === 400 || status === 401 || status === 403 || status === 404)
+            return false;
+        if (status === 429 || status >= 500)
+            return true;
+    }
+    const code = err?.code;
+    if (typeof code === "string" &&
+        ["ECONNRESET", "ETIMEDOUT", "ECONNREFUSED", "ENOTFOUND", "EPIPE"].includes(code)) {
+        return true;
+    }
+    const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+    return msg.includes("timeout") || msg.includes("connection") ||
+        msg.includes("network") || msg.includes("socket") ||
+        msg.includes("fetch failed");
+}
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+export async function withRetry(fn, opts = DEFAULT_RETRY_OPTIONS, label = "operation") {
+    for (let attempt = 0; attempt < opts.maxAttempts; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (err) {
+            const isLast = attempt === opts.maxAttempts - 1;
+            if (!isRetryable(err) || isLast)
+                throw err;
+            // Exponential backoff with ±10% jitter to avoid thundering herd
+            const base = opts.baseDelayMs * 2 ** attempt;
+            const jitter = base * 0.1 * (Math.random() * 2 - 1);
+            const wait = Math.min(base + jitter, opts.maxDelayMs);
+            console.warn(`[retry] ${label} — attempt ${attempt + 1}/${opts.maxAttempts} failed, ` +
+                `retrying in ${Math.round(wait)}ms — ${err instanceof Error ? err.message : err}`);
+            await sleep(wait);
+        }
+    }
+    // unreachable — loop always throws or returns
+    throw new Error("withRetry: exhausted attempts");
+}

package/dist/summarizer/topological.d.ts

@@ -0,0 +1,3 @@
+import type { CodeEdge, CodeNode } from "../types.js";
+import type { TopologicalResult } from "./types.js";
+export declare function buildTopologicalOrder(nodes: CodeNode[], edges: CodeEdge[]): TopologicalResult;

package/dist/summarizer/topological.js

@@ -0,0 +1,105 @@
+//Topo Sort
+export function buildTopologicalOrder(nodes, edges) {
+    //file nodes will be handled seperately. 
+    const fileNodes = [];
+    const regularNodesSet = new Set();
+    // const regularNodes: string[] = []; 
+    for (const node of nodes) {
+        if (node.type !== "FILE") {
+            // regularNodes.push(node.id);
+            regularNodesSet.add(node.id);
+        }
+        else {
+            fileNodes.push(node.id);
+        }
+    }
+    const inDegree = new Map();
+    const dependents = new Map(); // dependents[X] = list of nodes that depend on X (parent of X)
+    // When X is summarized → decrement in-degree of each dependent (parent node)
+    // If dependent's in-degree hits 0 → it's ready to summarize
+    for (const nodeId of regularNodesSet) {
+        inDegree.set(nodeId, 0);
+    }
+    for (const edge of edges) {
+        if (!regularNodesSet.has(edge.from) || !regularNodesSet.has(edge.to))
+            continue; // Skip edges that involve FILE nodes — they're handled separately
+        const currentInDegree = inDegree.get(edge.from) || 0;
+        let currentDependents = dependents.get(edge.to) || [];
+        inDegree.set(edge.from, currentInDegree + 1);
+        currentDependents?.push(edge.from);
+        dependents.set(edge.to, currentDependents);
+    }
+    // Now our indegree map is ready — push nodes with 0 indegree into the queue.
+    const queue = [];
+    // nodeOrder is now a nested array — each inner array is one parallel level.
+    // All nodes at the same level are independent and can be summarized concurrently.
+    const nodeOrder = [];
+    for (const [nodeId, degreeVal] of inDegree) {
+        if (!degreeVal) {
+            queue.push(nodeId);
+        }
+    }
+    while (queue.length !== 0) {
+        // Snapshot the current queue size — these are exactly the nodes at this level.
+        // New nodes pushed during this loop belong to the NEXT level.
+        let sz = queue.length;
+        const currentLevel = [];
+        while (sz--) {
+            const nodeId = queue.shift();
+            if (!nodeId)
+                continue;
+            currentLevel.push(nodeId);
+            for (const parentNodeId of (dependents.get(nodeId) || [])) {
+                let currentIndegree = inDegree.get(parentNodeId) || 0;
+                inDegree.set(parentNodeId, Math.max(--currentIndegree, 0));
+                if (currentIndegree === 0) {
+                    // Parent's deps are all done — eligible for next level
+                    queue.push(parentNodeId);
+                }
+            }
+        }
+        nodeOrder.push(currentLevel);
+    }
+    // nodeOrder is now ready — but cyclic nodes won't appear in it.
+    // Count total processed nodes across all levels to detect cycles.
+    const totalProcessed = nodeOrder.reduce((sum, level) => sum + level.length, 0);
+    const cycleGroups = [];
+    if (totalProcessed !== regularNodesSet.size) {
+        const processedSet = new Set(nodeOrder.flat());
+        const unvisited = new Set();
+        for (const regNode of regularNodesSet) {
+            if (!processedSet.has(regNode)) {
+                unvisited.add(regNode);
+            }
+        }
+        //now we have the unvisited nodes.
+        // undirected adjacency among cyclic nodes only
+        const cycleAdj = new Map();
+        for (const id of unvisited)
+            cycleAdj.set(id, []);
+        for (const edge of edges) {
+            if (!unvisited.has(edge.from) || !unvisited.has(edge.to))
+                continue; //if any of the node is not in unvisited meaning it is not a cyclic. 
+            cycleAdj.get(edge.from).push(edge.to);
+            cycleAdj.get(edge.to).push(edge.from);
+        }
+        // DFS to find connected components — each = one CycleGroup
+        for (const startId of unvisited) {
+            const component = [];
+            dfs(startId, unvisited, cycleAdj, component);
+            if (component.length > 0) {
+                cycleGroups.push({ nodeIds: component, size: component.length });
+            }
+        }
+    }
+    return { nodeOrder, cycleGroups, fileNodes };
+}
+function dfs(nodeId, unvisited, adj, component) {
+    unvisited.delete(nodeId); // mark visited
+    component.push(nodeId);
+    for (const adjNode of (adj.get(nodeId) || [])) {
+        if (!unvisited.has(adjNode))
+            continue; //perform dfs only of adjacent node is not visited
+        dfs(adjNode, unvisited, adj, component);
+    }
+}

package/dist/summarizer/types.d.ts

@@ -0,0 +1,57 @@
+import type { PipelineResult } from "../pipeline/index.js";
+export interface NodeSummary {
+    technicalSummary: string;
+    businessSummary: string;
+    security: {
+        severity: "none" | "low" | "medium" | "high";
+        summary: string;
+    };
+    model: string;
+    summarizedAt: string;
+    tokensUsed?: number;
+}
+export interface CycleGroup {
+    nodeIds: string[];
+    size: number;
+}
+export interface SummaryCheckpoint {
+    graphId: string;
+    commitHash: string;
+    status: "running" | "paused" | "completed";
+    createdAt: string;
+    updatedAt: string;
+    nodeOrder: string[][];
+    cycleGroups: CycleGroup[];
+    fileNodes: string[];
+    lastCompletedLevel: number;
+    lastCompletedCycleGroup: number;
+    lastCompletedFileNode: number;
+    totalNodes: number;
+    completedNodes: number;
+}
+export interface SummarizationCallbacks {
+    onStarted: (totalNodes: number) => void;
+    onProgress: (completed: number, total: number, nodeName: string) => void;
+    onPause: () => void;
+    onCancel: (cleanedUp: boolean) => void;
+    onComplete: () => void;
+    onError: (error: string) => void;
+}
+export interface SummarizationInput {
+    job: import("../jobs/types.js").Job;
+    queue: import("../jobs/queue/interface.js").JobQueue;
+    graphId: string;
+    commitHash: string;
+    repoPath: string;
+    previousCommitHash?: string;
+    routes: PipelineResult["routes"];
+    callbacks: SummarizationCallbacks;
+}
+export interface TopologicalResult {
+    nodeOrder: string[][];
+    cycleGroups: CycleGroup[];
+    fileNodes: string[];
+}
+export declare const MAX_GROUP_SUMMARY_SIZE = 3;
+export declare const MAPREDUCE_TOKEN_THRESHOLD = 1200;
+export declare const FILE_BATCH_SIZE = 10;

package/dist/summarizer/types.js

@@ -0,0 +1,17 @@
+//storing types used for summarization flow
+// Edge types that drive topological sort order.
+// A node must wait for all nodes it has these edges TO before being summarized.
+// export const HARD_DEPENDENCY_EDGES = new Set([
+//   "CALLS",
+//   "READS_FROM",
+//   "WRITES_TO",
+//   "GUARDS",
+// ]);
+// Cycle groups at or below this size → summarize together in one LLM call
+// Above this → summarize individually
+export const MAX_GROUP_SUMMARY_SIZE = 3;
+// Nodes whose source code exceeds this token estimate → MapReduce
+// ~1200 tokens ≈ 900 lines — only very large files hit this
+export const MAPREDUCE_TOKEN_THRESHOLD = 1200;
+// Batch size for the files to be summarized at a time
+export const FILE_BATCH_SIZE = 10;

package/dist/types.d.ts

@@ -0,0 +1,78 @@
+export type Language = "javascript" | "typescript" | "python" | "unknown";
+export type Framework = "nextjs" | "react" | "express" | "fastify" | "koa" | "unknown";
+export type FrontendFramework = "nextjs" | "react";
+export type BackendFramework = "express" | "fastify" | "koa";
+export type RouterType = "app" | "pages" | "app+pages" | "react-router" | "none";
+export type ProjectType = "frontend" | "backend" | "fullstack" | "unknown";
+export type StateLibrary = "zustand" | "redux" | "recoil" | "jotai" | "context-only";
+export type DataFetchingLibrary = "react-query" | "swr" | "axios" | "fetch";
+export type DatabaseLibrary = "prisma" | "drizzle" | "mongodb" | "firebase" | "supabase" | "planetscale" | "postgres" | "mysql" | "sqlite";
+export interface ProjectFingerprint {
+    language: Language;
+    projectType: ProjectType;
+    framework: Framework;
+    router: RouterType;
+    stateManagement: StateLibrary[];
+    dataFetching: DataFetchingLibrary[];
+    databases: DatabaseLibrary[];
+    rawDependencies: Record<string, string>;
+}
+export type RouteNodeType = "PAGE" | "LAYOUT" | "API_ROUTE" | "LOADING" | "ERROR" | "MIDDLEWARE" | "NOT_FOUND";
+export interface RouteNode {
+    type: RouteNodeType;
+    nodeId?: string;
+    urlPath: string;
+    filePath: string;
+    isDynamic: boolean;
+    isCatchAll: boolean;
+    isGroupRoute: boolean;
+    layoutPath?: string;
+    params?: string[];
+    httpMethods?: string[];
+}
+export interface BackendRouteNode {
+    type: "BACKEND_ROUTE";
+    nodeId?: string;
+    urlPath: string;
+    filePath: string;
+    httpMethod: string;
+    handlerName?: string;
+    framework: BackendFramework;
+    isDynamic: boolean;
+    params?: string[];
+    inlineHandler?: {
+        rawCode: string;
+        startLine: number;
+        endLine: number;
+    };
+}
+export type NodeType = "COMPONENT" | "HOOK" | "FUNCTION" | "STATE_STORE" | "UTILITY" | "FILE" | "GHOST" | "ROUTE" | "TEST" | "STORY" | "THIRD_PARTY";
+export interface CodeNode {
+    id: string;
+    name: string;
+    type: NodeType;
+    filePath: string;
+    startLine: number;
+    endLine: number;
+    rawCode?: string;
+    codeHash?: string;
+    technicalSummary?: string;
+    businessSummary?: string;
+    security?: {
+        severity: "none" | "low" | "medium" | "high";
+        summary: string;
+    };
+    summaryModel?: string;
+    summarizedAt?: string;
+    isEmbedded?: boolean;
+    parentFile?: string;
+    score?: Number;
+    metadata: Record<string, unknown>;
+}
+export type EdgeType = "CALLS" | "IMPORTS" | "READS_FROM" | "WRITES_TO" | "PROP_PASS" | "EMITS" | "LISTENS" | "WRAPPED_BY" | "GUARDS" | "HANDLES" | "TESTS" | "USES";
+export interface CodeEdge {
+    from: string;
+    to: string;
+    type: EdgeType;
+    metadata?: Record<string, unknown>;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "devlensio",
+  "version": "0.2.0",
+  "description": "Codebase intelligence engine for TypeScript/JavaScript/Reactjs/Nextjs repositories.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "scripts": {
+    "dev": "bun --watch src/server/index.ts",
+    "start": "bun src/server/index.ts",
+    "build": "tsc --project tsconfig.build.json",
+    "test": "bun test",
+    "export-graph": "bun src/debug/exportGraph.ts"
+  },
+  "keywords": ["codebase", "graph", "ast", "typescript", "javascript", "devtools"],
+  "author": "",
+  "license": "AGPL-3.0",
+  "files": [
+    "dist/**/*"
+  ],
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.104.2",
+    "@google/genai": "^2.8.0",
+    "dotenv": "^17.4.2",
+    "openai": "^6.42.0",
+    "simple-git": "^3.36.0",
+    "ts-morph": "^28.0.0",
+    "uuid": "^11.0.0"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.9",
+    "@types/jest": "^30.0.0",
+    "@types/node": "^25.3.0",
+    "@types/uuid": "^11.0.0",
+    "jest": "^30.2.0",
+    "ts-jest": "^29.4.6",
+    "ts-node": "^10.9.2",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.9.3"
+  }
+}