devlensio 0.2.0

package/dist/graph/thirdPartyLibs.js

@@ -0,0 +1,162 @@
+import fs from "fs";
+import path from "path";
+// ─── Exclusion / inclusion sets ───────────────────────────────────────────────
+// Packages whose primary purpose is UI rendering, styling, or testing —
+// they appear in almost every repo and add no analytical signal.
+const UI_EXCLUSION_SET = new Set([
+    // Core UI frameworks
+    "react", "react-dom", "next", "gatsby", "nuxt", "@sveltejs/kit",
+    "solid-js", "preact", "remix", "@remix-run/react", "@remix-run/node",
+    // Styling
+    "tailwindcss", "styled-components", "@emotion/react", "@emotion/styled",
+    "sass", "less", "postcss", "autoprefixer",
+    "clsx", "class-variance-authority", "tailwind-merge", "cva",
+    // Animation / transition
+    "framer-motion", "react-spring", "react-transition-group", "motion",
+    // UI component kits (prefix-matched below for @radix-ui/* etc.)
+    "@headlessui/react", "lucide-react", "react-icons", "@heroicons/react",
+    "@phosphor-icons/react", "react-feather",
+    // Testing
+    "jest", "vitest", "@testing-library/react", "@testing-library/dom",
+    "@testing-library/jest-dom", "@testing-library/user-event",
+    "@playwright/test", "cypress", "mocha", "chai", "supertest",
+    // Dev tools
+    "typescript", "eslint", "prettier", "webpack", "vite", "esbuild",
+    "rollup", "parcel", "turbopack",
+    "@types/node", "@types/react", "@types/react-dom",
+    "ts-node", "tsx", "nodemon", "concurrently", "husky", "lint-staged",
+    "rimraf", "cross-env", "dotenv-cli",
+]);
+// Packages that ARE runtime logic — they represent real architectural decisions.
+const RUNTIME_INCLUSION_SET = new Set([
+    // HTTP clients
+    "axios", "ky", "node-fetch", "got", "superagent", "undici",
+    // Auth
+    "next-auth", "@auth/core", "jsonwebtoken", "bcryptjs", "bcrypt",
+    "passport", "clerk", "@clerk/nextjs", "@clerk/clerk-react", "lucia",
+    // State management
+    "zustand", "jotai", "recoil", "redux", "@reduxjs/toolkit", "mobx",
+    "xstate", "valtio", "nanostores",
+    // Data fetching / caching
+    "@tanstack/react-query", "@tanstack/query-core",
+    "swr", "@apollo/client", "urql", "graphql",
+    // Validation / schema
+    "zod", "yup", "joi", "valibot", "superstruct", "ajv",
+    // ORM / DB
+    "@prisma/client", "drizzle-orm", "mongoose", "sequelize", "typeorm",
+    "pg", "mysql2", "better-sqlite3", "ioredis", "redis",
+    "@neondatabase/serverless", "@vercel/postgres",
+    // Payments
+    "stripe", "@stripe/stripe-js", "@stripe/react-stripe-js",
+    // Email
+    "nodemailer", "@sendgrid/mail", "resend", "@aws-sdk/client-ses",
+    // BaaS
+    "@supabase/supabase-js", "firebase", "firebase-admin",
+    // Cloud SDKs
+    "aws-sdk", "@aws-sdk/client-s3", "@aws-sdk/client-dynamodb",
+    "@aws-sdk/client-lambda", "@aws-sdk/client-sqs",
+    // Realtime
+    "socket.io", "socket.io-client", "ws", "pusher", "pusher-js",
+    "@ably/ably",
+    // Queues / jobs
+    "bull", "bullmq", "node-cron", "agenda", "bee-queue",
+    // File processing
+    "sharp", "multer", "formidable",
+    // CMS
+    "contentful", "@sanity/client", "@sanity/image-url",
+    // Observability
+    "@sentry/nextjs", "@sentry/node", "posthog-js", "@posthog/node",
+    "newrelic", "dd-trace",
+    // Utilities
+    "lodash", "date-fns", "dayjs", "moment", "uuid", "nanoid",
+    "slugify", "cheerio", "marked", "gray-matter",
+    // Backend frameworks
+    "express", "fastify", "koa", "hono", "nestjs", "@nestjs/core",
+    // AI / LLM
+    "openai", "@anthropic-ai/sdk", "langchain", "@langchain/core",
+    "ai", "@ai-sdk/openai", "@ai-sdk/anthropic",
+]);
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+export function readPackageDependencies(repoPath) {
+    const pkgPath = path.join(repoPath, "package.json");
+    if (!fs.existsSync(pkgPath)) {
+        return { dependencies: {}, devDependencies: {} };
+    }
+    try {
+        const raw = fs.readFileSync(pkgPath, "utf-8");
+        const pkg = JSON.parse(raw);
+        return {
+            dependencies: pkg.dependencies ?? {},
+            devDependencies: pkg.devDependencies ?? {},
+        };
+    }
+    catch {
+        return { dependencies: {}, devDependencies: {} };
+    }
+}
+export function categorizeLibrary(packageName, isDev) {
+    // Check exact match in UI set
+    if (UI_EXCLUSION_SET.has(packageName))
+        return "ui";
+    // Check prefix matches for scoped UI packages like @radix-ui/*, @types/*
+    if (packageName.startsWith("@radix-ui/") ||
+        packageName.startsWith("@types/") ||
+        packageName.startsWith("@testing-library/") ||
+        packageName.startsWith("@storybook/") ||
+        packageName.startsWith("eslint-") ||
+        packageName.startsWith("prettier-") ||
+        packageName.startsWith("babel-") ||
+        packageName.startsWith("@babel/"))
+        return "ui";
+    // Check runtime set
+    if (RUNTIME_INCLUSION_SET.has(packageName))
+        return "runtime";
+    // Check prefix matches for scoped runtime packages
+    if (packageName.startsWith("@aws-sdk/") ||
+        packageName.startsWith("@tanstack/") ||
+        packageName.startsWith("@sentry/") ||
+        packageName.startsWith("@clerk/") ||
+        packageName.startsWith("@auth/") ||
+        packageName.startsWith("@ai-sdk/") ||
+        packageName.startsWith("@nestjs/"))
+        return "runtime";
+    // devDependencies that didn't match anything above
+    if (isDev)
+        return "devtool";
+    return "unknown";
+}
+export function extractPackageName(importSpecifier) {
+    // Scoped package: @org/pkg/sub/path → @org/pkg
+    if (importSpecifier.startsWith("@")) {
+        const parts = importSpecifier.split("/");
+        return parts.slice(0, 2).join("/");
+    }
+    // Regular: axios/lib/core → axios
+    return importSpecifier.split("/")[0];
+}
+export function buildThirdPartyNodes(repoPath, includedLibs) {
+    if (!includedLibs.length)
+        return [];
+    const { dependencies, devDependencies } = readPackageDependencies(repoPath);
+    const nodes = [];
+    for (const name of includedLibs) {
+        const version = dependencies[name] ?? devDependencies[name] ?? "unknown";
+        const isDev = name in devDependencies && !(name in dependencies);
+        nodes.push({
+            id: `[npm]/${name}`,
+            name,
+            type: "THIRD_PARTY",
+            filePath: `[npm]/${name}`,
+            startLine: 0,
+            endLine: 0,
+            rawCode: undefined,
+            codeHash: undefined,
+            metadata: {
+                isThirdParty: true,
+                packageVersion: version,
+                category: categorizeLibrary(name, isDev),
+            },
+        });
+    }
+    return nodes;
+}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+export * from "./types.js";
+export * from "./jobs/types.js";
+export * from "./storage/fileStorage.js";
+export * from "./storage/interface.js";
+export * from "./jobs/queue/interface.js";
+export * from "./summarizer/types.js";
+export * from "./clustering/index.js";
+export * from "./pipeline/index.js";
+export * from "./config/types.js";
+export { analyzePipeline } from "./pipeline/index.js";
+export { runSummarization } from "./summarizer/index.js";
+export { resolveConfig } from "./config/index.js";
+export { computeClusters } from "./clustering/index.js";
+export { EDGE_LABELS } from "./summarizer/prompts.js";
+export { CycleGroup, TopologicalResult } from "./summarizer/types.js";

package/dist/index.js

@@ -0,0 +1,15 @@
+export * from "./types.js";
+export * from "./jobs/types.js";
+export * from "./storage/fileStorage.js";
+export * from "./storage/interface.js";
+export * from "./jobs/queue/interface.js";
+export * from "./summarizer/types.js";
+export * from "./clustering/index.js";
+export * from "./pipeline/index.js";
+export * from "./config/types.js";
+// Functions that cloud backend will need
+export { analyzePipeline } from "./pipeline/index.js";
+export { runSummarization } from "./summarizer/index.js";
+export { resolveConfig } from "./config/index.js";
+export { computeClusters } from "./clustering/index.js";
+export { EDGE_LABELS } from "./summarizer/prompts.js";

package/dist/jobs/index.d.ts

@@ -0,0 +1,5 @@
+import { JobQueue } from "./queue/interface.js";
+export declare const queue: JobQueue;
+export type { JobQueue } from "./queue/interface.js";
+export type { Job, JobSummary, JobStatus, JobPhase, ProgressEvent, JobInput } from "./types.js";
+export { isTerminal, isResumable, toJobSummary } from "./types.js";

package/dist/jobs/index.js

@@ -0,0 +1,11 @@
+import { InMemoryQueue } from "./queue/memory.js";
+function createQueue() {
+    return new InMemoryQueue();
+}
+// Singleton Queue
+//
+// One queue instance for the entire server process.
+// All handlers import this — never instantiate their own queue.
+// This is what ensures job deduplication works across requests.
+export const queue = createQueue();
+export { isTerminal, isResumable, toJobSummary } from "./types.js";

package/dist/jobs/queue/interface.d.ts

@@ -0,0 +1,13 @@
+import type { Job, JobInput, JobSummary, ProgressEvent } from "../types.js";
+export interface JobQueue {
+    enqueue(input: JobInput): Job;
+    getJob(jobId: string): Job | undefined;
+    listJobs(): JobSummary[];
+    findActiveJob(repoPath: string): Job | undefined;
+    pauseJob(jobId: string): boolean;
+    resumeJob(jobId: string): boolean;
+    cancelJob(jobId: string): boolean;
+    emitEvent(jobId: string, event: ProgressEvent): void;
+    subscribe(jobId: string, onEvent: (event: ProgressEvent) => void, onCompleted: () => void): () => void;
+    updateJob(jobId: string, updates: Partial<Job>): void;
+}

package/dist/jobs/queue/interface.js

@@ -0,0 +1 @@
+export {};

package/dist/jobs/queue/memory.d.ts

@@ -0,0 +1,24 @@
+import { Job, JobInput, JobSummary, ProgressEvent } from "../types.js";
+import { JobQueue } from "./interface.js";
+export declare class InMemoryQueue implements JobQueue {
+    private jobs;
+    private subscribers;
+    private waitingQueue;
+    enqueue(input: JobInput): Job;
+    getJob(jobId: string): Job | undefined;
+    listJobs(): JobSummary[];
+    findActiveJob(repoPath: string): Job | undefined;
+    pauseJob(jobId: string): boolean;
+    resumeJob(jobId: string): boolean;
+    cancelJob(jobId: string): boolean;
+    subscribe(jobId: string, onEvent: (event: ProgressEvent) => void, onCompleted: () => void): () => void;
+    updateJob(jobId: string, updates: Partial<Job>): void;
+    emitEvent(jobId: string, event: ProgressEvent): void;
+    private getRunningCount;
+    _markFailed(jobId: string, error: string): void;
+    private onJobTerminated;
+    _markPaused(jobId: string): void;
+    _markCancelled(jobId: string, cleanedUp: boolean): void;
+    _markCompleted(jobId: string, graphId: string): void;
+    private startJob;
+}

package/dist/jobs/queue/memory.js

@@ -0,0 +1,291 @@
+import { randomUUIDv7 } from "bun";
+import { isTerminal, toJobSummary } from "../types.js";
+const MAX_CONCURRENT = parseInt(process.env.MAX_CONCURRENT_JOBS ?? "3", 10);
+export class InMemoryQueue {
+    constructor() {
+        this.jobs = new Map();
+        this.subscribers = new Map(); //Active Subscribers per job
+        this.waitingQueue = []; //job Ids waiting for their turn
+    }
+    enqueue(input) {
+        const existing = this.findActiveJob(input.repoPath);
+        if (existing) {
+            console.log(`Job is already active for ${input.repoPath} - returning ${existing.jobId}`);
+            return existing;
+        }
+        const jobId = randomUUIDv7();
+        const now = new Date().toISOString();
+        const job = {
+            jobId,
+            status: "queued",
+            phase: null,
+            repoPath: input.repoPath,
+            isGithubRepo: input.isGithubRepo ?? false,
+            thresholds: input.thresholds,
+            config: input.config,
+            skipSummarization: input.skipSummarization ?? false,
+            forceSummarize: input.forceSummarize ?? false,
+            includedThirdPartyLibs: input.includedThirdPartyLibs ?? [],
+            graphId: undefined,
+            events: [],
+            pauseRequested: false,
+            cancelRequested: false,
+            createdAt: now,
+        };
+        this.jobs.set(jobId, job);
+        const position = this.waitingQueue.length + 1;
+        this.emitEvent(jobId, { event: "queued", jobId, position });
+        // If we have capacity, start immediately — no need to wait
+        const runningCount = this.getRunningCount();
+        if (runningCount < MAX_CONCURRENT) {
+            // Start async — does not block enqueue()
+            // The runner import is deferred to avoid circular dependency
+            // jobs/index.ts wires the runner in after both are initialized
+            setImmediate(() => this.startJob(jobId));
+        }
+        else {
+            console.log(`⏳ Job ${jobId} queued at position ${position} (${runningCount}/${MAX_CONCURRENT} slots used)`);
+            this.waitingQueue.push(jobId);
+        }
+        return job;
+    }
+    getJob(jobId) {
+        return this.jobs.get(jobId);
+    }
+    listJobs() {
+        return Array.from(this.jobs.values())
+            .map(toJobSummary)
+            .sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+    }
+    findActiveJob(repoPath) {
+        for (const job of this.jobs.values()) {
+            if (job.repoPath === repoPath &&
+                (job.status === "queued" ||
+                    job.status === "running" ||
+                    job.status === "paused")) {
+                return job;
+            }
+        }
+        return undefined;
+    }
+    // Only valid during summarization phase.
+    // Analysis phase cannot be paused — it's too fast and atomic.
+    // Sets the pauseRequested signal — runner checks between batches.
+    pauseJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return false;
+        // Can only pause a running job that is in summarization phase
+        if (job.status !== "running" || job.phase !== "summarization") {
+            return false;
+        }
+        job.pauseRequested = true;
+        console.log(`⏸️  Pause requested for job ${jobId}`);
+        return true;
+    }
+    resumeJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job || job.status !== "paused")
+            return false;
+        job.pauseRequested = false;
+        job.cancelRequested = false;
+        job.status = "running";
+        job.pausedAt = undefined;
+        this.emitEvent(jobId, {
+            event: "resumed",
+            jobId,
+            completedNodes: job.summarizationCompleted ?? 0,
+            totalNodes: job.summarizationTotal ?? 0,
+        });
+        // Re-start the job from checkpoint
+        setImmediate(() => this.startJob(jobId));
+        return true;
+    }
+    // ── cancelJob ───────────────────────────────────────────────────────────────
+    //
+    // Works from any non-terminal state.
+    // Queued jobs are cancelled immediately.
+    // Running/paused jobs set cancelRequested — runner handles cleanup.
+    cancelJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job || isTerminal(job.status))
+            return false;
+        if (job.status === "queued") {
+            // Remove from waiting queue immediately
+            this.waitingQueue = this.waitingQueue.filter(id => id !== jobId);
+            this._markCancelled(jobId, true);
+            return true;
+        }
+        // Running or paused — signal the runner
+        job.cancelRequested = true;
+        job.pauseRequested = false; // clear pause signal if both were set
+        console.log(`🚫 Cancel requested for job ${jobId}`);
+        return true;
+    }
+    // ── subscribe ───────────────────────────────────────────────────────────────
+    //
+    // Attaches an SSE subscriber to a job.
+    // Replays all past events immediately, then streams live.
+    // Returns unsubscribe function — call when SSE connection closes.
+    subscribe(jobId, onEvent, onCompleted) {
+        const job = this.jobs.get(jobId);
+        if (!job) {
+            // Job not found — call onCompleted immediately so handler closes stream
+            onCompleted();
+            return () => { };
+        }
+        // Replay all past events for catch-up
+        for (const event of job.events) {
+            try {
+                onEvent(event);
+            }
+            catch { /* ignore replay errors */ }
+        }
+        // If job is already terminal, close immediately after replay
+        if (isTerminal(job.status)) {
+            onCompleted();
+            return () => { };
+        }
+        // Register subscriber for live events
+        const sub = { onEvent, onCompleted };
+        if (!this.subscribers.has(jobId)) {
+            this.subscribers.set(jobId, new Set());
+        }
+        this.subscribers.get(jobId).add(sub);
+        // Return unsubscribe function
+        return () => {
+            const subs = this.subscribers.get(jobId);
+            if (subs) {
+                subs.delete(sub);
+                if (subs.size === 0)
+                    this.subscribers.delete(jobId);
+            }
+        };
+    }
+    // ── updateJob ───────────────────────────────────────────────────────────────
+    //
+    // Direct field updates — only called by runner.
+    // Uses Object.assign for a clean partial update.
+    updateJob(jobId, updates) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        Object.assign(job, updates);
+    }
+    //broadcasts event to every subscribers
+    emitEvent(jobId, event) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        // Store in history — enables SSE replay on reconnect
+        job.events.push(event);
+        // Broadcast to all active subscribers
+        const subs = this.subscribers.get(jobId);
+        if (!subs || subs.size === 0)
+            return;
+        for (const sub of subs) {
+            try {
+                sub.onEvent(event);
+            }
+            catch (err) {
+                // Subscriber errored (e.g. connection dropped) — remove it
+                console.warn(`SSE subscriber error for job ${jobId}:`, err);
+                subs.delete(sub);
+            }
+        }
+    }
+    //private helpers
+    getRunningCount() {
+        let count = 0;
+        for (const job of this.jobs.values()) {
+            if (job.status === "running")
+                count++;
+        }
+        return count;
+    }
+    _markFailed(jobId, error) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        job.status = "failed";
+        job.error = error;
+        job.failedAt = new Date().toISOString();
+        this.emitEvent(jobId, { event: "failed", jobId, error });
+        this.onJobTerminated(jobId);
+        console.error(`❌ Job ${jobId} failed: ${error}`);
+    }
+    // Called when a job reaches a terminal state.
+    // Notifies all subscribers to close their SSE streams.
+    // Then promotes the next waiting job if a slot opened up.
+    onJobTerminated(jobId) {
+        const subs = this.subscribers.get(jobId);
+        if (subs) {
+            for (const sub of subs) {
+                try {
+                    sub.onCompleted();
+                }
+                catch { /* ignore */ }
+            }
+            this.subscribers.delete(jobId);
+        }
+        // Promote next waiting job if capacity available
+        if (this.waitingQueue.length > 0) {
+            const nextJobId = this.waitingQueue.shift();
+            console.log(`▶️  Promoting queued job ${nextJobId}`);
+            setImmediate(() => this.startJob(nextJobId));
+        }
+    }
+    // Marks a job as completed and notifies subscribers + queue.
+    _markPaused(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        job.status = "paused";
+        job.pausedAt = new Date().toISOString();
+        this.emitEvent(jobId, { event: "paused", jobId, completedNodes: job.summarizationCompleted ?? 0, totalNodes: job.summarizationTotal ?? 0 });
+        console.log(`⏸️  Job ${jobId} paused at ${job.summarizationCompleted}/${job.summarizationTotal} nodes`);
+    }
+    // Marks a job as cancelled and notifies subscribers + queue.
+    // cleanedUp = whether checkpoint file was deleted from disk.
+    // Called directly for queued jobs, called by runner for running/paused jobs.
+    _markCancelled(jobId, cleanedUp) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        job.status = "cancelled";
+        job.cancelledAt = new Date().toISOString();
+        this.emitEvent(jobId, { event: "cancelled", jobId, cleanedUp });
+        this.onJobTerminated(jobId);
+        console.log(`🚫 Job ${jobId} cancelled (cleanedUp: ${cleanedUp})`);
+    }
+    // Marks a job as completed and notifies subscribers + queue.
+    _markCompleted(jobId, graphId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        job.status = "completed";
+        job.graphId = graphId;
+        job.completedAt = new Date().toISOString();
+        this.emitEvent(jobId, { event: "completed", jobId, graphId });
+        this.onJobTerminated(jobId);
+        console.log(`✅ Job ${jobId} completed — graph ${graphId}`);
+    }
+    async startJob(jobId) {
+        const job = this.jobs.get(jobId);
+        if (!job)
+            return;
+        // Job may have been cancelled while waiting in queue
+        if (job.status === "cancelled")
+            return;
+        try {
+            // Dynamic import breaks circular dependency:
+            // queue → runner → queue (for emitEvent/updateJob)
+            const { runJob } = await import("../runner.js");
+            await runJob(job, this);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : "Unknown error";
+            this._markFailed(jobId, message);
+        }
+    }
+}

package/dist/jobs/runner.d.ts

@@ -0,0 +1,3 @@
+import { Job } from "./types.js";
+import { JobQueue } from "./queue/interface.js";
+export declare function runJob(job: Job, queue: JobQueue): Promise<void>;