arceus-s 1.6.4

package/dist/core/ingestion/pipeline-phases/parse.js

@@ -0,0 +1,33 @@
+/**
+ * Phase: parse
+ *
+ * Chunked parse + resolve loop: reads source in byte-budget chunks,
+ * parses via worker pool (or sequential fallback), resolves imports,
+ * heritage, and calls, synthesizes wildcard bindings.
+ *
+ * This phase encapsulates the entire `runChunkedParseAndResolve` function
+ * from the original pipeline. The chunk loop is a memory optimization
+ * internal to this phase, not a phase boundary.
+ *
+ * @deps    structure, markdown, cobol
+ * @reads   scannedFiles, allPaths, totalFiles (from structure)
+ * @writes  graph (Symbol nodes, IMPORTS/CALLS/EXTENDS/IMPLEMENTS/ACCESSES edges)
+ * @output  exportedTypeMap, allFetchCalls, allExtractedRoutes, allDecoratorRoutes,
+ *          allToolDefs, allORMQueries, bindingAccumulator
+ */
+import { getPhaseOutput } from './types.js';
+import { runChunkedParseAndResolve } from './parse-impl.js';
+export const parsePhase = {
+    name: 'parse',
+    deps: ['structure', 'markdown', 'cobol'],
+    async execute(ctx, deps) {
+        const { scannedFiles, allPaths, allPathSet, totalFiles } = getPhaseOutput(deps, 'structure');
+        const result = await runChunkedParseAndResolve(ctx.graph, scannedFiles, allPaths, totalFiles, ctx.repoPath, ctx.pipelineStart, ctx.onProgress, ctx.options);
+        return {
+            ...result,
+            allPaths,
+            allPathSet,
+            totalFiles,
+        };
+    },
+};

package/dist/core/ingestion/pipeline-phases/processes.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Phase: processes
+ *
+ * Detects execution flows (processes) and creates Process nodes +
+ * STEP_IN_PROCESS edges. Also links Route/Tool nodes to processes.
+ *
+ * @deps    communities, routes, tools
+ * @reads   graph (all nodes and relationships), communityResult, routeRegistry, toolDefs
+ * @writes  graph (Process nodes, STEP_IN_PROCESS edges, ENTRY_POINT_OF edges)
+ */
+import type { PipelinePhase } from './types.js';
+import { type ProcessDetectionResult } from '../process-processor.js';
+export interface ProcessesOutput {
+    processResult: ProcessDetectionResult;
+}
+export declare const processesPhase: PipelinePhase<ProcessesOutput>;

package/dist/core/ingestion/pipeline-phases/processes.js

@@ -0,0 +1,149 @@
+/**
+ * Phase: processes
+ *
+ * Detects execution flows (processes) and creates Process nodes +
+ * STEP_IN_PROCESS edges. Also links Route/Tool nodes to processes.
+ *
+ * @deps    communities, routes, tools
+ * @reads   graph (all nodes and relationships), communityResult, routeRegistry, toolDefs
+ * @writes  graph (Process nodes, STEP_IN_PROCESS edges, ENTRY_POINT_OF edges)
+ */
+import { getPhaseOutput } from './types.js';
+import { processProcesses } from '../process-processor.js';
+import { generateId } from '../../../lib/utils.js';
+import { isDev } from '../utils/env.js';
+import { logger } from '../../logger.js';
+export const processesPhase = {
+    name: 'processes',
+    // `structure` supplies `totalFiles` (progress counter) without the spurious
+    // structural data dependency on `parse`.
+    deps: ['communities', 'routes', 'tools', 'structure'],
+    async execute(ctx, deps) {
+        const { totalFiles } = getPhaseOutput(deps, 'structure');
+        const { communityResult } = getPhaseOutput(deps, 'communities');
+        const { routeRegistry } = getPhaseOutput(deps, 'routes');
+        const { toolDefs } = getPhaseOutput(deps, 'tools');
+        ctx.onProgress({
+            phase: 'processes',
+            percent: 94,
+            message: 'Detecting execution flows...',
+            stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+        });
+        let symbolCount = 0;
+        ctx.graph.forEachNode((n) => {
+            if (n.label !== 'File')
+                symbolCount++;
+        });
+        const dynamicMaxProcesses = Math.max(20, Math.min(300, Math.round(symbolCount / 10)));
+        const processResult = await processProcesses(ctx.graph, communityResult.memberships, (message, progress) => {
+            const processProgress = 94 + progress * 0.05;
+            ctx.onProgress({
+                phase: 'processes',
+                percent: Math.round(processProgress),
+                message,
+                stats: { filesProcessed: totalFiles, totalFiles, nodesCreated: ctx.graph.nodeCount },
+            });
+        }, { maxProcesses: dynamicMaxProcesses, minSteps: 3 });
+        if (isDev) {
+            logger.info(`🔄 Process detection: ${processResult.stats.totalProcesses} processes found (${processResult.stats.crossCommunityCount} cross-community)`);
+        }
+        processResult.processes.forEach((proc) => {
+            ctx.graph.addNode({
+                id: proc.id,
+                label: 'Process',
+                properties: {
+                    name: proc.label,
+                    filePath: '',
+                    heuristicLabel: proc.heuristicLabel,
+                    processType: proc.processType,
+                    stepCount: proc.stepCount,
+                    communities: proc.communities,
+                    entryPointId: proc.entryPointId,
+                    terminalId: proc.terminalId,
+                },
+            });
+        });
+        processResult.steps.forEach((step) => {
+            ctx.graph.addRelationship({
+                id: `${step.nodeId}_step_${step.step}_${step.processId}`,
+                type: 'STEP_IN_PROCESS',
+                sourceId: step.nodeId,
+                targetId: step.processId,
+                confidence: 1.0,
+                reason: 'trace-detection',
+                step: step.step,
+            });
+        });
+        // Link Route and Tool nodes to Processes
+        if (routeRegistry.size > 0 || toolDefs.length > 0) {
+            const routesByFile = new Map();
+            for (const [url, entry] of routeRegistry) {
+                let list = routesByFile.get(entry.filePath);
+                if (!list) {
+                    list = [];
+                    routesByFile.set(entry.filePath, list);
+                }
+                list.push(url);
+            }
+            const toolsByHandlerId = new Map();
+            const toolsWithoutHandlerByFile = new Map();
+            for (const td of toolDefs) {
+                const key = td.handlerNodeId ?? td.filePath;
+                const targetMap = td.handlerNodeId ? toolsByHandlerId : toolsWithoutHandlerByFile;
+                let list = targetMap.get(key);
+                if (!list) {
+                    list = [];
+                    targetMap.set(key, list);
+                }
+                list.push(td.name);
+            }
+            let linked = 0;
+            for (const proc of processResult.processes) {
+                if (!proc.entryPointId)
+                    continue;
+                const entryNode = ctx.graph.getNode(proc.entryPointId);
+                if (!entryNode)
+                    continue;
+                const entryFile = entryNode.properties.filePath;
+                if (!entryFile)
+                    continue;
+                const routeURLs = routesByFile.get(entryFile);
+                if (routeURLs) {
+                    for (const routeURL of routeURLs) {
+                        const routeNodeId = generateId('Route', routeURL);
+                        ctx.graph.addRelationship({
+                            id: generateId('ENTRY_POINT_OF', `${routeNodeId}->${proc.id}`),
+                            sourceId: routeNodeId,
+                            targetId: proc.id,
+                            type: 'ENTRY_POINT_OF',
+                            confidence: 0.85,
+                            reason: 'route-handler-entry-point',
+                        });
+                        linked++;
+                    }
+                }
+                const exactToolNames = toolsByHandlerId.get(proc.entryPointId);
+                const fallbackToolNames = toolsWithoutHandlerByFile.get(entryFile);
+                const toolNames = exactToolNames ?? fallbackToolNames;
+                if (toolNames) {
+                    for (const toolName of toolNames) {
+                        const toolNodeId = generateId('Tool', toolName);
+                        ctx.graph.addRelationship({
+                            id: generateId('ENTRY_POINT_OF', `${toolNodeId}->${proc.id}`),
+                            sourceId: toolNodeId,
+                            targetId: proc.id,
+                            type: 'ENTRY_POINT_OF',
+                            confidence: 0.85,
+                            reason: 'tool-handler-entry-point',
+                        });
+                        linked++;
+                    }
+                }
+            }
+            if (isDev && linked > 0) {
+                logger.info(`🔗 Linked ${linked} Route/Tool nodes to execution flows`);
+            }
+        }
+        return { processResult };
+    },
+};

package/dist/core/ingestion/pipeline-phases/routes.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Phase: routes
+ *
+ * Builds the route registry (Next.js, Expo, PHP, Laravel, decorator-based)
+ * and creates Route graph nodes + HANDLES_ROUTE edges.
+ * Also links middleware, processes fetch() calls, and scans HTML templates.
+ *
+ * @deps    parse
+ * @reads   allPaths, allExtractedRoutes, allDecoratorRoutes, allFetchCalls
+ * @writes  graph (Route nodes, HANDLES_ROUTE, FETCHES_FROM edges)
+ * @output  routeRegistry, handlerContents
+ */
+import type { PipelinePhase } from './types.js';
+export interface RouteEntry {
+    filePath: string;
+    source: string;
+}
+export interface RoutesOutput {
+    routeRegistry: Map<string, RouteEntry>;
+}
+export declare const routesPhase: PipelinePhase<RoutesOutput>;

package/dist/core/ingestion/pipeline-phases/routes.js

@@ -0,0 +1,244 @@
+/**
+ * Phase: routes
+ *
+ * Builds the route registry (Next.js, Expo, PHP, Laravel, decorator-based)
+ * and creates Route graph nodes + HANDLES_ROUTE edges.
+ * Also links middleware, processes fetch() calls, and scans HTML templates.
+ *
+ * @deps    parse
+ * @reads   allPaths, allExtractedRoutes, allDecoratorRoutes, allFetchCalls
+ * @writes  graph (Route nodes, HANDLES_ROUTE, FETCHES_FROM edges)
+ * @output  routeRegistry, handlerContents
+ */
+import { getPhaseOutput } from './types.js';
+import { nextjsFileToRouteURL, normalizeFetchURL } from '../route-extractors/nextjs.js';
+import { expoFileToRouteURL } from '../route-extractors/expo.js';
+import { phpFileToRouteURL } from '../route-extractors/php.js';
+import { extractResponseShapes, extractPHPResponseShapes, } from '../route-extractors/response-shapes.js';
+import { extractMiddlewareChain, extractNextjsMiddlewareConfig, compileMatcher, compiledMatcherMatchesRoute, } from '../route-extractors/middleware.js';
+import { processNextjsFetchRoutes } from '../call-processor.js';
+import { generateId } from '../../../lib/utils.js';
+import { readFileContents } from '../filesystem-walker.js';
+import { isDev } from '../utils/env.js';
+import { logger } from '../../logger.js';
+const EXPO_NAV_PATTERNS = [
+    /router\.(push|replace|navigate)\(\s*['"`]([^'"`]+)['"`]/g,
+    /<Link\s+[^>]*href=\s*['"`]([^'"`]+)['"`]/g,
+];
+export const routesPhase = {
+    name: 'routes',
+    deps: ['parse'],
+    async execute(ctx, deps) {
+        const { allPaths, allFetchCalls: parseFetchCalls, allExtractedRoutes, allDecoratorRoutes, } = getPhaseOutput(deps, 'parse');
+        // Local copy — routes phase must not mutate upstream ParseOutput
+        const allFetchCalls = [...parseFetchCalls];
+        const routeRegistry = new Map();
+        // Detect Expo Router app/ roots vs Next.js app/ roots (monorepo-safe)
+        const expoAppRoots = new Set();
+        const nextjsAppRoots = new Set();
+        const expoAppPaths = new Set();
+        for (const p of allPaths) {
+            const norm = p.replace(/\\/g, '/');
+            const appIdx = norm.lastIndexOf('app/');
+            if (appIdx < 0)
+                continue;
+            const root = norm.slice(0, appIdx + 4);
+            if (/\/_layout\.(tsx?|jsx?)$/.test(norm))
+                expoAppRoots.add(root);
+            if (/\/page\.(tsx?|jsx?)$/.test(norm))
+                nextjsAppRoots.add(root);
+        }
+        for (const root of nextjsAppRoots)
+            expoAppRoots.delete(root);
+        if (expoAppRoots.size > 0) {
+            for (const p of allPaths) {
+                const norm = p.replace(/\\/g, '/');
+                const appIdx = norm.lastIndexOf('app/');
+                if (appIdx >= 0 && expoAppRoots.has(norm.slice(0, appIdx + 4)))
+                    expoAppPaths.add(p);
+            }
+        }
+        for (const p of allPaths) {
+            if (expoAppPaths.has(p)) {
+                const expoURL = expoFileToRouteURL(p);
+                if (expoURL && !routeRegistry.has(expoURL)) {
+                    routeRegistry.set(expoURL, { filePath: p, source: 'expo-filesystem-route' });
+                    continue;
+                }
+            }
+            const nextjsURL = nextjsFileToRouteURL(p);
+            if (nextjsURL && !routeRegistry.has(nextjsURL)) {
+                routeRegistry.set(nextjsURL, { filePath: p, source: 'nextjs-filesystem-route' });
+                continue;
+            }
+            if (p.endsWith('.php')) {
+                const phpURL = phpFileToRouteURL(p);
+                if (phpURL && !routeRegistry.has(phpURL)) {
+                    routeRegistry.set(phpURL, { filePath: p, source: 'php-file-route' });
+                }
+            }
+        }
+        const ensureSlash = (path) => (path.startsWith('/') ? path : '/' + path);
+        let duplicateRoutes = 0;
+        const addRoute = (url, entry) => {
+            if (routeRegistry.has(url)) {
+                duplicateRoutes++;
+                return;
+            }
+            routeRegistry.set(url, entry);
+        };
+        for (const route of allExtractedRoutes) {
+            if (!route.routePath)
+                continue;
+            addRoute(ensureSlash(route.routePath), {
+                filePath: route.filePath,
+                source: 'framework-route',
+            });
+        }
+        for (const dr of allDecoratorRoutes) {
+            addRoute(ensureSlash(dr.routePath), {
+                filePath: dr.filePath,
+                source: `decorator-${dr.decoratorName}`,
+            });
+        }
+        let handlerContents;
+        if (routeRegistry.size > 0) {
+            const handlerPaths = [...routeRegistry.values()].map((e) => e.filePath);
+            handlerContents = await readFileContents(ctx.repoPath, handlerPaths);
+            for (const [routeURL, entry] of routeRegistry) {
+                const { filePath: handlerPath, source: routeSource } = entry;
+                const content = handlerContents.get(handlerPath);
+                const { responseKeys, errorKeys } = content
+                    ? handlerPath.endsWith('.php')
+                        ? extractPHPResponseShapes(content)
+                        : extractResponseShapes(content)
+                    : { responseKeys: undefined, errorKeys: undefined };
+                const mwResult = content ? extractMiddlewareChain(content) : undefined;
+                const middleware = mwResult?.chain;
+                const routeNodeId = generateId('Route', routeURL);
+                ctx.graph.addNode({
+                    id: routeNodeId,
+                    label: 'Route',
+                    properties: {
+                        name: routeURL,
+                        filePath: handlerPath,
+                        ...(responseKeys ? { responseKeys } : {}),
+                        ...(errorKeys ? { errorKeys } : {}),
+                        ...(middleware && middleware.length > 0 ? { middleware } : {}),
+                    },
+                });
+                const handlerFileId = generateId('File', handlerPath);
+                ctx.graph.addRelationship({
+                    id: generateId('HANDLES_ROUTE', `${handlerFileId}->${routeNodeId}`),
+                    sourceId: handlerFileId,
+                    targetId: routeNodeId,
+                    type: 'HANDLES_ROUTE',
+                    confidence: 1.0,
+                    reason: routeSource,
+                });
+            }
+            if (isDev) {
+                logger.info(`🗺️ Route registry: ${routeRegistry.size} routes${duplicateRoutes > 0 ? ` (${duplicateRoutes} duplicate URLs skipped)` : ''}`);
+            }
+        }
+        // ── Link Next.js project-level middleware.ts to routes ──
+        if (routeRegistry.size > 0) {
+            const middlewareCandidates = allPaths.filter((p) => p === 'middleware.ts' ||
+                p === 'middleware.js' ||
+                p === 'middleware.tsx' ||
+                p === 'middleware.jsx' ||
+                p === 'src/middleware.ts' ||
+                p === 'src/middleware.js' ||
+                p === 'src/middleware.tsx' ||
+                p === 'src/middleware.jsx');
+            if (middlewareCandidates.length > 0) {
+                const mwContents = await readFileContents(ctx.repoPath, middlewareCandidates);
+                for (const [mwPath, mwContent] of mwContents) {
+                    const config = extractNextjsMiddlewareConfig(mwContent);
+                    if (!config)
+                        continue;
+                    const mwLabel = config.wrappedFunctions.length > 0 ? config.wrappedFunctions : [config.exportedName];
+                    const compiled = config.matchers
+                        .map(compileMatcher)
+                        .filter((m) => m !== null);
+                    let linkedCount = 0;
+                    for (const [routeURL] of routeRegistry) {
+                        const matches = compiled.length === 0 ||
+                            compiled.some((cm) => compiledMatcherMatchesRoute(cm, routeURL));
+                        if (!matches)
+                            continue;
+                        const routeNodeId = generateId('Route', routeURL);
+                        const existing = ctx.graph.getNode(routeNodeId);
+                        if (!existing)
+                            continue;
+                        const currentMw = existing.properties.middleware ?? [];
+                        existing.properties.middleware = [
+                            ...mwLabel,
+                            ...currentMw.filter((m) => !mwLabel.includes(m)),
+                        ];
+                        linkedCount++;
+                    }
+                    if (isDev && linkedCount > 0) {
+                        logger.info(`🛡️ Linked ${mwPath} middleware [${mwLabel.join(', ')}] to ${linkedCount} routes`);
+                    }
+                }
+            }
+        }
+        // Scan HTML/template files for form action and AJAX url patterns
+        const htmlCandidates = allPaths.filter((p) => p.endsWith('.html') ||
+            p.endsWith('.htm') ||
+            p.endsWith('.ejs') ||
+            p.endsWith('.hbs') ||
+            p.endsWith('.blade.php'));
+        if (htmlCandidates.length > 0 && routeRegistry.size > 0) {
+            const htmlContents = await readFileContents(ctx.repoPath, htmlCandidates);
+            const htmlPatterns = [/action=["']([^"']+)["']/g, /url:\s*["']([^"']+)["']/g];
+            for (const [filePath, content] of htmlContents) {
+                for (const pattern of htmlPatterns) {
+                    pattern.lastIndex = 0;
+                    let match;
+                    while ((match = pattern.exec(content)) !== null) {
+                        const normalized = normalizeFetchURL(match[1]);
+                        if (normalized) {
+                            allFetchCalls.push({ filePath, fetchURL: normalized, lineNumber: 0 });
+                        }
+                    }
+                }
+            }
+        }
+        // ── Extract Expo Router navigation patterns ──
+        if (expoAppPaths.size > 0 && routeRegistry.size > 0) {
+            const unreadExpoPaths = [...expoAppPaths].filter((p) => !handlerContents?.has(p));
+            const extraContents = unreadExpoPaths.length > 0
+                ? await readFileContents(ctx.repoPath, unreadExpoPaths)
+                : new Map();
+            const allExpoContents = new Map([...(handlerContents ?? new Map()), ...extraContents]);
+            for (const [filePath, content] of allExpoContents) {
+                if (!expoAppPaths.has(filePath))
+                    continue;
+                for (const pattern of EXPO_NAV_PATTERNS) {
+                    pattern.lastIndex = 0;
+                    let match;
+                    while ((match = pattern.exec(content)) !== null) {
+                        const url = match[2] ?? match[1];
+                        if (url && url.startsWith('/')) {
+                            allFetchCalls.push({ filePath, fetchURL: url, lineNumber: 0 });
+                        }
+                    }
+                }
+            }
+        }
+        if (routeRegistry.size > 0 && allFetchCalls.length > 0) {
+            const routeURLToFile = new Map();
+            for (const [url, entry] of routeRegistry)
+                routeURLToFile.set(url, entry.filePath);
+            const consumerPaths = [...new Set(allFetchCalls.map((c) => c.filePath))];
+            const consumerContents = await readFileContents(ctx.repoPath, consumerPaths);
+            processNextjsFetchRoutes(ctx.graph, allFetchCalls, routeURLToFile, consumerContents);
+            if (isDev) {
+                logger.info(`🔗 Processed ${allFetchCalls.length} fetch() calls against ${routeRegistry.size} routes`);
+            }
+        }
+        return { routeRegistry };
+    },
+};

package/dist/core/ingestion/pipeline-phases/runner.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Pipeline Phase Runner
+ *
+ * Executes pipeline phases in dependency order using Kahn's topological sort.
+ * Each phase receives typed outputs from its upstream dependencies.
+ *
+ * The runner is intentionally simple:
+ * - No dynamic phase loading
+ * - No plugin system
+ * - Static phase graph, compile-time type safety
+ * - Sequential execution (parallel support is architecturally possible
+ *   but most phases have linear dependencies)
+ */
+import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
+/**
+ * Execute a set of pipeline phases in dependency order.
+ *
+ * @param phases  All phases to execute (order doesn't matter — sorted internally)
+ * @param ctx     Shared pipeline context
+ * @returns       Map of phase name → PhaseResult (all completed phases)
+ */
+export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;