arceus-s 1.6.4

package/dist/core/ingestion/cobol/jcl-parser.d.ts

@@ -0,0 +1,68 @@
+/**
+ * JCL Parser — Regex single-pass extraction.
+ *
+ * Extracts JCL constructs from mainframe job streams:
+ * - JOB statements (job name, CLASS, MSGCLASS)
+ * - EXEC statements (step -> program or proc)
+ * - DD statements (dataset references, DISP)
+ * - PROC definitions (in-stream and catalogued)
+ * - INCLUDE MEMBER= directives
+ * - SET symbolic parameters
+ * - IF/ELSE/ENDIF conditional execution
+ * - JCLLIB ORDER= search paths
+ *
+ * Pattern follows cobol-preprocessor.ts — regex-only, no tree-sitter.
+ */
+export interface JclParseResults {
+    jobs: Array<{
+        name: string;
+        line: number;
+        class?: string;
+        msgclass?: string;
+    }>;
+    steps: Array<{
+        name: string;
+        jobName: string;
+        program?: string;
+        proc?: string;
+        line: number;
+    }>;
+    ddStatements: Array<{
+        ddName: string;
+        stepName: string;
+        dataset?: string;
+        disp?: string;
+        line: number;
+    }>;
+    procs: Array<{
+        name: string;
+        line: number;
+        isInStream: boolean;
+    }>;
+    includes: Array<{
+        member: string;
+        line: number;
+    }>;
+    sets: Array<{
+        variable: string;
+        value: string;
+        line: number;
+    }>;
+    jcllib: Array<{
+        order: string[];
+        line: number;
+    }>;
+    conditionals: Array<{
+        type: 'IF' | 'ELSE' | 'ENDIF';
+        condition?: string;
+        line: number;
+    }>;
+}
+/**
+ * Parse a JCL file and extract all constructs.
+ *
+ * @param content - Raw JCL file content
+ * @param filePath - Path for diagnostics (not used in extraction)
+ * @returns Parsed JCL results
+ */
+export declare function parseJcl(content: string, filePath: string): JclParseResults;

package/dist/core/ingestion/cobol/jcl-parser.js

@@ -0,0 +1,217 @@
+/**
+ * JCL Parser — Regex single-pass extraction.
+ *
+ * Extracts JCL constructs from mainframe job streams:
+ * - JOB statements (job name, CLASS, MSGCLASS)
+ * - EXEC statements (step -> program or proc)
+ * - DD statements (dataset references, DISP)
+ * - PROC definitions (in-stream and catalogued)
+ * - INCLUDE MEMBER= directives
+ * - SET symbolic parameters
+ * - IF/ELSE/ENDIF conditional execution
+ * - JCLLIB ORDER= search paths
+ *
+ * Pattern follows cobol-preprocessor.ts — regex-only, no tree-sitter.
+ */
+// ── JCL statement patterns ─────────────────────────────────────────────
+// JCL continuation: line ends with a non-blank in col 72, next line starts with //
+// We handle continuations by joining lines before matching.
+/** Match //jobname JOB ... */
+const JOB_RE = /^\/\/(\w{1,8})\s+JOB\s+(.*)/i;
+/** Match //stepname EXEC PGM=program or //stepname EXEC procname */
+const EXEC_RE = /^\/\/(\w{1,8})\s+EXEC\s+(.*)/i;
+/** Match //ddname DD ... */
+const DD_RE = /^\/\/(\w{1,8})\s+DD\s+(.*)/i;
+/** Match // JCLLIB ORDER=(lib1,lib2,...) */
+const JCLLIB_RE = /^\/\/\s+JCLLIB\s+ORDER=\(([^)]+)\)/i;
+/** Match // IF condition THEN */
+const IF_RE = /^\/\/\s+IF\s+(.+)\s+THEN/i;
+/** Match // ELSE */
+const ELSE_RE = /^\/\/\s+ELSE\b/i;
+/** Match // ENDIF */
+const ENDIF_RE = /^\/\/\s+ENDIF\b/i;
+/** Match // INCLUDE MEMBER=name */
+const INCLUDE_RE = /^\/\/\s+INCLUDE\s+MEMBER=(\w+)/i;
+/** Match // SET var=value */
+const SET_RE = /^\/\/\s+SET\s+(\w+)=(.+)/i;
+/** Match // PROC or //name PROC */
+const PROC_RE = /^\/\/(\w*)\s+PROC\b/i;
+/** Match // PEND */
+const PEND_RE = /^\/\/\s+PEND\b/i;
+// ── Parameter extractors ───────────────────────────────────────────────
+function extractParam(params, key) {
+    // Match KEY=VALUE or KEY='VALUE' in JCL parameter string
+    const re = new RegExp(`${key}=(?:'([^']*)'|(\\S+?))(?:[,\\s]|$)`, 'i');
+    const m = params.match(re);
+    return m ? (m[1] ?? m[2]) : undefined;
+}
+function extractPgm(params) {
+    return extractParam(params, 'PGM');
+}
+function extractProc(params) {
+    // If no PGM= keyword, the first positional parameter is the proc name
+    if (/PGM=/i.test(params))
+        return undefined;
+    const cleaned = params.replace(/,.*/, '').trim();
+    // Proc name is the first token (no = sign)
+    if (cleaned && !cleaned.includes('=')) {
+        return cleaned.replace(/[,\s].*/s, '').toUpperCase();
+    }
+    return undefined;
+}
+function extractDsn(params) {
+    return extractParam(params, 'DSN') ?? extractParam(params, 'DSNAME');
+}
+function extractDisp(params) {
+    const m = params.match(/DISP=\(?\s*([^),\s]+)/i);
+    return m ? m[1] : undefined;
+}
+/**
+ * Parse a JCL file and extract all constructs.
+ *
+ * @param content - Raw JCL file content
+ * @param filePath - Path for diagnostics (not used in extraction)
+ * @returns Parsed JCL results
+ */
+export function parseJcl(content, filePath) {
+    const results = {
+        jobs: [],
+        steps: [],
+        ddStatements: [],
+        procs: [],
+        includes: [],
+        sets: [],
+        jcllib: [],
+        conditionals: [],
+    };
+    const rawLines = content.split(/\r?\n/);
+    // Join continuation lines: a line ending with non-blank in col 71 (0-indexed)
+    // followed by a line starting with // is a continuation.
+    const lines = [];
+    let i = 0;
+    while (i < rawLines.length) {
+        let line = rawLines[i];
+        const lineNum = i + 1;
+        // JCL continuation: if line is exactly 72+ chars and col 72 is non-blank
+        // and the next line starts with //, join them.
+        while (i + 1 < rawLines.length &&
+            line.length >= 72 &&
+            line[71] !== ' ' &&
+            rawLines[i + 1].startsWith('//')) {
+            i++;
+            // Continuation text starts after // and leading spaces
+            const contText = rawLines[i].substring(2).replace(/^\s+/, ' ');
+            // Remove the continuation marker (col 72+) from current line
+            line = line.substring(0, 71).trimEnd() + contText;
+        }
+        lines.push({ text: line, lineNum });
+        i++;
+    }
+    let currentJobName = '';
+    let currentStepName = '';
+    let inStreamProcName = '';
+    for (const { text, lineNum } of lines) {
+        // Skip JCL comments (starting with //* )
+        if (text.startsWith('//*'))
+            continue;
+        // Skip non-JCL lines (don't start with //)
+        if (!text.startsWith('//'))
+            continue;
+        // PROC definition (in-stream)
+        const procMatch = text.match(PROC_RE);
+        if (procMatch) {
+            const procName = procMatch[1] || inStreamProcName;
+            if (procName) {
+                results.procs.push({ name: procName.toUpperCase(), line: lineNum, isInStream: true });
+            }
+            inStreamProcName = procName?.toUpperCase() || '';
+            continue;
+        }
+        // PEND (end of in-stream proc)
+        if (PEND_RE.test(text)) {
+            inStreamProcName = '';
+            continue;
+        }
+        // JCLLIB ORDER=
+        const jcllibMatch = text.match(JCLLIB_RE);
+        if (jcllibMatch) {
+            const libs = jcllibMatch[1].split(',').map((s) => s.trim().replace(/'/g, ''));
+            results.jcllib.push({ order: libs, line: lineNum });
+            continue;
+        }
+        // IF/ELSE/ENDIF
+        const ifMatch = text.match(IF_RE);
+        if (ifMatch) {
+            results.conditionals.push({ type: 'IF', condition: ifMatch[1].trim(), line: lineNum });
+            continue;
+        }
+        if (ELSE_RE.test(text)) {
+            results.conditionals.push({ type: 'ELSE', line: lineNum });
+            continue;
+        }
+        if (ENDIF_RE.test(text)) {
+            results.conditionals.push({ type: 'ENDIF', line: lineNum });
+            continue;
+        }
+        // INCLUDE MEMBER=
+        const includeMatch = text.match(INCLUDE_RE);
+        if (includeMatch) {
+            results.includes.push({ member: includeMatch[1].toUpperCase(), line: lineNum });
+            continue;
+        }
+        // SET var=value
+        const setMatch = text.match(SET_RE);
+        if (setMatch) {
+            results.sets.push({
+                variable: setMatch[1].toUpperCase(),
+                value: setMatch[2].trim().replace(/,\s*$/, ''),
+                line: lineNum,
+            });
+            continue;
+        }
+        // JOB statement
+        const jobMatch = text.match(JOB_RE);
+        if (jobMatch) {
+            currentJobName = jobMatch[1].toUpperCase();
+            const params = jobMatch[2];
+            results.jobs.push({
+                name: currentJobName,
+                line: lineNum,
+                class: extractParam(params, 'CLASS'),
+                msgclass: extractParam(params, 'MSGCLASS'),
+            });
+            continue;
+        }
+        // EXEC statement
+        const execMatch = text.match(EXEC_RE);
+        if (execMatch) {
+            currentStepName = execMatch[1].toUpperCase();
+            const params = execMatch[2];
+            const pgm = extractPgm(params);
+            const proc = pgm ? undefined : extractProc(params);
+            results.steps.push({
+                name: currentStepName,
+                jobName: currentJobName,
+                program: pgm?.toUpperCase(),
+                proc: proc?.toUpperCase(),
+                line: lineNum,
+            });
+            continue;
+        }
+        // DD statement
+        const ddMatch = text.match(DD_RE);
+        if (ddMatch) {
+            const ddName = ddMatch[1].toUpperCase();
+            const params = ddMatch[2];
+            results.ddStatements.push({
+                ddName,
+                stepName: currentStepName,
+                dataset: extractDsn(params)?.toUpperCase(),
+                disp: extractDisp(params)?.toUpperCase(),
+                line: lineNum,
+            });
+            continue;
+        }
+    }
+    return results;
+}

package/dist/core/ingestion/cobol/jcl-processor.d.ts

@@ -0,0 +1,33 @@
+/**
+ * JCL Processor — Converts JCL parse results into graph nodes and edges.
+ *
+ * Maps JCL entities to existing graph types (no new tables):
+ * - Job    -> CodeElement (description: "jcl-job class:A msgclass:X")
+ * - Step   -> CodeElement (description: "jcl-step pgm:PROGRAMNAME")
+ * - Dataset -> CodeElement (description: "jcl-dataset disp:SHR")
+ * - PROC   -> Module
+ *
+ * Edges:
+ * - Job CONTAINS Step
+ * - Step CALLS Module (when PGM= matches an indexed program)
+ * - Step references Dataset (CALLS edge with reason "jcl-dd")
+ * - Job/Step IMPORTS PROC
+ *
+ * Pattern follows detectCrossProgamContracts() in pipeline.ts.
+ */
+import type { KnowledgeGraph } from '../../graph/types.js';
+export interface JclProcessResult {
+    jobCount: number;
+    stepCount: number;
+    datasetCount: number;
+    programLinks: number;
+}
+/**
+ * Process JCL files and integrate into the knowledge graph.
+ *
+ * @param graph - The in-memory knowledge graph
+ * @param jclPaths - File paths of JCL files
+ * @param jclContents - Map of path -> file content
+ * @returns Summary of what was added
+ */
+export declare function processJclFiles(graph: KnowledgeGraph, jclPaths: string[], jclContents: Map<string, string>): JclProcessResult;

package/dist/core/ingestion/cobol/jcl-processor.js

@@ -0,0 +1,229 @@
+/**
+ * JCL Processor — Converts JCL parse results into graph nodes and edges.
+ *
+ * Maps JCL entities to existing graph types (no new tables):
+ * - Job    -> CodeElement (description: "jcl-job class:A msgclass:X")
+ * - Step   -> CodeElement (description: "jcl-step pgm:PROGRAMNAME")
+ * - Dataset -> CodeElement (description: "jcl-dataset disp:SHR")
+ * - PROC   -> Module
+ *
+ * Edges:
+ * - Job CONTAINS Step
+ * - Step CALLS Module (when PGM= matches an indexed program)
+ * - Step references Dataset (CALLS edge with reason "jcl-dd")
+ * - Job/Step IMPORTS PROC
+ *
+ * Pattern follows detectCrossProgamContracts() in pipeline.ts.
+ */
+import { parseJcl } from './jcl-parser.js';
+import { generateId } from '../../../lib/utils.js';
+/**
+ * Process JCL files and integrate into the knowledge graph.
+ *
+ * @param graph - The in-memory knowledge graph
+ * @param jclPaths - File paths of JCL files
+ * @param jclContents - Map of path -> file content
+ * @returns Summary of what was added
+ */
+export function processJclFiles(graph, jclPaths, jclContents) {
+    let jobCount = 0;
+    let stepCount = 0;
+    let datasetCount = 0;
+    let programLinks = 0;
+    // Collect all Module names for step -> program linking
+    const moduleNames = new Map(); // uppercase name -> node id
+    graph.forEachNode((node) => {
+        if (node.label === 'Module') {
+            const nodeName = node.properties.name;
+            if (typeof nodeName === 'string') {
+                moduleNames.set(nodeName.toUpperCase(), node.id);
+            }
+        }
+    });
+    for (const filePath of jclPaths) {
+        const content = jclContents.get(filePath);
+        if (!content)
+            continue;
+        const parsed = parseJcl(content, filePath);
+        const result = integrateJclResults(graph, parsed, filePath, moduleNames);
+        jobCount += result.jobCount;
+        stepCount += result.stepCount;
+        datasetCount += result.datasetCount;
+        programLinks += result.programLinks;
+    }
+    return { jobCount, stepCount, datasetCount, programLinks };
+}
+function integrateJclResults(graph, parsed, filePath, moduleNames) {
+    let jobCount = 0;
+    let stepCount = 0;
+    let datasetCount = 0;
+    let programLinks = 0;
+    // Track step node IDs for DD -> step linking
+    const stepNodeIds = new Map(); // stepName -> nodeId
+    // 1. Create Job nodes
+    for (const job of parsed.jobs) {
+        const jobId = generateId('CodeElement', `${filePath}:job:${job.name}`);
+        const classPart = job.class ? ` class:${job.class}` : '';
+        const msgPart = job.msgclass ? ` msgclass:${job.msgclass}` : '';
+        graph.addNode({
+            id: jobId,
+            label: 'CodeElement',
+            properties: {
+                name: job.name,
+                filePath,
+                startLine: job.line,
+                endLine: job.line,
+                description: `jcl-job${classPart}${msgPart}`,
+            },
+        });
+        // Link File -> Job (CONTAINS)
+        const fileId = generateId('File', filePath);
+        graph.addRelationship({
+            id: `${fileId}_contains_${jobId}`,
+            type: 'CONTAINS',
+            sourceId: fileId,
+            targetId: jobId,
+            confidence: 1.0,
+            reason: 'jcl-job',
+        });
+        jobCount++;
+    }
+    // 1.5 Pre-register in-stream PROCs so steps can reference them
+    // (fixes ordering bug: steps processed before PROCs were registered)
+    for (const proc of parsed.procs) {
+        const procId = generateId('Module', `${filePath}:proc:${proc.name}`);
+        moduleNames.set(proc.name.toUpperCase(), procId);
+    }
+    // 2. Create Step nodes and link to programs
+    for (const step of parsed.steps) {
+        const stepId = generateId('CodeElement', `${filePath}:step:${step.jobName}:${step.name}`);
+        const pgmPart = step.program ? ` pgm:${step.program}` : '';
+        const procPart = step.proc ? ` proc:${step.proc}` : '';
+        graph.addNode({
+            id: stepId,
+            label: 'CodeElement',
+            properties: {
+                name: step.name,
+                filePath,
+                startLine: step.line,
+                endLine: step.line,
+                description: `jcl-step${pgmPart}${procPart}`,
+            },
+        });
+        stepNodeIds.set(step.name, stepId);
+        // Link Job -> Step (CONTAINS)
+        if (step.jobName) {
+            const jobId = generateId('CodeElement', `${filePath}:job:${step.jobName}`);
+            graph.addRelationship({
+                id: `${jobId}_contains_${stepId}`,
+                type: 'CONTAINS',
+                sourceId: jobId,
+                targetId: stepId,
+                confidence: 1.0,
+                reason: 'jcl-step',
+            });
+        }
+        // Link Step -> Module (CALLS) when PGM= matches an indexed program
+        if (step.program) {
+            const moduleId = moduleNames.get(step.program.toUpperCase());
+            if (moduleId) {
+                graph.addRelationship({
+                    id: `${stepId}_calls_${moduleId}`,
+                    type: 'CALLS',
+                    sourceId: stepId,
+                    targetId: moduleId,
+                    confidence: 0.95,
+                    reason: 'jcl-exec-pgm',
+                });
+                programLinks++;
+            }
+        }
+        // Link Step -> PROC (CALLS) — PROC as Module
+        if (step.proc) {
+            const procModuleId = moduleNames.get(step.proc.toUpperCase());
+            if (procModuleId) {
+                graph.addRelationship({
+                    id: `${stepId}_calls_proc_${procModuleId}`,
+                    type: 'CALLS',
+                    sourceId: stepId,
+                    targetId: procModuleId,
+                    confidence: 0.9,
+                    reason: 'jcl-exec-proc',
+                });
+            }
+        }
+        stepCount++;
+    }
+    // 3. Create Dataset nodes from DD statements
+    const seenDatasets = new Set();
+    for (const dd of parsed.ddStatements) {
+        if (!dd.dataset)
+            continue;
+        // Create dataset node (deduplicated per file)
+        const datasetKey = `${filePath}:dataset:${dd.dataset}`;
+        const datasetId = generateId('CodeElement', datasetKey);
+        if (!seenDatasets.has(dd.dataset)) {
+            const dispPart = dd.disp ? ` disp:${dd.disp}` : '';
+            graph.addNode({
+                id: datasetId,
+                label: 'CodeElement',
+                properties: {
+                    name: dd.dataset,
+                    filePath,
+                    startLine: dd.line,
+                    endLine: dd.line,
+                    description: `jcl-dataset${dispPart}`,
+                },
+            });
+            seenDatasets.add(dd.dataset);
+            datasetCount++;
+        }
+        // Link Step -> Dataset (CALLS with reason jcl-dd)
+        const stepId = stepNodeIds.get(dd.stepName);
+        if (stepId) {
+            graph.addRelationship({
+                id: `${stepId}_dd_${dd.ddName}_${datasetId}`,
+                type: 'CALLS',
+                sourceId: stepId,
+                targetId: datasetId,
+                confidence: 0.85,
+                reason: `jcl-dd:${dd.ddName}`,
+            });
+        }
+    }
+    // 4. Create PROC nodes (in-stream procs as Module)
+    for (const proc of parsed.procs) {
+        if (!proc.isInStream)
+            continue;
+        const procId = generateId('Module', `${filePath}:proc:${proc.name}`);
+        graph.addNode({
+            id: procId,
+            label: 'Module',
+            properties: {
+                name: proc.name,
+                filePath,
+                startLine: proc.line,
+                endLine: proc.line,
+                description: 'jcl-proc-instream',
+            },
+        });
+        // Register for step linking
+        moduleNames.set(proc.name.toUpperCase(), procId);
+    }
+    // 5. INCLUDE directives -> IMPORTS edges
+    for (const inc of parsed.includes) {
+        const moduleId = moduleNames.get(inc.member.toUpperCase());
+        if (moduleId) {
+            const fileId = generateId('File', filePath);
+            graph.addRelationship({
+                id: `${fileId}_includes_${moduleId}`,
+                type: 'IMPORTS',
+                sourceId: fileId,
+                targetId: moduleId,
+                confidence: 0.9,
+                reason: 'jcl-include',
+            });
+        }
+    }
+    return { jobCount, stepCount, datasetCount, programLinks };
+}

package/dist/core/ingestion/cobol-processor.d.ts

@@ -0,0 +1,54 @@
+/**
+ * COBOL Processor
+ *
+ * Standalone regex-based processor for COBOL and JCL files.
+ * Follows the markdown-processor.ts pattern: takes (graph, files, allPathSet),
+ * does its own extraction, and writes directly to the graph.
+ *
+ * Pipeline:
+ *   1. Separate programs from copybooks
+ *   2. Build copybook map (name -> content)
+ *   3. For each program: expand COPY statements, then run regex extraction
+ *   4. Map CobolRegexResults to graph nodes and relationships
+ *   5. Optionally process JCL files for job-step cross-references
+ */
+import type { KnowledgeGraph } from '../graph/types.js';
+interface CobolFile {
+    path: string;
+    content: string;
+}
+export interface CobolProcessResult {
+    programs: number;
+    paragraphs: number;
+    sections: number;
+    dataItems: number;
+    calls: number;
+    copies: number;
+    execSqlBlocks: number;
+    execCicsBlocks: number;
+    entryPoints: number;
+    moves: number;
+    fileDeclarations: number;
+    jclJobs: number;
+    jclSteps: number;
+    sqlIncludes: number;
+    execDliBlocks: number;
+    declaratives: number;
+    sets: number;
+    inspects: number;
+    initializes: number;
+}
+/** Returns true if the file is a COBOL or copybook file. */
+export declare function isCobolFile(filePath: string): boolean;
+/** Returns true if the file is a JCL file. */
+export declare function isJclFile(filePath: string): boolean;
+/**
+ * Process COBOL and JCL files into the knowledge graph.
+ *
+ * @param graph    - The in-memory knowledge graph
+ * @param files    - Array of { path, content } for COBOL/JCL files
+ * @param allPathSet - Set of all file paths in the repository
+ * @returns Summary of what was extracted
+ */
+export declare const processCobol: (graph: KnowledgeGraph, files: CobolFile[], allPathSet: ReadonlySet<string>) => CobolProcessResult;
+export {};