cobolx 1.0.0

package/src/agent.rs

@@ -0,0 +1 @@
+pub mod client;

package/src/cobol/copybook.rs

@@ -0,0 +1,71 @@
+use crate::cobol::scanner::{CobolFileEntry, CobolFileType};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+pub(crate) fn build_copybook_index(files: &[CobolFileEntry]) -> HashMap<String, Vec<PathBuf>> {
+    let mut index = HashMap::<String, Vec<PathBuf>>::new();
+    for file in files {
+        if file.file_type != CobolFileType::Copybook {
+            continue;
+        }
+        if let Some(stem) = file.path.file_stem().and_then(|s| s.to_str()) {
+            index
+                .entry(stem.to_ascii_uppercase())
+                .or_default()
+                .push(file.path.clone());
+        }
+    }
+    for paths in index.values_mut() {
+        paths.sort();
+    }
+    index
+}
+
+pub(crate) fn resolve_copybook(
+    root: &Path,
+    from_file: &Path,
+    name: &str,
+    copybook_index: &HashMap<String, Vec<PathBuf>>,
+) -> Option<PathBuf> {
+    let mut dirs = Vec::with_capacity(2);
+    if let Some(parent) = from_file.parent() {
+        dirs.push(parent.to_path_buf());
+    }
+    dirs.push(root.to_path_buf());
+
+    for dir in dirs {
+        for candidate in candidate_copybook_names(name) {
+            let path = dir.join(&candidate);
+            if path.is_file() {
+                return Some(path);
+            }
+            if let Some(found) = find_case_insensitive(&dir, &candidate) {
+                return Some(found);
+            }
+        }
+    }
+
+    copybook_index
+        .get(&name.to_ascii_uppercase())
+        .and_then(|paths| paths.first().cloned())
+}
+
+fn candidate_copybook_names(name: &str) -> Vec<String> {
+    if Path::new(name).extension().is_some() {
+        vec![name.to_string()]
+    } else {
+        vec![name.to_string(), format!("{}.cpy", name)]
+    }
+}
+
+fn find_case_insensitive(dir: &Path, file_name: &str) -> Option<PathBuf> {
+    let target = file_name.to_ascii_uppercase();
+    let entries = std::fs::read_dir(dir).ok()?;
+    for entry in entries.flatten() {
+        let name = entry.file_name();
+        if name.to_string_lossy().to_ascii_uppercase() == target {
+            return Some(entry.path());
+        }
+    }
+    None
+}

package/src/cobol/data_parser.rs

@@ -0,0 +1,290 @@
+use crate::cobol::copybook::resolve_copybook;
+use crate::cobol::lexer::{clean_name, logical_lines, tokenize};
+use crate::cobol::model::{LogicalLine, ParsedDataItem, Token};
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+#[derive(Debug, Default)]
+struct DataParseState {
+    in_data_division: bool,
+    section: Option<String>,
+    parent_stack: Vec<(u16, String)>,
+    items: Vec<ParsedDataItem>,
+}
+
+pub(crate) fn collect_data_items(
+    root: &Path,
+    path: &Path,
+    copybook_index: &HashMap<String, Vec<PathBuf>>,
+    depth: usize,
+) -> std::io::Result<Vec<ParsedDataItem>> {
+    let mut state = DataParseState::default();
+    parse_data_file(root, path, copybook_index, depth, false, &mut state)?;
+    Ok(state.items)
+}
+
+fn parse_data_file(
+    root: &Path,
+    path: &Path,
+    copybook_index: &HashMap<String, Vec<PathBuf>>,
+    depth: usize,
+    is_copybook: bool,
+    state: &mut DataParseState,
+) -> std::io::Result<()> {
+    if depth > 16 {
+        return Ok(());
+    }
+
+    let content = std::fs::read_to_string(path)?;
+    let mut copybook_local_state;
+    let state = if is_copybook && !state.in_data_division {
+        copybook_local_state = DataParseState {
+            in_data_division: true,
+            section: None,
+            parent_stack: Vec::new(),
+            items: Vec::new(),
+        };
+        &mut copybook_local_state
+    } else {
+        state
+    };
+
+    for line in logical_lines(&content) {
+        let tokens = tokenize(&line.text);
+        if tokens.is_empty() {
+            continue;
+        }
+
+        if has_two_tokens(&tokens, "DATA", "DIVISION") {
+            state.in_data_division = true;
+            state.parent_stack.clear();
+            continue;
+        }
+        if has_two_tokens(&tokens, "PROCEDURE", "DIVISION") {
+            if !is_copybook {
+                break;
+            }
+            continue;
+        }
+        if !state.in_data_division && !is_copybook {
+            continue;
+        }
+        if is_section_line(&tokens) {
+            state.section = Some(tokens[0].text.clone());
+            state.parent_stack.clear();
+            continue;
+        }
+        if tokens[0].text == "COPY" {
+            if let Some(copy_name) = tokens.get(1).map(|t| clean_name(&t.text)) {
+                if let Some(copy_path) = resolve_copybook(root, path, &copy_name, copybook_index) {
+                    parse_data_file(root, &copy_path, copybook_index, depth + 1, true, state)?;
+                }
+            }
+            continue;
+        }
+
+        if let Some(item) = parse_data_item_line(path, &line, &tokens, state) {
+            state.items.push(item);
+        }
+    }
+
+    Ok(())
+}
+
+fn parse_data_item_line(
+    path: &Path,
+    line: &LogicalLine,
+    tokens: &[Token],
+    state: &mut DataParseState,
+) -> Option<ParsedDataItem> {
+    let level = tokens.first()?.text.parse::<u16>().ok()?;
+    if !is_data_level(level) {
+        return None;
+    }
+    let name = tokens.get(1).map(|t| clean_name(&t.text))?;
+    if name.is_empty() {
+        return None;
+    }
+
+    while state
+        .parent_stack
+        .last()
+        .is_some_and(|(parent_level, _)| *parent_level >= level)
+    {
+        state.parent_stack.pop();
+    }
+    let parent_name = if matches!(level, 1 | 66 | 77) {
+        None
+    } else {
+        state.parent_stack.last().map(|(_, name)| name.clone())
+    };
+
+    let pic = extract_clause_text(
+        &line.text,
+        tokens,
+        &["PIC", "PICTURE"],
+        &[
+            "USAGE",
+            "OCCURS",
+            "REDEFINES",
+            "VALUE",
+            "VALUES",
+            "SIGN",
+            "SYNC",
+            "SYNCHRONIZED",
+            "JUST",
+            "JUSTIFIED",
+            "DISPLAY",
+            "BINARY",
+            "COMP",
+            "COMP-1",
+            "COMP-2",
+            "COMP-3",
+            "COMP-4",
+            "COMP-5",
+            "COMPUTATIONAL",
+            "COMPUTATIONAL-1",
+            "COMPUTATIONAL-2",
+            "COMPUTATIONAL-3",
+            "COMPUTATIONAL-4",
+            "COMPUTATIONAL-5",
+            "PACKED-DECIMAL",
+            "INDEX",
+            "POINTER",
+            "NATIONAL",
+        ],
+    );
+    let usage_clause = extract_usage_clause(&line.text, tokens);
+    let occurs = extract_occurs(tokens);
+    let redefines = extract_next_name(tokens, "REDEFINES");
+
+    if !matches!(level, 66 | 88) {
+        state.parent_stack.push((level, name.clone()));
+    }
+
+    Some(ParsedDataItem {
+        source_path: path.to_path_buf(),
+        name,
+        level,
+        parent_name,
+        pic,
+        usage_clause,
+        occurs,
+        redefines,
+        section: state.section.clone(),
+        byte_offset: None,
+        byte_size: None,
+        storage_kind: None,
+        layout_status: None,
+        start_offset: line.start_offset + tokens[0].start,
+        byte_len: line.byte_len,
+    })
+}
+
+fn is_data_level(level: u16) -> bool {
+    (1..=49).contains(&level) || matches!(level, 66 | 77 | 88)
+}
+
+fn has_two_tokens(tokens: &[Token], first: &str, second: &str) -> bool {
+    tokens.len() >= 2 && tokens[0].text == first && tokens[1].text == second
+}
+
+fn is_section_line(tokens: &[Token]) -> bool {
+    tokens.len() >= 2
+        && tokens[1].text == "SECTION"
+        && matches!(
+            tokens[0].text.as_str(),
+            "FILE" | "WORKING-STORAGE" | "LOCAL-STORAGE" | "LINKAGE"
+        )
+}
+
+fn extract_clause_text(
+    line: &str,
+    tokens: &[Token],
+    names: &[&str],
+    stop_keywords: &[&str],
+) -> Option<String> {
+    let idx = tokens
+        .iter()
+        .position(|t| names.iter().any(|name| t.text == *name))?;
+    let mut start_idx = idx + 1;
+    if tokens.get(start_idx).is_some_and(|t| t.text == "IS") {
+        start_idx += 1;
+    }
+    let start = tokens.get(start_idx)?.start;
+    let end = tokens[start_idx..]
+        .iter()
+        .find(|t| stop_keywords.iter().any(|keyword| t.text == *keyword))
+        .map(|t| t.start)
+        .unwrap_or_else(|| line.len());
+    Some(
+        line[start..end]
+            .trim()
+            .trim_end_matches('.')
+            .trim()
+            .to_ascii_uppercase(),
+    )
+    .filter(|s| !s.is_empty())
+}
+
+fn extract_usage_clause(line: &str, tokens: &[Token]) -> Option<String> {
+    let explicit = extract_clause_text(
+        line,
+        tokens,
+        &["USAGE"],
+        &[
+            "OCCURS",
+            "REDEFINES",
+            "VALUE",
+            "VALUES",
+            "SIGN",
+            "SYNC",
+            "SYNCHRONIZED",
+            "JUST",
+            "JUSTIFIED",
+        ],
+    );
+    if explicit.is_some() {
+        return explicit;
+    }
+
+    tokens
+        .iter()
+        .find(|t| {
+            matches!(
+                t.text.as_str(),
+                "DISPLAY"
+                    | "BINARY"
+                    | "COMP"
+                    | "COMP-1"
+                    | "COMP-2"
+                    | "COMP-3"
+                    | "COMP-4"
+                    | "COMP-5"
+                    | "COMPUTATIONAL"
+                    | "COMPUTATIONAL-1"
+                    | "COMPUTATIONAL-2"
+                    | "COMPUTATIONAL-3"
+                    | "COMPUTATIONAL-4"
+                    | "COMPUTATIONAL-5"
+                    | "PACKED-DECIMAL"
+                    | "INDEX"
+                    | "POINTER"
+                    | "NATIONAL"
+            )
+        })
+        .map(|t| t.text.clone())
+}
+
+fn extract_occurs(tokens: &[Token]) -> Option<i64> {
+    let idx = tokens.iter().position(|t| t.text == "OCCURS")?;
+    tokens
+        .iter()
+        .skip(idx + 1)
+        .find_map(|t| t.text.parse::<i64>().ok())
+}
+
+fn extract_next_name(tokens: &[Token], keyword: &str) -> Option<String> {
+    let idx = tokens.iter().position(|t| t.text == keyword)?;
+    tokens.get(idx + 1).map(|t| clean_name(&t.text))
+}

package/src/cobol/indexer.rs

@@ -0,0 +1,256 @@
+use crate::cobol::copybook::{build_copybook_index, resolve_copybook};
+use crate::cobol::data_parser::collect_data_items;
+use crate::cobol::layout::compute_physical_layout;
+pub use crate::cobol::model::{
+    CallKind, CallSummary, CopybookSummary, IndexReport, ProgramSummary,
+};
+use crate::cobol::scanner::{CobolFileType, scan_sandbox};
+use crate::cobol::source_parser::parse_source_file;
+use crate::memory::MemoryStore;
+use rusqlite::params;
+use std::collections::HashMap;
+use std::error::Error;
+use std::path::Path;
+use std::time::UNIX_EPOCH;
+
+type IndexResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
+
+pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexReport> {
+    let files = scan_sandbox(root)?;
+    let source_count = files
+        .iter()
+        .filter(|f| f.file_type == CobolFileType::Source)
+        .count();
+    let copybook_count = files.len() - source_count;
+
+    let parsed = files
+        .iter()
+        .filter(|f| f.file_type == CobolFileType::Source)
+        .map(|f| parse_source_file(&f.path))
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let tx = store.connection_mut().transaction()?;
+    tx.execute_batch(
+        r#"
+        DELETE FROM call_edges;
+        DELETE FROM copybook_uses;
+        DELETE FROM data_items;
+        DELETE FROM programs;
+        DELETE FROM files;
+        "#,
+    )?;
+
+    let mut file_ids = HashMap::with_capacity(files.len());
+    for file in &files {
+        let rel = relative_path(root, &file.path);
+        let kind = match file.file_type {
+            CobolFileType::Source => "source",
+            CobolFileType::Copybook => "copybook",
+        };
+        tx.execute(
+            "INSERT INTO files(path, kind, size_bytes, mtime_unix, sha256) VALUES (?1, ?2, ?3, ?4, ?5)",
+            params![
+                rel,
+                kind,
+                file.size_bytes as i64,
+                mtime_unix(&file.path),
+                Option::<Vec<u8>>::None
+            ],
+        )?;
+        file_ids.insert(file.path.clone(), tx.last_insert_rowid());
+    }
+
+    let mut program_ids = HashMap::new();
+    let mut program_file = HashMap::new();
+    for file in &parsed {
+        let Some(file_id) = file_ids.get(&file.path).copied() else {
+            continue;
+        };
+        for program in &file.programs {
+            tx.execute(
+                "INSERT INTO programs(name, file_id, start_offset, byte_len) VALUES (?1, ?2, ?3, ?4)",
+                params![
+                    program.name,
+                    file_id,
+                    program.start_offset as i64,
+                    program.byte_len as i64
+                ],
+            )?;
+            let id = tx.last_insert_rowid();
+            program_ids.insert(program.name.clone(), id);
+            program_file.insert(program.name.clone(), file.path.clone());
+        }
+    }
+
+    let mut report_programs = HashMap::<String, ProgramSummary>::new();
+    for (name, path) in &program_file {
+        report_programs.insert(
+            name.clone(),
+            ProgramSummary {
+                name: name.clone(),
+                path: path.clone(),
+                copybooks: Vec::new(),
+                calls: Vec::new(),
+                data_items: 0,
+            },
+        );
+    }
+
+    let copybook_index = build_copybook_index(&files);
+    let mut copybook_uses = 0;
+    let mut resolved_copybooks = 0;
+    let mut unresolved_copybooks = Vec::new();
+    let mut static_calls = 0;
+    let mut dynamic_calls = 0;
+    let mut data_items = 0usize;
+
+    for file in &parsed {
+        let Some(from_file_id) = file_ids.get(&file.path).copied() else {
+            continue;
+        };
+        let default_program = file.programs.first().map(|p| p.name.as_str());
+
+        for copy in &file.copies {
+            copybook_uses += 1;
+            let resolved = resolve_copybook(root, &file.path, &copy.name, &copybook_index);
+            if resolved.is_some() {
+                resolved_copybooks += 1;
+            } else {
+                unresolved_copybooks.push(format!(
+                    "{} from {}",
+                    copy.name,
+                    relative_path(root, &file.path)
+                ));
+            }
+
+            let resolved_file_id = resolved.as_ref().and_then(|p| file_ids.get(p).copied());
+            tx.execute(
+                "INSERT INTO copybook_uses(from_file_id, copybook_name, start_offset, byte_len, resolved_file_id, resolve_status, replacing_text) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
+                params![
+                    from_file_id,
+                    copy.name,
+                    copy.start_offset as i64,
+                    copy.byte_len as i64,
+                    resolved_file_id,
+                    if resolved_file_id.is_some() { "resolved" } else { "missing" },
+                    copy.replacing_text,
+                ],
+            )?;
+
+            if let Some(program_name) = default_program {
+                if let Some(summary) = report_programs.get_mut(program_name) {
+                    summary.copybooks.push(CopybookSummary {
+                        name: copy.name.clone(),
+                        resolved_path: resolved,
+                        has_replacing: copy.replacing_text.is_some(),
+                    });
+                }
+            }
+        }
+
+        if let Some(program_name) = default_program {
+            let Some(program_id) = program_ids.get(program_name).copied() else {
+                continue;
+            };
+            let mut expanded_items = collect_data_items(root, &file.path, &copybook_index, 0)?;
+            compute_physical_layout(&mut expanded_items);
+            data_items += expanded_items.len();
+
+            for item in expanded_items {
+                tx.execute(
+                    "INSERT INTO data_items(program_id, source_file_id, name, level, parent_name, pic, usage_clause, occurs, redefines, section, byte_offset, byte_size, storage_kind, layout_status, start_offset, byte_len) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
+                    params![
+                        program_id,
+                        file_ids.get(&item.source_path).copied(),
+                        item.name,
+                        item.level as i64,
+                        item.parent_name,
+                        item.pic,
+                        item.usage_clause,
+                        item.occurs,
+                        item.redefines,
+                        item.section,
+                        item.byte_offset,
+                        item.byte_size,
+                        item.storage_kind,
+                        item.layout_status,
+                        item.start_offset as i64,
+                        item.byte_len as i64,
+                    ],
+                )?;
+                if let Some(summary) = report_programs.get_mut(program_name) {
+                    summary.data_items += 1;
+                }
+            }
+        }
+
+        for call in &file.calls {
+            match call.kind {
+                CallKind::Static => static_calls += 1,
+                CallKind::Dynamic => dynamic_calls += 1,
+            }
+            let caller_name = call.caller_name.as_deref().or(default_program);
+            let Some(caller_name) = caller_name else {
+                continue;
+            };
+            let Some(caller_program_id) = program_ids.get(caller_name).copied() else {
+                continue;
+            };
+            tx.execute(
+                "INSERT INTO call_edges(caller_program_id, callee_name, start_offset, byte_len, kind, using_count) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+                params![
+                    caller_program_id,
+                    call.target,
+                    call.start_offset as i64,
+                    call.byte_len as i64,
+                    call.kind.as_str(),
+                    call.using_count as i64,
+                ],
+            )?;
+
+            if let Some(summary) = report_programs.get_mut(caller_name) {
+                summary.calls.push(CallSummary {
+                    target: call.target.clone(),
+                    kind: call.kind,
+                    using_count: call.using_count,
+                });
+            }
+        }
+    }
+
+    tx.commit()?;
+
+    let mut programs = report_programs.into_values().collect::<Vec<_>>();
+    programs.sort_by(|a, b| a.name.cmp(&b.name));
+    unresolved_copybooks.sort();
+    unresolved_copybooks.dedup();
+
+    Ok(IndexReport {
+        files,
+        source_count,
+        copybook_count,
+        programs,
+        copybook_uses,
+        resolved_copybooks,
+        unresolved_copybooks,
+        static_calls,
+        dynamic_calls,
+        data_items,
+    })
+}
+
+fn relative_path(root: &Path, path: &Path) -> String {
+    path.strip_prefix(root)
+        .unwrap_or(path)
+        .to_string_lossy()
+        .replace('\\', "/")
+}
+
+fn mtime_unix(path: &Path) -> i64 {
+    std::fs::metadata(path)
+        .and_then(|m| m.modified())
+        .ok()
+        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
+        .map(|d| d.as_secs() as i64)
+        .unwrap_or(0)
+}