cobolx 1.0.3 → 1.0.4

package/src/cobol/indexer.rs

@@ -1,6 +1,8 @@
 use crate::cobol::copybook::{build_copybook_index, resolve_copybook};
 use crate::cobol::data_parser::collect_data_items;
 use crate::cobol::layout::compute_physical_layout;
+use crate::cobol::lexer::clean_name;
+use crate::cobol::model::ParsedCodeBlock;
 pub use crate::cobol::model::{
     CallKind, CallSummary, CopybookSummary, IndexReport, ProgramSummary,
 };
@@ -8,8 +10,9 @@ use crate::cobol::scanner::{CobolFileType, scan_sandbox};
 use crate::cobol::source_parser::parse_source_file;
 use crate::memory::MemoryStore;
 use rusqlite::params;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::error::Error;
+use std::hash::Hash;
 use std::path::Path;
 use std::time::UNIX_EPOCH;
 
@@ -32,6 +35,12 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
     let tx = store.connection_mut().transaction()?;
     tx.execute_batch(
         r#"
+        DELETE FROM external_ops;
+        DELETE FROM code_blocks;
+        DELETE FROM literals;
+        DELETE FROM identifiers;
+        DELETE FROM program_features;
+        DELETE FROM copybook_features;
         DELETE FROM call_edges;
         DELETE FROM copybook_uses;
         DELETE FROM data_items;
@@ -104,6 +113,25 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
     let mut dynamic_calls = 0;
     let mut data_items = 0usize;
 
+    let mut incoming_calls = HashMap::<i64, usize>::new();
+    let mut outgoing_calls = HashMap::<i64, usize>::new();
+    let mut static_calls_by_program = HashMap::<i64, usize>::new();
+    let mut dynamic_calls_by_program = HashMap::<i64, usize>::new();
+    let mut copybook_use_count_by_program = HashMap::<i64, usize>::new();
+    let mut distinct_copybooks_by_program = HashMap::<i64, HashSet<i64>>::new();
+    let mut referenced_by_files = HashMap::<i64, HashSet<i64>>::new();
+    let mut data_item_count_by_program = HashMap::<i64, usize>::new();
+    let mut paragraph_count_by_program = HashMap::<i64, usize>::new();
+    let mut external_op_count_by_program = HashMap::<i64, usize>::new();
+    let mut identifier_count_by_program = HashMap::<i64, usize>::new();
+    let mut literal_count_by_program = HashMap::<i64, usize>::new();
+    let mut copybook_programs = HashMap::<i64, HashSet<i64>>::new();
+    let mut copybook_files = HashMap::<i64, HashSet<i64>>::new();
+    let mut copybook_replacing_counts = HashMap::<i64, usize>::new();
+    let mut copybook_item_names = HashMap::<i64, Vec<String>>::new();
+    let mut identifier_mentions = HashMap::<MentionKey, MentionAggregate>::new();
+    let mut literal_mentions = HashMap::<MentionKey, MentionAggregate>::new();
+
     for file in &parsed {
         let Some(from_file_id) = file_ids.get(&file.path).copied() else {
             continue;
@@ -137,14 +165,35 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
                 ],
             )?;
 
-            if let Some(program_name) = default_program {
-                if let Some(summary) = report_programs.get_mut(program_name) {
+            let caller_name = copy.caller_name.as_deref().or(default_program);
+            if let Some(caller_name) = caller_name {
+                if let Some(summary) = report_programs.get_mut(caller_name) {
                     summary.copybooks.push(CopybookSummary {
                         name: copy.name.clone(),
-                        resolved_path: resolved,
+                        resolved_path: resolved.clone(),
                         has_replacing: copy.replacing_text.is_some(),
                     });
                 }
+                if let Some(program_id) = program_ids.get(caller_name).copied() {
+                    increment_count(&mut copybook_use_count_by_program, program_id);
+                    if let Some(copybook_file_id) = resolved_file_id {
+                        distinct_copybooks_by_program
+                            .entry(program_id)
+                            .or_default()
+                            .insert(copybook_file_id);
+                        copybook_programs
+                            .entry(copybook_file_id)
+                            .or_default()
+                            .insert(program_id);
+                        copybook_files
+                            .entry(copybook_file_id)
+                            .or_default()
+                            .insert(from_file_id);
+                        if copy.replacing_text.is_some() {
+                            increment_count(&mut copybook_replacing_counts, copybook_file_id);
+                        }
+                    }
+                }
             }
         }
 
@@ -157,11 +206,12 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
             data_items += expanded_items.len();
 
             for item in expanded_items {
+                let source_file_id = file_ids.get(&item.source_path).copied();
                 tx.execute(
                     "INSERT INTO data_items(program_id, source_file_id, name, level, parent_name, pic, usage_clause, occurs, redefines, section, byte_offset, byte_size, storage_kind, layout_status, start_offset, byte_len) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
                     params![
                         program_id,
-                        file_ids.get(&item.source_path).copied(),
+                        source_file_id,
                         item.name,
                         item.level as i64,
                         item.parent_name,
@@ -178,12 +228,122 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
                         item.byte_len as i64,
                     ],
                 )?;
+
+                increment_count(&mut data_item_count_by_program, program_id);
+                increment_count(&mut identifier_count_by_program, program_id);
+                if let Some(source_file_id) = source_file_id {
+                    aggregate_mention(
+                        &mut identifier_mentions,
+                        program_id,
+                        source_file_id,
+                        "data_name",
+                        &item.name,
+                        item.start_offset,
+                    );
+                    if source_file_id != from_file_id {
+                        copybook_item_names
+                            .entry(source_file_id)
+                            .or_default()
+                            .push(item.name.clone());
+                    }
+                }
                 if let Some(summary) = report_programs.get_mut(program_name) {
                     summary.data_items += 1;
                 }
             }
         }
 
+        for code_block in &file.code_blocks {
+            let Some(program_id) = resolve_program_id(
+                &program_ids,
+                code_block.caller_name.as_deref(),
+                default_program,
+            ) else {
+                continue;
+            };
+            tx.execute(
+                "INSERT INTO code_blocks(program_id, source_file_id, name, kind, parent_section, sequence_no, statement_count, start_offset, byte_len) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
+                params![
+                    program_id,
+                    from_file_id,
+                    code_block.name,
+                    code_block.kind.as_str(),
+                    code_block.parent_section,
+                    code_block_sequence(code_block, &file.code_blocks) as i64,
+                    code_block.statement_count as i64,
+                    code_block.start_offset as i64,
+                    code_block.byte_len as i64,
+                ],
+            )?;
+            if matches!(
+                code_block.kind,
+                crate::cobol::model::CodeBlockKind::Paragraph
+            ) {
+                increment_count(&mut paragraph_count_by_program, program_id);
+            }
+        }
+
+        for external_op in &file.external_ops {
+            let Some(program_id) = resolve_program_id(
+                &program_ids,
+                external_op.caller_name.as_deref(),
+                default_program,
+            ) else {
+                continue;
+            };
+            tx.execute(
+                "INSERT INTO external_ops(program_id, source_file_id, kind, verb, target, start_offset, byte_len) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
+                params![
+                    program_id,
+                    from_file_id,
+                    external_op.kind.as_str(),
+                    external_op.verb,
+                    external_op.target,
+                    external_op.start_offset as i64,
+                    external_op.byte_len as i64,
+                ],
+            )?;
+            increment_count(&mut external_op_count_by_program, program_id);
+        }
+
+        for identifier in &file.identifiers {
+            let Some(program_id) = resolve_program_id(
+                &program_ids,
+                identifier.caller_name.as_deref(),
+                default_program,
+            ) else {
+                continue;
+            };
+            increment_count(&mut identifier_count_by_program, program_id);
+            aggregate_mention(
+                &mut identifier_mentions,
+                program_id,
+                from_file_id,
+                &identifier.kind,
+                &identifier.value,
+                identifier.start_offset,
+            );
+        }
+
+        for literal in &file.literals {
+            let Some(program_id) = resolve_program_id(
+                &program_ids,
+                literal.caller_name.as_deref(),
+                default_program,
+            ) else {
+                continue;
+            };
+            increment_count(&mut literal_count_by_program, program_id);
+            aggregate_mention(
+                &mut literal_mentions,
+                program_id,
+                from_file_id,
+                &literal.kind,
+                &literal.value,
+                literal.start_offset,
+            );
+        }
+
         for call in &file.calls {
             match call.kind {
                 CallKind::Static => static_calls += 1,
@@ -208,6 +368,26 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
                 ],
             )?;
 
+            increment_count(&mut outgoing_calls, caller_program_id);
+            match call.kind {
+                CallKind::Static => {
+                    increment_count(&mut static_calls_by_program, caller_program_id)
+                }
+                CallKind::Dynamic => {
+                    increment_count(&mut dynamic_calls_by_program, caller_program_id)
+                }
+            }
+
+            if matches!(call.kind, CallKind::Static) {
+                if let Some(callee_program_id) = program_ids.get(&call.target).copied() {
+                    increment_count(&mut incoming_calls, callee_program_id);
+                    referenced_by_files
+                        .entry(callee_program_id)
+                        .or_default()
+                        .insert(from_file_id);
+                }
+            }
+
             if let Some(summary) = report_programs.get_mut(caller_name) {
                 summary.calls.push(CallSummary {
                     target: call.target.clone(),
@@ -218,6 +398,114 @@ pub fn index_sandbox(root: &Path, store: &mut MemoryStore) -> IndexResult<IndexR
         }
     }
 
+    for (key, aggregate) in identifier_mentions {
+        tx.execute(
+            "INSERT INTO identifiers(program_id, source_file_id, kind, value, occurrences, first_offset) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            params![
+                key.program_id,
+                key.source_file_id,
+                key.kind,
+                key.value,
+                aggregate.occurrences as i64,
+                aggregate.first_offset as i64,
+            ],
+        )?;
+    }
+
+    for (key, aggregate) in literal_mentions {
+        tx.execute(
+            "INSERT INTO literals(program_id, source_file_id, kind, value, occurrences, first_offset) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
+            params![
+                key.program_id,
+                key.source_file_id,
+                key.kind,
+                key.value,
+                aggregate.occurrences as i64,
+                aggregate.first_offset as i64,
+            ],
+        )?;
+    }
+
+    for file in files
+        .iter()
+        .filter(|f| f.file_type == CobolFileType::Copybook)
+    {
+        let Some(copybook_file_id) = file_ids.get(&file.path).copied() else {
+            continue;
+        };
+        let copybook_name = file
+            .path
+            .file_stem()
+            .and_then(|s| s.to_str())
+            .map(clean_name)
+            .filter(|s| !s.is_empty())
+            .unwrap_or_else(|| relative_path(root, &file.path).to_ascii_uppercase());
+        let item_names = copybook_item_names
+            .remove(&copybook_file_id)
+            .unwrap_or_default();
+        let contains_header_fields = item_names.iter().any(|name| looks_like_header_field(name));
+        let contains_error_fields = item_names.iter().any(|name| looks_like_error_field(name));
+        tx.execute(
+            "INSERT INTO copybook_features(copybook_file_id, copybook_name, used_by_program_count, used_by_file_count, replacing_use_count, data_item_count, contains_header_fields, contains_error_fields) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
+            params![
+                copybook_file_id,
+                copybook_name,
+                copybook_programs
+                    .get(&copybook_file_id)
+                    .map_or(0_i64, |ids| ids.len() as i64),
+                copybook_files
+                    .get(&copybook_file_id)
+                    .map_or(0_i64, |ids| ids.len() as i64),
+                copybook_replacing_counts
+                    .get(&copybook_file_id)
+                    .copied()
+                    .unwrap_or(0) as i64,
+                item_names.len() as i64,
+                bool_to_int(contains_header_fields),
+                bool_to_int(contains_error_fields),
+            ],
+        )?;
+    }
+
+    for (program_name, path) in &program_file {
+        let Some(program_id) = program_ids.get(program_name).copied() else {
+            continue;
+        };
+        let Some(source_file_id) = file_ids.get(path).copied() else {
+            continue;
+        };
+        let incoming = incoming_calls.get(&program_id).copied().unwrap_or(0);
+        let copybook_use_count = copybook_use_count_by_program
+            .get(&program_id)
+            .copied()
+            .unwrap_or(0);
+        tx.execute(
+            "INSERT INTO program_features(program_id, source_file_id, incoming_call_count, outgoing_call_count, static_call_count, dynamic_call_count, copybook_use_count, distinct_copybook_count, referenced_by_file_count, is_entrypoint, has_heavy_copy_usage, data_item_count, paragraph_count, external_op_count, identifier_count, literal_count) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)",
+            params![
+                program_id,
+                source_file_id,
+                incoming as i64,
+                outgoing_calls.get(&program_id).copied().unwrap_or(0) as i64,
+                static_calls_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                dynamic_calls_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                copybook_use_count as i64,
+                distinct_copybooks_by_program
+                    .get(&program_id)
+                    .map_or(0_i64, |ids| ids.len() as i64),
+                referenced_by_files
+                    .get(&program_id)
+                    .map_or(0_i64, |ids| ids.len() as i64),
+                bool_to_int(incoming == 0),
+                bool_to_int(copybook_use_count >= 3),
+                data_item_count_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                paragraph_count_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                external_op_count_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                identifier_count_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+                literal_count_by_program.get(&program_id).copied().unwrap_or(0) as i64,
+            ],
+        )?;
+    }
+
     tx.commit()?;
 
     let mut programs = report_programs.into_values().collect::<Vec<_>>();
@@ -254,3 +542,85 @@ fn mtime_unix(path: &Path) -> i64 {
         .map(|d| d.as_secs() as i64)
         .unwrap_or(0)
 }
+
+fn increment_count<K>(map: &mut HashMap<K, usize>, key: K)
+where
+    K: Eq + Hash,
+{
+    *map.entry(key).or_default() += 1;
+}
+
+fn resolve_program_id(
+    program_ids: &HashMap<String, i64>,
+    explicit_name: Option<&str>,
+    default_name: Option<&str>,
+) -> Option<i64> {
+    explicit_name
+        .or(default_name)
+        .and_then(|name| program_ids.get(name).copied())
+}
+
+fn aggregate_mention(
+    map: &mut HashMap<MentionKey, MentionAggregate>,
+    program_id: i64,
+    source_file_id: i64,
+    kind: &str,
+    value: &str,
+    start_offset: usize,
+) {
+    let key = MentionKey {
+        program_id,
+        source_file_id,
+        kind: kind.to_string(),
+        value: value.to_string(),
+    };
+    let entry = map.entry(key).or_insert(MentionAggregate {
+        occurrences: 0,
+        first_offset: start_offset,
+    });
+    entry.occurrences += 1;
+    entry.first_offset = entry.first_offset.min(start_offset);
+}
+
+fn code_block_sequence(code_block: &ParsedCodeBlock, all_blocks: &[ParsedCodeBlock]) -> usize {
+    all_blocks
+        .iter()
+        .position(|candidate| {
+            candidate.start_offset == code_block.start_offset
+                && candidate.name == code_block.name
+                && candidate.kind == code_block.kind
+        })
+        .map_or(1, |idx| idx + 1)
+}
+
+fn looks_like_header_field(name: &str) -> bool {
+    let upper = name.to_ascii_uppercase();
+    upper.contains("HEADER") || upper.contains("HDR")
+}
+
+fn looks_like_error_field(name: &str) -> bool {
+    let upper = name.to_ascii_uppercase();
+    upper.contains("ERROR")
+        || upper.contains("ERR")
+        || upper.contains("SQLCODE")
+        || upper.contains("RETURN-CODE")
+        || upper.contains("RESP-CODE")
+}
+
+fn bool_to_int(value: bool) -> i64 {
+    if value { 1 } else { 0 }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+struct MentionKey {
+    program_id: i64,
+    source_file_id: i64,
+    kind: String,
+    value: String,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct MentionAggregate {
+    occurrences: usize,
+    first_offset: usize,
+}

package/src/cobol/model.rs

@@ -15,6 +15,42 @@ impl CallKind {
     }
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum CodeBlockKind {
+    Section,
+    Paragraph,
+}
+
+impl CodeBlockKind {
+    pub(crate) fn as_str(self) -> &'static str {
+        match self {
+            CodeBlockKind::Section => "section",
+            CodeBlockKind::Paragraph => "paragraph",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ExternalOpKind {
+    ExecSql,
+    ExecCics,
+    FileIo,
+    CallLiteral,
+    CallIdentifier,
+}
+
+impl ExternalOpKind {
+    pub(crate) fn as_str(self) -> &'static str {
+        match self {
+            ExternalOpKind::ExecSql => "exec_sql",
+            ExternalOpKind::ExecCics => "exec_cics",
+            ExternalOpKind::FileIo => "file_io",
+            ExternalOpKind::CallLiteral => "call_literal",
+            ExternalOpKind::CallIdentifier => "call_identifier",
+        }
+    }
+}
+
 #[derive(Debug, Clone)]
 pub struct CallSummary {
     pub target: String,
@@ -138,6 +174,7 @@ pub(crate) struct ParsedProgram {
 
 #[derive(Debug)]
 pub(crate) struct ParsedCopy {
+    pub(crate) caller_name: Option<String>,
     pub(crate) name: String,
     pub(crate) start_offset: usize,
     pub(crate) byte_len: usize,
@@ -154,6 +191,43 @@ pub(crate) struct ParsedCall {
     pub(crate) using_count: usize,
 }
 
+#[derive(Debug)]
+pub(crate) struct ParsedCodeBlock {
+    pub(crate) caller_name: Option<String>,
+    pub(crate) name: String,
+    pub(crate) kind: CodeBlockKind,
+    pub(crate) parent_section: Option<String>,
+    pub(crate) start_offset: usize,
+    pub(crate) byte_len: usize,
+    pub(crate) statement_count: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct ParsedExternalOp {
+    pub(crate) caller_name: Option<String>,
+    pub(crate) kind: ExternalOpKind,
+    pub(crate) verb: String,
+    pub(crate) target: Option<String>,
+    pub(crate) start_offset: usize,
+    pub(crate) byte_len: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct ParsedIdentifier {
+    pub(crate) caller_name: Option<String>,
+    pub(crate) kind: String,
+    pub(crate) value: String,
+    pub(crate) start_offset: usize,
+}
+
+#[derive(Debug)]
+pub(crate) struct ParsedLiteral {
+    pub(crate) caller_name: Option<String>,
+    pub(crate) kind: String,
+    pub(crate) value: String,
+    pub(crate) start_offset: usize,
+}
+
 #[derive(Debug)]
 pub(crate) struct ParsedDataItem {
     pub(crate) source_path: PathBuf,
@@ -179,6 +253,10 @@ pub(crate) struct ParsedFile {
     pub(crate) programs: Vec<ParsedProgram>,
     pub(crate) copies: Vec<ParsedCopy>,
     pub(crate) calls: Vec<ParsedCall>,
+    pub(crate) code_blocks: Vec<ParsedCodeBlock>,
+    pub(crate) external_ops: Vec<ParsedExternalOp>,
+    pub(crate) identifiers: Vec<ParsedIdentifier>,
+    pub(crate) literals: Vec<ParsedLiteral>,
 }
 
 #[derive(Debug)]

package/src/cobol/scanner.rs

@@ -29,6 +29,8 @@ fn should_exclude_dir(name: &str) -> bool {
         || name == "node_modules"
         || name == "vendor"
         || name == "build"
+        || name == "tests"
+        || name == "test"
 }
 
 /// Scans entries within `dir`. Does NOT check exclusion on `dir` itself —