cobolx 1.0.3 → 1.0.4

package/src/cobol/source_parser.rs

@@ -1,5 +1,8 @@
 use crate::cobol::lexer::{clean_name, logical_lines, tokenize};
-use crate::cobol::model::{CallKind, ParsedCall, ParsedCopy, ParsedFile, ParsedProgram, Token};
+use crate::cobol::model::{
+    CallKind, CodeBlockKind, ExternalOpKind, ParsedCall, ParsedCodeBlock, ParsedCopy,
+    ParsedExternalOp, ParsedFile, ParsedIdentifier, ParsedLiteral, ParsedProgram, Token,
+};
 use std::path::Path;
 
 pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
@@ -7,7 +10,14 @@ pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
     let mut programs = Vec::new();
     let mut copies = Vec::new();
     let mut calls = Vec::new();
+    let mut code_blocks = Vec::new();
+    let mut external_ops = Vec::new();
+    let mut identifiers = Vec::new();
+    let mut literals = Vec::new();
     let mut current_program = None::<String>;
+    let mut in_procedure_division = false;
+    let mut current_section = None::<String>;
+    let mut current_block_idx = None::<usize>;
 
     for line in logical_lines(&content) {
         let tokens = tokenize(&line.text);
@@ -15,6 +25,54 @@ pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
             continue;
         }
 
+        if has_two_tokens(&tokens, "PROCEDURE", "DIVISION") {
+            in_procedure_division = true;
+            current_section = None;
+            current_block_idx = None;
+            continue;
+        }
+
+        if in_procedure_division {
+            if let Some(section_name) = parse_section_name(&tokens) {
+                code_blocks.push(ParsedCodeBlock {
+                    caller_name: current_program.clone(),
+                    name: section_name.clone(),
+                    kind: CodeBlockKind::Section,
+                    parent_section: None,
+                    start_offset: line.start_offset + tokens[0].start,
+                    byte_len: line.byte_len,
+                    statement_count: 0,
+                });
+                current_section = Some(section_name);
+                current_block_idx = Some(code_blocks.len() - 1);
+                continue;
+            }
+
+            if let Some(paragraph_name) = parse_paragraph_name(&line.text, &tokens) {
+                identifiers.push(ParsedIdentifier {
+                    caller_name: current_program.clone(),
+                    kind: "paragraph_name".to_string(),
+                    value: paragraph_name.clone(),
+                    start_offset: line.start_offset + tokens[0].start,
+                });
+                code_blocks.push(ParsedCodeBlock {
+                    caller_name: current_program.clone(),
+                    name: paragraph_name,
+                    kind: CodeBlockKind::Paragraph,
+                    parent_section: current_section.clone(),
+                    start_offset: line.start_offset + tokens[0].start,
+                    byte_len: line.byte_len,
+                    statement_count: 0,
+                });
+                current_block_idx = Some(code_blocks.len() - 1);
+                continue;
+            }
+
+            if let Some(block_idx) = current_block_idx {
+                code_blocks[block_idx].statement_count += 1;
+            }
+        }
+
         for idx in 0..tokens.len() {
             match tokens[idx].text.as_str() {
                 "PROGRAM-ID" => {
@@ -37,6 +95,7 @@ pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
                                 .find(|t| t.text == "REPLACING")
                                 .map(|t| line.text[t.start..].trim().to_string());
                             copies.push(ParsedCopy {
+                                caller_name: current_program.clone(),
                                 name,
                                 start_offset: line.start_offset + tokens[idx].start,
                                 byte_len: line.byte_len,
@@ -56,18 +115,89 @@ pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
                             };
                             calls.push(ParsedCall {
                                 caller_name: current_program.clone(),
-                                target: name,
+                                target: name.clone(),
                                 kind,
                                 start_offset: line.start_offset + tokens[idx].start,
                                 byte_len: line.byte_len,
                                 using_count: count_using_args(&tokens[idx + 2..]),
                             });
+                            external_ops.push(ParsedExternalOp {
+                                caller_name: current_program.clone(),
+                                kind: if target.quoted {
+                                    ExternalOpKind::CallLiteral
+                                } else {
+                                    ExternalOpKind::CallIdentifier
+                                },
+                                verb: "CALL".to_string(),
+                                target: Some(name.clone()),
+                                start_offset: line.start_offset + tokens[idx].start,
+                                byte_len: line.byte_len,
+                            });
+                            if target.quoted {
+                                literals.push(ParsedLiteral {
+                                    caller_name: current_program.clone(),
+                                    kind: "call_target".to_string(),
+                                    value: name,
+                                    start_offset: line.start_offset + target.start,
+                                });
+                            } else {
+                                identifiers.push(ParsedIdentifier {
+                                    caller_name: current_program.clone(),
+                                    kind: "call_target_identifier".to_string(),
+                                    value: name,
+                                    start_offset: line.start_offset + target.start,
+                                });
+                            }
                         }
                     }
                 }
                 _ => {}
             }
         }
+
+        if !in_procedure_division {
+            continue;
+        }
+
+        if let Some((op, mut ids, mut lits)) = parse_exec_sql(
+            &tokens,
+            line.start_offset,
+            line.byte_len,
+            current_program.clone(),
+        ) {
+            external_ops.push(op);
+            identifiers.append(&mut ids);
+            literals.append(&mut lits);
+            continue;
+        }
+
+        if let Some((op, mut lits)) = parse_exec_cics(
+            &tokens,
+            line.start_offset,
+            line.byte_len,
+            current_program.clone(),
+        ) {
+            external_ops.push(op);
+            literals.append(&mut lits);
+            continue;
+        }
+
+        if let Some((verb, target)) = parse_file_io(&tokens) {
+            external_ops.push(ParsedExternalOp {
+                caller_name: current_program.clone(),
+                kind: ExternalOpKind::FileIo,
+                verb: verb.to_string(),
+                target: Some(target.clone()),
+                start_offset: line.start_offset + tokens[0].start,
+                byte_len: line.byte_len,
+            });
+            identifiers.push(ParsedIdentifier {
+                caller_name: current_program.clone(),
+                kind: "file_name".to_string(),
+                value: target,
+                start_offset: line.start_offset + tokens[1].start,
+            });
+        }
     }
 
     Ok(ParsedFile {
@@ -75,6 +205,10 @@ pub(crate) fn parse_source_file(path: &Path) -> std::io::Result<ParsedFile> {
         programs,
         copies,
         calls,
+        code_blocks,
+        external_ops,
+        identifiers,
+        literals,
     })
 }
 
@@ -89,3 +223,208 @@ fn count_using_args(tokens: &[Token]) -> usize {
         .filter(|t| !matches!(t.text.as_str(), "BY" | "REFERENCE" | "CONTENT" | "VALUE"))
         .count()
 }
+
+fn has_two_tokens(tokens: &[Token], first: &str, second: &str) -> bool {
+    tokens.len() >= 2 && tokens[0].text == first && tokens[1].text == second
+}
+
+fn parse_section_name(tokens: &[Token]) -> Option<String> {
+    (tokens.len() >= 2 && tokens[1].text == "SECTION").then(|| clean_name(&tokens[0].text))
+}
+
+fn parse_paragraph_name(line: &str, tokens: &[Token]) -> Option<String> {
+    if tokens.len() != 1 || !line.trim_end().ends_with('.') {
+        return None;
+    }
+
+    let name = clean_name(&tokens[0].text);
+    (!name.is_empty() && !is_reserved_label(&name)).then_some(name)
+}
+
+fn is_reserved_label(name: &str) -> bool {
+    matches!(
+        name,
+        "ACCEPT"
+            | "ADD"
+            | "CALL"
+            | "CANCEL"
+            | "CLOSE"
+            | "COMPUTE"
+            | "CONTINUE"
+            | "DELETE"
+            | "DISPLAY"
+            | "DIVIDE"
+            | "ELSE"
+            | "END-CALL"
+            | "END-EVALUATE"
+            | "END-IF"
+            | "END-PERFORM"
+            | "ENTRY"
+            | "EVALUATE"
+            | "EXEC"
+            | "EXIT"
+            | "GOBACK"
+            | "GO"
+            | "IF"
+            | "INITIALIZE"
+            | "INSPECT"
+            | "MERGE"
+            | "MOVE"
+            | "MULTIPLY"
+            | "OPEN"
+            | "PERFORM"
+            | "PROCEDURE"
+            | "READ"
+            | "RELEASE"
+            | "RETURN"
+            | "REWRITE"
+            | "SEARCH"
+            | "SORT"
+            | "START"
+            | "STOP"
+            | "STRING"
+            | "SUBTRACT"
+            | "UNSTRING"
+            | "USE"
+            | "WHEN"
+            | "WRITE"
+    )
+}
+
+fn parse_exec_sql(
+    tokens: &[Token],
+    line_start_offset: usize,
+    byte_len: usize,
+    caller_name: Option<String>,
+) -> Option<(ParsedExternalOp, Vec<ParsedIdentifier>, Vec<ParsedLiteral>)> {
+    if !has_two_tokens(tokens, "EXEC", "SQL") {
+        return None;
+    }
+
+    let verb = tokens[2..]
+        .iter()
+        .find(|t| {
+            matches!(
+                t.text.as_str(),
+                "SELECT" | "INSERT" | "UPDATE" | "DELETE" | "MERGE"
+            )
+        })
+        .map(|t| t.text.clone())
+        .unwrap_or_else(|| "SQL".to_string());
+    let target = extract_sql_target(tokens, &verb);
+    let mut identifiers = Vec::new();
+    if let Some(ref table) = target {
+        identifiers.push(ParsedIdentifier {
+            caller_name: caller_name.clone(),
+            kind: "sql_table".to_string(),
+            value: table.clone(),
+            start_offset: line_start_offset,
+        });
+    }
+    let literals = tokens
+        .iter()
+        .filter(|t| t.quoted)
+        .map(|t| ParsedLiteral {
+            caller_name: caller_name.clone(),
+            kind: "string_literal".to_string(),
+            value: clean_name(&t.text),
+            start_offset: line_start_offset + t.start,
+        })
+        .collect::<Vec<_>>();
+
+    Some((
+        ParsedExternalOp {
+            caller_name,
+            kind: ExternalOpKind::ExecSql,
+            verb,
+            target,
+            start_offset: line_start_offset,
+            byte_len,
+        },
+        identifiers,
+        literals,
+    ))
+}
+
+fn extract_sql_target(tokens: &[Token], verb: &str) -> Option<String> {
+    match verb {
+        "SELECT" => token_after_keyword(tokens, "FROM"),
+        "INSERT" => token_after_keyword(tokens, "INTO"),
+        "UPDATE" => token_after_keyword(tokens, "UPDATE"),
+        "DELETE" => token_after_keyword(tokens, "FROM"),
+        "MERGE" => {
+            token_after_keyword(tokens, "INTO").or_else(|| token_after_keyword(tokens, "USING"))
+        }
+        _ => None,
+    }
+}
+
+fn parse_exec_cics(
+    tokens: &[Token],
+    line_start_offset: usize,
+    byte_len: usize,
+    caller_name: Option<String>,
+) -> Option<(ParsedExternalOp, Vec<ParsedLiteral>)> {
+    if !has_two_tokens(tokens, "EXEC", "CICS") {
+        return None;
+    }
+
+    let verb = tokens
+        .get(2)
+        .map(|t| clean_name(&t.text))
+        .filter(|s| !s.is_empty())
+        .unwrap_or_else(|| "CICS".to_string());
+    let target_idx = tokens.iter().position(|t| t.text == "PROGRAM");
+    let target = target_idx
+        .and_then(|idx| tokens.get(idx + 1))
+        .map(|t| clean_name(&t.text))
+        .filter(|s| !s.is_empty());
+    let literals = target_idx
+        .and_then(|idx| tokens.get(idx + 1))
+        .filter(|t| t.quoted)
+        .map(|t| ParsedLiteral {
+            caller_name: caller_name.clone(),
+            kind: "exec_cics_program".to_string(),
+            value: clean_name(&t.text),
+            start_offset: line_start_offset + t.start,
+        })
+        .into_iter()
+        .collect::<Vec<_>>();
+
+    Some((
+        ParsedExternalOp {
+            caller_name,
+            kind: ExternalOpKind::ExecCics,
+            verb,
+            target,
+            start_offset: line_start_offset,
+            byte_len,
+        },
+        literals,
+    ))
+}
+
+fn parse_file_io(tokens: &[Token]) -> Option<(String, String)> {
+    match tokens.first()?.text.as_str() {
+        "OPEN" => {
+            let mode_idx = tokens
+                .iter()
+                .position(|t| matches!(t.text.as_str(), "INPUT" | "OUTPUT" | "I-O" | "EXTEND"))?;
+            let target = tokens.get(mode_idx + 1).map(|t| clean_name(&t.text))?;
+            (!target.is_empty()).then_some(("OPEN".to_string(), target))
+        }
+        "READ" | "WRITE" | "REWRITE" | "DELETE" | "START" | "CLOSE" | "RETURN" | "RELEASE" => {
+            let target = tokens.get(1).map(|t| clean_name(&t.text))?;
+            (!target.is_empty()).then_some((tokens[0].text.clone(), target))
+        }
+        _ => None,
+    }
+}
+
+fn token_after_keyword(tokens: &[Token], keyword: &str) -> Option<String> {
+    let idx = tokens.iter().position(|t| t.text == keyword)?;
+    tokens
+        .get(idx + 1)
+        .map(|t| clean_name(&t.text))
+        .filter(|s| !s.is_empty())
+}

package/src/lib.rs

@@ -3,4 +3,5 @@ pub mod agent;
 pub mod cobol;
 pub mod config;
 pub mod memory;
+pub mod path_safety;
 pub mod ui;

package/src/main.rs

@@ -2,6 +2,7 @@ mod agent;
 mod cobol;
 mod config;
 mod memory;
+mod path_safety;
 mod ui;
 
 #[tokio::main]

package/src/memory/memories.rs

@@ -0,0 +1,208 @@
+use std::error::Error;
+use std::path::{Path, PathBuf};
+
+type MemResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
+
+/// Codex-style: consolidate when cumulative session tokens exceed this.
+pub const TOKEN_SUMMARY_THRESHOLD: u32 = 8_000;
+
+/// Codex injects ~5k tokens of `memory_summary.md`; ~4 chars/token.
+pub const MEMORY_SUMMARY_INJECT_MAX_CHARS: usize = 20_000;
+
+pub const MEMORY_SUMMARY_FILE: &str = "memory_summary.md";
+pub const MEMORY_HANDBOOK_FILE: &str = "MEMORY.md";
+pub const RAW_MEMORIES_FILE: &str = "raw_memories.md";
+pub const AGENTS_FILE: &str = "AGENTS.md";
+pub const LEGACY_SUMMARY_FILE: &str = "SUMMARY_MEM.md";
+
+pub const CONSOLIDATION_SUMMARY_MARKER: &str = "---COBOLX_MEMORY_SUMMARY---";
+pub const CONSOLIDATION_HANDBOOK_MARKER: &str = "---COBOLX_MEMORY_HANDBOOK---";
+
+/// Codex-aligned memory layout under `.cobolx/memories/`.
+pub struct CodexMemories {
+    memories_dir: PathBuf,
+    rollout_summaries_dir: PathBuf,
+    project_root: PathBuf,
+    legacy_summary_path: PathBuf,
+}
+
+impl CodexMemories {
+    pub fn for_project(base_dir: &Path, project_root: &Path) -> Self {
+        let memories_dir = base_dir.join("memories");
+        let rollout_summaries_dir = memories_dir.join("rollout_summaries");
+        Self {
+            memories_dir,
+            rollout_summaries_dir,
+            project_root: project_root.to_path_buf(),
+            legacy_summary_path: base_dir.join(LEGACY_SUMMARY_FILE),
+        }
+    }
+
+    pub fn memories_dir(&self) -> &Path {
+        &self.memories_dir
+    }
+
+    pub fn rollout_summaries_dir(&self) -> &Path {
+        &self.rollout_summaries_dir
+    }
+
+    pub fn ensure_layout(&self) -> MemResult<()> {
+        std::fs::create_dir_all(&self.memories_dir)?;
+        std::fs::create_dir_all(&self.rollout_summaries_dir)?;
+
+        let summary_path = self.memory_summary_path();
+        if !summary_path.exists() {
+            if self.legacy_summary_path.exists() {
+                let legacy = std::fs::read_to_string(&self.legacy_summary_path)?;
+                std::fs::write(&summary_path, legacy)?;
+            } else {
+                std::fs::write(&summary_path, default_memory_summary())?;
+            }
+        }
+
+        let handbook_path = self.memory_handbook_path();
+        if !handbook_path.exists() {
+            std::fs::write(&handbook_path, default_memory_handbook())?;
+        }
+
+        Ok(())
+    }
+
+    pub fn read_agents_instructions(&self) -> Option<String> {
+        let path = self.project_root.join(AGENTS_FILE);
+        if path.exists() {
+            std::fs::read_to_string(path).ok()
+        } else {
+            None
+        }
+    }
+
+    /// Short summary injected each prompt (Codex: `memory_summary.md`, capped).
+    pub fn read_memory_summary_for_injection(&self) -> MemResult<String> {
+        let raw = std::fs::read_to_string(self.memory_summary_path())?;
+        Ok(truncate_utf8_prefix(&raw, MEMORY_SUMMARY_INJECT_MAX_CHARS))
+    }
+
+    pub fn read_memory_summary(&self) -> MemResult<String> {
+        Ok(std::fs::read_to_string(self.memory_summary_path())?)
+    }
+
+    pub fn read_memory_handbook(&self) -> MemResult<String> {
+        Ok(std::fs::read_to_string(self.memory_handbook_path())?)
+    }
+
+    /// Phase 1: per-run recap (Codex: `rollout_summaries/{id}.md`).
+    pub fn write_rollout_summary(&self, run_id: &str, content: &str) -> MemResult<PathBuf> {
+        let path = self.rollout_summaries_dir.join(format!("{}.md", run_id));
+        std::fs::write(&path, content)?;
+        Ok(path)
+    }
+
+    pub fn write_raw_memories(&self, content: &str) -> MemResult<()> {
+        std::fs::write(self.memories_dir.join(RAW_MEMORIES_FILE), content)?;
+        Ok(())
+    }
+
+    pub fn write_consolidated(&self, memory_summary: &str, memory_handbook: &str) -> MemResult<()> {
+        std::fs::write(self.memory_summary_path(), memory_summary)?;
+        std::fs::write(self.memory_handbook_path(), memory_handbook)?;
+        Ok(())
+    }
+
+    pub fn parse_consolidation_output(output: &str) -> Option<(String, String)> {
+        let summary_start = output.find(CONSOLIDATION_SUMMARY_MARKER)?;
+        let handbook_start = output.find(CONSOLIDATION_HANDBOOK_MARKER)?;
+        if handbook_start <= summary_start {
+            return None;
+        }
+        let summary = output[summary_start + CONSOLIDATION_SUMMARY_MARKER.len()..handbook_start]
+            .trim()
+            .to_string();
+        let handbook = output[handbook_start + CONSOLIDATION_HANDBOOK_MARKER.len()..]
+            .trim()
+            .to_string();
+        if summary.is_empty() || handbook.is_empty() {
+            return None;
+        }
+        Some((summary, handbook))
+    }
+
+    fn memory_summary_path(&self) -> PathBuf {
+        self.memories_dir.join(MEMORY_SUMMARY_FILE)
+    }
+
+    fn memory_handbook_path(&self) -> PathBuf {
+        self.memories_dir.join(MEMORY_HANDBOOK_FILE)
+    }
+}
+
+pub fn default_memory_summary() -> String {
+    "# COBOLX Memory Summary\n\n\
+     Navigational summary for the next session (Codex-style `memory_summary.md`).\n\n\
+     - last_updated: (none)\n\
+     - tokens_summarized: 0\n\n\
+     ## Context\n\n\
+     (none yet)\n\n\
+     ## Key findings\n\n\
+     (none yet)\n"
+        .to_string()
+}
+
+pub fn default_memory_handbook() -> String {
+    "# COBOLX Memory Handbook\n\n\
+     Long-form project memory (Codex-style `MEMORY.md`). You may edit manually.\n\n\
+     ## Project\n\n\
+     (none yet)\n\n\
+     ## COBOL programs\n\n\
+     (none yet)\n\n\
+     ## Open questions\n\n\
+     (none yet)\n"
+        .to_string()
+}
+
+fn truncate_utf8_prefix(content: &str, max_bytes: usize) -> String {
+    if content.len() <= max_bytes {
+        return content.to_string();
+    }
+    let mut end = max_bytes;
+    while end > 0 && !content.is_char_boundary(end) {
+        end -= 1;
+    }
+    format!("{}…\n\n[truncated for context budget]", &content[..end])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile::tempdir;
+
+    #[test]
+    fn codex_memories_layout_and_rollout_summary() {
+        let dir = tempdir().unwrap();
+        let base = dir.path().join(".cobolx");
+        let mem = CodexMemories::for_project(&base, dir.path());
+        mem.ensure_layout().unwrap();
+
+        assert!(mem.memories_dir().join(MEMORY_SUMMARY_FILE).exists());
+        assert!(mem.memories_dir().join(MEMORY_HANDBOOK_FILE).exists());
+
+        mem.write_rollout_summary("20250626T120000", "# rollout recap\n")
+            .unwrap();
+        assert!(
+            mem.rollout_summaries_dir()
+                .join("20250626T120000.md")
+                .exists()
+        );
+    }
+
+    #[test]
+    fn parses_consolidation_markers() {
+        let out = format!(
+            "preamble\n{}\nsummary body\n{}\nhandbook body",
+            CONSOLIDATION_SUMMARY_MARKER, CONSOLIDATION_HANDBOOK_MARKER
+        );
+        let (s, h) = CodexMemories::parse_consolidation_output(&out).unwrap();
+        assert_eq!(s, "summary body");
+        assert_eq!(h, "handbook body");
+    }
+}