cobolx 1.0.3 → 1.0.4

package/Cargo.lock

@@ -1098,7 +1098,7 @@ dependencies = [
 
 [[package]]
 name = "rdo"
-version = "1.0.2"
+version = "1.0.4"
 dependencies = [
  "chrono",
  "clap",

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rdo"
-version = "1.0.3"
+version = "1.0.4"
 edition = "2024"
 
 [dependencies]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cobolx",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "CobolX CLI",
   "main": "bin/cobolx.js",
   "bin": {

package/src/agent/client.rs

@@ -3,9 +3,12 @@ use super::clients::{DeepSeekClient, GlmClient};
 // Re-exports — tui.rs and other in-crate code uses `use crate::agent::client::{...}`
 pub use super::types::{ChatMessage, Route, Usage};
 use crate::config::ConfigManager;
+use crate::memory::{CodexMemories, MemoryStore};
 use crate::ui::tui::{Message, Sender};
 use std::path::Path;
 
+const SUMMARY_LOG_MAX_BYTES: usize = 12_000;
+
 pub struct AgentRouter {
     pub(crate) deepseek: Option<DeepSeekClient>,
     pub(crate) glm: Option<GlmClient>,
@@ -91,14 +94,44 @@ impl AgentRouter {
         }
     }
 
-    fn build_messages(history: &[Message]) -> Vec<ChatMessage> {
+    fn load_prompt_memory(sandbox_path: Option<&Path>) -> (Option<String>, Option<String>) {
+        match sandbox_path {
+            None => (None, None),
+            Some(p) => MemoryStore::open_or_create(p)
+                .ok()
+                .map(|store| {
+                    let mem = store.codex_memories();
+                    let agents = mem.read_agents_instructions();
+                    let summary = mem.read_memory_summary_for_injection().ok();
+                    (agents, summary)
+                })
+                .unwrap_or((None, None)),
+        }
+    }
+
+    fn build_messages(
+        history: &[Message],
+        agents_instructions: Option<&str>,
+        memory_summary: Option<&str>,
+    ) -> Vec<ChatMessage> {
+        let mut system_text = String::from(
+            "You are COBOLX, a helpful assistant. COBOLX is a migration agent for legacy \
+            COBOL systems based on DeepSeek.",
+        );
+        if let Some(agents) = agents_instructions {
+            system_text.push_str("\n\n## Project instructions (AGENTS.md)\n\n");
+            system_text.push_str(agents);
+        }
+        if let Some(summary) = memory_summary {
+            system_text.push_str(
+                "\n\n## Persisted memory summary (memory_summary.md)\n\
+                Codex-style navigational memory for continuity:\n\n",
+            );
+            system_text.push_str(summary);
+        }
         let mut messages = vec![ChatMessage {
             role: "system".to_string(),
-            content: Some(
-                "You are COBOLX, a helpful assistant. COBOLX is a migration agent for legacy \
-                COBOL systems based on DeepSeek."
-                    .to_string(),
-            ),
+            content: Some(system_text),
             tool_call_id: None,
             tool_calls: None,
         }];
@@ -130,6 +163,59 @@ impl AgentRouter {
         messages
     }
 
+    /// Codex Phase 2: consolidate rollout log into `memory_summary.md` + `MEMORY.md`.
+    pub async fn consolidate_codex_memories(
+        &self,
+        memory_summary: &str,
+        memory_handbook: &str,
+        rollout_log: &str,
+        tokens_summarized: u32,
+    ) -> Result<(String, String), String> {
+        let truncated_log = truncate_utf8_tail(rollout_log, SUMMARY_LOG_MAX_BYTES);
+        let system_msg = ChatMessage {
+            role: "system".to_string(),
+            content: Some(
+                "You are the COBOLX memory consolidation agent (Codex Phase 2). \
+                Merge existing memory with a new rollout log. Output ONLY two markdown \
+                documents separated by exact markers:\n\
+                ---COBOLX_MEMORY_SUMMARY---\n\
+                (short navigational summary, like Codex memory_summary.md)\n\
+                ---COBOLX_MEMORY_HANDBOOK---\n\
+                (long-form handbook, like Codex MEMORY.md)\n\
+                Include last_updated (ISO8601 UTC) and tokens_summarized (cumulative) in the summary. \
+                Be concise. Preserve useful manual edits. Focus on COBOL programs, user goals, \
+                sandbox facts, and decisions."
+                    .to_string(),
+            ),
+            tool_call_id: None,
+            tool_calls: None,
+        };
+        let user_msg = ChatMessage {
+            role: "user".to_string(),
+            content: Some(format!(
+                "tokens_to_add: {}\n\n\
+                 Existing memory_summary.md:\n\n{}\n\n---\n\n\
+                 Existing MEMORY.md:\n\n{}\n\n---\n\n\
+                 New rollout log:\n\n{}",
+                tokens_summarized, memory_summary, memory_handbook, truncated_log
+            )),
+            tool_call_id: None,
+            tool_calls: None,
+        };
+        let messages = vec![system_msg, user_msg];
+
+        let output = if let Some(ref ds) = self.deepseek {
+            ds.call_api(&messages, Some(0.2)).await?
+        } else if let Some(ref g) = self.glm {
+            g.call_api(&messages, Some(0.2)).await?
+        } else {
+            return Err("No API client available for memory consolidation.".to_string());
+        };
+
+        CodexMemories::parse_consolidation_output(&output)
+            .ok_or_else(|| "Consolidation output missing required markers.".to_string())
+    }
+
     #[allow(dead_code)]
     pub async fn execute_chat(
         &self,
@@ -137,7 +223,8 @@ impl AgentRouter {
         route: Route,
         _sandbox_path: Option<&Path>,
     ) -> Result<(String, &'static str), String> {
-        let messages = Self::build_messages(history);
+        let (agents, memory_summary) = Self::load_prompt_memory(_sandbox_path);
+        let messages = Self::build_messages(history, agents.as_deref(), memory_summary.as_deref());
         match route {
             Route::Light => {
                 if let Some(ref ds) = self.deepseek {
@@ -174,7 +261,8 @@ impl AgentRouter {
         sandbox_path: Option<&Path>,
         tx: tokio::sync::mpsc::UnboundedSender<String>,
     ) -> Result<(Option<Usage>, &'static str), String> {
-        let messages = Self::build_messages(history);
+        let (agents, memory_summary) = Self::load_prompt_memory(sandbox_path);
+        let messages = Self::build_messages(history, agents.as_deref(), memory_summary.as_deref());
         match route {
             Route::Light => {
                 if let Some(ref ds) = self.deepseek {
@@ -248,6 +336,17 @@ impl AgentRouter {
     }
 }
 
+fn truncate_utf8_tail(content: &str, max_bytes: usize) -> &str {
+    if content.len() <= max_bytes {
+        return content;
+    }
+    let mut start = content.len() - max_bytes;
+    while start < content.len() && !content.is_char_boundary(start) {
+        start += 1;
+    }
+    &content[start..]
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

package/src/agent/db_agent.rs

@@ -1,4 +1,5 @@
 use super::AgentRouter;
+use super::skills::{AgentKind, append_agent_skills};
 use super::types::merge_tool_call_deltas;
 use super::types::{
     ChatMessage, ChatRequest, FunctionDefinition, StreamOptions, Tool, ToolCall, Usage,
@@ -6,6 +7,27 @@ use super::types::{
 use crate::memory::MemoryStore;
 use std::path::Path;
 
+fn build_database_query_tool() -> Tool {
+    Tool {
+        r#type: "function".to_string(),
+        function: FunctionDefinition {
+            name: "query_sqlite".to_string(),
+            description: "Run one read-only SELECT query against the indexed project SQLite database. Use this for project facts from files, programs, data_items, call_edges, copybook_uses, program_features, code_blocks, external_ops, identifiers, literals, copybook_features, and other indexed COBOL metadata. Do not use it for writes, DDL, or guesses."
+                .to_string(),
+            parameters: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "sql": {
+                        "type": "string",
+                        "description": "A single SQLite SELECT statement that reads indexed project data."
+                    }
+                },
+                "required": ["sql"]
+            }),
+        },
+    }
+}
+
 impl AgentRouter {
     pub(crate) async fn run_database_agent_stream(
         &self,
@@ -34,7 +56,7 @@ impl AgentRouter {
 
         if let Some(first_msg) = messages.get_mut(0) {
             if first_msg.role == "system" {
-                first_msg.content = Some(
+                let mut system_prompt =
                     "You are the COBOLX Database Sub-Agent. Your task is to help the user analyze \
                     their COBOL codebase by querying the local SQLite database. You have access to \
                     the `query_sqlite` tool to execute read-only SELECT queries.\n\
@@ -47,33 +69,35 @@ impl AgentRouter {
                          kind 'static'|'dynamic', using_count)\n\
                     5. `data_items` (id, program_id, source_file_id, name, level, parent_name, \
                          pic, usage_clause, occurs, redefines, section, byte_offset, byte_size, \
-                         storage_kind, layout_status, start_offset, byte_len)\n\n\
+                         storage_kind, layout_status, start_offset, byte_len)\n\
+                    6. `program_features` (program_id, source_file_id, incoming_call_count, \
+                         outgoing_call_count, static_call_count, dynamic_call_count, \
+                         copybook_use_count, distinct_copybook_count, referenced_by_file_count, \
+                         is_entrypoint, has_heavy_copy_usage, data_item_count, paragraph_count, \
+                         external_op_count, identifier_count, literal_count)\n\
+                    7. `code_blocks` (id, program_id, source_file_id, name, kind \
+                         'section'|'paragraph', parent_section, sequence_no, statement_count, \
+                         start_offset, byte_len)\n\
+                    8. `external_ops` (id, program_id, source_file_id, kind, verb, target, \
+                         start_offset, byte_len)\n\
+                    9. `identifiers` (id, program_id, source_file_id, kind, value, occurrences, \
+                         first_offset)\n\
+                    10. `literals` (id, program_id, source_file_id, kind, value, occurrences, \
+                         first_offset)\n\
+                    11. `copybook_features` (copybook_file_id, copybook_name, \
+                         used_by_program_count, used_by_file_count, replacing_use_count, \
+                         data_item_count, contains_header_fields, contains_error_fields)\n\n\
                     GUIDELINES:\n\
                     - Write standard SELECT queries only (read-only).\n\
                     - If unsure about columns, query the schema first.\n\
                     - Explain answers clearly; if no data matches, say so."
-                        .to_string(),
-                );
+                        .to_string();
+                append_agent_skills(&mut system_prompt, sandbox_path, AgentKind::Database)?;
+                first_msg.content = Some(system_prompt);
             }
         }
 
-        let query_sqlite_tool = Tool {
-            r#type: "function".to_string(),
-            function: FunctionDefinition {
-                name: "query_sqlite".to_string(),
-                description: "Run a read-only SELECT query against the local SQLite database \
-                    indexing the COBOL project structure."
-                    .to_string(),
-                parameters: serde_json::json!({
-                    "type": "object",
-                    "properties": {
-                        "sql": { "type": "string", "description": "The SQLite SELECT statement." }
-                    },
-                    "required": ["sql"]
-                }),
-            },
-        };
-        let tools = vec![query_sqlite_tool];
+        let tools = vec![build_database_query_tool()];
         let mut final_usage = Usage::default();
 
         for _turn in 0..10 {
@@ -185,8 +209,15 @@ impl AgentRouter {
                         .get("sql")
                         .and_then(|v| v.as_str())
                         .unwrap_or("");
-                    let db_result = match store.query_readonly(sql) {
-                        Ok(json_val) => json_val.to_string(),
+                    let db_result = match store.project_index_is_empty() {
+                        Ok(true) => serde_json::json!({
+                            "error": "Project index is empty. Run /init before asking database questions."
+                        })
+                        .to_string(),
+                        Ok(false) => match store.query_readonly(sql) {
+                            Ok(json_val) => json_val.to_string(),
+                            Err(err) => serde_json::json!({ "error": err.to_string() }).to_string(),
+                        },
                         Err(err) => serde_json::json!({ "error": err.to_string() }).to_string(),
                     };
                     messages.push(ChatMessage {
@@ -204,3 +235,20 @@ impl AgentRouter {
         Ok(Some(final_usage))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn database_query_tool_description_spells_out_readonly_scope() {
+        let tool = build_database_query_tool();
+        assert!(tool.function.description.contains("SELECT"));
+        assert!(tool.function.description.contains("read-only"));
+        assert!(
+            tool.function.description.contains(
+                "files, programs, data_items, call_edges, copybook_uses, program_features"
+            )
+        );
+    }
+}

package/src/agent/explain_agent.rs

@@ -1,10 +1,12 @@
 use super::AgentRouter;
+use super::skills::{AgentKind, append_agent_skills};
 use super::types::merge_tool_call_deltas;
 use super::types::{
-    ChatMessage, ChatRequest, ChatResponse, FunctionDefinition, StreamOptions, Tool, ToolCall,
-    Usage,
+    ChatMessage, ChatRequest, ChatResponse, FunctionDefinition, SharedWriteBuffer, StreamOptions,
+    Tool, ToolCall, Usage, WriteBuffer,
 };
 use std::path::Path;
+use std::sync::Arc;
 
 impl AgentRouter {
     /// Verify agent: reviews a draft answer and returns (passed, feedback).
@@ -13,6 +15,7 @@ impl AgentRouter {
         user_question: &str,
         gathered_data: &str,
         draft_content: &str,
+        sandbox_path: &Path,
     ) -> Result<(bool, String), String> {
         let (api_key, api_url, model_name) = if let Some(ref g) = self.glm {
             (
@@ -30,12 +33,15 @@ impl AgentRouter {
             return Err("No API client for Verify Agent.".to_string());
         };
 
-        let system_prompt = "You are the COBOLX Verify Agent. Review the draft answer.\n\
-            Look for: incomplete paragraphs, missing values, TODOs, incorrect analysis, \
-            grammar/logic issues, missing files/variables the user asked about.\n\
+        let mut system_prompt = "You are the COBOLX Verify Agent. Review the draft answer.\n\
+            Look for:\n\
+            1. Incomplete paragraphs, missing values, TODOs, incorrect analysis, grammar/logic issues.\n\
+            2. Missing files/variables the user asked about.\n\
+            3. File naming preservation: ensure any documentation written via write_file uses the original base name of the COBOL source file (e.g. docs/XYZ.md for source XYZ.cbl) instead of generic names like entity.md or model.md.\n\
             Return a JSON object ONLY (no markdown wrapping):\n\
             { \"passed\": bool, \"feedback\": \"...\" }"
             .to_string();
+        append_agent_skills(&mut system_prompt, sandbox_path, AgentKind::Verify)?;
 
         let user_prompt = format!(
             "User Question:\n{}\n\nGathered Data:\n{}\n\nDraft Answer:\n{}",
@@ -139,6 +145,7 @@ impl AgentRouter {
                         gathered_data,
                         sandbox_path,
                         tx.clone(),
+                        None,
                     )
                     .await?;
                 if let Some(u) = usage {
@@ -157,6 +164,8 @@ impl AgentRouter {
 
             let (temp_tx, mut temp_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
             let real_tx_clone = tx.clone();
+            let write_buf: SharedWriteBuffer = Arc::new(WriteBuffer::new(Vec::new()));
+            let write_buf_clone = write_buf.clone();
 
             let collect_handle = tokio::spawn(async move {
                 let mut accumulated_text = String::new();
@@ -174,7 +183,13 @@ impl AgentRouter {
             });
 
             let res = self
-                .run_explain_agent_stream_internal(&messages, gathered_data, sandbox_path, temp_tx)
+                .run_explain_agent_stream_internal(
+                    &messages,
+                    gathered_data,
+                    sandbox_path,
+                    temp_tx,
+                    Some(write_buf_clone),
+                )
                 .await;
 
             let (accumulated_text, accumulated_reasoning) = match collect_handle.await {
@@ -192,12 +207,25 @@ impl AgentRouter {
 
             let _ = tx.send("\x01STATUS:Verify Agent: Reviewing draft...".to_string());
             match self
-                .run_verify_agent(&user_question, gathered_data, &accumulated_text)
+                .run_verify_agent(
+                    &user_question,
+                    gathered_data,
+                    &accumulated_text,
+                    sandbox_path,
+                )
                 .await
             {
                 Ok((passed, feedback)) => {
                     if passed {
                         let _ = tx.send("\x01STATUS:Verify Agent: Validation passed!".to_string());
+
+                        if let Ok(lock) = write_buf.lock() {
+                            if let Err(e) = self.commit_write_buffer(&lock) {
+                                let _ =
+                                    tx.send(format!("\x01STATUS:Failed to commit files: {}", e));
+                            }
+                        }
+
                         if !accumulated_reasoning.is_empty() {
                             let _ = tx.send(format!("\x01REASONING:{}", accumulated_reasoning));
                         }
@@ -232,6 +260,13 @@ impl AgentRouter {
                 }
                 Err(e) => {
                     let _ = tx.send(format!("\x01STATUS:Verify Agent error: {}, skipping...", e));
+
+                    if let Ok(lock) = write_buf.lock() {
+                        if let Err(e) = self.commit_write_buffer(&lock) {
+                            let _ = tx.send(format!("\x01STATUS:Failed to commit files: {}", e));
+                        }
+                    }
+
                     if !accumulated_reasoning.is_empty() {
                         let _ = tx.send(format!("\x01REASONING:{}", accumulated_reasoning));
                     }
@@ -268,6 +303,7 @@ impl AgentRouter {
         gathered_data: &str,
         sandbox_path: &Path,
         tx: tokio::sync::mpsc::UnboundedSender<String>,
+        write_buffer: Option<SharedWriteBuffer>,
     ) -> Result<(Option<Usage>, &'static str), String> {
         let (api_key, api_url, model_name_static) = if let Some(ref g) = self.glm {
             (
@@ -300,7 +336,7 @@ impl AgentRouter {
             .unwrap_or("")
             .to_string();
 
-        let system_prompt = format!(
+        let mut system_prompt = format!(
             "You are COBOLX, an expert COBOL systems analyst and legacy migration specialist.\n\
             A retrieval agent has already gathered this structured data from the COBOL project:\n\n\
             ---\n{gathered_data}\n---\n\n\
@@ -308,9 +344,11 @@ impl AgentRouter {
             - Explanation/analysis: Markdown document covering program purpose, data structures, \
               business logic, CALL graph, COPY dependencies, and migration notes.\n\
             - Documentation (e.g. /docs): use write_file to write Markdown docs to docs/.\n\
+              CRITICAL: You MUST preserve the original base name of the COBOL source file and replace its extension with `.md` (e.g., write to `docs/XYZ.md` for a source file named `XYZ.cbl` or `XYZ.cob`). Do NOT use generic names like `entity.md` or `model.md` unless explicitly asked.\n\
             Sandbox root for write_file: {sandbox_display}",
             sandbox_display = sandbox_path.to_string_lossy()
         );
+        append_agent_skills(&mut system_prompt, sandbox_path, AgentKind::Explain)?;
 
         let mut messages = vec![
             ChatMessage {
@@ -455,21 +493,14 @@ impl AgentRouter {
                     let path_str = args.get("path").and_then(|v| v.as_str()).unwrap_or("");
                     let content = args.get("content").and_then(|v| v.as_str()).unwrap_or("");
                     let _ = tx.send(format!("\x01STATUS:Writing file: {path_str}"));
-                    match Self::validate_sandbox_path(sandbox_path, path_str) {
+                    match self.write_file(sandbox_path, path_str, content, write_buffer.as_deref())
+                    {
+                        Ok(full_path) => serde_json::json!({
+                            "ok": true,
+                            "path": full_path.to_string_lossy()
+                        })
+                        .to_string(),
                         Err(e) => serde_json::json!({ "error": e }).to_string(),
-                        Ok(full_path) => {
-                            if let Some(parent) = full_path.parent() {
-                                let _ = std::fs::create_dir_all(parent);
-                            }
-                            match std::fs::write(&full_path, content) {
-                                Ok(_) => serde_json::json!({
-                                    "ok": true,
-                                    "path": full_path.to_string_lossy()
-                                })
-                                .to_string(),
-                                Err(e) => serde_json::json!({ "error": e.to_string() }).to_string(),
-                            }
-                        }
                     }
                 } else {
                     serde_json::json!({ "error": format!("Unknown tool: {}", tc.function.name) })