cobolx 1.0.3 → 1.0.4

package/src/ui/tui.rs

@@ -1,4 +1,5 @@
 use crate::agent::client::AgentRouter;
+use crate::memory::{RunJournal, TOKEN_SUMMARY_THRESHOLD};
 use crate::ui::draw;
 use chrono::Local;
 use crossterm::{
@@ -7,6 +8,7 @@ use crossterm::{
     terminal::{EnterAlternateScreen, LeaveAlternateScreen, disable_raw_mode, enable_raw_mode},
 };
 use ratatui::{Terminal, backend::CrosstermBackend};
+use serde_json::json;
 use std::io;
 use std::sync::Arc;
 
@@ -52,6 +54,10 @@ pub enum TaskUpdate {
         Result<Option<crate::agent::client::Usage>, String>,
         &'static str,
     ),
+    MemorySummarized {
+        ok: bool,
+        detail: String,
+    },
 }
 
 pub struct App {
@@ -80,6 +86,8 @@ pub struct App {
     pub spinner_tick: usize,
     pub console_scroll_offset: u16,
     pub custom_prompt_override: Option<String>,
+    pub run_journal: Option<RunJournal>,
+    pub tokens_since_summary: u32,
 }
 
 impl App {
@@ -142,6 +150,38 @@ impl App {
             spinner_tick: 0,
             console_scroll_offset: 0,
             custom_prompt_override: None,
+            run_journal: None,
+            tokens_since_summary: 0,
+        }
+    }
+
+    fn start_run_journal(&mut self, sandbox: &std::path::Path) {
+        if let Ok(store) = crate::memory::MemoryStore::open_or_create(sandbox) {
+            let _ = store.codex_memories().ensure_layout();
+            if let Ok(journal) = RunJournal::start(store.runs_dir()) {
+                self.run_journal = Some(journal);
+            }
+        }
+    }
+
+    fn log_run(&mut self, kind: &str, payload: serde_json::Value) {
+        if let Some(journal) = &mut self.run_journal {
+            journal.log(kind, payload);
+        }
+    }
+
+    fn finish_run_journal(&mut self, status: &str) {
+        if let Some(journal) = &self.run_journal {
+            journal.finish(status);
+            if let Some(sandbox) = &self.sandbox_path {
+                if let Ok(store) = crate::memory::MemoryStore::open_or_create(sandbox) {
+                    let mem = store.codex_memories();
+                    if let Ok(log) = journal.read_log() {
+                        let _ = mem.write_rollout_summary(journal.run_id(), &log);
+                        let _ = mem.write_raw_memories(&log);
+                    }
+                }
+            }
         }
     }
 
@@ -155,6 +195,7 @@ impl App {
             "/tokens".to_string(),
             "/init".to_string(),
             "/docs".to_string(),
+            "/skills".to_string(),
             "/exit".to_string(),
         ];
         if !self.input_text.starts_with('/') {
@@ -235,6 +276,14 @@ impl App {
             timestamp: Local::now().format("%H:%M:%S").to_string(),
         });
 
+        self.log_run(
+            "user_message",
+            json!({
+                "text": text,
+                "is_command": text.starts_with('/'),
+            }),
+        );
+
         let mut should_exit = false;
         let mut is_command = false;
 
@@ -246,6 +295,7 @@ impl App {
                 let cmd = parts[0][1..].to_lowercase();
                 match cmd.as_str() {
                     "exit" | "quit" => {
+                        self.log_run("session_exit", json!({ "reason": "user_command" }));
                         should_exit = true;
                     }
                     "clear" => {
@@ -276,6 +326,7 @@ impl App {
                                             /tokens           - Show model routing and token consumption statistics\n\
                                             /init             - Scan the sandbox directory for COBOL files\n\
                                             /docs             - Generate Markdown documentation for all COBOL files into docs/\n\
+                                            /skills           - Check which agent skills (.cobolx/skills) are actually loaded\n\
                                             /exit             - Close the interactive console";
                         self.messages.push(Message {
                             sender: Sender::Cobolx,
@@ -302,6 +353,13 @@ impl App {
                             ) {
                                 Ok((report, db_path)) => {
                                     self.discovered_files = report.files.clone();
+                                    self.log_run(
+                                        "index_completed",
+                                        json!({
+                                            "file_count": report.files.len(),
+                                            "db_path": db_path.to_string_lossy(),
+                                        }),
+                                    );
                                     if self.discovered_files.is_empty() {
                                         self.messages.push(Message {
                                             sender: Sender::Cobolx,
@@ -317,6 +375,7 @@ impl App {
                                     }
                                 }
                                 Err(e) => {
+                                    self.log_run("index_failed", json!({ "error": e.to_string() }));
                                     self.messages.push(Message {
                                         sender: Sender::Cobolx,
                                         text: format!("Error indexing sandbox: {}", e),
@@ -590,6 +649,73 @@ impl App {
                             });
                         }
                     }
+                    "skills" => {
+                        if let Some(ref sandbox_path) = self.sandbox_path {
+                            let checks = crate::agent::skills::check_all_agent_skills(sandbox_path);
+                            let mut report = String::from(
+                                "Agent Skills Check (.cobolx/skills):\n\
+                                                             -------------------------------------\n",
+                            );
+                            for check in &checks {
+                                let dir = format!(".cobolx/skills/{}", check.agent.skill_dir());
+                                if let Some(err) = &check.error {
+                                    report.push_str(&format!(
+                                        "[{}] {}  →  ERROR: {}\n",
+                                        check.agent.label(),
+                                        dir,
+                                        err
+                                    ));
+                                    continue;
+                                }
+                                if !check.dir_exists {
+                                    report.push_str(&format!(
+                                        "[{}] {}  →  not configured (no directory)\n",
+                                        check.agent.label(),
+                                        dir
+                                    ));
+                                    continue;
+                                }
+                                if check.files.is_empty() {
+                                    report.push_str(&format!(
+                                        "[{}] {}  →  directory exists, no .md skill files found\n",
+                                        check.agent.label(),
+                                        dir
+                                    ));
+                                    continue;
+                                }
+                                let status = if check.is_active() {
+                                    "ACTIVE (loaded into system prompt)"
+                                } else {
+                                    "INACTIVE (failed to load)"
+                                };
+                                report.push_str(&format!(
+                                    "[{}] {}  →  {}\n  files: {} ({} bytes source, {} bytes injected{})\n",
+                                    check.agent.label(),
+                                    dir,
+                                    status,
+                                    check.files.len(),
+                                    check.source_bytes,
+                                    check.injected_bytes,
+                                    if check.truncated { ", TRUNCATED at 8KB cap" } else { "" }
+                                ));
+                                for f in &check.files {
+                                    report.push_str(&format!("    - {}\n", f));
+                                }
+                            }
+                            self.messages.push(Message {
+                                sender: Sender::Cobolx,
+                                text: report.trim_end().to_string(),
+                                timestamp: Local::now().format("%H:%M:%S").to_string(),
+                            });
+                        } else {
+                            self.messages.push(Message {
+                                sender: Sender::Cobolx,
+                                text: "No sandbox directory set. Please select a sandbox first."
+                                    .to_string(),
+                                timestamp: Local::now().format("%H:%M:%S").to_string(),
+                            });
+                        }
+                    }
                     _ => {
                         self.messages.push(Message {
                             sender: Sender::Cobolx,
@@ -703,6 +829,66 @@ fn trigger_chat_task(app: &mut App, tx: &tokio::sync::mpsc::UnboundedSender<Task
     false
 }
 
+fn spawn_memory_consolidation(
+    app: &App,
+    tx: &tokio::sync::mpsc::UnboundedSender<TaskUpdate>,
+    tokens_summarized: u32,
+) {
+    let sandbox = app.sandbox_path.clone();
+    let router = Arc::clone(&app.router);
+    let run_log = app
+        .run_journal
+        .as_ref()
+        .and_then(|j| j.read_log().ok())
+        .unwrap_or_default();
+    let tx = tx.clone();
+
+    tokio::spawn(async move {
+        let _ = tx.send(TaskUpdate::Status(
+            "Consolidating memories (Codex Phase 2)...".to_string(),
+        ));
+        let result: Result<(), String> = async {
+            let sandbox = sandbox.ok_or("No sandbox path for memory consolidation.")?;
+            let store =
+                crate::memory::MemoryStore::open_or_create(&sandbox).map_err(|e| e.to_string())?;
+            let mem = store.codex_memories();
+            let summary = mem.read_memory_summary().map_err(|e| e.to_string())?;
+            let handbook = mem.read_memory_handbook().map_err(|e| e.to_string())?;
+            let (new_summary, new_handbook) = router
+                .consolidate_codex_memories(&summary, &handbook, &run_log, tokens_summarized)
+                .await?;
+            mem.write_consolidated(&new_summary, &new_handbook)
+                .map_err(|e| e.to_string())?;
+            Ok(())
+        }
+        .await;
+        let _ = tx.send(TaskUpdate::Status("".to_string()));
+        match result {
+            Ok(()) => {
+                let _ = tx.send(TaskUpdate::MemorySummarized {
+                    ok: true,
+                    detail: format!("~{tokens_summarized} tokens → memory_summary.md + MEMORY.md"),
+                });
+            }
+            Err(err) => {
+                let _ = tx.send(TaskUpdate::MemorySummarized {
+                    ok: false,
+                    detail: err,
+                });
+            }
+        }
+    });
+}
+
+fn route_label(route: &crate::agent::client::Route) -> &'static str {
+    match route {
+        crate::agent::client::Route::Light => "LIGHT",
+        crate::agent::client::Route::Heavy => "HEAVY",
+        crate::agent::client::Route::Database => "DATABASE",
+        crate::agent::client::Route::Filesystem => "FILESYSTEM",
+    }
+}
+
 pub fn run_tui() -> Result<(), io::Error> {
     enable_raw_mode()?;
     let mut stdout = io::stdout();
@@ -718,6 +904,13 @@ pub fn run_tui() -> Result<(), io::Error> {
         while let Ok(update) = rx.try_recv() {
             match update {
                 TaskUpdate::Routed(ref route, model_used) => {
+                    app.log_run(
+                        "route_selected",
+                        json!({
+                            "route": route_label(route),
+                            "model": model_used,
+                        }),
+                    );
                     app.active_agent = Some(model_used.to_string());
                     if matches!(
                         route,
@@ -769,6 +962,9 @@ pub fn run_tui() -> Result<(), io::Error> {
                     }
                 }
                 TaskUpdate::Status(status) => {
+                    if !status.is_empty() {
+                        app.log_run("agent_status", json!({ "status": status }));
+                    }
                     if status.is_empty() {
                         app.agent_status = None;
                     } else {
@@ -776,6 +972,25 @@ pub fn run_tui() -> Result<(), io::Error> {
                     }
                 }
                 TaskUpdate::Finished(res, model_used) => {
+                    let payload = match &res {
+                        Ok(Some(usage)) => json!({
+                            "model": model_used,
+                            "ok": true,
+                            "prompt_tokens": usage.prompt_tokens,
+                            "completion_tokens": usage.completion_tokens,
+                            "total_tokens": usage.total_tokens,
+                        }),
+                        Ok(None) => json!({
+                            "model": model_used,
+                            "ok": true,
+                        }),
+                        Err(err) => json!({
+                            "model": model_used,
+                            "ok": false,
+                            "error": err,
+                        }),
+                    };
+                    app.log_run("agent_finished", payload);
                     app.active_agent = None;
                     app.agent_status = None;
                     if let Some(msg) = app.messages.iter_mut().last() {
@@ -802,6 +1017,13 @@ pub fn run_tui() -> Result<(), io::Error> {
                                 app.glm_prompt_tokens += usage.prompt_tokens;
                                 app.glm_completion_tokens += usage.completion_tokens;
                             }
+
+                            app.tokens_since_summary += usage.total_tokens;
+                            if app.tokens_since_summary >= TOKEN_SUMMARY_THRESHOLD {
+                                let tokens_batch = app.tokens_since_summary;
+                                app.tokens_since_summary = 0;
+                                spawn_memory_consolidation(&app, &tx, tokens_batch);
+                            }
                         }
                         Ok(None) => {
                             app.last_model = Some(model_used.to_string());
@@ -820,6 +1042,16 @@ pub fn run_tui() -> Result<(), io::Error> {
                         }
                     }
                 }
+                TaskUpdate::MemorySummarized { ok, detail } => {
+                    app.log_run("memory_summarized", json!({ "ok": ok, "detail": detail }));
+                    if ok {
+                        app.messages.push(Message {
+                            sender: Sender::Cobolx,
+                            text: format!("Memory consolidated ({detail})."),
+                            timestamp: Local::now().format("%H:%M:%S").to_string(),
+                        });
+                    }
+                }
             }
         }
 
@@ -928,7 +1160,12 @@ pub fn run_tui() -> Result<(), io::Error> {
                                     .and_then(|p| p.parent().map(|parent| parent.to_path_buf()))
                             };
                             if let Some(path) = resolved {
+                                app.start_run_journal(&path);
                                 app.sandbox_path = Some(path.clone());
+                                app.log_run(
+                                    "sandbox_selected",
+                                    json!({ "path": path.to_string_lossy() }),
+                                );
                                 app.view_mode = ViewMode::Chat;
                                 app.messages.push(Message {
                                     sender: Sender::Cobolx,
@@ -1048,6 +1285,8 @@ pub fn run_tui() -> Result<(), io::Error> {
         }
     }
 
+    app.finish_run_journal("completed");
+
     disable_raw_mode()?;
     execute!(
         terminal.backend_mut(),

package/tests/indexer_tests.rs

@@ -190,3 +190,264 @@ fn init_indexer_persists_program_copybook_and_call_edges() {
     assert_eq!(alias_layout, (8, 10));
     assert_eq!(occurs_layout, (23, 60));
 }
+
+#[test]
+fn init_indexer_persists_richer_semantic_index_data() {
+    let dir = tempdir().unwrap();
+    let copy_dir = dir.path().join("copy");
+    fs::create_dir_all(&copy_dir).unwrap();
+
+    File::create(dir.path().join("MAIN.cbl"))
+        .unwrap()
+        .write_all(
+            br#"
+       IDENTIFICATION DIVISION.
+       PROGRAM-ID. MAIN.
+       DATA DIVISION.
+       WORKING-STORAGE SECTION.
+       01 WS-NEXT-PGM PIC X(8).
+       01 CUSTOMER-FILE PIC X(8).
+       01 CUSTOMER-RECORD PIC X(80).
+       COPY COMMONHDR.
+       COPY ERRCODES.
+       PROCEDURE DIVISION.
+       MAIN-SECTION SECTION.
+       MAIN-START.
+           OPEN INPUT CUSTOMER-FILE.
+           EXEC SQL
+              SELECT CUST_ID
+                INTO :WS-NEXT-PGM
+                FROM CUSTOMER_TABLE
+               WHERE STATUS = 'ER'
+           END-EXEC.
+           CALL "SUB001"
+               USING WS-NEXT-PGM.
+           CALL WS-NEXT-PGM.
+           PERFORM HANDLE-ERROR.
+           READ CUSTOMER-FILE.
+       HANDLE-ERROR.
+           EXEC CICS
+               LINK PROGRAM('ERRHNDL')
+               COMMAREA(WS-NEXT-PGM)
+           END-EXEC.
+           WRITE CUSTOMER-RECORD.
+           CLOSE CUSTOMER-FILE.
+           STOP RUN.
+"#,
+        )
+        .unwrap();
+
+    File::create(dir.path().join("SUB001.cbl"))
+        .unwrap()
+        .write_all(
+            br#"
+       IDENTIFICATION DIVISION.
+       PROGRAM-ID. SUB001.
+       PROCEDURE DIVISION.
+           EXIT PROGRAM.
+"#,
+        )
+        .unwrap();
+
+    File::create(copy_dir.join("COMMONHDR.cpy"))
+        .unwrap()
+        .write_all(
+            br#"
+       01 COMMON-HDR.
+          05 HDR-REQUEST-ID PIC X(16).
+          05 HDR-DATE PIC X(8).
+"#,
+        )
+        .unwrap();
+
+    File::create(copy_dir.join("ERRCODES.cpy"))
+        .unwrap()
+        .write_all(
+            br#"
+       01 ERROR-INFO.
+          05 ERR-CODE PIC X(4).
+          05 ERR-MESSAGE PIC X(40).
+"#,
+        )
+        .unwrap();
+
+    let mut store = rdo::memory::MemoryStore::open_or_create(dir.path()).unwrap();
+    rdo::cobol::indexer::index_sandbox(dir.path(), &mut store).unwrap();
+
+    let conn = store.connection();
+
+    let main_program_features: (i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) = conn
+        .query_row(
+            "SELECT incoming_call_count, outgoing_call_count, static_call_count, dynamic_call_count, \
+                    copybook_use_count, referenced_by_file_count, is_entrypoint, paragraph_count, \
+                    external_op_count, literal_count \
+             FROM program_features pf \
+             JOIN programs p ON p.id = pf.program_id \
+             WHERE p.name = 'MAIN'",
+            [],
+            |row| {
+                Ok((
+                    row.get(0)?,
+                    row.get(1)?,
+                    row.get(2)?,
+                    row.get(3)?,
+                    row.get(4)?,
+                    row.get(5)?,
+                    row.get(6)?,
+                    row.get(7)?,
+                    row.get(8)?,
+                    row.get(9)?,
+                ))
+            },
+        )
+        .unwrap();
+
+    let sub_program_features: (i64, i64, i64) = conn
+        .query_row(
+            "SELECT incoming_call_count, referenced_by_file_count, is_entrypoint \
+             FROM program_features pf \
+             JOIN programs p ON p.id = pf.program_id \
+             WHERE p.name = 'SUB001'",
+            [],
+            |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)),
+        )
+        .unwrap();
+
+    let code_blocks: Vec<(String, String, i64)> = {
+        let mut stmt = conn
+            .prepare(
+                "SELECT cb.name, cb.kind, cb.statement_count \
+                 FROM code_blocks cb \
+                 JOIN programs p ON p.id = cb.program_id \
+                 WHERE p.name = 'MAIN' \
+                 ORDER BY sequence_no",
+            )
+            .unwrap();
+        stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap()
+    };
+
+    let external_ops: Vec<(String, String, Option<String>)> = {
+        let mut stmt = conn
+            .prepare(
+                "SELECT kind, verb, target \
+                 FROM external_ops eo \
+                 JOIN programs p ON p.id = eo.program_id \
+                 WHERE p.name = 'MAIN' \
+                 ORDER BY eo.id",
+            )
+            .unwrap();
+        stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap()
+    };
+
+    let identifiers: Vec<(String, String, i64)> = {
+        let mut stmt = conn
+            .prepare(
+                "SELECT kind, value, occurrences \
+                 FROM identifiers i \
+                 JOIN programs p ON p.id = i.program_id \
+                 WHERE p.name = 'MAIN' \
+                 ORDER BY kind, value",
+            )
+            .unwrap();
+        stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap()
+    };
+
+    let literals: Vec<(String, String, i64)> = {
+        let mut stmt = conn
+            .prepare(
+                "SELECT kind, value, occurrences \
+                 FROM literals l \
+                 JOIN programs p ON p.id = l.program_id \
+                 WHERE p.name = 'MAIN' \
+                 ORDER BY kind, value",
+            )
+            .unwrap();
+        stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)))
+            .unwrap()
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap()
+    };
+
+    let copybook_features: Vec<(String, i64, i64, i64, i64, i64)> = {
+        let mut stmt = conn
+            .prepare(
+                "SELECT cf.copybook_name, cf.used_by_program_count, cf.used_by_file_count, \
+                        cf.data_item_count, cf.contains_header_fields, cf.contains_error_fields \
+                 FROM copybook_features cf \
+                 ORDER BY cf.copybook_name",
+            )
+            .unwrap();
+        stmt.query_map([], |row| {
+            Ok((
+                row.get(0)?,
+                row.get(1)?,
+                row.get(2)?,
+                row.get(3)?,
+                row.get(4)?,
+                row.get(5)?,
+            ))
+        })
+        .unwrap()
+        .collect::<Result<Vec<_>, _>>()
+        .unwrap()
+    };
+
+    assert_eq!(main_program_features, (0, 2, 1, 1, 2, 0, 1, 2, 8, 3));
+    assert_eq!(sub_program_features, (1, 1, 0));
+    assert_eq!(
+        code_blocks,
+        vec![
+            ("MAIN-SECTION".to_string(), "section".to_string(), 0),
+            ("MAIN-START".to_string(), "paragraph".to_string(), 6),
+            ("HANDLE-ERROR".to_string(), "paragraph".to_string(), 4),
+        ]
+    );
+    assert!(external_ops.contains(&(
+        "file_io".to_string(),
+        "OPEN".to_string(),
+        Some("CUSTOMER-FILE".to_string())
+    )));
+    assert!(external_ops.contains(&(
+        "exec_sql".to_string(),
+        "SELECT".to_string(),
+        Some("CUSTOMER_TABLE".to_string())
+    )));
+    assert!(external_ops.contains(&(
+        "call_literal".to_string(),
+        "CALL".to_string(),
+        Some("SUB001".to_string())
+    )));
+    assert!(external_ops.contains(&(
+        "call_identifier".to_string(),
+        "CALL".to_string(),
+        Some("WS-NEXT-PGM".to_string())
+    )));
+    assert!(external_ops.contains(&(
+        "exec_cics".to_string(),
+        "LINK".to_string(),
+        Some("ERRHNDL".to_string())
+    )));
+    assert!(identifiers.contains(&("sql_table".to_string(), "CUSTOMER_TABLE".to_string(), 1)));
+    assert!(identifiers.contains(&("file_name".to_string(), "CUSTOMER-FILE".to_string(), 3)));
+    assert!(identifiers.contains(&("paragraph_name".to_string(), "HANDLE-ERROR".to_string(), 1)));
+    assert!(identifiers.contains(&("data_name".to_string(), "ERR-CODE".to_string(), 1)));
+    assert!(literals.contains(&("call_target".to_string(), "SUB001".to_string(), 1)));
+    assert!(literals.contains(&("exec_cics_program".to_string(), "ERRHNDL".to_string(), 1)));
+    assert!(literals.contains(&("string_literal".to_string(), "ER".to_string(), 1)));
+    assert_eq!(
+        copybook_features,
+        vec![
+            ("COMMONHDR".to_string(), 1, 1, 3, 1, 0),
+            ("ERRCODES".to_string(), 1, 1, 3, 0, 1),
+        ]
+    );
+}