@sesamespace/hivemind 0.8.12 → 0.10.0

package/packages/memory/src/src/tasks.rs

@@ -0,0 +1,321 @@
+use anyhow::Result;
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use chrono::Utc;
+use futures::stream::TryStreamExt;
+use lancedb::{connection::Connection, query::ExecutableQuery, query::QueryBase, Table};
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+const TASKS_TABLE: &str = "tasks";
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct TaskRecord {
+    pub id: String,
+    pub context_name: String,
+    pub title: String,
+    pub description: String,
+    pub status: String,
+    pub blocked_by: String, // JSON array stored as string
+    pub created_at: String,
+    pub updated_at: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TaskInput {
+    pub context_name: String,
+    pub title: String,
+    pub description: String,
+    #[serde(default = "default_status")]
+    pub status: String,
+    #[serde(default)]
+    pub blocked_by: Vec<String>,
+}
+
+fn default_status() -> String {
+    "planned".to_string()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TaskUpdate {
+    pub status: Option<String>,
+    pub title: Option<String>,
+    pub description: Option<String>,
+    pub blocked_by: Option<Vec<String>>,
+}
+
+pub struct TaskStore {
+    db: Connection,
+}
+
+impl TaskStore {
+    pub async fn new(db: Connection) -> Result<Self> {
+        let store = Self { db };
+        store.ensure_table().await?;
+        Ok(store)
+    }
+
+    fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Utf8, false),
+            Field::new("context_name", DataType::Utf8, false),
+            Field::new("title", DataType::Utf8, false),
+            Field::new("description", DataType::Utf8, false),
+            Field::new("status", DataType::Utf8, false),
+            Field::new("blocked_by", DataType::Utf8, false),
+            Field::new("created_at", DataType::Utf8, false),
+            Field::new("updated_at", DataType::Utf8, false),
+        ]))
+    }
+
+    async fn ensure_table(&self) -> Result<()> {
+        let names = self.db.table_names().execute().await?;
+        if !names.contains(&TASKS_TABLE.to_string()) {
+            let schema = Self::schema();
+            let batch = RecordBatch::new_empty(schema.clone());
+            let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+            self.db
+                .create_table(TASKS_TABLE, Box::new(batches))
+                .execute()
+                .await?;
+            tracing::info!("Created tasks table");
+        }
+        Ok(())
+    }
+
+    pub async fn create_task(&self, input: TaskInput) -> Result<TaskRecord> {
+        let id = uuid::Uuid::new_v4().to_string();
+        let now = Utc::now().to_rfc3339();
+        let blocked_by_json = serde_json::to_string(&input.blocked_by)?;
+
+        let task = TaskRecord {
+            id: id.clone(),
+            context_name: input.context_name.clone(),
+            title: input.title.clone(),
+            description: input.description.clone(),
+            status: input.status,
+            blocked_by: blocked_by_json.clone(),
+            created_at: now.clone(),
+            updated_at: now.clone(),
+        };
+
+        let schema = Self::schema();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![task.id.as_str()])),
+                Arc::new(StringArray::from(vec![task.context_name.as_str()])),
+                Arc::new(StringArray::from(vec![task.title.as_str()])),
+                Arc::new(StringArray::from(vec![task.description.as_str()])),
+                Arc::new(StringArray::from(vec![task.status.as_str()])),
+                Arc::new(StringArray::from(vec![blocked_by_json.as_str()])),
+                Arc::new(StringArray::from(vec![task.created_at.as_str()])),
+                Arc::new(StringArray::from(vec![task.updated_at.as_str()])),
+            ],
+        )?;
+
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        table.add(Box::new(batches)).execute().await?;
+
+        tracing::debug!("Created task {} in context {}", task.id, task.context_name);
+        Ok(task)
+    }
+
+    pub async fn list_tasks(
+        &self,
+        context: &str,
+        status_filter: Option<&str>,
+    ) -> Result<Vec<TaskRecord>> {
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+
+        let filter = match status_filter {
+            Some(status) => format!("context_name = '{}' AND status = '{}'", context, status),
+            None => format!("context_name = '{}'", context),
+        };
+
+        let results = table.query().only_if(filter).execute().await?;
+
+        let mut tasks = Vec::new();
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let ctx_names = batch
+                .column_by_name("context_name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let titles = batch
+                .column_by_name("title")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let descriptions = batch
+                .column_by_name("description")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let statuses = batch
+                .column_by_name("status")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let blocked_bys = batch
+                .column_by_name("blocked_by")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let created_ats = batch
+                .column_by_name("created_at")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let updated_ats = batch
+                .column_by_name("updated_at")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+
+            for i in 0..batch.num_rows() {
+                tasks.push(TaskRecord {
+                    id: ids.value(i).to_string(),
+                    context_name: ctx_names.value(i).to_string(),
+                    title: titles.value(i).to_string(),
+                    description: descriptions.value(i).to_string(),
+                    status: statuses.value(i).to_string(),
+                    blocked_by: blocked_bys.value(i).to_string(),
+                    created_at: created_ats.value(i).to_string(),
+                    updated_at: updated_ats.value(i).to_string(),
+                });
+            }
+        }
+
+        // Sort by created_at
+        tasks.sort_by(|a, b| a.created_at.cmp(&b.created_at));
+
+        Ok(tasks)
+    }
+
+    pub async fn get_task(&self, id: &str) -> Result<Option<TaskRecord>> {
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        let results = table
+            .query()
+            .only_if(format!("id = '{}'", id))
+            .execute()
+            .await?;
+
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+        for batch in &batches {
+            if batch.num_rows() > 0 {
+                let ids = batch.column_by_name("id").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let ctx_names = batch.column_by_name("context_name").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let titles = batch.column_by_name("title").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let descriptions = batch.column_by_name("description").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let statuses = batch.column_by_name("status").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let blocked_bys = batch.column_by_name("blocked_by").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let created_ats = batch.column_by_name("created_at").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let updated_ats = batch.column_by_name("updated_at").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+
+                return Ok(Some(TaskRecord {
+                    id: ids.value(0).to_string(),
+                    context_name: ctx_names.value(0).to_string(),
+                    title: titles.value(0).to_string(),
+                    description: descriptions.value(0).to_string(),
+                    status: statuses.value(0).to_string(),
+                    blocked_by: blocked_bys.value(0).to_string(),
+                    created_at: created_ats.value(0).to_string(),
+                    updated_at: updated_ats.value(0).to_string(),
+                }));
+            }
+        }
+
+        Ok(None)
+    }
+
+    pub async fn update_task(&self, id: &str, update: TaskUpdate) -> Result<Option<TaskRecord>> {
+        let existing = self.get_task(id).await?;
+        let Some(mut task) = existing else {
+            return Ok(None);
+        };
+
+        // Apply updates
+        if let Some(status) = update.status {
+            task.status = status;
+        }
+        if let Some(title) = update.title {
+            task.title = title;
+        }
+        if let Some(description) = update.description {
+            task.description = description;
+        }
+        if let Some(blocked_by) = update.blocked_by {
+            task.blocked_by = serde_json::to_string(&blocked_by)?;
+        }
+        task.updated_at = Utc::now().to_rfc3339();
+
+        // Delete and reinsert
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        table.delete(&format!("id = '{}'", id)).await?;
+
+        let schema = Self::schema();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![task.id.as_str()])),
+                Arc::new(StringArray::from(vec![task.context_name.as_str()])),
+                Arc::new(StringArray::from(vec![task.title.as_str()])),
+                Arc::new(StringArray::from(vec![task.description.as_str()])),
+                Arc::new(StringArray::from(vec![task.status.as_str()])),
+                Arc::new(StringArray::from(vec![task.blocked_by.as_str()])),
+                Arc::new(StringArray::from(vec![task.created_at.as_str()])),
+                Arc::new(StringArray::from(vec![task.updated_at.as_str()])),
+            ],
+        )?;
+
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        table.add(Box::new(batches)).execute().await?;
+
+        Ok(Some(task))
+    }
+
+    /// Get the next available task for a context:
+    /// - Status is "planned" (not active/complete/archived)
+    /// - Not blocked by any incomplete tasks
+    pub async fn get_next_task(&self, context: &str) -> Result<Option<TaskRecord>> {
+        let planned = self.list_tasks(context, Some("planned")).await?;
+        let all_tasks = self.list_tasks(context, None).await?;
+
+        // Build a set of complete task IDs
+        let complete_ids: std::collections::HashSet<String> = all_tasks
+            .iter()
+            .filter(|t| t.status == "complete" || t.status == "archived")
+            .map(|t| t.id.clone())
+            .collect();
+
+        for task in planned {
+            let blocked_by: Vec<String> =
+                serde_json::from_str(&task.blocked_by).unwrap_or_default();
+
+            // Task is available if all blockers are complete
+            let is_blocked = blocked_by.iter().any(|b| !complete_ids.contains(b));
+            if !is_blocked {
+                return Ok(Some(task));
+            }
+        }
+
+        Ok(None)
+    }
+}

package/.pnpmrc.json

@@ -1 +0,0 @@
-{"onlyBuiltDependencies":["better-sqlite3"]}

package/DASHBOARD-PLAN.md

@@ -1,206 +0,0 @@
-# Hivemind Dashboard — Implementation Plan
-
-**Goal:** Local web dashboard for debugging memory, context routing, and LLM request formation.
-**Access:** `http://localhost:9485` on the Mac mini (local access only for now).
-**Priority:** LLM Request Inspector first, then Memory Browser, then Context Overview.
-
----
-
-## Phase 1: LLM Request Logger + Inspector UI
-
-### Backend: Request Logging
-
-**Where:** Instrument `buildMessages()` in `prompt.ts` and `processMessage()` in `agent.ts`.
-
-Each logged request captures:
-```typescript
-interface RequestLog {
-  id: string;                    // uuid
-  timestamp: string;             // ISO-8601
-  // Routing
-  context: string;               // which context was used
-  contextSwitched: boolean;      // explicit switch?
-  routingReason: string;         // "pattern_match:X" | "inferred:X" | "active:X"
-  // Sender
-  channelId: string;
-  channelKind: "dm" | "group";
-  senderHandle: string;
-  rawMessage: string;            // as received (with prefix)
-  // Prompt components (broken out for UI)
-  systemPrompt: {
-    identity: string;            // workspace files section
-    l3Knowledge: string[];       // individual L3 entries
-    l2Episodes: Array<{
-      id: string;
-      content: string;
-      score: number;
-      timestamp: string;
-      context_name: string;
-      role: string;
-    }>;
-    contextInfo: string;         // active context section
-    fullText: string;            // complete system prompt as sent
-  };
-  conversationHistory: Array<{ role: string; content: string }>;  // L1 turns included
-  userMessage: string;           // final user message
-  // Response
-  response: {
-    content: string;
-    model: string;
-    latencyMs: number;
-    skipped: boolean;            // was it __SKIP__?
-  };
-  // Config snapshot
-  config: {
-    topK: number;
-    model: string;
-    maxTokens: number;
-    temperature: number;
-  };
-  // Approximate token counts (char-based estimate: chars/4)
-  tokenEstimates: {
-    systemPrompt: number;
-    conversationHistory: number;
-    userMessage: number;
-    total: number;
-  };
-}
-```
-
-**Storage:** SQLite database at `data/dashboard.db`. 
-- Single `request_logs` table with JSON columns for complex fields.
-- Auto-prune: keep last 7 days or 10,000 entries (whichever is smaller).
-- Why SQLite over ring buffer: survives restarts, queryable, minimal overhead.
-
-**Token estimation:** Use chars/4 approximation. Good enough for relative sizing. Avoid tokenizer dependency.
-
-**Logging approach:** Eager logging. Serialize at request time. The overhead is minimal (~1ms for JSON.stringify) compared to LLM latency (~1-10s). Capturing the exact state at request time is more valuable than lazy reconstruction.
-
-### Backend: Dashboard HTTP Server
-
-**Where:** New file `packages/runtime/src/dashboard.ts`.
-
-Extend the existing health server (or create a sibling on port 9485):
-- `GET /` — serve the SPA (single HTML file)
-- `GET /api/requests` — list recent requests (paginated, filterable)
-- `GET /api/requests/:id` — single request detail
-- `GET /api/contexts` — proxy to memory daemon's context list
-- `GET /api/contexts/:name/episodes` — proxy L2 episodes
-- `GET /api/contexts/:name/l3` — proxy L3 knowledge
-- `GET /api/stats` — memory stats (episode counts, last promotion, etc.)
-- `DELETE /api/l3/:id` — delete a bad L3 entry (write op from day 1)
-- `POST /api/l3/:id/edit` — edit L3 entry content
-
-Bind to `127.0.0.1:9485` only.
-
-### Frontend: Single-File SPA
-
-**Why single file:** No build step, no React, no dependencies. Ship as one HTML file with embedded CSS/JS. Can always upgrade later.
-
-**Layout:**
-- Left sidebar: navigation (Requests, Memory, Contexts)
-- Main area: content
-
-**Request Inspector view:**
-- Reverse-chronological list of requests
-- Each row: timestamp, sender, context, model, latency, token estimate
-- Click to expand → shows all sections:
-  - **Identity files** (collapsible, usually not interesting)
-  - **L3 Knowledge** (list of entries with metadata)
-  - **L2 Episodes** (with similarity scores, timestamps, source context)
-  - **L1 History** (conversation turns)
-  - **User Message** (raw with prefix)
-  - **Response** (with model, latency)
-  - **Config** (top_k, model, temperature)
-  - **Token breakdown** (bar chart showing proportion per section)
-- Filters: by context, by sender, by time range
-- Search: full-text search across messages
-
-**Memory Browser view (Phase 2):**
-- L2: searchable episode list, filterable by context/role/time
-- L3: per-context knowledge entries with edit/delete buttons
-- Promotion log (if we add logging for it)
-
-**Context Overview (Phase 2):**
-- List of contexts with episode counts, last active
-- Active context highlighted
-- Click to drill into episodes/L3
-
----
-
-## Phase 2: Memory Browser + Context Overview
-
-After Phase 1 is working and useful, add:
-- Full L2 browsing with semantic search UI
-- L3 management (view, edit, delete)
-- Context explorer with stats
-- Promotion history logging
-
----
-
-## Implementation Steps (Phase 1)
-
-### Step 1: Request logging infrastructure
-- [ ] Create `packages/runtime/src/request-logger.ts`
-  - SQLite setup (using better-sqlite3)
-  - `logRequest()` method
-  - `getRequests()` with pagination/filters
-  - `getRequest(id)` for detail view
-  - Auto-pruning on startup
-- [ ] Add better-sqlite3 dependency
-
-### Step 2: Instrument the pipeline
-- [ ] Modify `agent.ts` `processMessage()` to capture routing decision + timing
-- [ ] Modify `prompt.ts` `buildSystemPrompt()` to return structured components (not just string)
-- [ ] Log each request after LLM response arrives
-- [ ] Capture config snapshot with each log entry
-
-### Step 3: Dashboard HTTP server
-- [ ] Create `packages/runtime/src/dashboard.ts`
-  - Express-free: use Node's built-in `http` module (like health server)
-  - Serve SPA at `/`
-  - JSON APIs for request logs and memory proxy
-- [ ] Wire into `pipeline.ts` startup
-
-### Step 4: Frontend SPA
-- [ ] Single HTML file at `packages/runtime/src/dashboard.html`
-  - Vanilla JS, no framework
-  - CSS grid layout
-  - Fetch-based API calls
-  - Expandable request cards
-  - Token breakdown visualization
-  - Basic filtering
-
-### Step 5: Memory proxy + write ops
-- [ ] Proxy endpoints to memory daemon for L2/L3 browsing
-- [ ] DELETE/PATCH endpoints for L3 management
-
----
-
-## Design Decisions
-
-| Question | Decision | Rationale |
-|----------|----------|-----------|
-| Storage | SQLite | Survives restarts, queryable, lightweight |
-| Token counting | chars/4 estimate | Good enough, no tokenizer dep |
-| Logging | Eager | Captures exact state, overhead negligible vs LLM latency |
-| Bind address | 127.0.0.1 only | Local access, no auth needed |
-| Framework | None (vanilla) | Single HTML file, no build step |
-| Read-only or read-write? | Read-write from start | Ryan will want to delete bad L3 entries immediately |
-| Persist request logs? | Yes, 7 days | Need to compare across memory config changes |
-| Multi-agent? | Single agent for now | Don't over-engineer, but use agent name in logs |
-| Port | 9485 | Next to health port (9484), easy to remember |
-
----
-
-## Sesame Command Fix (Bonus)
-
-While we're in the code, fix the sender prefix issue:
-- In `pipeline.ts` `startSesameLoop()`, before calling `agent.processMessage()`, strip the sender prefix for command parsing
-- Or better: in `agent.ts` `handleSpecialCommand()`, strip known prefix patterns before regex matching
-- This unblocks context switching, task commands, and cross-context search over Sesame
-
----
-
-*Created: 2026-02-28*
-*Status: Ready to implement*