@sesamespace/hivemind 0.2.0 → 0.3.0

package/packages/memory/src/scoring.rs

@@ -0,0 +1,131 @@
+use chrono::{DateTime, Utc};
+use std::collections::HashMap;
+use std::sync::RwLock;
+
+/// Default half-life in hours for recency decay
+const DEFAULT_HALF_LIFE_HOURS: f64 = 48.0;
+
+/// Per-context scoring configuration
+pub struct ScoringConfig {
+    /// Half-life per context (in hours). Contexts not in this map use the default.
+    context_half_lives: RwLock<HashMap<String, f64>>,
+}
+
+impl ScoringConfig {
+    pub fn new() -> Self {
+        Self {
+            context_half_lives: RwLock::new(HashMap::new()),
+        }
+    }
+
+    pub fn set_half_life(&self, context: &str, hours: f64) {
+        let mut map = self.context_half_lives.write().unwrap();
+        map.insert(context.to_string(), hours);
+    }
+
+    pub fn get_half_life(&self, context: &str) -> f64 {
+        let map = self.context_half_lives.read().unwrap();
+        map.get(context).copied().unwrap_or(DEFAULT_HALF_LIFE_HOURS)
+    }
+}
+
+/// Compute a recency weight for an episode based on its timestamp.
+/// Uses exponential decay: weight = e^(-lambda * hours_ago)
+/// Half-life of ~48 hours means recent episodes are strongly preferred
+/// but older ones still surface if semantically relevant.
+pub fn recency_weight(timestamp: &str) -> f64 {
+    recency_weight_with_half_life(timestamp, DEFAULT_HALF_LIFE_HOURS)
+}
+
+/// Recency weight with a custom half-life (per-context independent decay)
+pub fn recency_weight_with_half_life(timestamp: &str, half_life_hours: f64) -> f64 {
+    let lambda = (2.0_f64).ln() / half_life_hours;
+
+    let parsed = DateTime::parse_from_rfc3339(timestamp)
+        .map(|dt| dt.with_timezone(&Utc))
+        .unwrap_or_else(|_| Utc::now());
+
+    let hours_ago = (Utc::now() - parsed).num_seconds() as f64 / 3600.0;
+    let hours_ago = hours_ago.max(0.0);
+
+    (-lambda * hours_ago).exp()
+}
+
+/// Combined relevance score: recency * semantic similarity
+/// Distance is the vector distance (lower = more similar)
+pub fn combined_score(timestamp: &str, distance: f32, half_life_hours: f64) -> f64 {
+    let recency = recency_weight_with_half_life(timestamp, half_life_hours);
+    recency * (1.0 / (1.0 + distance as f64))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::Utc;
+
+    #[test]
+    fn recent_episode_has_high_weight() {
+        let now = Utc::now().to_rfc3339();
+        let w = recency_weight(&now);
+        assert!(w > 0.99, "Weight for now should be ~1.0, got {}", w);
+    }
+
+    #[test]
+    fn old_episode_has_lower_weight() {
+        let two_days_ago = (Utc::now() - chrono::Duration::hours(48)).to_rfc3339();
+        let w = recency_weight(&two_days_ago);
+        assert!(
+            (w - 0.5).abs() < 0.05,
+            "Weight at half-life should be ~0.5, got {}",
+            w
+        );
+    }
+
+    #[test]
+    fn custom_half_life_works() {
+        let one_hour_ago = (Utc::now() - chrono::Duration::hours(1)).to_rfc3339();
+
+        // With 1-hour half-life, should be ~0.5
+        let w = recency_weight_with_half_life(&one_hour_ago, 1.0);
+        assert!(
+            (w - 0.5).abs() < 0.05,
+            "Weight at 1h with 1h half-life should be ~0.5, got {}",
+            w
+        );
+
+        // With 100-hour half-life, should still be close to 1.0
+        let w2 = recency_weight_with_half_life(&one_hour_ago, 100.0);
+        assert!(
+            w2 > 0.99,
+            "Weight at 1h with 100h half-life should be ~1.0, got {}",
+            w2
+        );
+    }
+
+    #[test]
+    fn scoring_config_per_context() {
+        let config = ScoringConfig::new();
+
+        // Default
+        assert_eq!(config.get_half_life("unknown"), DEFAULT_HALF_LIFE_HOURS);
+
+        // Set custom
+        config.set_half_life("project-a", 24.0);
+        assert_eq!(config.get_half_life("project-a"), 24.0);
+        assert_eq!(config.get_half_life("project-b"), DEFAULT_HALF_LIFE_HOURS);
+    }
+
+    #[test]
+    fn combined_score_works() {
+        let now = Utc::now().to_rfc3339();
+        let score = combined_score(&now, 0.0, DEFAULT_HALF_LIFE_HOURS);
+        assert!(score > 0.99, "Perfect match + recent should score ~1.0, got {}", score);
+
+        let score2 = combined_score(&now, 1.0, DEFAULT_HALF_LIFE_HOURS);
+        assert!(
+            (score2 - 0.5).abs() < 0.05,
+            "Perfect recency + distance 1.0 should score ~0.5, got {}",
+            score2
+        );
+    }
+}

package/packages/memory/src/store.rs

@@ -0,0 +1,460 @@
+use anyhow::Result;
+use arrow_array::{
+    types::Float32Type, Array, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator,
+    StringArray, UInt64Array,
+};
+use arrow_schema::{DataType, Field, Schema};
+use chrono::Utc;
+use futures::stream::TryStreamExt;
+use lancedb::{connect, connection::Connection, query::ExecutableQuery, query::QueryBase, Table};
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+const EMBEDDING_DIM: i32 = 768;
+const TABLE_NAME: &str = "episodes";
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct Episode {
+    pub id: String,
+    pub timestamp: String,
+    pub context_name: String,
+    pub role: String,
+    pub content: String,
+    #[serde(default)]
+    pub layer: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct EpisodeInput {
+    pub context_name: Option<String>,
+    pub role: String,
+    pub content: String,
+}
+
+pub struct MemoryStore {
+    db: Connection,
+}
+
+impl MemoryStore {
+    pub async fn new(path: &str) -> Result<Self> {
+        std::fs::create_dir_all(path)?;
+        let db = connect(path).execute().await?;
+        let store = Self { db };
+        store.ensure_table().await?;
+        Ok(store)
+    }
+
+    pub fn get_connection(&self) -> Connection {
+        self.db.clone()
+    }
+
+    fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Utf8, false),
+            Field::new("timestamp", DataType::Utf8, false),
+            Field::new("context_name", DataType::Utf8, false),
+            Field::new("role", DataType::Utf8, false),
+            Field::new("content", DataType::Utf8, false),
+            Field::new("layer", DataType::Utf8, false),
+            Field::new(
+                "embedding",
+                DataType::FixedSizeList(
+                    Arc::new(Field::new("item", DataType::Float32, true)),
+                    EMBEDDING_DIM,
+                ),
+                false,
+            ),
+        ]))
+    }
+
+    async fn ensure_table(&self) -> Result<()> {
+        let names = self.db.table_names().execute().await?;
+        if !names.contains(&TABLE_NAME.to_string()) {
+            let schema = Self::schema();
+            let batch = RecordBatch::new_empty(schema.clone());
+            let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+            self.db
+                .create_table(TABLE_NAME, Box::new(batches))
+                .execute()
+                .await?;
+            tracing::info!("Created episodes table");
+        }
+        Ok(())
+    }
+
+    pub async fn insert_episode(
+        &self,
+        input: EpisodeInput,
+        embedding: Vec<f32>,
+    ) -> Result<Episode> {
+        let id = uuid::Uuid::new_v4().to_string();
+        let timestamp = Utc::now().to_rfc3339();
+        let context_name = input.context_name.unwrap_or_else(|| "global".to_string());
+        let layer = "L2".to_string();
+
+        let episode = Episode {
+            id: id.clone(),
+            timestamp: timestamp.clone(),
+            context_name: context_name.clone(),
+            role: input.role.clone(),
+            content: input.content.clone(),
+            layer: layer.clone(),
+        };
+
+        let schema = Self::schema();
+        let embedding_array = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+            vec![Some(embedding.into_iter().map(Some).collect::<Vec<_>>())],
+            EMBEDDING_DIM,
+        );
+
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![id.as_str()])),
+                Arc::new(StringArray::from(vec![timestamp.as_str()])),
+                Arc::new(StringArray::from(vec![context_name.as_str()])),
+                Arc::new(StringArray::from(vec![input.role.as_str()])),
+                Arc::new(StringArray::from(vec![input.content.as_str()])),
+                Arc::new(StringArray::from(vec![layer.as_str()])),
+                Arc::new(embedding_array) as Arc<dyn Array>,
+            ],
+        )?;
+
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        table.add(Box::new(batches)).execute().await?;
+
+        tracing::debug!(
+            "Stored episode {} in context {}",
+            episode.id,
+            episode.context_name
+        );
+        Ok(episode)
+    }
+
+    pub async fn search(
+        &self,
+        query_embedding: Vec<f32>,
+        context: &str,
+        limit: usize,
+    ) -> Result<Vec<(Episode, f32)>> {
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+
+        let results = table
+            .vector_search(query_embedding)?
+            .limit(limit * 3) // over-fetch then filter by context
+            .execute()
+            .await?;
+
+        let mut episodes = Vec::new();
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let timestamps = batch
+                .column_by_name("timestamp")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contexts = batch
+                .column_by_name("context_name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let roles = batch
+                .column_by_name("role")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contents = batch
+                .column_by_name("content")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let layers = batch.column_by_name("layer");
+            let distances = batch
+                .column_by_name("_distance")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap();
+
+            for i in 0..batch.num_rows() {
+                let ctx = contexts.value(i);
+                if ctx != context && ctx != "global" {
+                    continue;
+                }
+
+                let layer = layers
+                    .and_then(|l| l.as_any().downcast_ref::<StringArray>())
+                    .map(|l| l.value(i).to_string())
+                    .unwrap_or_else(|| "L2".to_string());
+
+                let episode = Episode {
+                    id: ids.value(i).to_string(),
+                    timestamp: timestamps.value(i).to_string(),
+                    context_name: ctx.to_string(),
+                    role: roles.value(i).to_string(),
+                    content: contents.value(i).to_string(),
+                    layer,
+                };
+
+                episodes.push((episode, distances.value(i)));
+
+                if episodes.len() >= limit {
+                    break;
+                }
+            }
+
+            if episodes.len() >= limit {
+                break;
+            }
+        }
+
+        Ok(episodes)
+    }
+
+    /// Search across all contexts without filtering
+    pub async fn search_all(
+        &self,
+        query_embedding: Vec<f32>,
+        limit: usize,
+    ) -> Result<Vec<(Episode, f32)>> {
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+
+        let results = table
+            .vector_search(query_embedding)?
+            .limit(limit)
+            .execute()
+            .await?;
+
+        let mut episodes = Vec::new();
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let timestamps = batch
+                .column_by_name("timestamp")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contexts = batch
+                .column_by_name("context_name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let roles = batch
+                .column_by_name("role")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contents = batch
+                .column_by_name("content")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let layers = batch.column_by_name("layer");
+            let distances = batch
+                .column_by_name("_distance")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Float32Array>()
+                .unwrap();
+
+            for i in 0..batch.num_rows() {
+                let layer = layers
+                    .and_then(|l| l.as_any().downcast_ref::<StringArray>())
+                    .map(|l| l.value(i).to_string())
+                    .unwrap_or_else(|| "L2".to_string());
+
+                let episode = Episode {
+                    id: ids.value(i).to_string(),
+                    timestamp: timestamps.value(i).to_string(),
+                    context_name: contexts.value(i).to_string(),
+                    role: roles.value(i).to_string(),
+                    content: contents.value(i).to_string(),
+                    layer,
+                };
+
+                episodes.push((episode, distances.value(i)));
+            }
+        }
+
+        Ok(episodes)
+    }
+
+    pub async fn get_by_id(&self, id: &str) -> Result<Option<Episode>> {
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+        let results = table
+            .query()
+            .only_if(format!("id = '{}'", id))
+            .execute()
+            .await?;
+
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+        for batch in &batches {
+            if batch.num_rows() > 0 {
+                let ids = batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                let timestamps = batch
+                    .column_by_name("timestamp")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                let contexts = batch
+                    .column_by_name("context_name")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                let roles = batch
+                    .column_by_name("role")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                let contents = batch
+                    .column_by_name("content")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap();
+                let layers = batch.column_by_name("layer");
+
+                let layer = layers
+                    .and_then(|l| l.as_any().downcast_ref::<StringArray>())
+                    .map(|l| l.value(0).to_string())
+                    .unwrap_or_else(|| "L2".to_string());
+
+                return Ok(Some(Episode {
+                    id: ids.value(0).to_string(),
+                    timestamp: timestamps.value(0).to_string(),
+                    context_name: contexts.value(0).to_string(),
+                    role: roles.value(0).to_string(),
+                    content: contents.value(0).to_string(),
+                    layer,
+                }));
+            }
+        }
+
+        Ok(None)
+    }
+
+    pub async fn get_by_context(&self, context: &str) -> Result<Vec<Episode>> {
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+
+        let results = table
+            .query()
+            .only_if(format!("context_name = '{}'", context))
+            .execute()
+            .await?;
+
+        let mut episodes = Vec::new();
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let timestamps = batch
+                .column_by_name("timestamp")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contexts = batch
+                .column_by_name("context_name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let roles = batch
+                .column_by_name("role")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let contents = batch
+                .column_by_name("content")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let layers = batch.column_by_name("layer");
+
+            for i in 0..batch.num_rows() {
+                let layer = layers
+                    .and_then(|l| l.as_any().downcast_ref::<StringArray>())
+                    .map(|l| l.value(i).to_string())
+                    .unwrap_or_else(|| "L2".to_string());
+
+                episodes.push(Episode {
+                    id: ids.value(i).to_string(),
+                    timestamp: timestamps.value(i).to_string(),
+                    context_name: contexts.value(i).to_string(),
+                    role: roles.value(i).to_string(),
+                    content: contents.value(i).to_string(),
+                    layer,
+                });
+            }
+        }
+
+        Ok(episodes)
+    }
+
+    pub async fn count_by_context(&self, context: &str) -> Result<usize> {
+        let episodes = self.get_by_context(context).await?;
+        Ok(episodes.len())
+    }
+
+    pub async fn delete_by_context(&self, context: &str) -> Result<()> {
+        let table = self.db.open_table(TABLE_NAME).execute().await?;
+        table
+            .delete(&format!("context_name = '{}'", context))
+            .await?;
+        tracing::info!("Deleted all episodes in context: {}", context);
+        Ok(())
+    }
+
+    /// Update the layer field for an episode (used by promotion engine)
+    pub async fn promote_to_l3(&self, id: &str) -> Result<()> {
+        // LanceDB doesn't support in-place updates easily.
+        // We'll use a delete-and-reinsert approach via the promotion engine.
+        // For now, this is a no-op marker — the promotion engine tracks L3 status
+        // in its own metadata table.
+        tracing::debug!("Marking episode {} as L3", id);
+        Ok(())
+    }
+
+    /// Get all episode IDs for a context
+    pub async fn get_ids_by_context(&self, context: &str) -> Result<Vec<String>> {
+        let episodes = self.get_by_context(context).await?;
+        Ok(episodes.into_iter().map(|e| e.id).collect())
+    }
+}