@sesamespace/hivemind 0.2.0 → 0.3.0

package/packages/memory/src/tasks.rs

@@ -0,0 +1,321 @@
+use anyhow::Result;
+use arrow_array::{RecordBatch, RecordBatchIterator, StringArray};
+use arrow_schema::{DataType, Field, Schema};
+use chrono::Utc;
+use futures::stream::TryStreamExt;
+use lancedb::{connection::Connection, query::ExecutableQuery, query::QueryBase, Table};
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+const TASKS_TABLE: &str = "tasks";
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct TaskRecord {
+    pub id: String,
+    pub context_name: String,
+    pub title: String,
+    pub description: String,
+    pub status: String,
+    pub blocked_by: String, // JSON array stored as string
+    pub created_at: String,
+    pub updated_at: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TaskInput {
+    pub context_name: String,
+    pub title: String,
+    pub description: String,
+    #[serde(default = "default_status")]
+    pub status: String,
+    #[serde(default)]
+    pub blocked_by: Vec<String>,
+}
+
+fn default_status() -> String {
+    "planned".to_string()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct TaskUpdate {
+    pub status: Option<String>,
+    pub title: Option<String>,
+    pub description: Option<String>,
+    pub blocked_by: Option<Vec<String>>,
+}
+
+pub struct TaskStore {
+    db: Connection,
+}
+
+impl TaskStore {
+    pub async fn new(db: Connection) -> Result<Self> {
+        let store = Self { db };
+        store.ensure_table().await?;
+        Ok(store)
+    }
+
+    fn schema() -> Arc<Schema> {
+        Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Utf8, false),
+            Field::new("context_name", DataType::Utf8, false),
+            Field::new("title", DataType::Utf8, false),
+            Field::new("description", DataType::Utf8, false),
+            Field::new("status", DataType::Utf8, false),
+            Field::new("blocked_by", DataType::Utf8, false),
+            Field::new("created_at", DataType::Utf8, false),
+            Field::new("updated_at", DataType::Utf8, false),
+        ]))
+    }
+
+    async fn ensure_table(&self) -> Result<()> {
+        let names = self.db.table_names().execute().await?;
+        if !names.contains(&TASKS_TABLE.to_string()) {
+            let schema = Self::schema();
+            let batch = RecordBatch::new_empty(schema.clone());
+            let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+            self.db
+                .create_table(TASKS_TABLE, Box::new(batches))
+                .execute()
+                .await?;
+            tracing::info!("Created tasks table");
+        }
+        Ok(())
+    }
+
+    pub async fn create_task(&self, input: TaskInput) -> Result<TaskRecord> {
+        let id = uuid::Uuid::new_v4().to_string();
+        let now = Utc::now().to_rfc3339();
+        let blocked_by_json = serde_json::to_string(&input.blocked_by)?;
+
+        let task = TaskRecord {
+            id: id.clone(),
+            context_name: input.context_name.clone(),
+            title: input.title.clone(),
+            description: input.description.clone(),
+            status: input.status,
+            blocked_by: blocked_by_json.clone(),
+            created_at: now.clone(),
+            updated_at: now.clone(),
+        };
+
+        let schema = Self::schema();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![task.id.as_str()])),
+                Arc::new(StringArray::from(vec![task.context_name.as_str()])),
+                Arc::new(StringArray::from(vec![task.title.as_str()])),
+                Arc::new(StringArray::from(vec![task.description.as_str()])),
+                Arc::new(StringArray::from(vec![task.status.as_str()])),
+                Arc::new(StringArray::from(vec![blocked_by_json.as_str()])),
+                Arc::new(StringArray::from(vec![task.created_at.as_str()])),
+                Arc::new(StringArray::from(vec![task.updated_at.as_str()])),
+            ],
+        )?;
+
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        table.add(Box::new(batches)).execute().await?;
+
+        tracing::debug!("Created task {} in context {}", task.id, task.context_name);
+        Ok(task)
+    }
+
+    pub async fn list_tasks(
+        &self,
+        context: &str,
+        status_filter: Option<&str>,
+    ) -> Result<Vec<TaskRecord>> {
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+
+        let filter = match status_filter {
+            Some(status) => format!("context_name = '{}' AND status = '{}'", context, status),
+            None => format!("context_name = '{}'", context),
+        };
+
+        let results = table.query().only_if(filter).execute().await?;
+
+        let mut tasks = Vec::new();
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+
+        for batch in &batches {
+            let ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let ctx_names = batch
+                .column_by_name("context_name")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let titles = batch
+                .column_by_name("title")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let descriptions = batch
+                .column_by_name("description")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let statuses = batch
+                .column_by_name("status")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let blocked_bys = batch
+                .column_by_name("blocked_by")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let created_ats = batch
+                .column_by_name("created_at")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+            let updated_ats = batch
+                .column_by_name("updated_at")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<StringArray>()
+                .unwrap();
+
+            for i in 0..batch.num_rows() {
+                tasks.push(TaskRecord {
+                    id: ids.value(i).to_string(),
+                    context_name: ctx_names.value(i).to_string(),
+                    title: titles.value(i).to_string(),
+                    description: descriptions.value(i).to_string(),
+                    status: statuses.value(i).to_string(),
+                    blocked_by: blocked_bys.value(i).to_string(),
+                    created_at: created_ats.value(i).to_string(),
+                    updated_at: updated_ats.value(i).to_string(),
+                });
+            }
+        }
+
+        // Sort by created_at
+        tasks.sort_by(|a, b| a.created_at.cmp(&b.created_at));
+
+        Ok(tasks)
+    }
+
+    pub async fn get_task(&self, id: &str) -> Result<Option<TaskRecord>> {
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        let results = table
+            .query()
+            .only_if(format!("id = '{}'", id))
+            .execute()
+            .await?;
+
+        let batches: Vec<RecordBatch> = results.try_collect().await?;
+        for batch in &batches {
+            if batch.num_rows() > 0 {
+                let ids = batch.column_by_name("id").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let ctx_names = batch.column_by_name("context_name").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let titles = batch.column_by_name("title").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let descriptions = batch.column_by_name("description").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let statuses = batch.column_by_name("status").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let blocked_bys = batch.column_by_name("blocked_by").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let created_ats = batch.column_by_name("created_at").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+                let updated_ats = batch.column_by_name("updated_at").unwrap().as_any().downcast_ref::<StringArray>().unwrap();
+
+                return Ok(Some(TaskRecord {
+                    id: ids.value(0).to_string(),
+                    context_name: ctx_names.value(0).to_string(),
+                    title: titles.value(0).to_string(),
+                    description: descriptions.value(0).to_string(),
+                    status: statuses.value(0).to_string(),
+                    blocked_by: blocked_bys.value(0).to_string(),
+                    created_at: created_ats.value(0).to_string(),
+                    updated_at: updated_ats.value(0).to_string(),
+                }));
+            }
+        }
+
+        Ok(None)
+    }
+
+    pub async fn update_task(&self, id: &str, update: TaskUpdate) -> Result<Option<TaskRecord>> {
+        let existing = self.get_task(id).await?;
+        let Some(mut task) = existing else {
+            return Ok(None);
+        };
+
+        // Apply updates
+        if let Some(status) = update.status {
+            task.status = status;
+        }
+        if let Some(title) = update.title {
+            task.title = title;
+        }
+        if let Some(description) = update.description {
+            task.description = description;
+        }
+        if let Some(blocked_by) = update.blocked_by {
+            task.blocked_by = serde_json::to_string(&blocked_by)?;
+        }
+        task.updated_at = Utc::now().to_rfc3339();
+
+        // Delete and reinsert
+        let table = self.db.open_table(TASKS_TABLE).execute().await?;
+        table.delete(&format!("id = '{}'", id)).await?;
+
+        let schema = Self::schema();
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(StringArray::from(vec![task.id.as_str()])),
+                Arc::new(StringArray::from(vec![task.context_name.as_str()])),
+                Arc::new(StringArray::from(vec![task.title.as_str()])),
+                Arc::new(StringArray::from(vec![task.description.as_str()])),
+                Arc::new(StringArray::from(vec![task.status.as_str()])),
+                Arc::new(StringArray::from(vec![task.blocked_by.as_str()])),
+                Arc::new(StringArray::from(vec![task.created_at.as_str()])),
+                Arc::new(StringArray::from(vec![task.updated_at.as_str()])),
+            ],
+        )?;
+
+        let batches = RecordBatchIterator::new(vec![Ok(batch)], schema);
+        table.add(Box::new(batches)).execute().await?;
+
+        Ok(Some(task))
+    }
+
+    /// Get the next available task for a context:
+    /// - Status is "planned" (not active/complete/archived)
+    /// - Not blocked by any incomplete tasks
+    pub async fn get_next_task(&self, context: &str) -> Result<Option<TaskRecord>> {
+        let planned = self.list_tasks(context, Some("planned")).await?;
+        let all_tasks = self.list_tasks(context, None).await?;
+
+        // Build a set of complete task IDs
+        let complete_ids: std::collections::HashSet<String> = all_tasks
+            .iter()
+            .filter(|t| t.status == "complete" || t.status == "archived")
+            .map(|t| t.id.clone())
+            .collect();
+
+        for task in planned {
+            let blocked_by: Vec<String> =
+                serde_json::from_str(&task.blocked_by).unwrap_or_default();
+
+            // Task is available if all blockers are complete
+            let is_blocked = blocked_by.iter().any(|b| !complete_ids.contains(b));
+            if !is_blocked {
+                return Ok(Some(task));
+            }
+        }
+
+        Ok(None)
+    }
+}

package/packages/runtime/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@hivemind/runtime",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "dev": "tsx src/start.ts",
+    "start": "node dist/start.js",
+    "test": "echo \"No tests yet\""
+  },
+  "dependencies": {
+    "@iarna/toml": "^2.2.5",
+    "ws": "^8.18.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.0",
+    "@types/ws": "^8.5.10",
+    "tsx": "^4.16.0",
+    "typescript": "^5.5.0"
+  }
+}

package/packages/runtime/src/__tests__/fleet-integration.test.ts

@@ -0,0 +1,235 @@
+/**
+ * Fleet integration test (Phase 3, Task 3.5).
+ *
+ * Tests multi-worker fleet lifecycle: two WorkerServers, a PrimaryClient
+ * coordinating both, FleetManager dashboard, context migration, and
+ * memory sync round-trips.
+ */
+
+import { describe, it, before, after } from "node:test";
+import assert from "node:assert/strict";
+import { PrimaryClient } from "../fleet/primary-client.js";
+import { WorkerServer } from "../fleet/worker-server.js";
+import { FleetManager } from "../fleet/fleet-manager.js";
+import type {
+  WorkerRegistrationRequest,
+  WorkerStatusReport,
+  SyncPullRequest,
+  SyncPushRequest,
+} from "../fleet/worker-protocol.js";
+
+const PORT_A = 19900;
+const PORT_B = 19901;
+const URL_A = `http://localhost:${PORT_A}`;
+const URL_B = `http://localhost:${PORT_B}`;
+
+describe("Fleet Integration — Multi-Worker", () => {
+  let workerA: WorkerServer;
+  let workerB: WorkerServer;
+  let primary: PrimaryClient;
+  let fleet: FleetManager;
+  let workerIdA: string;
+  let workerIdB: string;
+
+  before(async () => {
+    workerA = new WorkerServer({
+      workerId: "worker-a",
+      port: PORT_A,
+      maxContexts: 3,
+      memoryDaemonUrl: "http://localhost:9999",
+      ollamaUrl: "http://localhost:11434",
+    });
+    workerB = new WorkerServer({
+      workerId: "worker-b",
+      port: PORT_B,
+      maxContexts: 2,
+      memoryDaemonUrl: "http://localhost:9999",
+      ollamaUrl: "http://localhost:11434",
+    });
+    await workerA.start();
+    await workerB.start();
+
+    primary = new PrimaryClient();
+    fleet = new FleetManager(primary);
+  });
+
+  after(async () => {
+    primary.stopHealthPolling();
+    await workerA.stop();
+    await workerB.stop();
+  });
+
+  describe("Registration", () => {
+    it("should register worker A", () => {
+      const req: WorkerRegistrationRequest = {
+        url: URL_A,
+        capabilities: { max_contexts: 3, has_ollama: true, has_memory_daemon: true, available_models: [] },
+      };
+      const resp = primary.handleRegistration(req);
+      workerIdA = resp.worker_id;
+      assert.ok(workerIdA);
+    });
+
+    it("should register worker B", () => {
+      const req: WorkerRegistrationRequest = {
+        url: URL_B,
+        capabilities: { max_contexts: 2, has_ollama: true, has_memory_daemon: true, available_models: [] },
+      };
+      const resp = primary.handleRegistration(req);
+      workerIdB = resp.worker_id;
+      assert.ok(workerIdB);
+    });
+
+    it("should list both workers", () => {
+      assert.equal(primary.getWorkers().length, 2);
+    });
+  });
+
+  describe("Health Checks — Both Workers", () => {
+    it("should get healthy responses from both", async () => {
+      const healthA = await primary.checkHealth(workerIdA);
+      const healthB = await primary.checkHealth(workerIdB);
+      assert.ok(healthA);
+      assert.ok(healthB);
+      assert.equal(healthA.status, "healthy");
+      assert.equal(healthB.status, "healthy");
+    });
+
+    it("should poll all and get two healthy", async () => {
+      const results = await primary.checkAllHealth();
+      assert.equal(results.size, 2);
+      for (const [, status] of results) {
+        assert.equal(status, "healthy");
+      }
+    });
+  });
+
+  describe("Context Distribution Across Workers", () => {
+    it("should assign project-alpha to worker A", async () => {
+      const resp = await primary.assignContext(workerIdA, "project-alpha", "Alpha");
+      assert.equal(resp.accepted, true);
+    });
+
+    it("should assign project-beta to worker B", async () => {
+      const resp = await primary.assignContext(workerIdB, "project-beta", "Beta");
+      assert.equal(resp.accepted, true);
+    });
+
+    it("should assign project-gamma to worker A", async () => {
+      const resp = await primary.assignContext(workerIdA, "project-gamma", "Gamma");
+      assert.equal(resp.accepted, true);
+    });
+
+    it("should find correct worker for each context", () => {
+      assert.equal(primary.findWorkerForContext("project-alpha")?.id, workerIdA);
+      assert.equal(primary.findWorkerForContext("project-beta")?.id, workerIdB);
+      assert.equal(primary.findWorkerForContext("project-gamma")?.id, workerIdA);
+    });
+
+    it("should reject when worker B is at capacity (max 2)", async () => {
+      // B already has project-beta (1/2), add one more
+      const resp1 = await primary.assignContext(workerIdB, "project-delta", "Delta");
+      assert.equal(resp1.accepted, true);
+      // Now at 2/2 — next should fail
+      const resp2 = await primary.assignContext(workerIdB, "project-epsilon", "Epsilon");
+      assert.equal(resp2.accepted, false);
+      assert.ok(resp2.reason?.includes("capacity"));
+    });
+  });
+
+  describe("Status Reporting", () => {
+    it("should report working status on worker A", async () => {
+      workerA.setActiveContext("project-alpha");
+      workerA.setCurrentTask("task-1");
+
+      const report: WorkerStatusReport = { activity: "working", current_context: "project-alpha", current_task: "task-1" };
+      const status = primary.handleStatusReport(workerIdA, report);
+      assert.ok(status);
+      assert.equal(status.activity, "working");
+      assert.equal(status.current_context, "project-alpha");
+    });
+
+    it("should report idle status on worker B", async () => {
+      const resp = await fetch(`${URL_B}/status`);
+      const body = (await resp.json()) as WorkerStatusReport;
+      assert.equal(body.activity, "idle");
+    });
+  });
+
+  describe("Memory Sync Endpoints", () => {
+    it("should return 501 when no sync handler registered", async () => {
+      const resp = await fetch(`${URL_A}/sync/push`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          entries: [],
+          episodes: [],
+        } satisfies SyncPushRequest),
+      });
+      // 501 = handler not registered (expected without MemorySync wiring)
+      assert.equal(resp.status, 501);
+    });
+
+    it("should accept sync push when handler is registered", async () => {
+      workerA.onSyncPush(async (_req) => ({
+        l3_accepted: 0,
+        l2_appended: 0,
+      }));
+
+      const resp = await fetch(`${URL_A}/sync/push`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          entries: [],
+          episodes: [],
+        } satisfies SyncPushRequest),
+      });
+      assert.equal(resp.status, 200);
+      const body = await resp.json() as any;
+      assert.equal(body.l3_accepted, 0);
+    });
+  });
+
+  describe("Context Unassignment and Reassignment", () => {
+    it("should unassign project-gamma from worker A", async () => {
+      const ok = await primary.unassignContext(workerIdA, "project-gamma");
+      assert.equal(ok, true);
+    });
+
+    it("should no longer find worker for project-gamma", () => {
+      assert.equal(primary.findWorkerForContext("project-gamma"), undefined);
+    });
+
+    it("should reassign project-gamma to worker B (after freeing a slot)", async () => {
+      // Unassign delta from B first
+      await primary.unassignContext(workerIdB, "project-delta");
+      const resp = await primary.assignContext(workerIdB, "project-gamma", "Gamma moved");
+      assert.equal(resp.accepted, true);
+      assert.equal(primary.findWorkerForContext("project-gamma")?.id, workerIdB);
+    });
+  });
+
+  describe("Fleet Dashboard", () => {
+    it("should produce a dashboard with current state", async () => {
+      const dashboard = await fleet.getDashboard();
+      assert.ok(dashboard);
+      assert.equal(dashboard.total_workers, 2);
+      // Workers should have contexts assigned
+      assert.ok(dashboard.workers.length === 2);
+    });
+  });
+
+  describe("Deregistration", () => {
+    it("should deregister worker A", () => {
+      assert.equal(primary.deregister(workerIdA), true);
+    });
+
+    it("should deregister worker B", () => {
+      assert.equal(primary.deregister(workerIdB), true);
+    });
+
+    it("should have no workers left", () => {
+      assert.equal(primary.getWorkers().length, 0);
+    });
+  });
+});