@johpaz/hive-sdk 0.0.14 → 0.0.16

package/packages/core/src/workers/WorkerPool.ts

@@ -0,0 +1,167 @@
+/**
+ * WorkerPool — Manages a pool of Bun Workers for parallel task execution.
+ *
+ * Creates workers on demand, reuses idle workers, and handles task queuing.
+ */
+
+import { logger } from "../utils/logger.ts";
+import { createWorker, type WorkerConfig, type WorkerInstance } from "./createWorker.ts";
+
+const log = logger.child("worker-pool");
+
+export interface WorkerPoolConfig {
+  maxWorkers?: number;
+  taskTimeoutMs?: number;
+  workerConfig?: WorkerConfig;
+}
+
+export interface PoolTask {
+  id: string;
+  message: string;
+  agentId?: string;
+  threadId?: string;
+  channel?: string;
+  systemPrompt?: string;
+}
+
+export interface PoolTaskResult {
+  taskId: string;
+  result: string;
+  error?: string;
+  durationMs: number;
+}
+
+export class WorkerPool {
+  private maxWorkers: number;
+  private taskTimeoutMs: number;
+  private workerConfig: WorkerConfig;
+  private workers: Map<string, WorkerInstance> = new Map();
+  private idleWorkers: string[] = [];
+  private taskQueue: Array<{ task: PoolTask; resolve: (result: PoolTaskResult) => void }> = [];
+  private busyWorkers: Set<string> = new Set();
+
+  constructor(config: WorkerPoolConfig = {}) {
+    this.maxWorkers = config.maxWorkers ?? 4;
+    this.taskTimeoutMs = config.taskTimeoutMs ?? 120000;
+    this.workerConfig = config.workerConfig ?? { name: "pool-worker" };
+  }
+
+  /**
+   * Execute a single task in a worker.
+   */
+  async execute(task: PoolTask): Promise<PoolTaskResult> {
+    const worker = await this.acquireWorker();
+    const startedAt = performance.now();
+
+    try {
+      const result = await worker.run(task.message, {
+        threadId: task.threadId,
+        channel: task.channel,
+      });
+      return {
+        taskId: task.id,
+        result,
+        durationMs: Math.round(performance.now() - startedAt),
+      };
+    } catch (err) {
+      return {
+        taskId: task.id,
+        result: "",
+        error: (err as Error).message,
+        durationMs: Math.round(performance.now() - startedAt),
+      };
+    } finally {
+      this.releaseWorker(worker.id);
+    }
+  }
+
+  /**
+   * Execute multiple tasks in parallel, up to maxWorkers at a time.
+   */
+  async executeBatch(tasks: PoolTask[]): Promise<PoolTaskResult[]> {
+    return Promise.all(tasks.map((task) => this.execute(task)));
+  }
+
+  /**
+   * Execute tasks with a limit on concurrency.
+   */
+  async executeWithConcurrency(tasks: PoolTask[], concurrency: number): Promise<PoolTaskResult[]> {
+    const results: PoolTaskResult[] = [];
+    const executing = new Set<Promise<void>>();
+
+    for (const task of tasks) {
+      const promise = this.execute(task).then((result) => {
+        results.push(result);
+      });
+      executing.add(promise);
+
+      if (executing.size >= concurrency) {
+        await Promise.race(executing);
+        executing.delete(promise);
+      }
+    }
+
+    await Promise.all(executing);
+    return results;
+  }
+
+  /**
+   * Get or create a worker.
+   */
+  private acquireWorker(): Promise<WorkerInstance> {
+    return new Promise((resolve) => {
+      // Try to reuse an idle worker
+      if (this.idleWorkers.length > 0) {
+        const workerId = this.idleWorkers.pop()!;
+        this.busyWorkers.add(workerId);
+        resolve(this.workers.get(workerId)!);
+        return;
+      }
+
+      // Create a new worker if under limit
+      if (this.workers.size < this.maxWorkers) {
+        const worker = createWorker(this.workerConfig);
+        this.workers.set(worker.id, worker);
+        this.busyWorkers.add(worker.id);
+        resolve(worker);
+        return;
+      }
+
+      // Queue the request
+      const checkInterval = setInterval(() => {
+        if (this.idleWorkers.length > 0) {
+          clearInterval(checkInterval);
+          const workerId = this.idleWorkers.pop()!;
+          this.busyWorkers.add(workerId);
+          resolve(this.workers.get(workerId)!);
+        }
+      }, 50);
+    });
+  }
+
+  private releaseWorker(workerId: string): void {
+    this.busyWorkers.delete(workerId);
+    this.idleWorkers.push(workerId);
+  }
+
+  /**
+   * Terminate all workers in the pool.
+   */
+  shutdown(): void {
+    log.info(`[WorkerPool] Shutting down ${this.workers.size} workers`);
+    for (const worker of this.workers.values()) {
+      worker.terminate();
+    }
+    this.workers.clear();
+    this.idleWorkers = [];
+    this.busyWorkers.clear();
+  }
+
+  get stats(): { total: number; busy: number; idle: number } {
+    return {
+      total: this.workers.size,
+      busy: this.busyWorkers.size,
+      idle: this.idleWorkers.length,
+    };
+  }
+}

package/packages/core/src/workers/agent.worker.ts

@@ -0,0 +1,68 @@
+/**
+ * Agent Worker — Bun Worker that runs an agent task in isolation.
+ *
+ * Receives: { type: "AGENT_TASK", taskId, message, agentId, threadId, channel, systemPrompt }
+ * Sends:    { type: "AGENT_RESULT", taskId, result } | { type: "AGENT_CHUNK", taskId, chunk }
+ */
+
+import { runAgent } from "../agent/AgentRunner.ts";
+import type { StreamChunk } from "../agent/AgentRunner.ts";
+
+declare var self: {
+  onmessage: ((event: { data: WorkerMessage }) => void) | null;
+  postMessage(message: WorkerResponse): void;
+};
+
+type WorkerMessage = {
+  type: "AGENT_TASK";
+  taskId: string;
+  message: string;
+  agentId: string;
+  threadId?: string;
+  channel?: string;
+  systemPrompt?: string;
+};
+
+type WorkerResponse =
+  | { type: "AGENT_CHUNK"; taskId: string; chunk: StreamChunk }
+  | { type: "AGENT_RESULT"; taskId: string; result: string; error?: string };
+
+self.onmessage = async (event) => {
+  const { type, taskId, message, agentId, threadId, channel, systemPrompt } = event.data;
+
+  if (type !== "AGENT_TASK") return;
+
+  try {
+    const stream = runAgent({
+      agentId,
+      userMessage: message,
+      threadId: threadId ?? `worker-${taskId}`,
+      channel: channel ?? "worker",
+      isolated: true,
+      systemPromptOverride: systemPrompt,
+    });
+
+    let fullResponse = "";
+
+    for await (const chunk of stream) {
+      self.postMessage({ type: "AGENT_CHUNK", taskId, chunk });
+
+      if (chunk.agent?.messages) {
+        for (const msg of chunk.agent.messages) {
+          if (msg.content && typeof msg.content === "string") {
+            fullResponse = msg.content;
+          }
+        }
+      }
+    }
+
+    self.postMessage({ type: "AGENT_RESULT", taskId, result: fullResponse });
+  } catch (err) {
+    self.postMessage({
+      type: "AGENT_RESULT",
+      taskId,
+      result: "",
+      error: (err as Error).message,
+    });
+  }
+};

package/packages/core/src/workers/createWorker.ts

@@ -0,0 +1,144 @@
+/**
+ * createWorker — Factory for creating specialized Bun Workers.
+ *
+ * Creates a dedicated worker thread that runs an agent with a custom system prompt.
+ * Useful for parallel task execution and specialized agent roles.
+ */
+
+import { logger } from "../utils/logger.ts";
+
+const log = logger.child("createWorker");
+
+export interface WorkerConfig {
+  name: string;
+  agentId?: string;
+  systemPrompt?: string;
+  model?: string;
+  provider?: string;
+}
+
+export interface WorkerInstance {
+  readonly name: string;
+  readonly id: string;
+  run(message: string, opts?: { threadId?: string; channel?: string }): Promise<string>;
+  runStream(message: string, opts?: { threadId?: string; channel?: string }): AsyncGenerator<WorkerChunk>;
+  terminate(): void;
+}
+
+export interface WorkerChunk {
+  type: "chunk" | "result" | "error";
+  content?: string;
+  chunk?: any;
+  error?: string;
+}
+
+const WORKER_EXT = import.meta.url.endsWith(".ts") ? ".worker.ts" : ".worker.js";
+
+function resolveWorkerPath(): string {
+  return new URL(`./agent${WORKER_EXT}`, import.meta.url).pathname;
+}
+
+export function createWorker(config: WorkerConfig): WorkerInstance {
+  const workerId = `${config.name}-${crypto.randomUUID().slice(0, 8)}`;
+  const workerPath = resolveWorkerPath();
+
+  log.info(`[createWorker] Spawning worker ${workerId} from ${workerPath}`);
+
+  const worker = new Worker(workerPath, { smol: true });
+
+  return {
+    name: config.name,
+    id: workerId,
+
+    async run(message, opts) {
+      return new Promise((resolve, reject) => {
+        const taskId = crypto.randomUUID();
+        const timeout = setTimeout(() => {
+          reject(new Error(`Worker ${workerId} timed out`));
+        }, 120000);
+
+        const handler = (event: MessageEvent) => {
+          const data = event.data;
+          if (data.taskId !== taskId) return;
+
+          if (data.type === "AGENT_RESULT") {
+            clearTimeout(timeout);
+            worker.removeEventListener("message", handler);
+            if (data.error) {
+              reject(new Error(data.error));
+            } else {
+              resolve(data.result);
+            }
+          }
+        };
+
+        worker.addEventListener("message", handler);
+        worker.postMessage({
+          type: "AGENT_TASK",
+          taskId,
+          message,
+          agentId: config.agentId ?? config.name,
+          threadId: opts?.threadId,
+          channel: opts?.channel ?? config.name,
+          systemPrompt: config.systemPrompt,
+        });
+      });
+    },
+
+    async *runStream(message, opts) {
+      const taskId = crypto.randomUUID();
+
+      worker.postMessage({
+        type: "AGENT_TASK",
+        taskId,
+        message,
+        agentId: config.agentId ?? config.name,
+        threadId: opts?.threadId,
+        channel: opts?.channel ?? config.name,
+        systemPrompt: config.systemPrompt,
+      });
+
+      let resolved = false;
+      const pending: WorkerResponse[] = [];
+
+      const handler = (event: MessageEvent) => {
+        const data = event.data as WorkerResponse;
+        if (data.taskId !== taskId) return;
+        pending.push(data);
+      };
+
+      worker.addEventListener("message", handler);
+
+      try {
+        while (!resolved) {
+          while (pending.length === 0) {
+            await new Promise((r) => setTimeout(r, 10));
+          }
+          const data = pending.shift()!;
+
+          if (data.type === "AGENT_CHUNK") {
+            yield { type: "chunk" as const, chunk: data.chunk };
+          } else if (data.type === "AGENT_RESULT") {
+            resolved = true;
+            if (data.error) {
+              yield { type: "error" as const, error: data.error };
+            } else {
+              yield { type: "result" as const, content: data.result };
+            }
+          }
+        }
+      } finally {
+        worker.removeEventListener("message", handler);
+      }
+    },
+
+    terminate() {
+      log.info(`[createWorker] Terminating worker ${workerId}`);
+      worker.terminate();
+    },
+  };
+}
+
+type WorkerResponse =
+  | { type: "AGENT_CHUNK"; taskId: string; chunk: any }
+  | { type: "AGENT_RESULT"; taskId: string; result: string; error?: string };

package/packages/core/src/workers/index.ts

@@ -0,0 +1,5 @@
+export { createWorker } from "./createWorker.ts";
+export type { WorkerConfig, WorkerInstance, WorkerChunk } from "./createWorker.ts";
+
+export { WorkerPool } from "./WorkerPool.ts";
+export type { WorkerPoolConfig, PoolTask, PoolTaskResult } from "./WorkerPool.ts";

package/packages/core/src/workers/workers.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, it, beforeAll, afterAll } from "bun:test";
+import { createWorker, WorkerPool } from "./index.ts";
+import { setupTestDb, teardownTestDb, insertTestAgent, insertTestProvider } from "../../../../test/setup-db.ts";
+
+describe("createWorker", () => {
+  beforeAll(() => {
+    setupTestDb();
+  });
+
+  afterAll(() => {
+    teardownTestDb();
+  });
+
+  it("creates a worker instance with config", () => {
+    const worker = createWorker({
+      name: "test-worker",
+      systemPrompt: "You are a test worker.",
+    });
+
+    expect(worker.name).toBe("test-worker");
+    expect(worker.id).toContain("test-worker-");
+    worker.terminate();
+  });
+
+  it("WorkerPool creates and manages workers", () => {
+    const pool = new WorkerPool({ maxWorkers: 2 });
+    expect(pool.stats.total).toBe(0);
+    expect(pool.stats.idle).toBe(0);
+    expect(pool.stats.busy).toBe(0);
+    pool.shutdown();
+  });
+
+  it("WorkerPool executes tasks with limited concurrency", async () => {
+    const pool = new WorkerPool({
+      maxWorkers: 2,
+      workerConfig: { name: "pool-test" },
+    });
+
+    const tasks = Array.from({ length: 4 }, (_, i) => ({
+      id: `task-${i}`,
+      message: `Test message ${i}`,
+    }));
+
+    // Since we don't have a real LLM in tests, we just verify the pool structure
+    expect(tasks).toHaveLength(4);
+    pool.shutdown();
+  });
+});

package/test/setup-db.ts

@@ -0,0 +1,216 @@
+/**
+ * Test Database Setup with Real SQLite
+ * 
+ * Uses the project's real database if available, or creates an in-memory one.
+ */
+
+import { Database } from "bun:sqlite";
+import * as path from "node:path";
+import { existsSync, mkdirSync } from "node:fs";
+import { SCHEMA, PROJECTS_SCHEMA, CONTEXT_ENGINE_SCHEMA, MEETING_SCHEMA } from "../packages/core/src/storage/schema.ts";
+
+let testDb: Database | null = null;
+
+export function getTestDb(): Database {
+  if (!testDb) {
+    throw new Error("Test DB not initialized. Call setupTestDb() first.");
+  }
+  return testDb;
+}
+
+export function setupTestDb(options?: { useRealDb?: boolean; dbPath?: string }): Database {
+  const useRealDb = options?.useRealDb ?? false;
+  
+  if (useRealDb) {
+    // Use real database from project
+    try {
+      const hiveDir = process.env.HIVE_DATA_DIR || path.join(process.cwd(), ".hive");
+      const dataDir = path.join(hiveDir, "data");
+      
+      if (!existsSync(dataDir)) {
+        mkdirSync(dataDir, { recursive: true });
+      }
+      
+      const dbPath = options?.dbPath || path.join(dataDir, "hive.db");
+      testDb = new Database(dbPath);
+      
+      // Initialize schema if needed
+      testDb.run(SCHEMA);
+      testDb.run(PROJECTS_SCHEMA);
+      testDb.run(CONTEXT_ENGINE_SCHEMA);
+      testDb.run(MEETING_SCHEMA);
+      
+      console.log("[test] Using real database at:", dbPath);
+    } catch (err) {
+      console.warn("[test] Could not use real DB, falling back to in-memory:", err);
+      testDb = createInMemoryDb();
+    }
+  } else {
+    testDb = createInMemoryDb();
+  }
+  
+  return testDb;
+}
+
+function createInMemoryDb(): Database {
+  testDb = new Database(":memory:");
+  
+  testDb.run(SCHEMA);
+  testDb.run(PROJECTS_SCHEMA);
+  testDb.run(CONTEXT_ENGINE_SCHEMA);
+  testDb.run(MEETING_SCHEMA);
+  
+  console.log("[test] Using in-memory database");
+  
+  return testDb;
+}
+
+export function teardownTestDb() {
+  if (testDb) {
+    testDb.close();
+    testDb = null;
+  }
+}
+
+export function insertTestAgent(overrides: Record<string, any> = {}): string {
+  const db = getTestDb();
+  const id = overrides.id ?? crypto.randomUUID();
+  
+  db.query(`
+    INSERT INTO agents (id, name, description, role, status, enabled, max_iterations, user_id)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    id,
+    overrides.name ?? "Test Agent",
+    overrides.description ?? "Test agent description",
+    overrides.role ?? "worker",
+    overrides.status ?? "idle",
+    overrides.enabled ?? 1,
+    overrides.maxIterations ?? 10,
+    overrides.userId ?? "test-user"
+  );
+  
+  return id;
+}
+
+export function insertTestProvider(overrides: Record<string, any> = {}): string {
+  const db = getTestDb();
+  const id = overrides.id ?? crypto.randomUUID();
+  
+  db.query(`
+    INSERT INTO providers (id, name, category, enabled, active)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(
+    id,
+    overrides.name ?? "test-provider",
+    overrides.category ?? "llm",
+    overrides.enabled ?? 1,
+    overrides.active ?? 0
+  );
+  
+  return id;
+}
+
+export function insertTestTool(overrides: Record<string, any> = {}): string {
+  const db = getTestDb();
+  const id = overrides.id ?? crypto.randomUUID();
+  
+  db.query(`
+    INSERT INTO tools (id, name, description, category, enabled, active)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `).run(
+    id,
+    overrides.name ?? "test_tool",
+    overrides.description ?? "Test tool description",
+    overrides.category ?? "bundled",
+    overrides.enabled ?? 1,
+    overrides.active ?? 1
+  );
+  
+  try {
+    db.query(`
+      INSERT OR REPLACE INTO tools_fts (tool_name, name, description, category)
+      VALUES (?, ?, ?, ?)
+    `).run(
+      overrides.name ?? "test_tool",
+      overrides.name ?? "test_tool",
+      overrides.description ?? "Test tool description",
+      overrides.category ?? "bundled"
+    );
+  } catch {
+    // FTS table might not exist in minimal schema
+  }
+  
+  return id;
+}
+
+export function insertTestSkill(overrides: Record<string, any> = {}): string {
+  const db = getTestDb();
+  const id = overrides.id ?? crypto.randomUUID();
+  
+  db.query(`
+    INSERT INTO skills (id, name, description, category, tools, triggers, body, active)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(
+    id,
+    overrides.name ?? "test_skill",
+    overrides.description ?? "Test skill description",
+    overrides.category ?? "core",
+    overrides.tools ?? "[]",
+    overrides.triggers ?? "[]",
+    overrides.body ?? "# Test Skill",
+    overrides.active ?? 1
+  );
+  
+  try {
+    db.query(`
+      INSERT OR REPLACE INTO skills_fts (id, name, description, category, tools, triggers, body)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      id,
+      overrides.name ?? "test_skill",
+      overrides.description ?? "Test skill description",
+      overrides.category ?? "core",
+      overrides.tools ?? "[]",
+      overrides.triggers ?? "[]",
+      overrides.body ?? "# Test Skill"
+    );
+  } catch {
+    // FTS table might not exist
+  }
+  
+  return id;
+}
+
+export function insertTestPlaybookRule(overrides: Record<string, any> = {}): number {
+  const db = getTestDb();
+  
+  const result = db.query(`
+    INSERT INTO playbook (rule, category, applicable_to, active)
+    VALUES (?, ?, ?, ?)
+  `).run(
+    overrides.rule ?? "Test rule",
+    overrides.category ?? "tool_selection",
+    overrides.applicableTo ?? null,
+    overrides.active ?? 1
+  );
+  
+  return result.lastInsertRowid as number;
+}
+
+export function insertTestConversation(threadId: string, role: string, content: string): number {
+  const db = getTestDb();
+  
+  const result = db.query(`
+    INSERT INTO conversations (thread_id, channel, role, content, token_count)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(
+    threadId,
+    "test",
+    role,
+    content,
+    Math.floor(content.length / 4)
+  );
+  
+  return result.lastInsertRowid as number;
+}

package/tsconfig.json

@@ -0,0 +1,40 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "lib": [
+      "ESNext",
+      "DOM",
+      "DOM.Iterable"
+    ],
+    "strict": false,
+    "noImplicitAny": false,
+    "strictNullChecks": false,
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "allowImportingTsExtensions": true,
+    "allowSyntheticDefaultImports": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    "paths": {
+      "@hive/core": ["./packages/core/src/index.ts"],
+      "@hive/core/*": ["./packages/core/src/*"]
+    }
+  },
+  "include": [
+    "packages/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "packages/cli/templates"
+  ]
+}

package/src/agents.ts

@@ -1 +0,0 @@
-export * from "@hive-sdk/agent";

package/src/canvas.ts

@@ -1 +0,0 @@
-export * from "@hive-sdk/canvas";

package/src/channels.ts

@@ -1 +0,0 @@
-export * from "@hive-sdk/channels";

package/src/config.ts

@@ -1 +0,0 @@
-export * from "@hive-sdk/config";

package/src/events.ts

@@ -1 +0,0 @@
-export * from "@hive-sdk/events";