@johpaz/hive-sdk 0.0.18 → 0.1.4

package/packages/core/src/agent/selectors/ToolSelector.ts

@@ -40,8 +40,9 @@
  *    - core (notify, report_progress, save_note)
  */
 
-import { getDb } from "../../storage/SQLiteStorage.ts"
+import { getHiveDB } from "../../storage/HiveDBStorage.ts"
 import { logger } from "../../utils/logger.ts"
+import type { HiveDB, IndexDoc } from "@johpaz/hive-db";
 
 const log = logger.child("tool-selector")
 
@@ -302,11 +303,11 @@ function getAbstractionPreference(): "atomic" | "orchestration" {
  * 5. If ambiguous → prefer atomic over orchestration
  * 6. Return top maxTools results (default: MAX_TOOLS_PER_TURN)
  */
-export function selectTools(
+export async function selectTools(
     userMessage: string,
     fullToolList: ToolDescriptor[] = CORE_TOOL_CATALOG,
     maxTools: number = MAX_TOOLS_PER_TURN
-): ToolDescriptor[] {
+): Promise<ToolDescriptor[]> {
     const startTime = performance.now()
 
     // Log incoming user message for debugging/validation
@@ -318,78 +319,58 @@ export function selectTools(
         return []
     }
 
-    // Step 2: Build FTS5 query
-    const ftsQuery = buildFTSQuery(userMessage)
-    if (!ftsQuery) {
-        log.debug(`[tool-selector] No valid FTS query terms, returning empty array`)
+    // Step 2: Build search query
+    const searchQuery = buildFTSQuery(userMessage)
+    if (!searchQuery) {
+        log.debug(`[tool-selector] No valid query terms, returning empty array`)
         return []
     }
 
-    log.debug(`[tool-selector] FTS query: "${ftsQuery}"`)
-
-    // Step 3: Execute FTS5 query with bm25 scoring
-    const db = getDb()
-
-    // Use bm25() with column weights for relevance scoring
-    // FTS5 table columns: tool_name, name, description, category
-    // Weights: tool_name=1.0, name=5.0, description=3.0, category=1.0
-    // Higher weight on name (5.0) for exact tool name matching
-    // Get more initially (maxTools * 2) for filtering, then limit to maxTools
-    const ftsResults = db.query(`
-      SELECT tool_name, bm25(tools_fts, 1.0, 5.0, 3.0, 1.0) as bm25_score
-      FROM tools_fts
-      WHERE tools_fts MATCH ?
-      ORDER BY bm25_score ASC
-      LIMIT ?
-    `).all(ftsQuery, maxTools * 2) as { tool_name: string; bm25_score: number }[]
-
-    if (ftsResults.length === 0) {
-        log.debug(`[tool-selector] No FTS matches, returning empty array`)
-        return []
-    }
+    log.debug(`[tool-selector] Search query: "${searchQuery}"`)
 
-    // Log raw scores for debugging
-    log.info(`[tool-selector] Raw FTS scores: ${ftsResults.slice(0, 10).map(r => `${r.tool_name}=${r.bm25_score.toFixed(2)}`).join(", ")}`)
+    // Step 3: Execute hybrid search over the HiveDB semantic index
+    const db = await getHiveDB()
 
-    // Step 4: Apply relevance threshold filter
-    // bm25() returns negative scores; threshold is -0.5 (loosened from typical -5)
-    const relevantResults = ftsResults.filter(r => r.bm25_score >= MIN_RELEVANCE_THRESHOLD)
+    // HiveDB text-only BM25 returns positive scores where higher is better.
+    const hits = await db.queryHybrid({
+        text: searchQuery,
+        k: maxTools * 2,
+        boosts: { name: 5.0, body: 3.0, tags: 1.0 },
+    })
 
-    if (relevantResults.length === 0) {
-        log.debug(`[tool-selector] All results below threshold ${MIN_RELEVANCE_THRESHOLD}, returning empty`)
+    if (hits.length === 0) {
+        log.debug(`[tool-selector] No index matches, returning empty array`)
         return []
     }
 
-    // Step 5: Map to tool descriptors with additional metadata
+    // Log raw scores for debugging
+    log.info(`[tool-selector] Raw scores: ${hits.slice(0, 10).map(r => `${r.id}=${r.score.toFixed(2)}`).join(", ")}`)
+
+    // Step 4: Map to tool descriptors with additional metadata
     const toolMap = new Map(fullToolList.map(t => [t.name, t]))
 
     const scoredTools: SelectedTool[] = []
 
-    for (const result of relevantResults) {
-        const tool = toolMap.get(result.tool_name)
+    for (const hit of hits) {
+        const tool = toolMap.get(hit.id)
         if (tool) {
             scoredTools.push({
                 name: tool.name,
-                score: result.bm25_score,
+                score: hit.score,
                 category: tool.category,
             })
         }
     }
 
-    // Step 6: Prefer atomic over orchestration when ambiguous
-    // If we have more than MAX_TOOLS_PER_TURN, prioritize by abstraction level
+    // Step 5: Prefer atomic over orchestration when ambiguous
     const abstractionPref = getAbstractionPreference()
 
     if (scoredTools.length > MAX_TOOLS_PER_TURN) {
-        // Sort by score first, then by abstraction level preference
-        // CRITICAL FIX: bm25() returns NEGATIVE scores where closer to 0 = more relevant
-        // So we sort ASCENDING (a.score - b.score) to put -8.02 before -5.11
+        // Sort by score descending (higher HiveDB score = more relevant)
         scoredTools.sort((a, b) => {
-            // First by score (ascending for bm25 - closer to 0 is better)
             if (Math.abs(a.score - b.score) > 0.1) {
-                return a.score - b.score  // ✅ Fixed: ascending for negative bm25 scores
+                return b.score - a.score
             }
-            // Then by abstraction preference (preferred type first)
             const aTool = toolMap.get(a.name)
             const bTool = toolMap.get(b.name)
             const aLevel = aTool?.abstractionLevel ?? "atomic"
@@ -403,15 +384,14 @@ export function selectTools(
         })
     }
 
-    // Step 7: Take top N tools
+    // Step 6: Take top N tools
     const topTools = scoredTools.slice(0, maxTools)
 
-    // Step 8: Return as ToolDescriptor array
+    // Step 7: Return as ToolDescriptor array
     const result = topTools.map(t => toolMap.get(t.name)!).filter(Boolean)
 
     const timing = performance.now() - startTime
 
-    // Log final selected tools with info level (important for tracking tool selection process)
     if (result.length > 0) {
         log.info(`[tool-selector] Selected ${result.length} tools in ${timing.toFixed(2)}ms:`,
             result.map(t => ({ name: t.name, category: t.category })))
@@ -434,29 +414,29 @@ export function selectTools(
  * @param tools - Optional array of tools to sync. If not provided, fetches from DB.
  */
 export async function syncToolCatalogToFTS(tools?: ToolDescriptor[]): Promise<void> {
-    const db = getDb()
+    const db = await getHiveDB()
 
     try {
-        // Step 1: Build full catalog = CORE_TOOL_CATALOG + any tools in DB not already covered
-        // CORE_TOOL_CATALOG has bilingual keywords; DB tools may be dynamically registered
+        // Step 1: Build full catalog = CORE_TOOL_CATALOG + any explicitly passed tools
         const catalogByName = new Map<string, ToolDescriptor>(
             CORE_TOOL_CATALOG.map(t => [t.name, t])
         )
 
-        // Merge in any tools from the DB that are missing from the static catalog
-        const dbTools = db.query("SELECT name, description, category FROM tools").all() as Array<{ name: string; description: string | null; category: string | null }>
-        for (const row of dbTools) {
+        // Merge in any tools persisted in HiveDB that are missing from the static catalog
+        const toolsCol = db.collection<ToolDescriptor>("tools")
+        const dbTools = await toolsCol.scan()
+        for (const entry of dbTools) {
+            const row = entry.doc
             if (!catalogByName.has(row.name)) {
                 catalogByName.set(row.name, {
                     name: row.name,
                     description: row.description ?? row.name,
-                    category: (row.category ?? "core") as any,
-                    abstractionLevel: "atomic",
+                    category: row.category ?? "core",
+                    abstractionLevel: row.abstractionLevel ?? "atomic",
                 })
             }
         }
 
-        // Also merge any explicitly passed tools (e.g. from initializer)
         for (const t of (tools || [])) {
             if (!catalogByName.has(t.name)) {
                 catalogByName.set(t.name, t)
@@ -465,39 +445,23 @@ export async function syncToolCatalogToFTS(tools?: ToolDescriptor[]): Promise<vo
 
         const toolCatalog = Array.from(catalogByName.values())
 
-        // Step 2: Atomic transaction for FTS5 sync
-        // We use a transaction to ensure that if sync fails, we don't end up with an empty FTS table
-        const syncTransaction = db.transaction(() => {
-            // Verify table exists inside transaction (optional but safer)
-            const tableCheck = db.query("SELECT name FROM sqlite_master WHERE type='table' AND name='tools_fts'").get()
-            if (!tableCheck) {
-                throw new Error("tools_fts table does not exist!")
-            }
-
-            // A: Clear existing data
-            db.run("DELETE FROM tools_fts")
-
-            // B: Prepare insertion
-            const insert = db.prepare(`
-                INSERT INTO tools_fts(tool_name, name, description, category)
-                VALUES (?, ?, ?, ?)
-            `)
-
-            // C: Re-populate
-            for (const tool of toolCatalog) {
-                const enriched = enrichToolDescription(tool)
-                insert.run(tool.name, tool.name, enriched, tool.category)
-            }
-        })
+        // Step 2: Build index documents and upsert atomically via batch
+        const docs: IndexDoc[] = toolCatalog.map(tool => ({
+            id: tool.name,
+            name: tool.name,
+            body: enrichToolDescription(tool),
+            tags: tool.category,
+            filters: [{ field: "type", value: "tool" }],
+        }))
 
-        // Execute transaction
-        syncTransaction()
+        await db.clearIndex()
+        await db.upsertBatch(docs)
 
-        log.info(`[tool-selector] Atomic sync complete: ${toolCatalog.length} tools indexed in FTS5`)
+        log.info(`[tool-selector] Atomic sync complete: ${toolCatalog.length} tools indexed in HiveDB`)
 
     } catch (err) {
         log.error(`[tool-selector] Transactional sync failed:`, err)
-        throw err // Re-throw to inform initializer
+        throw err
     }
 }
 
@@ -506,7 +470,7 @@ export async function syncToolCatalogToFTS(tools?: ToolDescriptor[]): Promise<vo
  * 
  * This improves FTS5 matching for both English and Spanish queries.
  */
-function enrichToolDescription(tool: ToolDescriptor): string {
+export function enrichToolDescription(tool: ToolDescriptor): string {
     const keywordsByCategory: Record<string, string> = {
         scheduling: "programar recordatorio alarma cron schedule reminder task future tiempo",
         projects: "proyecto tarea plan organizer milestone backlog sprint work",

package/packages/core/src/auth/auth.ts

@@ -1,6 +1,6 @@
 import jwt from "jsonwebtoken";
 import { hashString } from "../utils/crypto.ts";
-import { getDb } from "../storage/SQLiteStorage.ts";
+import { getHiveDB } from "../storage/HiveDBStorage.ts";
 
 const JWT_SECRET = process.env.JWT_SECRET || "hive-default-jwt-secret-change-in-production";
 const ACCESS_TOKEN_EXPIRY = "15m";
@@ -19,6 +19,17 @@ interface JwtPayload {
   type: "access" | "refresh";
 }
 
+interface RefreshTokenDoc {
+  userId: string;
+  tokenHash: string;
+  expiresAt: number;
+  revoked: boolean;
+}
+
+function tokensCollection() {
+  return getHiveDB().then(db => db.collection<RefreshTokenDoc>("refresh_tokens"));
+}
+
 export async function generateTokens(userId: string): Promise<AuthTokens> {
   const accessToken = jwt.sign({ userId, type: "access" } satisfies JwtPayload, JWT_SECRET, {
     expiresIn: ACCESS_TOKEN_EXPIRY,
@@ -31,12 +42,8 @@ export async function generateTokens(userId: string): Promise<AuthTokens> {
   const refreshTokenHash = hashString(refreshToken);
   const expiresAt = Math.floor(Date.now() / 1000) + REFRESH_TOKEN_EXPIRY_SECONDS;
 
-  const db = getDb();
-  db.run(
-    `INSERT INTO refresh_tokens (user_id, token_hash, expires_at, revoked) 
-     VALUES (?, ?, ?, 0)`,
-    [userId, refreshTokenHash, expiresAt]
-  );
+  const col = await tokensCollection();
+  await col.put(refreshTokenHash, { userId, tokenHash: refreshTokenHash, expiresAt, revoked: false });
 
   return {
     accessToken,
@@ -59,27 +66,25 @@ export async function refreshAccessToken(refreshToken: string): Promise<AuthToke
   }
 
   const refreshTokenHash = hashString(refreshToken);
-  const db = getDb();
-  const tokenRow = db
-    .query(
-      `SELECT user_id, expires_at, revoked FROM refresh_tokens WHERE token_hash = ?`
-    )
-    .get(refreshTokenHash) as { user_id: string; expires_at: number; revoked: number } | undefined;
-
-  if (!tokenRow) {
+  const col = await tokensCollection();
+  const entry = await col.get(refreshTokenHash);
+
+  if (!entry) {
     throw new Error("Refresh token not found");
   }
 
-  if (tokenRow.revoked === 1) {
+  const tokenRow = entry.doc;
+
+  if (tokenRow.revoked) {
     throw new Error("Refresh token has been revoked");
   }
 
-  if (tokenRow.expires_at < Math.floor(Date.now() / 1000)) {
-    db.run(`DELETE FROM refresh_tokens WHERE token_hash = ?`, [refreshTokenHash]);
+  if (tokenRow.expiresAt < Math.floor(Date.now() / 1000)) {
+    await col.delete(refreshTokenHash);
     throw new Error("Refresh token has expired");
   }
 
-  db.run(`DELETE FROM refresh_tokens WHERE token_hash = ?`, [refreshTokenHash]);
+  await col.delete(refreshTokenHash);
 
   return generateTokens(payload.userId);
 }
@@ -98,11 +103,19 @@ export async function validateAccessToken(token: string): Promise<{ userId: stri
 
 export async function revokeRefreshToken(refreshToken: string): Promise<void> {
   const refreshTokenHash = hashString(refreshToken);
-  const db = getDb();
-  db.run(`UPDATE refresh_tokens SET revoked = 1 WHERE token_hash = ?`, [refreshTokenHash]);
+  const col = await tokensCollection();
+  const entry = await col.get(refreshTokenHash);
+  if (entry) {
+    await col.put(refreshTokenHash, { ...entry.doc, revoked: true });
+  }
 }
 
 export async function revokeAllUserTokens(userId: string): Promise<void> {
-  const db = getDb();
-  db.run(`UPDATE refresh_tokens SET revoked = 1 WHERE user_id = ?`, [userId]);
+  const col = await tokensCollection();
+  const entries = await col.scan();
+  for (const e of entries) {
+    if (e.doc.userId === userId && !e.doc.revoked) {
+      await col.put(e.id, { ...e.doc, revoked: true });
+    }
+  }
 }

package/packages/core/src/harness/boot-id.ts

@@ -0,0 +1,20 @@
+/**
+ * boot-id — a unique id generated on every process start so that durable
+ * leases (harness runs / jobs) can detect which rows belong to a dead
+ * process after a crash/restart.
+ */
+
+import { randomBytes } from "node:crypto";
+
+let currentBootId: string | null = null;
+
+export function getBootId(): string {
+  if (!currentBootId) {
+    currentBootId = randomBytes(8).toString("hex");
+  }
+  return currentBootId;
+}
+
+export function resetBootId(): void {
+  currentBootId = null;
+}

package/packages/core/src/harness/collections.ts

@@ -0,0 +1,98 @@
+/**
+ * Document shapes for the harness's HiveDB collections. Ported from `hive`'s
+ * durable-task harness, generalized for SDK consumers: `JobDoc.type` and
+ * `HarnessRunDoc.kind` are plain strings (not a fixed union) so a host app
+ * (hive-cloud, a custom hive-sdk app, etc.) can define its own job/run
+ * vocabulary and register executors for it via `registerExecutor()`.
+ */
+
+export interface HarnessRunDoc {
+  id: string
+  thread_id: string
+  agent_id: string
+  user_id: string
+  channel: string | null
+  /** Host-defined run kind, e.g. "chat" | "worker" | "goal". */
+  kind: string
+  status: "running" | "completed" | "failed" | "interrupted" | "aborted"
+
+  iterations_used: number
+  max_iterations: number
+  turns_used: number
+  max_turns: number | null
+  tokens_used: number
+  max_tokens: number | null
+
+  goal: string | null
+  goal_check_tool: string | null
+  goal_attempts: number
+
+  state_json: string
+  state_bytes: number
+  pending_tool_calls_json: string | null
+  checkpointed_at: number
+
+  boot_id: string
+  lease_expires_at: number
+  resume_policy: "resume" | "mark_interrupted" | "discard"
+
+  /** Whole-job acceptance criteria (harness-engineering "proof" concept): JSON array of AcceptanceCriterion. */
+  acceptance_json: string | null
+  /** Fixed-worker epoch recorded at run creation: RunEpoch JSON. */
+  epoch_json: string | null
+
+  error: string | null
+  created_at: number
+  updated_at: number
+  finished_at: number | null
+}
+
+export interface JobDoc {
+  id: string
+  lane: string
+  /** Host-defined job type, e.g. "chat_turn" | "worker_task" | "goal_run". */
+  type: string
+  status: "pending" | "running" | "completed" | "failed" | "cancelled" | "interrupted"
+  priority: number
+  payload_json: string
+  run_id: string
+  attempts: number
+  max_attempts: number
+  not_before: number
+  boot_id: string | null
+  lease_expires_at: number | null
+  result_json: string | null
+  error: string | null
+  created_at: number
+  started_at: number | null
+  finished_at: number | null
+  /** Logical-failure retries (executor returned {ok:false, retryable:true}). Separate from `attempts` (crash/lease-expiry only). */
+  retry_count: number
+  /** Error from the most recent logical-failure retry; `error` stays null until the job is terminal. */
+  last_error: string | null
+  /** `toIndexable`-encoded — sentinel when unset. Client-supplied dedup key for job creation. */
+  idempotency_key: string
+}
+
+/**
+ * Compressed evidence artifact for a completed run — the "proof packet"
+ * concept from harness-engineering's proof/verification practice: what was
+ * intended, what was checked, what evidence backs the verdict, known limits.
+ */
+export interface ProofPacketDoc {
+  id: string
+  run_id: string
+  agent_id: string
+  intended_outcome: string
+  /** Per-acceptance-criterion verdicts: [{id, description, met, evidence}]. */
+  acceptance_results_json: string
+  /** Names of checks executed (tool ids, LLM verifier, etc). */
+  checks_run_json: string
+  /** Free-form evidence snippets backing the verdict (tool outputs, verifier reasons). */
+  evidence_json: string
+  known_limits: string | null
+  /** Fixed-worker epoch this run executed under — copied from HarnessRunDoc.epoch_json. */
+  epoch_json: string | null
+  met: boolean
+  created_at: number
+}

package/packages/core/src/harness/db-helpers.ts

@@ -0,0 +1,87 @@
+/**
+ * Reusable HiveDB collection helpers for the harness module — ported from
+ * `hive`'s `storage/hive.ts`. Provides the primitives HiveDB's `Collection`
+ * API doesn't have directly: autoincrement ids (`nextId`), whitelisted
+ * partial UPDATEs (`updateDoc`), and `WHERE field IN (...)` (`findByAny`).
+ */
+
+import { hiveCollection } from "../storage/HiveDBStorage.ts";
+
+const MAX_RETRIES = 5;
+
+/** Sentinel for nullable FK-like fields used in equality indexes (`findBy`/`createIndex` reject `null`). */
+export const NO_PARENT = "__none__";
+
+/** Encode a nullable FK-like value for storage in an indexed field. */
+export function toIndexable(value: string | null | undefined): string {
+  return value ?? NO_PARENT;
+}
+
+/** Decode a value stored via {@link toIndexable} back to its nullable form. */
+export function fromIndexable(value: string): string | null {
+  return value === NO_PARENT ? null : value;
+}
+
+export async function col<T>(name: string) {
+  return hiveCollection<T>(name);
+}
+
+/**
+ * Monotonic counter, formatted as a zero-padded string so lexicographic
+ * `scan()` order matches numeric order. Retries on optimistic-concurrency
+ * conflicts (another writer bumped the same counter concurrently).
+ */
+export async function nextId(counterName: string): Promise<string> {
+  const counters = await col<{ value: number }>("harness_counters");
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const cur = await counters.get(counterName);
+    const next = (cur?.doc.value ?? 0) + 1;
+    try {
+      await counters.put(counterName, { value: next }, { expectedVersion: cur?.version ?? 0 });
+      return String(next).padStart(15, "0");
+    } catch {
+      // Version conflict — another writer won the race, retry.
+    }
+  }
+  throw new Error(`nextId: too much contention on counter "${counterName}"`);
+}
+
+/**
+ * Read-modify-write a document, replacing the whole-document `put()` with
+ * whitelisted-field UPDATE semantics. Retries on optimistic-concurrency
+ * conflicts.
+ */
+export async function updateDoc<T extends object>(
+  collection: string,
+  id: string,
+  patch: Partial<T>
+): Promise<T> {
+  const c = await col<T>(collection);
+  for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+    const existing = await c.get(id);
+    if (!existing) throw new Error(`${collection}/${id} not found`);
+    const merged = { ...existing.doc, ...patch };
+    try {
+      await c.put(id, merged, { expectedVersion: existing.version });
+      return merged;
+    } catch {
+      // Version conflict — retry with a fresh read.
+    }
+  }
+  throw new Error(`updateDoc: too much contention on ${collection}/${id}`);
+}
+
+/**
+ * Fetch documents whose indexed `field` matches any of `values` — emulates
+ * `WHERE field IN (...)`. Requires a prior `createIndex(field)`.
+ */
+export async function findByAny<T>(
+  collection: string,
+  field: string,
+  values: Array<string | number | boolean>
+): Promise<Array<{ id: string; version: number; doc: T }>> {
+  const c = await col<T>(collection);
+  const uniq = [...new Set(values)];
+  const chunks = await Promise.all(uniq.map((v) => c.findBy(field, v)));
+  return chunks.flat();
+}