@desplega.ai/agent-swarm 1.92.0 → 1.92.2

package/src/be/db.ts

@@ -64,6 +64,7 @@ import type {
   ScheduledTaskSummary,
   ScriptRun,
   ScriptRunJournalEntry,
+  ScriptRunKind,
   ScriptRunStatus,
   Service,
   ServiceStatus,
@@ -71,6 +72,7 @@ import type {
   SessionCostSource,
   SessionLog,
   Skill,
+  SkillFile,
   SkillScope,
   SkillType,
   SkillWithInstallInfo,
@@ -113,6 +115,26 @@ export function isSqliteVecAvailable(): boolean {
   return sqliteVecAvailable;
 }
 
+function loadSqliteVec(database: Database): void {
+  sqliteVecAvailable = false;
+  try {
+    const extensionPath = process.env.SQLITE_VEC_EXTENSION_PATH;
+    if (extensionPath) {
+      database.loadExtension(extensionPath);
+    } else {
+      const sqliteVec = require("sqlite-vec");
+      sqliteVec.load(database);
+    }
+    sqliteVecAvailable = true;
+    console.log(`[db] sqlite-vec loaded${extensionPath ? ` from ${extensionPath}` : ""}`);
+  } catch (err) {
+    console.warn(
+      "[db] sqlite-vec not available, falling back to in-memory cosine:",
+      (err as Error).message,
+    );
+  }
+}
+
 export function initDb(dbPath = "./agent-swarm-db.sqlite"): Database {
   if (db) {
     return db;
@@ -129,6 +151,7 @@ export function initDb(dbPath = "./agent-swarm-db.sqlite"): Database {
     db = Database.deserialize(templateBytes);
     db.run("PRAGMA busy_timeout = 5000;");
     db.run("PRAGMA foreign_keys = ON;");
+    loadSqliteVec(db);
     configureDbResolver(resolvePromptTemplate);
     // Ensure the encryption key is resolved even when restoring from the test
     // template. The cache may have been cleared via __resetEncryptionKeyForTests
@@ -154,22 +177,7 @@ export function initDb(dbPath = "./agent-swarm-db.sqlite"): Database {
   // `require.resolve("sqlite-vec-<platform>/vec0.so")` can't find the native
   // asset — so we prefer an explicit filesystem path when set, and only fall
   // back to the npm resolver for normal dev runs.
-  try {
-    const extensionPath = process.env.SQLITE_VEC_EXTENSION_PATH;
-    if (extensionPath) {
-      database.loadExtension(extensionPath);
-    } else {
-      const sqliteVec = require("sqlite-vec");
-      sqliteVec.load(database);
-    }
-    sqliteVecAvailable = true;
-    console.log(`[db] sqlite-vec loaded${extensionPath ? ` from ${extensionPath}` : ""}`);
-  } catch (err) {
-    console.warn(
-      "[db] sqlite-vec not available, falling back to in-memory cosine:",
-      (err as Error).message,
-    );
-  }
+  loadSqliteVec(database);
 
   // Run database migrations (schema creation + incremental changes)
   runMigrations(database);
@@ -347,6 +355,7 @@ export function closeDb(): void {
     db.close();
     db = null;
   }
+  sqliteVecAvailable = false;
 }
 
 // ============================================================================
@@ -2108,6 +2117,14 @@ export function failTask(id: string, reason: string): AgentTask | null {
         });
       });
     } catch {}
+
+    // Cascade-fail any non-terminal tasks that depend on this one.
+    // The cascade is recursive (transitive closure) and cycle-safe.
+    try {
+      cascadeFailDependents(id, "failed");
+    } catch (err) {
+      console.error("[failTask] cascade-fail dependents error:", err);
+    }
   }
   return row ? rowToAgentTask(row) : null;
 }
@@ -2146,6 +2163,12 @@ export function cancelTask(id: string, reason?: string): AgentTask | null {
         });
       });
     } catch {}
+
+    try {
+      cascadeFailDependents(id, "cancelled");
+    } catch (err) {
+      console.error("[cancelTask] cascade-fail dependents error:", err);
+    }
   }
 
   return row ? rowToAgentTask(row) : null;
@@ -2209,6 +2232,12 @@ export function supersedeTask(
         });
       });
     } catch {}
+
+    try {
+      cascadeFailDependents(id, "superseded");
+    } catch (err) {
+      console.error("[supersedeTask] cascade-fail dependents error:", err);
+    }
   }
 
   return row ? rowToAgentTask(row) : null;
@@ -3381,6 +3410,75 @@ export function checkDependencies(taskId: string): {
   return { ready: blockedBy.length === 0, blockedBy };
 }
 
+/**
+ * Reverse-lookup: find all tasks whose `dependsOn` JSON array contains `parentId`.
+ * Uses SQLite `json_each` to scan the dependsOn column efficiently.
+ * Returns only non-terminal tasks by default (the callers want to cascade-fail
+ * live dependents, not re-process already-finished ones).
+ */
+export function getDependentTasks(
+  parentId: string,
+  opts?: { includeTerminal?: boolean },
+): AgentTask[] {
+  const database = getDb();
+  const rows = database
+    .prepare<AgentTaskRow, [string]>(
+      `SELECT t.*
+       FROM agent_tasks t, json_each(t.dependsOn) AS dep
+       WHERE dep.value = ?`,
+    )
+    .all(parentId);
+
+  const tasks = rows.map(rowToAgentTask);
+  if (opts?.includeTerminal) return tasks;
+  return tasks.filter((t) => !isTerminalTaskStatus(t.status));
+}
+
+export interface CascadeFailResult {
+  taskId: string;
+  taskSubject: string;
+}
+
+/**
+ * Recursively cascade-fail all transitive dependents of a parent task.
+ * Walks the full dependency graph: if A fails, and B depends on A, and C
+ * depends on B, then both B and C are failed.
+ *
+ * Guards against cycles with a visited set. Skips already-terminal tasks.
+ * Returns the list of tasks that were actually cascade-failed (for follow-up
+ * enrichment).
+ */
+export function cascadeFailDependents(
+  parentId: string,
+  parentStatus: string,
+  visited?: Set<string>,
+): CascadeFailResult[] {
+  const seen = visited ?? new Set<string>();
+  if (seen.has(parentId)) return [];
+  seen.add(parentId);
+
+  const dependents = getDependentTasks(parentId);
+  const results: CascadeFailResult[] = [];
+
+  for (const dep of dependents) {
+    if (seen.has(dep.id)) continue;
+
+    const reason = `Blocked dependency ${parentId.slice(0, 8)} was ${parentStatus}`;
+    const failed = failTask(dep.id, reason);
+    if (failed) {
+      results.push({
+        taskId: failed.id,
+        taskSubject: failed.task.slice(0, 120),
+      });
+      // Recurse: this dependent may itself have dependents
+      const transitive = cascadeFailDependents(dep.id, "failed (cascade)", seen);
+      results.push(...transitive);
+    }
+  }
+
+  return results;
+}
+
 // ============================================================================
 // Agent Profile Operations
 // ============================================================================
@@ -7091,7 +7189,7 @@ export function createPage(data: {
   title: string;
   description?: string;
   contentType: PageContentType;
-  authMode: PageAuthMode;
+  authMode?: PageAuthMode;
   passwordHash?: string;
   body: string;
   needsCredentials?: string[];
@@ -7110,7 +7208,7 @@ export function createPage(data: {
       data.title,
       data.description ?? null,
       data.contentType,
-      data.authMode,
+      data.authMode ?? "authed",
       data.passwordHash ?? null,
       data.body,
       data.needsCredentials ? JSON.stringify(data.needsCredentials) : null,
@@ -8692,6 +8790,119 @@ function rowToSkillWithInstall(row: SkillWithInstallRow): SkillWithInstallInfo {
   };
 }
 
+type SkillFileRow = {
+  id: string;
+  skillId: string;
+  path: string;
+  content: string;
+  mimeType: string;
+  isBinary: number;
+  size: number | null;
+  createdAt: string;
+  lastUpdatedAt: string;
+};
+
+function rowToSkillFile(row: SkillFileRow): SkillFile {
+  return {
+    id: row.id,
+    skillId: row.skillId,
+    path: row.path,
+    content: row.content,
+    mimeType: row.mimeType,
+    isBinary: row.isBinary === 1,
+    size: row.size,
+    createdAt: row.createdAt,
+    lastUpdatedAt: row.lastUpdatedAt,
+  };
+}
+
+export type SkillFileInput = {
+  path: string;
+  content: string;
+  mimeType?: string;
+  isBinary?: boolean;
+  size?: number | null;
+};
+
+export type SkillFileManifestEntry = Omit<SkillFile, "content">;
+type NormalizedSkillFileInput = {
+  path: string;
+  content: string;
+  mimeType: string;
+  isBinary: boolean;
+  size: number;
+};
+
+export const SKILL_FILE_LIMITS = {
+  maxCount: Number(process.env.SKILL_FILES_MAX_COUNT ?? 100),
+  maxTotalBytes: Number(process.env.SKILL_FILES_MAX_TOTAL_BYTES ?? 10 * 1024 * 1024),
+  maxFileBytes: Number(process.env.SKILL_FILES_MAX_FILE_BYTES ?? 500 * 1024),
+};
+
+const BINARY_SKILL_FILE_PLACEHOLDER = "[binary file - not synced]";
+
+export function normalizeSkillFilePath(path: string): string {
+  const raw = path.trim().replace(/\\/g, "/");
+  if (!raw) throw new Error("File path is required");
+  if (raw.startsWith("/")) throw new Error("File path must be relative");
+
+  const parts = raw.split("/").filter(Boolean);
+  if (parts.length === 0) throw new Error("File path is required");
+  if (parts.some((part) => part === "." || part === "..")) {
+    throw new Error("File path cannot contain traversal segments");
+  }
+
+  const normalized = parts.join("/");
+  if (normalized === "SKILL.md") {
+    throw new Error("SKILL.md is stored on the skill record, not in skill_files");
+  }
+  return normalized;
+}
+
+function byteSize(content: string): number {
+  return Buffer.byteLength(content, "utf8");
+}
+
+function normalizeSkillFileInput(input: SkillFileInput): NormalizedSkillFileInput {
+  const path = normalizeSkillFilePath(input.path);
+  const isBinary = input.isBinary === true;
+  const content = isBinary ? input.content || BINARY_SKILL_FILE_PLACEHOLDER : input.content;
+  const size = input.size ?? byteSize(content);
+  if (!Number.isFinite(size) || size < 0) {
+    throw new Error("File size must be a non-negative number");
+  }
+  if (size > SKILL_FILE_LIMITS.maxFileBytes) {
+    throw new Error(`File ${path} exceeds max size ${SKILL_FILE_LIMITS.maxFileBytes}`);
+  }
+
+  return {
+    path,
+    content,
+    mimeType: input.mimeType ?? "text/plain",
+    isBinary,
+    size,
+  };
+}
+
+function assertSkillFileLimits(skillId: string, incoming: SkillFileInput[], replaceAll: boolean) {
+  const existing = replaceAll ? [] : listSkillFileManifest(skillId);
+  const byPath = new Map(existing.map((file) => [file.path, file.size ?? 0]));
+
+  for (const input of incoming) {
+    const normalized = normalizeSkillFileInput(input);
+    byPath.set(normalized.path, normalized.size);
+  }
+
+  if (byPath.size > SKILL_FILE_LIMITS.maxCount) {
+    throw new Error(`Skill file count exceeds max ${SKILL_FILE_LIMITS.maxCount}`);
+  }
+
+  const total = [...byPath.values()].reduce((sum, size) => sum + size, 0);
+  if (total > SKILL_FILE_LIMITS.maxTotalBytes) {
+    throw new Error(`Skill files exceed max total size ${SKILL_FILE_LIMITS.maxTotalBytes}`);
+  }
+}
+
 export interface SkillInsert {
   name: string;
   description: string;
@@ -8865,6 +9076,124 @@ export function updateSkill(
   return row ? rowToSkill(row) : null;
 }
 
+function bumpSkillVersion(skillId: string, now = new Date().toISOString()) {
+  getDb()
+    .prepare("UPDATE skills SET version = version + 1, lastUpdatedAt = ? WHERE id = ?")
+    .run(now, skillId);
+}
+
+export function listSkillFileManifest(skillId: string): SkillFileManifestEntry[] {
+  return getDb()
+    .prepare<SkillFileRow, [string]>(
+      `SELECT id, skillId, path, content, mimeType, isBinary, size, createdAt, lastUpdatedAt
+       FROM skill_files
+       WHERE skillId = ?
+       ORDER BY path ASC`,
+    )
+    .all(skillId)
+    .map((row) => {
+      const { content: _content, ...manifest } = rowToSkillFile(row);
+      return manifest;
+    });
+}
+
+export function getSkillFiles(skillId: string): SkillFile[] {
+  return getDb()
+    .prepare<SkillFileRow, [string]>(
+      `SELECT id, skillId, path, content, mimeType, isBinary, size, createdAt, lastUpdatedAt
+       FROM skill_files
+       WHERE skillId = ?
+       ORDER BY path ASC`,
+    )
+    .all(skillId)
+    .map(rowToSkillFile);
+}
+
+export function getSkillFile(skillId: string, path: string): SkillFile | null {
+  const normalizedPath = normalizeSkillFilePath(path);
+  const row = getDb()
+    .prepare<SkillFileRow, [string, string]>(
+      `SELECT id, skillId, path, content, mimeType, isBinary, size, createdAt, lastUpdatedAt
+       FROM skill_files
+       WHERE skillId = ? AND path = ?`,
+    )
+    .get(skillId, normalizedPath);
+  return row ? rowToSkillFile(row) : null;
+}
+
+export function upsertSkillFile(skillId: string, input: SkillFileInput): SkillFile {
+  const payload = normalizeSkillFileInput(input);
+  assertSkillFileLimits(skillId, [payload], false);
+
+  const id = crypto.randomUUID();
+  const now = new Date().toISOString();
+  return upsertSkillFileUnchecked(skillId, payload, id, now, true);
+}
+
+function upsertSkillFileUnchecked(
+  skillId: string,
+  payload: NormalizedSkillFileInput,
+  id: string,
+  now: string,
+  bumpVersion: boolean,
+): SkillFile {
+  const row = getDb()
+    .prepare<SkillFileRow, (string | number | null)[]>(
+      `INSERT INTO skill_files (
+        id, skillId, path, content, mimeType, isBinary, size, createdAt, lastUpdatedAt
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(skillId, path) DO UPDATE SET
+        content = excluded.content,
+        mimeType = excluded.mimeType,
+        isBinary = excluded.isBinary,
+        size = excluded.size,
+        lastUpdatedAt = excluded.lastUpdatedAt
+      RETURNING *`,
+    )
+    .get(
+      id,
+      skillId,
+      payload.path,
+      payload.content,
+      payload.mimeType,
+      payload.isBinary ? 1 : 0,
+      payload.size,
+      now,
+      now,
+    );
+
+  if (!row) throw new Error("Failed to upsert skill file");
+  if (bumpVersion) bumpSkillVersion(skillId, now);
+  return rowToSkillFile(row);
+}
+
+export function upsertSkillFiles(skillId: string, files: SkillFileInput[]): SkillFile[] {
+  if (files.length === 0) return [];
+  const normalized = files.map(normalizeSkillFileInput);
+  assertSkillFileLimits(skillId, normalized, false);
+
+  const now = new Date().toISOString();
+  return getDb().transaction(() => {
+    const rows = normalized.map((file) =>
+      upsertSkillFileUnchecked(skillId, file, crypto.randomUUID(), now, false),
+    );
+    bumpSkillVersion(skillId, now);
+    return rows;
+  })();
+}
+
+export function deleteSkillFile(skillId: string, path: string): boolean {
+  const normalizedPath = normalizeSkillFilePath(path);
+  const result = getDb()
+    .prepare("DELETE FROM skill_files WHERE skillId = ? AND path = ?")
+    .run(skillId, normalizedPath);
+  if (result.changes > 0) {
+    bumpSkillVersion(skillId);
+    return true;
+  }
+  return false;
+}
+
 export function deleteSkill(id: string): boolean {
   const result = getDb().prepare("DELETE FROM skills WHERE id = ?").run(id);
   return result.changes > 0;
@@ -11231,6 +11560,7 @@ type ScriptRunRow = {
   scriptName: string | null;
   source: string;
   args: string;
+  kind: string;
   status: string;
   pid: number | null;
   startedAt: string;
@@ -11256,6 +11586,7 @@ function rowToScriptRun(row: ScriptRunRow): ScriptRun {
     scriptName: row.scriptName ?? undefined,
     source: row.source,
     args: JSON.parse(row.args),
+    kind: row.kind as ScriptRunKind,
     status: row.status as ScriptRunStatus,
     pid: row.pid ?? undefined,
     startedAt: row.startedAt,
@@ -11322,6 +11653,67 @@ export function createScriptRun(data: {
   return { run: rowToScriptRun(row), existing: false };
 }
 
+// Persist a synchronous inline run (POST /api/scripts/run) as an already-terminal
+// row. Unlike createScriptRun these never get a journal and never use the
+// idempotencyKey column (inline idempotency lives in the kv table).
+export function recordInlineScriptRun(data: {
+  id: string;
+  agentId: string;
+  source: string;
+  args: unknown;
+  scriptName?: string;
+  status: "completed" | "failed";
+  output?: unknown;
+  error?: string;
+  startedAt: string;
+  finishedAt: string;
+  requestedByUserId?: string;
+  createdBy?: string;
+}): ScriptRun {
+  const row = getDb()
+    .prepare<
+      ScriptRunRow,
+      [
+        string,
+        string,
+        string | null,
+        string,
+        string,
+        string,
+        string | null,
+        string | null,
+        string,
+        string,
+        string | null,
+        string | null,
+        string | null,
+      ]
+    >(
+      `INSERT INTO script_runs
+        (id, agentId, scriptName, source, args, kind, status, output, error,
+         startedAt, finishedAt, requestedByUserId, created_by, updated_by)
+       VALUES (?, ?, ?, ?, ?, 'inline', ?, ?, ?, ?, ?, ?, ?, ?)
+       RETURNING *`,
+    )
+    .get(
+      data.id,
+      data.agentId,
+      data.scriptName ?? null,
+      data.source,
+      JSON.stringify(data.args ?? null),
+      data.status,
+      data.output === undefined ? null : JSON.stringify(data.output),
+      data.error ?? null,
+      data.startedAt,
+      data.finishedAt,
+      data.requestedByUserId ?? null,
+      data.createdBy ?? null,
+      data.createdBy ?? null,
+    );
+  if (!row) throw new Error("Failed to record inline script run");
+  return rowToScriptRun(row);
+}
+
 export function getScriptRun(id: string): ScriptRun | null {
   const row = getDb()
     .prepare<ScriptRunRow, [string]>("SELECT * FROM script_runs WHERE id = ?")
@@ -11459,6 +11851,7 @@ type ScriptRunJournalRow = {
   error: string | null;
   startedAt: string;
   completedAt: string | null;
+  durationMs: number | null;
   created_by: string | null;
   updated_by: string | null;
 };
@@ -11475,6 +11868,7 @@ function rowToScriptRunJournalEntry(row: ScriptRunJournalRow): ScriptRunJournalE
     error: row.error ?? undefined,
     startedAt: row.startedAt,
     completedAt: row.completedAt ?? undefined,
+    durationMs: row.durationMs ?? undefined,
   };
 }
 
@@ -11498,11 +11892,12 @@ export function upsertScriptRunJournalStep(data: {
   status: "completed" | "failed";
   result?: unknown;
   error?: string;
+  durationMs?: number;
 }): void {
   getDb().run(
     `INSERT OR IGNORE INTO script_run_journal
-      (id, runId, stepKey, stepType, config, status, result, error, completedAt)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
+      (id, runId, stepKey, stepType, config, status, result, error, durationMs, completedAt)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))`,
     [
       crypto.randomUUID(),
       data.runId,
@@ -11512,6 +11907,7 @@ export function upsertScriptRunJournalStep(data: {
       data.status,
       data.result !== undefined ? JSON.stringify(data.result) : null,
       data.error ?? null,
+      data.durationMs ?? null,
     ],
   );
 }

package/src/be/memory/boot-reembed.ts

@@ -0,0 +1,85 @@
+/**
+ * Startup backfill: detect agent_memory rows with wrong-dimension embeddings
+ * (not 512d) and re-embed them in the background. Runs once per boot,
+ * async/non-blocking, idempotent, no-op when the DB is clean.
+ *
+ * This is the app-level equivalent of a forward-only migration — SQL can't
+ * call OpenAI, so the backfill runs at startup instead.
+ */
+
+import { getDb } from "@/be/db";
+import { EMBEDDING_DIMENSIONS } from "./constants";
+import { getEmbeddingProvider, getMemoryStore } from "./index";
+
+const VECTOR_BYTES = EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
+const BATCH_SIZE = 20;
+const BACKFILL_KV_KEY = "memory:reembed:backfill_complete";
+
+export async function runBootReembed(): Promise<void> {
+  const db = getDb();
+
+  const invalidCount =
+    db
+      .prepare<{ count: number }, []>(
+        `SELECT COUNT(*) as count FROM agent_memory
+       WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}`,
+      )
+      .get()?.count ?? 0;
+
+  if (invalidCount === 0) {
+    return;
+  }
+
+  const provider = getEmbeddingProvider();
+  const testEmbed = await provider.embed("test");
+  if (!testEmbed) {
+    console.warn(
+      `[boot-reembed] skipped: ${invalidCount} wrong-dimension rows found but no OpenAI key configured`,
+    );
+    return;
+  }
+
+  console.log(`[boot-reembed] starting: ${invalidCount} rows with wrong embedding dimensions`);
+
+  const store = getMemoryStore();
+  const rows = db
+    .prepare<{ id: string; content: string }, []>(
+      `SELECT id, content FROM agent_memory
+       WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}`,
+    )
+    .all();
+
+  let reembedded = 0;
+  let failed = 0;
+
+  for (let i = 0; i < rows.length; i += BATCH_SIZE) {
+    const batch = rows.slice(i, i + BATCH_SIZE);
+    try {
+      const embeddings = await provider.embedBatch(batch.map((m) => m.content));
+      for (let j = 0; j < embeddings.length; j++) {
+        if (embeddings[j]) {
+          store.updateEmbedding(batch[j]!.id, embeddings[j]!, provider.name);
+          reembedded++;
+        }
+      }
+    } catch (err) {
+      failed += batch.length;
+      console.error(
+        `[boot-reembed] batch ${Math.floor(i / BATCH_SIZE) + 1} failed:`,
+        (err as Error).message,
+      );
+    }
+  }
+
+  const afterInvalid =
+    db
+      .prepare<{ count: number }, []>(
+        `SELECT COUNT(*) as count FROM agent_memory
+       WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}`,
+      )
+      .get()?.count ?? 0;
+
+  console.log(
+    `[boot-reembed] complete: reembedded=${reembedded} failed=${failed} remaining_invalid=${afterInvalid}`,
+  );
+}