@desplega.ai/agent-swarm 1.20.0 → 1.51.2

package/src/be/chunking.ts

@@ -0,0 +1,193 @@
+export interface MemoryChunk {
+  content: string;
+  chunkIndex: number;
+  totalChunks: number;
+  headings: string[];
+}
+
+const MAX_CHUNK_SIZE = 2000; // ~500 tokens
+const CHUNK_OVERLAP = 100; // chars
+const MIN_CHUNK_SIZE = 50; // skip trivially small chunks
+
+/**
+ * Two-stage markdown-aware content chunker.
+ * Stage 1: Split by markdown headers (#, ##, ###) to preserve document structure.
+ * Stage 2: If any section exceeds MAX_CHUNK_SIZE, apply recursive character splitting.
+ * Small files (< MAX_CHUNK_SIZE) are returned as a single chunk.
+ */
+export function chunkContent(text: string): MemoryChunk[] {
+  const trimmed = text.trim();
+  if (!trimmed || trimmed.length < MIN_CHUNK_SIZE) {
+    return [];
+  }
+
+  // Small content: single chunk
+  if (trimmed.length <= MAX_CHUNK_SIZE) {
+    return [{ content: trimmed, chunkIndex: 0, totalChunks: 1, headings: [] }];
+  }
+
+  // Stage 1: Split by markdown headers
+  const sections = splitByHeaders(trimmed);
+
+  // Stage 2: Split oversized sections
+  const rawChunks: { content: string; headings: string[] }[] = [];
+  for (const section of sections) {
+    if (section.content.length <= MAX_CHUNK_SIZE) {
+      rawChunks.push(section);
+    } else {
+      const subChunks = recursiveSplit(section.content);
+      for (const sub of subChunks) {
+        rawChunks.push({ content: sub, headings: section.headings });
+      }
+    }
+  }
+
+  // Filter out trivially small chunks and build final result
+  const filtered = rawChunks.filter((c) => c.content.trim().length >= MIN_CHUNK_SIZE);
+  if (filtered.length === 0) {
+    return [{ content: trimmed, chunkIndex: 0, totalChunks: 1, headings: [] }];
+  }
+
+  return filtered.map((c, i) => ({
+    content: c.headings.length > 0 ? `${c.headings.join(" > ")}\n\n${c.content}` : c.content,
+    chunkIndex: i,
+    totalChunks: filtered.length,
+    headings: c.headings,
+  }));
+}
+
+interface Section {
+  content: string;
+  headings: string[];
+}
+
+function splitByHeaders(text: string): Section[] {
+  const lines = text.split("\n");
+  const sections: Section[] = [];
+  let currentContent: string[] = [];
+  const headingStack: string[] = [];
+
+  for (const line of lines) {
+    const headingMatch = line.match(/^(#{1,3})\s+(.+)$/);
+    if (headingMatch) {
+      // Flush current section
+      if (currentContent.length > 0) {
+        const content = currentContent.join("\n").trim();
+        if (content) {
+          sections.push({ content, headings: [...headingStack] });
+        }
+        currentContent = [];
+      }
+
+      const level = headingMatch[1]!.length;
+      const title = headingMatch[2]!.trim();
+
+      // Update heading stack: pop everything at or below this level
+      while (headingStack.length >= level) {
+        headingStack.pop();
+      }
+      headingStack.push(`${"#".repeat(level)} ${title}`);
+    } else {
+      currentContent.push(line);
+    }
+  }
+
+  // Flush remaining content
+  if (currentContent.length > 0) {
+    const content = currentContent.join("\n").trim();
+    if (content) {
+      sections.push({ content, headings: [...headingStack] });
+    }
+  }
+
+  // If no headers found, return as single section
+  if (sections.length === 0) {
+    return [{ content: text, headings: [] }];
+  }
+
+  return sections;
+}
+
+const SEPARATORS = ["\n\n", "\n", ". ", " "];
+
+function recursiveSplit(text: string): string[] {
+  return recursiveSplitWithSeparators(text, 0);
+}
+
+function recursiveSplitWithSeparators(text: string, separatorIndex: number): string[] {
+  if (text.length <= MAX_CHUNK_SIZE) {
+    return [text.trim()].filter((s) => s.length > 0);
+  }
+
+  if (separatorIndex >= SEPARATORS.length) {
+    // Last resort: hard split at MAX_CHUNK_SIZE with overlap
+    return hardSplit(text);
+  }
+
+  const separator = SEPARATORS[separatorIndex]!;
+  const parts = text.split(separator);
+
+  if (parts.length <= 1) {
+    // This separator doesn't split; try next
+    return recursiveSplitWithSeparators(text, separatorIndex + 1);
+  }
+
+  const chunks: string[] = [];
+  let current = "";
+
+  for (const part of parts) {
+    const candidate = current ? current + separator + part : part;
+    if (candidate.length <= MAX_CHUNK_SIZE) {
+      current = candidate;
+    } else {
+      if (current) {
+        chunks.push(current.trim());
+      }
+      // If part itself is too big, recursively split it
+      if (part.length > MAX_CHUNK_SIZE) {
+        chunks.push(...recursiveSplitWithSeparators(part, separatorIndex + 1));
+        current = "";
+      } else {
+        current = part;
+      }
+    }
+  }
+
+  if (current.trim()) {
+    chunks.push(current.trim());
+  }
+
+  // Add overlap between chunks
+  return addOverlap(chunks, text);
+}
+
+function hardSplit(text: string): string[] {
+  const chunks: string[] = [];
+  let start = 0;
+  while (start < text.length) {
+    const end = Math.min(start + MAX_CHUNK_SIZE, text.length);
+    chunks.push(text.slice(start, end).trim());
+    start = end - CHUNK_OVERLAP;
+    if (start >= text.length - MIN_CHUNK_SIZE) break;
+  }
+  return chunks.filter((c) => c.length > 0);
+}
+
+function addOverlap(chunks: string[], _originalText: string): string[] {
+  if (chunks.length <= 1) return chunks;
+
+  const result: string[] = [chunks[0]!];
+  for (let i = 1; i < chunks.length; i++) {
+    const prevChunk = chunks[i - 1]!;
+    const chunk = chunks[i]!;
+    // Take the last CHUNK_OVERLAP chars from previous chunk as prefix
+    const overlapText = prevChunk.slice(-CHUNK_OVERLAP);
+    // Only add overlap if it doesn't make the chunk too long
+    if (overlapText.length + chunk.length <= MAX_CHUNK_SIZE + CHUNK_OVERLAP) {
+      result.push(overlapText + chunk);
+    } else {
+      result.push(chunk);
+    }
+  }
+  return result;
+}

package/src/be/db-queries/oauth.ts

@@ -0,0 +1,90 @@
+import type { OAuthApp, OAuthTokens } from "../../tracker/types";
+import { getDb } from "../db";
+
+// ── OAuth Apps ──
+
+export function getOAuthApp(provider: string): OAuthApp | null {
+  return getDb()
+    .query("SELECT * FROM oauth_apps WHERE provider = ?")
+    .get(provider) as OAuthApp | null;
+}
+
+export function upsertOAuthApp(
+  provider: string,
+  data: {
+    clientId: string;
+    clientSecret: string;
+    authorizeUrl: string;
+    tokenUrl: string;
+    redirectUri: string;
+    scopes: string;
+    metadata?: string;
+  },
+): void {
+  getDb()
+    .query(
+      `INSERT INTO oauth_apps (provider, clientId, clientSecret, authorizeUrl, tokenUrl, redirectUri, scopes, metadata)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(provider) DO UPDATE SET
+         clientId = excluded.clientId,
+         clientSecret = excluded.clientSecret,
+         authorizeUrl = excluded.authorizeUrl,
+         tokenUrl = excluded.tokenUrl,
+         redirectUri = excluded.redirectUri,
+         scopes = excluded.scopes,
+         metadata = excluded.metadata,
+         updatedAt = datetime('now')`,
+    )
+    .run(
+      provider,
+      data.clientId,
+      data.clientSecret,
+      data.authorizeUrl,
+      data.tokenUrl,
+      data.redirectUri,
+      data.scopes,
+      data.metadata ?? "{}",
+    );
+}
+
+// ── OAuth Tokens ──
+
+export function getOAuthTokens(provider: string): OAuthTokens | null {
+  return getDb()
+    .query("SELECT * FROM oauth_tokens WHERE provider = ?")
+    .get(provider) as OAuthTokens | null;
+}
+
+export function storeOAuthTokens(
+  provider: string,
+  data: {
+    accessToken: string;
+    refreshToken?: string | null;
+    expiresAt: string;
+    scope?: string | null;
+  },
+): void {
+  getDb()
+    .query(
+      `INSERT INTO oauth_tokens (provider, accessToken, refreshToken, expiresAt, scope)
+       VALUES (?, ?, ?, ?, ?)
+       ON CONFLICT(provider) DO UPDATE SET
+         accessToken = excluded.accessToken,
+         refreshToken = COALESCE(excluded.refreshToken, oauth_tokens.refreshToken),
+         expiresAt = excluded.expiresAt,
+         scope = COALESCE(excluded.scope, oauth_tokens.scope),
+         updatedAt = datetime('now')`,
+    )
+    .run(provider, data.accessToken, data.refreshToken ?? null, data.expiresAt, data.scope ?? null);
+}
+
+export function deleteOAuthTokens(provider: string): void {
+  getDb().query("DELETE FROM oauth_tokens WHERE provider = ?").run(provider);
+}
+
+export function isTokenExpiringSoon(provider: string, bufferMs = 5 * 60 * 1000): boolean {
+  const tokens = getOAuthTokens(provider);
+  if (!tokens) return true;
+  const expiresAt = new Date(tokens.expiresAt).getTime();
+  return expiresAt - Date.now() < bufferMs;
+}

package/src/be/db-queries/tracker.ts

@@ -0,0 +1,182 @@
+import type { TrackerAgentMapping, TrackerSync } from "../../tracker/types";
+import { getDb } from "../db";
+
+// ── Tracker Sync ──
+
+export function getTrackerSync(
+  provider: string,
+  entityType: "task" | "epic",
+  swarmId: string,
+): TrackerSync | null {
+  return getDb()
+    .query("SELECT * FROM tracker_sync WHERE provider = ? AND entityType = ? AND swarmId = ?")
+    .get(provider, entityType, swarmId) as TrackerSync | null;
+}
+
+export function getTrackerSyncByExternalId(
+  provider: string,
+  entityType: "task" | "epic",
+  externalId: string,
+): TrackerSync | null {
+  return getDb()
+    .query("SELECT * FROM tracker_sync WHERE provider = ? AND entityType = ? AND externalId = ?")
+    .get(provider, entityType, externalId) as TrackerSync | null;
+}
+
+export function createTrackerSync(data: {
+  provider: string;
+  entityType: "task" | "epic";
+  providerEntityType?: string | null;
+  swarmId: string;
+  externalId: string;
+  externalIdentifier?: string | null;
+  externalUrl?: string | null;
+  lastSyncOrigin?: "swarm" | "external" | null;
+  lastDeliveryId?: string | null;
+  syncDirection?: "inbound" | "outbound" | "bidirectional";
+}): TrackerSync {
+  const result = getDb()
+    .query(
+      `INSERT INTO tracker_sync (provider, entityType, providerEntityType, swarmId, externalId, externalIdentifier, externalUrl, lastSyncOrigin, lastDeliveryId, syncDirection)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       RETURNING *`,
+    )
+    .get(
+      data.provider,
+      data.entityType,
+      data.providerEntityType ?? null,
+      data.swarmId,
+      data.externalId,
+      data.externalIdentifier ?? null,
+      data.externalUrl ?? null,
+      data.lastSyncOrigin ?? null,
+      data.lastDeliveryId ?? null,
+      data.syncDirection ?? "inbound",
+    ) as TrackerSync;
+  return result;
+}
+
+export function updateTrackerSync(
+  id: string,
+  data: Partial<
+    Pick<
+      TrackerSync,
+      | "lastSyncedAt"
+      | "lastSyncOrigin"
+      | "lastDeliveryId"
+      | "syncDirection"
+      | "externalUrl"
+      | "externalIdentifier"
+    >
+  >,
+): void {
+  const sets: string[] = [];
+  const values: (string | null)[] = [];
+
+  if (data.lastSyncedAt !== undefined) {
+    sets.push("lastSyncedAt = ?");
+    values.push(data.lastSyncedAt);
+  }
+  if (data.lastSyncOrigin !== undefined) {
+    sets.push("lastSyncOrigin = ?");
+    values.push(data.lastSyncOrigin);
+  }
+  if (data.lastDeliveryId !== undefined) {
+    sets.push("lastDeliveryId = ?");
+    values.push(data.lastDeliveryId);
+  }
+  if (data.syncDirection !== undefined) {
+    sets.push("syncDirection = ?");
+    values.push(data.syncDirection);
+  }
+  if (data.externalUrl !== undefined) {
+    sets.push("externalUrl = ?");
+    values.push(data.externalUrl);
+  }
+  if (data.externalIdentifier !== undefined) {
+    sets.push("externalIdentifier = ?");
+    values.push(data.externalIdentifier);
+  }
+
+  if (sets.length === 0) return;
+
+  values.push(id);
+  getDb()
+    .query(`UPDATE tracker_sync SET ${sets.join(", ")} WHERE id = ?`)
+    .run(...values);
+}
+
+export function deleteTrackerSync(id: string): void {
+  getDb().query("DELETE FROM tracker_sync WHERE id = ?").run(id);
+}
+
+export function getAllTrackerSyncs(provider?: string, entityType?: "task" | "epic"): TrackerSync[] {
+  const conditions: string[] = [];
+  const values: string[] = [];
+
+  if (provider) {
+    conditions.push("provider = ?");
+    values.push(provider);
+  }
+  if (entityType) {
+    conditions.push("entityType = ?");
+    values.push(entityType);
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+  return getDb()
+    .query(`SELECT * FROM tracker_sync ${where} ORDER BY createdAt DESC`)
+    .all(...values) as TrackerSync[];
+}
+
+// ── Tracker Agent Mapping ──
+
+export function getTrackerAgentMapping(
+  provider: string,
+  agentId: string,
+): TrackerAgentMapping | null {
+  return getDb()
+    .query("SELECT * FROM tracker_agent_mapping WHERE provider = ? AND agentId = ?")
+    .get(provider, agentId) as TrackerAgentMapping | null;
+}
+
+export function getTrackerAgentMappingByExternalUser(
+  provider: string,
+  externalUserId: string,
+): TrackerAgentMapping | null {
+  return getDb()
+    .query("SELECT * FROM tracker_agent_mapping WHERE provider = ? AND externalUserId = ?")
+    .get(provider, externalUserId) as TrackerAgentMapping | null;
+}
+
+export function createTrackerAgentMapping(data: {
+  provider: string;
+  agentId: string;
+  externalUserId: string;
+  agentName: string;
+}): TrackerAgentMapping {
+  return getDb()
+    .query(
+      `INSERT INTO tracker_agent_mapping (provider, agentId, externalUserId, agentName)
+       VALUES (?, ?, ?, ?)
+       RETURNING *`,
+    )
+    .get(data.provider, data.agentId, data.externalUserId, data.agentName) as TrackerAgentMapping;
+}
+
+export function deleteTrackerAgentMapping(provider: string, agentId: string): void {
+  getDb()
+    .query("DELETE FROM tracker_agent_mapping WHERE provider = ? AND agentId = ?")
+    .run(provider, agentId);
+}
+
+export function getAllTrackerAgentMappings(provider?: string): TrackerAgentMapping[] {
+  if (provider) {
+    return getDb()
+      .query("SELECT * FROM tracker_agent_mapping WHERE provider = ? ORDER BY createdAt DESC")
+      .all(provider) as TrackerAgentMapping[];
+  }
+  return getDb()
+    .query("SELECT * FROM tracker_agent_mapping ORDER BY createdAt DESC")
+    .all() as TrackerAgentMapping[];
+}