@swarmclawai/swarmclaw 0.6.7 → 0.7.0

package/src/lib/server/integrity-monitor.ts

@@ -0,0 +1,208 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import crypto from 'node:crypto'
+import { DATA_DIR } from './data-dir'
+import { resolveOpenClawWorkspace } from './openclaw-sync'
+import { loadIntegrityBaselines, saveIntegrityBaselines } from './storage'
+
+export interface IntegrityBaselineEntry {
+  id: string
+  filePath: string
+  kind: 'identity' | 'config' | 'plugin'
+  present: boolean
+  hash: string | null
+  size: number | null
+  mtimeMs: number | null
+  updatedAt: number
+}
+
+export interface IntegrityDrift {
+  id: string
+  filePath: string
+  kind: IntegrityBaselineEntry['kind']
+  type: 'created' | 'modified' | 'deleted'
+  previousHash: string | null
+  nextHash: string | null
+  checkedAt: number
+}
+
+export interface IntegrityMonitorResult {
+  enabled: boolean
+  checkedAt: number
+  checkedFiles: number
+  drifts: IntegrityDrift[]
+}
+
+interface WatchTarget {
+  id: string
+  filePath: string
+  kind: IntegrityBaselineEntry['kind']
+}
+
+function parseBool(value: unknown, fallback: boolean): boolean {
+  if (typeof value === 'boolean') return value
+  if (typeof value === 'string') {
+    const normalized = value.trim().toLowerCase()
+    if (['1', 'true', 'yes', 'on'].includes(normalized)) return true
+    if (['0', 'false', 'no', 'off'].includes(normalized)) return false
+  }
+  return fallback
+}
+
+function fileHash(filePath: string): string {
+  const hasher = crypto.createHash('sha256')
+  const content = fs.readFileSync(filePath)
+  hasher.update(content)
+  return hasher.digest('hex')
+}
+
+function safeStat(filePath: string): fs.Stats | null {
+  try {
+    return fs.statSync(filePath)
+  } catch {
+    return null
+  }
+}
+
+function toId(filePath: string): string {
+  return crypto.createHash('sha1').update(path.resolve(filePath)).digest('hex')
+}
+
+function pushIfExists(targets: WatchTarget[], filePath: string, kind: WatchTarget['kind']): void {
+  if (!fs.existsSync(filePath)) return
+  targets.push({
+    id: toId(filePath),
+    filePath: path.resolve(filePath),
+    kind,
+  })
+}
+
+function collectWatchTargets(): WatchTarget[] {
+  const targets: WatchTarget[] = []
+  const cwd = process.cwd()
+
+  // Core workspace identity/config files.
+  pushIfExists(targets, path.join(cwd, 'AGENTS.md'), 'identity')
+  pushIfExists(targets, path.join(cwd, 'SOUL.md'), 'identity')
+  pushIfExists(targets, path.join(cwd, 'IDENTITY.md'), 'identity')
+  pushIfExists(targets, path.join(cwd, '.env.local'), 'config')
+
+  // Repo-level AGENTS.md (one level above app dir when present).
+  pushIfExists(targets, path.resolve(cwd, '..', 'AGENTS.md'), 'identity')
+
+  // Plugin files + plugin config.
+  pushIfExists(targets, path.join(DATA_DIR, 'plugins.json'), 'config')
+  const pluginDir = path.join(DATA_DIR, 'plugins')
+  if (fs.existsSync(pluginDir)) {
+    for (const entry of fs.readdirSync(pluginDir)) {
+      if (!entry.endsWith('.js') && !entry.endsWith('.mjs') && !entry.endsWith('.cjs')) continue
+      pushIfExists(targets, path.join(pluginDir, entry), 'plugin')
+    }
+  }
+
+  // OpenClaw agent identity files.
+  try {
+    const workspace = resolveOpenClawWorkspace()
+    const agentsDir = path.join(workspace, 'agents')
+    if (fs.existsSync(agentsDir)) {
+      for (const agentDirName of fs.readdirSync(agentsDir)) {
+        const dirPath = path.join(agentsDir, agentDirName)
+        if (!safeStat(dirPath)?.isDirectory()) continue
+        pushIfExists(targets, path.join(dirPath, 'SOUL.md'), 'identity')
+        pushIfExists(targets, path.join(dirPath, 'IDENTITY.md'), 'identity')
+        pushIfExists(targets, path.join(dirPath, 'TOOLS.md'), 'identity')
+        pushIfExists(targets, path.join(dirPath, 'AGENTS.md'), 'identity')
+      }
+    }
+  } catch {
+    // OpenClaw workspace is optional.
+  }
+
+  // Deduplicate path collisions.
+  const seen = new Set<string>()
+  return targets.filter((target) => {
+    if (seen.has(target.id)) return false
+    seen.add(target.id)
+    return true
+  })
+}
+
+function toBaseline(target: WatchTarget, checkedAt: number): IntegrityBaselineEntry {
+  const stat = safeStat(target.filePath)
+  const present = !!stat && stat.isFile()
+  return {
+    id: target.id,
+    filePath: target.filePath,
+    kind: target.kind,
+    present,
+    hash: present ? fileHash(target.filePath) : null,
+    size: present ? stat!.size : null,
+    mtimeMs: present ? Math.trunc(stat!.mtimeMs) : null,
+    updatedAt: checkedAt,
+  }
+}
+
+export function runIntegrityMonitor(settings?: Record<string, unknown> | null): IntegrityMonitorResult {
+  const enabled = parseBool(settings?.integrityMonitorEnabled, true)
+  const checkedAt = Date.now()
+  if (!enabled) {
+    return {
+      enabled: false,
+      checkedAt,
+      checkedFiles: 0,
+      drifts: [],
+    }
+  }
+
+  const targets = collectWatchTargets()
+  const stored = loadIntegrityBaselines() as Record<string, IntegrityBaselineEntry>
+  const nextBaselines: Record<string, IntegrityBaselineEntry> = { ...stored }
+  const drifts: IntegrityDrift[] = []
+  let dirty = false
+
+  for (const target of targets) {
+    const previous = stored[target.id]
+    const current = toBaseline(target, checkedAt)
+
+    if (!previous) {
+      nextBaselines[target.id] = current
+      dirty = true
+      continue
+    }
+
+    const changed = (
+      previous.present !== current.present
+      || previous.hash !== current.hash
+      || previous.filePath !== current.filePath
+      || previous.kind !== current.kind
+    )
+
+    if (changed) {
+      let type: IntegrityDrift['type'] = 'modified'
+      if (!previous.present && current.present) type = 'created'
+      else if (previous.present && !current.present) type = 'deleted'
+      drifts.push({
+        id: current.id,
+        filePath: current.filePath,
+        kind: current.kind,
+        type,
+        previousHash: previous.hash || null,
+        nextHash: current.hash || null,
+        checkedAt,
+      })
+      nextBaselines[target.id] = current
+      dirty = true
+    }
+  }
+
+  if (dirty) {
+    saveIntegrityBaselines(nextBaselines)
+  }
+
+  return {
+    enabled: true,
+    checkedAt,
+    checkedFiles: targets.length,
+    drifts,
+  }
+}

package/src/lib/server/langgraph-checkpoint.ts

@@ -266,6 +266,16 @@ export class SqliteCheckpointSaver extends BaseCheckpointSaver {
     this.db.prepare(`DELETE FROM langgraph_checkpoints WHERE thread_id = ?`).run(threadId)
     this.db.prepare(`DELETE FROM langgraph_writes WHERE thread_id = ?`).run(threadId)
   }
+
+  async deleteCheckpoint(threadId: string, checkpointId: string): Promise<void> {
+    this.db.prepare(`DELETE FROM langgraph_checkpoints WHERE thread_id = ? AND checkpoint_id = ?`).run(threadId, checkpointId)
+    this.db.prepare(`DELETE FROM langgraph_writes WHERE thread_id = ? AND checkpoint_id = ?`).run(threadId, checkpointId)
+  }
+
+  async deleteCheckpointsAfter(threadId: string, timestamp: number): Promise<void> {
+    this.db.prepare(`DELETE FROM langgraph_checkpoints WHERE thread_id = ? AND created_at > ?`).run(threadId, timestamp)
+    this.db.prepare(`DELETE FROM langgraph_writes WHERE thread_id = ? AND checkpoint_id NOT IN (SELECT checkpoint_id FROM langgraph_checkpoints WHERE thread_id = ?)`).run(threadId, threadId)
+  }
 }
 
 let _saver: SqliteCheckpointSaver | undefined

package/src/lib/server/link-understanding.ts

@@ -0,0 +1,55 @@
+import * as cheerio from 'cheerio'
+import { truncate } from './session-tools/context'
+
+const BARE_LINK_RE = /https?:\/\/\S+/gi
+
+/**
+ * Automatically fetch and summarize links found in user messages.
+ * This aligns SwarmClaw with OpenClaw's proactive link-understanding feature.
+ */
+export async function runLinkUnderstanding(message: string): Promise<string[]> {
+  const links = message.match(BARE_LINK_RE)
+  if (!links || links.length === 0) return []
+
+  const uniqueLinks = Array.from(new Set(links)).slice(0, 3) // Limit to first 3 links
+  const results: string[] = []
+
+  for (const url of uniqueLinks) {
+    try {
+      const res = await fetch(url, {
+        headers: { 'User-Agent': 'Mozilla/5.0 (compatible; SwarmClaw/1.0)' },
+        signal: AbortSignal.timeout(8000),
+      })
+      if (!res.ok) continue
+
+      const contentType = res.headers.get('content-type') || ''
+      if (contentType.includes('text/html')) {
+        const html = await res.text()
+        const $ = cheerio.load(html)
+        
+        // Handle YouTube specifically (OpenClaw favorite)
+        if (url.includes('youtube.com/') || url.includes('youtu.be/')) {
+          const title = $('meta[property="og:title"]').attr('content') || $('title').text()
+          const desc = $('meta[property="og:description"]').attr('content') || ''
+          results.push(`[Link Analysis: YouTube] ${url}\nTitle: ${title}\nDescription: ${desc}`)
+          continue
+        }
+
+        // General web page extraction
+        $('script, style, noscript, nav, footer, header').remove()
+        const title = $('title').text().trim()
+        const main = $('article, main, [role="main"]').first()
+        const bodyText = (main.length ? main.text() : $('body').text())
+          .replace(/\s+/g, ' ')
+          .trim()
+        
+        results.push(`[Link Analysis] ${url}\nTitle: ${title}\nContent: ${truncate(bodyText, 1000)}`)
+      }
+    } catch (err) {
+      // Fail silently for link understanding — don't block the main run
+      console.error(`Link understanding failed for ${url}:`, err)
+    }
+  }
+
+  return results
+}

package/src/lib/server/llm-response-cache.test.ts

@@ -0,0 +1,102 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import type { Message } from '@/types'
+import {
+  buildLlmResponseCacheKey,
+  clearLlmResponseCache,
+  getCachedLlmResponse,
+  resolveLlmResponseCacheConfig,
+  setCachedLlmResponse,
+} from './llm-response-cache.ts'
+
+const HISTORY: Message[] = [
+  { role: 'user', text: 'Plan a release.', time: 1 },
+  { role: 'assistant', text: 'Drafted plan.', time: 2 },
+]
+
+test('buildLlmResponseCacheKey is deterministic for equivalent payloads', () => {
+  const keyA = buildLlmResponseCacheKey({
+    provider: 'openai',
+    model: 'gpt-4o-mini',
+    apiEndpoint: 'https://api.openai.com/v1',
+    systemPrompt: 'System prompt',
+    message: 'hello',
+    history: HISTORY,
+    attachedFiles: ['a.txt', 'b.txt'],
+  })
+  const keyB = buildLlmResponseCacheKey({
+    provider: 'openai',
+    model: 'gpt-4o-mini',
+    apiEndpoint: 'https://api.openai.com/v1',
+    systemPrompt: '  System   prompt ',
+    message: 'hello',
+    history: [...HISTORY],
+    attachedFiles: ['a.txt', 'b.txt'],
+  })
+  assert.equal(keyA, keyB)
+})
+
+test('set/get cached responses returns hit and increments hit count', () => {
+  clearLlmResponseCache()
+  const config = { enabled: true, ttlMs: 60_000, maxEntries: 10 }
+  const keyInput = {
+    provider: 'openai',
+    model: 'gpt-4o',
+    message: 'status',
+    history: HISTORY,
+  }
+  setCachedLlmResponse(keyInput, 'cached answer', config, 1000)
+  const hit1 = getCachedLlmResponse(keyInput, config, 1500)
+  assert.ok(hit1)
+  assert.equal(hit1?.text, 'cached answer')
+  assert.equal(hit1?.hits, 1)
+  const hit2 = getCachedLlmResponse(keyInput, config, 1600)
+  assert.equal(hit2?.hits, 2)
+})
+
+test('expired cache entry is not returned', () => {
+  clearLlmResponseCache()
+  const config = { enabled: true, ttlMs: 1000, maxEntries: 10 }
+  const keyInput = {
+    provider: 'openai',
+    model: 'gpt-4o',
+    message: 'status',
+    history: HISTORY,
+  }
+  setCachedLlmResponse(keyInput, 'stale', config, 1000)
+  const miss = getCachedLlmResponse(keyInput, config, 3001)
+  assert.equal(miss, null)
+})
+
+test('cache evicts least recently used entries over maxEntries', () => {
+  clearLlmResponseCache()
+  const config = { enabled: true, ttlMs: 60_000, maxEntries: 2 }
+  const inputA = { provider: 'openai', model: 'gpt-4o', message: 'a', history: HISTORY }
+  const inputB = { provider: 'openai', model: 'gpt-4o', message: 'b', history: HISTORY }
+  const inputC = { provider: 'openai', model: 'gpt-4o', message: 'c', history: HISTORY }
+  setCachedLlmResponse(inputA, 'A', config, 1000)
+  setCachedLlmResponse(inputB, 'B', config, 1001)
+  // Touch A so B becomes LRU.
+  getCachedLlmResponse(inputA, config, 1002)
+  setCachedLlmResponse(inputC, 'C', config, 1003)
+
+  assert.equal(getCachedLlmResponse(inputB, config, 1004), null)
+  assert.equal(getCachedLlmResponse(inputA, config, 1004)?.text, 'A')
+  assert.equal(getCachedLlmResponse(inputC, config, 1004)?.text, 'C')
+})
+
+test('resolveLlmResponseCacheConfig applies defaults and bounds', () => {
+  const fallback = resolveLlmResponseCacheConfig({})
+  assert.equal(fallback.enabled, true)
+  assert.equal(fallback.ttlMs, 900_000)
+  assert.equal(fallback.maxEntries, 500)
+
+  const custom = resolveLlmResponseCacheConfig({
+    responseCacheEnabled: false,
+    responseCacheTtlSec: 1,
+    responseCacheMaxEntries: 999999,
+  })
+  assert.equal(custom.enabled, false)
+  assert.equal(custom.ttlMs, 5000)
+  assert.equal(custom.maxEntries, 20_000)
+})

package/src/lib/server/llm-response-cache.ts

@@ -0,0 +1,227 @@
+import crypto from 'node:crypto'
+import type { AppSettings, Message } from '@/types'
+
+export interface LlmResponseCacheConfig {
+  enabled: boolean
+  ttlMs: number
+  maxEntries: number
+}
+
+export interface LlmResponseCacheKeyInput {
+  provider: string
+  model: string
+  apiEndpoint?: string | null
+  systemPrompt?: string
+  message: string
+  imagePath?: string
+  imageUrl?: string
+  attachedFiles?: string[]
+  history: Message[]
+}
+
+export interface LlmResponseCacheHit {
+  key: string
+  text: string
+  provider: string
+  model: string
+  createdAt: number
+  ageMs: number
+  hits: number
+}
+
+interface LlmResponseCacheEntry {
+  key: string
+  text: string
+  provider: string
+  model: string
+  createdAt: number
+  expiresAt: number
+  hits: number
+}
+
+const DEFAULT_ENABLED = true
+const DEFAULT_TTL_SEC = 15 * 60
+const DEFAULT_MAX_ENTRIES = 500
+
+const MIN_TTL_SEC = 5
+const MAX_TTL_SEC = 7 * 24 * 3600
+const MIN_ENTRIES = 1
+const MAX_ENTRIES = 20_000
+
+const responseCache = new Map<string, LlmResponseCacheEntry>()
+
+function normalizeText(value: unknown): string {
+  return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : ''
+}
+
+function normalizeList(value: unknown): string[] {
+  if (!Array.isArray(value)) return []
+  return value
+    .filter((entry): entry is string => typeof entry === 'string')
+    .map((entry) => entry.trim())
+    .filter(Boolean)
+}
+
+function normalizeInt(value: unknown, fallback: number, min: number, max: number): number {
+  const parsed = typeof value === 'number'
+    ? value
+    : typeof value === 'string'
+      ? Number.parseInt(value, 10)
+      : Number.NaN
+  if (!Number.isFinite(parsed)) return fallback
+  return Math.max(min, Math.min(max, Math.trunc(parsed)))
+}
+
+function normalizeBool(value: unknown, fallback: boolean): boolean {
+  if (typeof value === 'boolean') return value
+  if (typeof value === 'string') {
+    const normalized = value.trim().toLowerCase()
+    if (['1', 'true', 'yes', 'on'].includes(normalized)) return true
+    if (['0', 'false', 'no', 'off'].includes(normalized)) return false
+  }
+  return fallback
+}
+
+function stableStringify(value: unknown): string {
+  if (value === null) return 'null'
+  const kind = typeof value
+  if (kind === 'number' || kind === 'boolean') return JSON.stringify(value)
+  if (kind === 'string') return JSON.stringify(value)
+  if (Array.isArray(value)) return `[${value.map((entry) => stableStringify(entry)).join(',')}]`
+  if (kind === 'object') {
+    const entries = Object.entries(value as Record<string, unknown>)
+      .filter(([, v]) => v !== undefined)
+      .sort(([a], [b]) => a.localeCompare(b))
+    return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`).join(',')}}`
+  }
+  return JSON.stringify(String(value))
+}
+
+function normalizeHistory(history: Message[]): Array<Record<string, unknown>> {
+  return history.map((entry) => ({
+    role: entry.role,
+    text: normalizeText(entry.text),
+    kind: entry.kind || null,
+    imagePath: entry.imagePath || null,
+    imageUrl: entry.imageUrl || null,
+    attachedFiles: normalizeList(entry.attachedFiles),
+    replyToId: entry.replyToId || null,
+  }))
+}
+
+function trimToCapacity(maxEntries: number): void {
+  while (responseCache.size > maxEntries) {
+    const oldestKey = responseCache.keys().next().value as string | undefined
+    if (!oldestKey) break
+    responseCache.delete(oldestKey)
+  }
+}
+
+function moveToMostRecent(key: string, entry: LlmResponseCacheEntry): void {
+  responseCache.delete(key)
+  responseCache.set(key, entry)
+}
+
+export function resolveLlmResponseCacheConfig(
+  settings?: AppSettings | Record<string, unknown> | null,
+): LlmResponseCacheConfig {
+  const raw = settings && typeof settings === 'object' ? settings as Record<string, unknown> : {}
+  const ttlSec = normalizeInt(raw.responseCacheTtlSec, DEFAULT_TTL_SEC, MIN_TTL_SEC, MAX_TTL_SEC)
+  const maxEntries = normalizeInt(raw.responseCacheMaxEntries, DEFAULT_MAX_ENTRIES, MIN_ENTRIES, MAX_ENTRIES)
+  const enabled = normalizeBool(raw.responseCacheEnabled, DEFAULT_ENABLED)
+  return {
+    enabled,
+    ttlMs: ttlSec * 1000,
+    maxEntries,
+  }
+}
+
+export function buildLlmResponseCacheKey(input: LlmResponseCacheKeyInput): string {
+  const payload = {
+    provider: normalizeText(input.provider).toLowerCase(),
+    model: normalizeText(input.model),
+    apiEndpoint: normalizeText(input.apiEndpoint || ''),
+    systemPrompt: normalizeText(input.systemPrompt || ''),
+    message: normalizeText(input.message),
+    imagePath: normalizeText(input.imagePath || ''),
+    imageUrl: normalizeText(input.imageUrl || ''),
+    attachedFiles: normalizeList(input.attachedFiles),
+    history: normalizeHistory(Array.isArray(input.history) ? input.history : []),
+  }
+  const stable = stableStringify(payload)
+  return crypto.createHash('sha256').update(stable).digest('hex')
+}
+
+export function getCachedLlmResponse(
+  input: LlmResponseCacheKeyInput,
+  config: LlmResponseCacheConfig,
+  now = Date.now(),
+): LlmResponseCacheHit | null {
+  if (!config.enabled) return null
+  const key = buildLlmResponseCacheKey(input)
+  const found = responseCache.get(key)
+  if (!found) return null
+  if (now >= found.expiresAt) {
+    responseCache.delete(key)
+    return null
+  }
+  const next = { ...found, hits: found.hits + 1 }
+  moveToMostRecent(key, next)
+  return {
+    key,
+    text: next.text,
+    provider: next.provider,
+    model: next.model,
+    createdAt: next.createdAt,
+    ageMs: Math.max(0, now - next.createdAt),
+    hits: next.hits,
+  }
+}
+
+export function setCachedLlmResponse(
+  input: LlmResponseCacheKeyInput,
+  text: string,
+  config: LlmResponseCacheConfig,
+  now = Date.now(),
+): void {
+  if (!config.enabled) return
+  const normalizedText = normalizeText(text)
+  if (!normalizedText) return
+  const key = buildLlmResponseCacheKey(input)
+  const existing = responseCache.get(key)
+  const createdAt = existing?.createdAt ?? now
+  const entry: LlmResponseCacheEntry = {
+    key,
+    text: normalizedText,
+    provider: normalizeText(input.provider).toLowerCase(),
+    model: normalizeText(input.model),
+    createdAt,
+    expiresAt: now + config.ttlMs,
+    hits: existing?.hits ?? 0,
+  }
+  moveToMostRecent(key, entry)
+  trimToCapacity(config.maxEntries)
+}
+
+export function getLlmResponseCacheStats(now = Date.now()): {
+  entries: number
+  expired: number
+  oldestAgeMs: number
+} {
+  let expired = 0
+  let oldestCreatedAt = Number.POSITIVE_INFINITY
+  for (const entry of responseCache.values()) {
+    if (entry.expiresAt <= now) expired++
+    oldestCreatedAt = Math.min(oldestCreatedAt, entry.createdAt)
+  }
+  const oldestAgeMs = Number.isFinite(oldestCreatedAt) ? Math.max(0, now - oldestCreatedAt) : 0
+  return {
+    entries: responseCache.size,
+    expired,
+    oldestAgeMs,
+  }
+}
+
+export function clearLlmResponseCache(): void {
+  responseCache.clear()
+}