@swarmclawai/swarmclaw 0.2.0

package/src/lib/server/embeddings.ts

@@ -0,0 +1,111 @@
+import { loadSettings, loadCredentials, decryptKey } from './storage'
+
+let localPipeline: any = null
+let localPipelineLoading: Promise<any> | null = null
+
+async function getLocalPipeline() {
+  if (localPipeline) return localPipeline
+  if (localPipelineLoading) return localPipelineLoading
+
+  localPipelineLoading = (async () => {
+    const { pipeline } = await import('@huggingface/transformers')
+    localPipeline = await pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2', {
+      dtype: 'fp32',
+    })
+    return localPipeline
+  })()
+
+  return localPipelineLoading
+}
+
+export async function getEmbedding(text: string): Promise<number[] | null> {
+  const settings = loadSettings()
+  const provider = settings.embeddingProvider
+  if (!provider) return null
+
+  const model = settings.embeddingModel || 'text-embedding-3-small'
+
+  let apiKey: string | null = null
+  if (settings.embeddingCredentialId) {
+    const creds = loadCredentials()
+    const cred = creds[settings.embeddingCredentialId]
+    if (cred?.encryptedKey) {
+      try { apiKey = decryptKey(cred.encryptedKey) } catch { /* ignore */ }
+    }
+  }
+
+  try {
+    if (provider === 'local') {
+      return await localEmbed(text)
+    } else if (provider === 'openai') {
+      return await openaiEmbed(text, model, apiKey)
+    } else if (provider === 'ollama') {
+      return await ollamaEmbed(text, model, settings.langGraphEndpoint)
+    }
+  } catch (err: any) {
+    console.error(`[embeddings] Error computing embedding:`, err.message)
+  }
+
+  return null
+}
+
+async function localEmbed(text: string): Promise<number[]> {
+  const pipe = await getLocalPipeline()
+  const output = await pipe(text.slice(0, 8000), { pooling: 'mean', normalize: true })
+  return Array.from(output.data as Float32Array)
+}
+
+async function openaiEmbed(text: string, model: string, apiKey: string | null): Promise<number[]> {
+  if (!apiKey) throw new Error('OpenAI API key required for embeddings')
+  const res = await fetch('https://api.openai.com/v1/embeddings', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      Authorization: `Bearer ${apiKey}`,
+    },
+    body: JSON.stringify({
+      model,
+      input: text.slice(0, 8000),
+    }),
+  })
+  if (!res.ok) throw new Error(`OpenAI embeddings API error: ${res.status}`)
+  const data = await res.json()
+  return data.data[0].embedding
+}
+
+async function ollamaEmbed(text: string, model: string, endpoint?: string | null): Promise<number[]> {
+  const baseUrl = endpoint || 'http://localhost:11434'
+  const res = await fetch(`${baseUrl}/api/embeddings`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model,
+      prompt: text.slice(0, 8000),
+    }),
+  })
+  if (!res.ok) throw new Error(`Ollama embeddings API error: ${res.status}`)
+  const data = await res.json()
+  return data.embedding
+}
+
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0
+  let dotProduct = 0
+  let normA = 0
+  let normB = 0
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i]
+    normA += a[i] * a[i]
+    normB += b[i] * b[i]
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB)
+  return denom === 0 ? 0 : dotProduct / denom
+}
+
+export function serializeEmbedding(embedding: number[]): Buffer {
+  return Buffer.from(new Float32Array(embedding).buffer)
+}
+
+export function deserializeEmbedding(buf: Buffer): number[] {
+  return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4))
+}

package/src/lib/server/execution-log.ts

@@ -0,0 +1,257 @@
+import fs from 'fs'
+import path from 'path'
+import crypto from 'crypto'
+import Database from 'better-sqlite3'
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type LogCategory =
+  | 'trigger'       // what kicked off the action
+  | 'decision'      // reasoning / model choice
+  | 'tool_call'     // tool invocation with input
+  | 'tool_result'   // tool output
+  | 'outbound'      // messages sent to users/platforms
+  | 'file_op'       // file read/write/delete with checksums
+  | 'commit'        // git commit activity
+  | 'error'         // errors during execution
+
+export interface ExecutionLogEntry {
+  id: string
+  sessionId: string
+  runId: string | null
+  agentId: string | null
+  category: LogCategory
+  summary: string
+  detail: Record<string, unknown> | null
+  ts: number
+}
+
+export interface LogQueryOpts {
+  sessionId?: string
+  agentId?: string
+  runId?: string
+  category?: LogCategory
+  since?: number
+  until?: number
+  limit?: number
+  offset?: number
+  search?: string
+}
+
+// ---------------------------------------------------------------------------
+// Database setup
+// ---------------------------------------------------------------------------
+
+import { DATA_DIR } from './data-dir'
+if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true })
+
+const DB_PATH = path.join(DATA_DIR, 'logs.db')
+
+let _db: Database.Database | null = null
+
+function getDb(): Database.Database {
+  if (_db) return _db
+  _db = new Database(DB_PATH)
+  _db.pragma('journal_mode = WAL')
+  _db.exec(`
+    CREATE TABLE IF NOT EXISTS execution_logs (
+      id         TEXT PRIMARY KEY,
+      session_id TEXT NOT NULL,
+      run_id     TEXT,
+      agent_id   TEXT,
+      category   TEXT NOT NULL,
+      summary    TEXT NOT NULL,
+      detail     TEXT,
+      ts         INTEGER NOT NULL
+    )
+  `)
+  _db.exec(`CREATE INDEX IF NOT EXISTS idx_logs_session ON execution_logs(session_id, ts)`)
+  _db.exec(`CREATE INDEX IF NOT EXISTS idx_logs_agent   ON execution_logs(agent_id, ts)`)
+  _db.exec(`CREATE INDEX IF NOT EXISTS idx_logs_run     ON execution_logs(run_id)`)
+  _db.exec(`CREATE INDEX IF NOT EXISTS idx_logs_cat     ON execution_logs(category)`)
+  return _db
+}
+
+// ---------------------------------------------------------------------------
+// Write
+// ---------------------------------------------------------------------------
+
+const insertStmt = () =>
+  getDb().prepare(
+    `INSERT INTO execution_logs (id, session_id, run_id, agent_id, category, summary, detail, ts)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+  )
+
+export function logExecution(
+  sessionId: string,
+  category: LogCategory,
+  summary: string,
+  opts?: {
+    runId?: string | null
+    agentId?: string | null
+    detail?: Record<string, unknown>
+  },
+): string {
+  const id = crypto.randomBytes(8).toString('hex')
+  const ts = Date.now()
+  try {
+    insertStmt().run(
+      id,
+      sessionId,
+      opts?.runId ?? null,
+      opts?.agentId ?? null,
+      category,
+      summary,
+      opts?.detail ? JSON.stringify(opts.detail) : null,
+      ts,
+    )
+  } catch {
+    // Non-critical — never block agent execution for logging failures
+  }
+  return id
+}
+
+// Batch insert for bulk writes (e.g. file ops)
+export function logExecutionBatch(
+  entries: Array<{
+    sessionId: string
+    category: LogCategory
+    summary: string
+    runId?: string | null
+    agentId?: string | null
+    detail?: Record<string, unknown>
+  }>,
+): void {
+  const db = getDb()
+  const stmt = db.prepare(
+    `INSERT INTO execution_logs (id, session_id, run_id, agent_id, category, summary, detail, ts)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+  )
+  const ts = Date.now()
+  const tx = db.transaction(() => {
+    for (const e of entries) {
+      stmt.run(
+        crypto.randomBytes(8).toString('hex'),
+        e.sessionId,
+        e.runId ?? null,
+        e.agentId ?? null,
+        e.category,
+        e.summary,
+        e.detail ? JSON.stringify(e.detail) : null,
+        ts,
+      )
+    }
+  })
+  try { tx() } catch { /* non-critical */ }
+}
+
+// ---------------------------------------------------------------------------
+// Read / Query
+// ---------------------------------------------------------------------------
+
+export function queryLogs(opts: LogQueryOpts): ExecutionLogEntry[] {
+  const conditions: string[] = []
+  const params: unknown[] = []
+
+  if (opts.sessionId) {
+    conditions.push('session_id = ?')
+    params.push(opts.sessionId)
+  }
+  if (opts.agentId) {
+    conditions.push('agent_id = ?')
+    params.push(opts.agentId)
+  }
+  if (opts.runId) {
+    conditions.push('run_id = ?')
+    params.push(opts.runId)
+  }
+  if (opts.category) {
+    conditions.push('category = ?')
+    params.push(opts.category)
+  }
+  if (opts.since) {
+    conditions.push('ts >= ?')
+    params.push(opts.since)
+  }
+  if (opts.until) {
+    conditions.push('ts <= ?')
+    params.push(opts.until)
+  }
+  if (opts.search) {
+    conditions.push('(summary LIKE ? OR detail LIKE ?)')
+    const pattern = `%${opts.search}%`
+    params.push(pattern, pattern)
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
+  const limit = opts.limit ?? 200
+  const offset = opts.offset ?? 0
+
+  const rows = getDb()
+    .prepare(`SELECT * FROM execution_logs ${where} ORDER BY ts DESC LIMIT ? OFFSET ?`)
+    .all(...params, limit, offset) as Array<{
+      id: string
+      session_id: string
+      run_id: string | null
+      agent_id: string | null
+      category: string
+      summary: string
+      detail: string | null
+      ts: number
+    }>
+
+  return rows.map((r) => ({
+    id: r.id,
+    sessionId: r.session_id,
+    runId: r.run_id,
+    agentId: r.agent_id,
+    category: r.category as LogCategory,
+    summary: r.summary,
+    detail: r.detail ? JSON.parse(r.detail) : null,
+    ts: r.ts,
+  }))
+}
+
+export function countLogs(opts: Omit<LogQueryOpts, 'limit' | 'offset'>): number {
+  const conditions: string[] = []
+  const params: unknown[] = []
+
+  if (opts.sessionId) { conditions.push('session_id = ?'); params.push(opts.sessionId) }
+  if (opts.agentId) { conditions.push('agent_id = ?'); params.push(opts.agentId) }
+  if (opts.runId) { conditions.push('run_id = ?'); params.push(opts.runId) }
+  if (opts.category) { conditions.push('category = ?'); params.push(opts.category) }
+  if (opts.since) { conditions.push('ts >= ?'); params.push(opts.since) }
+  if (opts.until) { conditions.push('ts <= ?'); params.push(opts.until) }
+  if (opts.search) {
+    conditions.push('(summary LIKE ? OR detail LIKE ?)')
+    const pattern = `%${opts.search}%`
+    params.push(pattern, pattern)
+  }
+
+  const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''
+  const row = getDb()
+    .prepare(`SELECT COUNT(*) as cnt FROM execution_logs ${where}`)
+    .get(...params) as { cnt: number }
+  return row.cnt
+}
+
+// ---------------------------------------------------------------------------
+// Clear
+// ---------------------------------------------------------------------------
+
+export function clearLogs(sessionId?: string): number {
+  if (sessionId) {
+    const result = getDb().prepare('DELETE FROM execution_logs WHERE session_id = ?').run(sessionId)
+    return result.changes
+  }
+  const result = getDb().prepare('DELETE FROM execution_logs').run()
+  return result.changes
+}
+
+export function clearLogsByAge(maxAgeMs: number): number {
+  const cutoff = Date.now() - maxAgeMs
+  const result = getDb().prepare('DELETE FROM execution_logs WHERE ts < ?').run(cutoff)
+  return result.changes
+}

package/src/lib/server/gateway/protocol.test.ts

@@ -0,0 +1,54 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import {
+  createGatewayRequestFrame,
+  parseGatewayFrame,
+  serializeGatewayFrame,
+} from './protocol.ts'
+
+test('gateway protocol parses request/response/event frames', () => {
+  const req = parseGatewayFrame('{"type":"req","id":"1","method":"connect","params":{"foo":"bar"}}')
+  assert.deepEqual(req, {
+    type: 'req',
+    id: '1',
+    method: 'connect',
+    params: { foo: 'bar' },
+  })
+
+  const res = parseGatewayFrame('{"type":"res","id":"1","ok":true,"payload":{"ok":true}}')
+  assert.deepEqual(res, {
+    type: 'res',
+    id: '1',
+    ok: true,
+    payload: { ok: true },
+    error: null,
+  })
+
+  const event = parseGatewayFrame('{"type":"event","event":"tick","payload":{"n":1}}')
+  assert.deepEqual(event, {
+    type: 'event',
+    event: 'tick',
+    payload: { n: 1 },
+  })
+})
+
+test('gateway protocol rejects malformed frames', () => {
+  assert.equal(parseGatewayFrame('not-json'), null)
+  assert.equal(parseGatewayFrame('{"type":"event"}'), null)
+  assert.equal(parseGatewayFrame('{"type":"req","id":"1"}'), null)
+  assert.equal(parseGatewayFrame({ nope: true }), null)
+})
+
+test('gateway request helper and serializer produce valid frame', () => {
+  const frame = createGatewayRequestFrame('abc', 'chat.send', { message: 'hello' })
+  assert.deepEqual(frame, {
+    type: 'req',
+    id: 'abc',
+    method: 'chat.send',
+    params: { message: 'hello' },
+  })
+
+  const encoded = serializeGatewayFrame(frame)
+  const decoded = parseGatewayFrame(encoded)
+  assert.deepEqual(decoded, frame)
+})

package/src/lib/server/gateway/protocol.ts

@@ -0,0 +1,114 @@
+export interface GatewayRequestFrame {
+  type: 'req'
+  id: string
+  method: string
+  params?: Record<string, unknown>
+}
+
+export interface GatewayResponseFrame {
+  type: 'res'
+  id: string
+  ok: boolean
+  payload?: unknown
+  error?: {
+    code?: string
+    message?: string
+  } | null
+}
+
+export interface GatewayEventFrame {
+  type: 'event'
+  event: string
+  payload?: unknown
+}
+
+export type GatewayFrame = GatewayRequestFrame | GatewayResponseFrame | GatewayEventFrame
+
+function toJsonString(raw: unknown): string | null {
+  if (typeof raw === 'string') return raw
+  if (raw instanceof Uint8Array) return Buffer.from(raw).toString('utf8')
+  if (Buffer.isBuffer(raw)) return raw.toString('utf8')
+  if (raw && typeof raw === 'object' && 'toString' in raw) {
+    try {
+      return String((raw as { toString: () => string }).toString())
+    } catch {
+      return null
+    }
+  }
+  return null
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return !!value && typeof value === 'object' && !Array.isArray(value)
+}
+
+function parseFrameObject(value: unknown): GatewayFrame | null {
+  if (!isRecord(value)) return null
+  const type = typeof value.type === 'string' ? value.type : ''
+  if (type === 'req') {
+    const id = typeof value.id === 'string' ? value.id : ''
+    const method = typeof value.method === 'string' ? value.method : ''
+    if (!id || !method) return null
+    const params = isRecord(value.params) ? value.params : undefined
+    return { type: 'req', id, method, params }
+  }
+  if (type === 'res') {
+    const id = typeof value.id === 'string' ? value.id : ''
+    const ok = value.ok === true
+    if (!id) return null
+    const error = isRecord(value.error)
+      ? {
+          code: typeof value.error.code === 'string' ? value.error.code : undefined,
+          message: typeof value.error.message === 'string' ? value.error.message : undefined,
+        }
+      : null
+    return {
+      type: 'res',
+      id,
+      ok,
+      payload: value.payload,
+      error,
+    }
+  }
+  if (type === 'event') {
+    const event = typeof value.event === 'string' ? value.event : ''
+    if (!event) return null
+    return {
+      type: 'event',
+      event,
+      payload: value.payload,
+    }
+  }
+  return null
+}
+
+export function parseGatewayFrame(raw: unknown): GatewayFrame | null {
+  if (isRecord(raw) && typeof raw.type === 'string') {
+    return parseFrameObject(raw)
+  }
+  const text = toJsonString(raw)
+  if (!text) return null
+  try {
+    const parsed = JSON.parse(text)
+    return parseFrameObject(parsed)
+  } catch {
+    return null
+  }
+}
+
+export function serializeGatewayFrame(frame: GatewayFrame): string {
+  return JSON.stringify(frame)
+}
+
+export function createGatewayRequestFrame(
+  id: string,
+  method: string,
+  params?: Record<string, unknown>,
+): GatewayRequestFrame {
+  return {
+    type: 'req',
+    id,
+    method,
+    params,
+  }
+}