@swarmclawai/swarmclaw 1.3.4 → 1.3.5

package/src/lib/server/execution-engine/task-attempt.ts

@@ -3,7 +3,7 @@ import { WORKSPACE_DIR } from '@/lib/server/data-dir'
 import { log } from '@/lib/server/logger'
 import { loadSettings } from '@/lib/server/settings/settings-repository'
 import { loadSessions } from '@/lib/server/sessions/session-repository'
-import { appendPersistedRunEvent, persistRun } from '@/lib/server/runtime/run-ledger'
+import { appendPersistedRunEvent, buildRetrievalSummary, persistRun } from '@/lib/server/runtime/run-ledger'
 import { notify } from '@/lib/server/ws-hub'
 import { captureGuardianCheckpoint } from '@/lib/server/agents/guardian'
 import {
@@ -68,6 +68,7 @@ function notifyExecutionState(sessionId: string): void {
 }
 
 function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Record<string, unknown>): void {
+  const { citations, retrievalTrace, ...eventExtra } = extra || {}
   appendPersistedRunEvent({
     runId: run.id,
     sessionId: run.sessionId,
@@ -78,6 +79,8 @@ function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Rec
     phase: 'status',
     status,
     summary: run.resultPreview || run.error || undefined,
+    citations: citations as import('@/types').KnowledgeCitation[] | undefined,
+    retrievalTrace: (retrievalTrace as import('@/types').KnowledgeRetrievalTrace | undefined) || undefined,
     event: {
       t: 'md',
       text: JSON.stringify({
@@ -90,7 +93,7 @@ function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Rec
           status,
           source: run.source,
           internal: run.internal,
-          ...extra,
+          ...eventExtra,
         },
       }),
     },
@@ -268,6 +271,7 @@ export function enqueueTaskAttemptExecution(
       run.endedAt = Date.now()
       run.error = controller.signal.aborted ? (run.error || 'Cancelled') : result.error
       run.resultPreview = result.text?.slice(0, 280)
+      run.retrievalSummary = buildRetrievalSummary(result.citations)
       if (typeof result.inputTokens === 'number') run.totalInputTokens = result.inputTokens
       if (typeof result.outputTokens === 'number') run.totalOutputTokens = result.outputTokens
       if (typeof result.estimatedCost === 'number') run.estimatedCost = result.estimatedCost
@@ -275,6 +279,8 @@ export function enqueueTaskAttemptExecution(
       emitStatus(run, run.status, {
         hasText: !!result.text,
         error: run.error || null,
+        citations: result.citations,
+        retrievalTrace: result.retrievalTrace,
       })
       return result
     } catch (err: unknown) {

package/src/lib/server/knowledge-import.ts

@@ -0,0 +1,159 @@
+import fs from 'fs'
+import path from 'path'
+import * as cheerio from 'cheerio'
+
+const TEXT_EXTS = new Set([
+  '.txt', '.md', '.markdown', '.csv', '.tsv', '.json', '.jsonl',
+  '.html', '.htm', '.xml', '.yaml', '.yml', '.toml', '.ini', '.cfg',
+  '.js', '.ts', '.tsx', '.jsx', '.py', '.go', '.rs', '.java', '.c', '.cpp', '.h',
+  '.rb', '.php', '.sh', '.bash', '.zsh', '.sql', '.r', '.swift', '.kt',
+  '.env', '.log', '.conf', '.properties', '.gitignore', '.dockerignore',
+])
+
+export const MAX_KNOWLEDGE_IMPORT_BYTES = 10 * 1024 * 1024
+export const MAX_KNOWLEDGE_CONTENT_CHARS = 500_000
+
+export function isKnowledgeTextFile(filename: string): boolean {
+  const ext = path.extname(filename).toLowerCase()
+  return TEXT_EXTS.has(ext) || ext === ''
+}
+
+export function deriveKnowledgeTitle(filename: string): string {
+  const name = path.basename(filename, path.extname(filename))
+  return name
+    .replace(/[-_]+/g, ' ')
+    .replace(/([a-z])([A-Z])/g, '$1 $2')
+    .replace(/\b\w/g, (char) => char.toUpperCase())
+    .trim() || 'Knowledge Source'
+}
+
+function normalizeKnowledgeContent(content: string): string {
+  const normalized = String(content || '')
+    .replace(/^\uFEFF/, '')
+    .replace(/\r\n/g, '\n')
+    .trim()
+
+  if (normalized.length <= MAX_KNOWLEDGE_CONTENT_CHARS) return normalized
+  return `${normalized.slice(0, MAX_KNOWLEDGE_CONTENT_CHARS)}\n\n[... truncated at 500k characters]`
+}
+
+async function extractPdfText(buffer: Buffer, filePathHint?: string): Promise<string> {
+  try {
+    const pdfParseModule = await import('pdf-parse') as unknown as {
+      default?: (input: Buffer) => Promise<{ text?: string }>
+    }
+    const pdfParse = pdfParseModule.default
+    if (typeof pdfParse !== 'function') throw new Error('pdf-parse loader unavailable')
+    const result = await pdfParse(buffer)
+    return normalizeKnowledgeContent(result.text || '')
+  } catch {
+    return normalizeKnowledgeContent(
+      `[PDF document]\n\nUnable to extract text automatically.${filePathHint ? `\n\nSaved at: ${filePathHint}` : ''}`,
+    )
+  }
+}
+
+function htmlToReadableText(html: string): { title: string | null; content: string } {
+  const $ = cheerio.load(html)
+  $('script, style, noscript, svg, nav, footer, header').remove()
+
+  const title = $('title').first().text().trim() || null
+  const root = $('main').first().length
+    ? $('main').first()
+    : $('article').first().length
+      ? $('article').first()
+      : $('body').first().length
+        ? $('body').first()
+        : $('html').first()
+
+  const text = root
+    .text()
+    .replace(/\u00a0/g, ' ')
+    .split('\n')
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .join('\n\n')
+
+  return {
+    title,
+    content: normalizeKnowledgeContent(text),
+  }
+}
+
+export async function extractKnowledgeTextFromBuffer(
+  buffer: Buffer,
+  filename: string,
+  filePathHint?: string,
+): Promise<string> {
+  if (buffer.length === 0) return ''
+  if (buffer.length > MAX_KNOWLEDGE_IMPORT_BYTES) {
+    throw new Error('File too large. Maximum 10MB.')
+  }
+
+  const ext = path.extname(filename).toLowerCase()
+  if (ext === '.pdf') {
+    return extractPdfText(buffer, filePathHint)
+  }
+
+  if (isKnowledgeTextFile(filename)) {
+    return normalizeKnowledgeContent(buffer.toString('utf-8'))
+  }
+
+  return normalizeKnowledgeContent(
+    `[Binary file: ${filename}]${filePathHint ? `\n\nSaved at: ${filePathHint}` : ''}`,
+  )
+}
+
+export async function extractKnowledgeTextFromFile(filePath: string, filename?: string): Promise<string> {
+  const buffer = await fs.promises.readFile(filePath)
+  return extractKnowledgeTextFromBuffer(buffer, filename || path.basename(filePath), filePath)
+}
+
+export async function extractKnowledgeTextFromUrl(sourceUrl: string): Promise<{
+  title: string | null
+  content: string
+  contentType: string | null
+}> {
+  const response = await fetch(sourceUrl, {
+    headers: {
+      'user-agent': 'SwarmClaw/knowledge-import',
+      accept: 'text/html, text/plain, application/json, application/pdf, */*',
+    },
+  })
+
+  if (!response.ok) {
+    throw new Error(`URL fetch failed (${response.status})`)
+  }
+
+  const contentType = response.headers.get('content-type')
+  const contentLength = Number.parseInt(response.headers.get('content-length') || '', 10)
+  if (Number.isFinite(contentLength) && contentLength > MAX_KNOWLEDGE_IMPORT_BYTES) {
+    throw new Error('Remote document is too large. Maximum 10MB.')
+  }
+
+  if ((contentType || '').includes('application/pdf') || sourceUrl.toLowerCase().endsWith('.pdf')) {
+    const buffer = Buffer.from(await response.arrayBuffer())
+    return {
+      title: null,
+      content: await extractPdfText(buffer, sourceUrl),
+      contentType,
+    }
+  }
+
+  const text = await response.text()
+  const looksLikeHtml = (contentType || '').includes('text/html') || /<html[\s>]|<body[\s>]/i.test(text)
+  if (looksLikeHtml) {
+    const parsed = htmlToReadableText(text)
+    return {
+      title: parsed.title,
+      content: parsed.content,
+      contentType,
+    }
+  }
+
+  return {
+    title: null,
+    content: normalizeKnowledgeContent(text),
+    contentType,
+  }
+}

package/src/lib/server/knowledge-sources.test.ts

@@ -0,0 +1,215 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { runWithTempDataDir } from '@/lib/server/test-utils/run-with-temp-data-dir'
+
+test('buildKnowledgeRetrievalTrace returns active hits and selectKnowledgeCitations marks empty replies as no_match', () => {
+  const output = runWithTempDataDir<{
+    sourceId: string | null
+    hitCount: number
+    firstHitSourceId: string | null
+    matchedStatus: string | null
+    matchedCitationCount: number
+    whyMatched: string | null
+    unmatchedStatus: string | null
+    unmatchedCitationCount: number
+  }>(`
+    const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
+    const knowledge = knowledgeMod.default || knowledgeMod
+
+    const detail = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Gateway Migration Runbook',
+      content: 'Use blue green deployment for gateway migrations so rollback stays simple and downtime stays low.',
+      tags: ['deploy'],
+    })
+
+    const trace = await knowledge.buildKnowledgeRetrievalTrace({
+      query: 'gateway blue green rollback',
+    })
+
+    const matched = knowledge.selectKnowledgeCitations({
+      responseText: 'Use blue green deployment for the gateway migration so rollback stays simple.',
+      retrievalTrace: trace,
+    })
+
+    const unmatched = knowledge.selectKnowledgeCitations({
+      responseText: '',
+      retrievalTrace: trace,
+    })
+
+    console.log(JSON.stringify({
+      sourceId: detail?.source?.id || null,
+      hitCount: trace?.hits?.length || 0,
+      firstHitSourceId: trace?.hits?.[0]?.sourceId || null,
+      matchedStatus: matched.retrievalTrace?.selectorStatus || null,
+      matchedCitationCount: matched.citations.length,
+      whyMatched: matched.citations[0]?.whyMatched || null,
+      unmatchedStatus: unmatched.retrievalTrace?.selectorStatus || null,
+      unmatchedCitationCount: unmatched.citations.length,
+    }))
+  `, { prefix: 'swarmclaw-knowledge-trace-' })
+
+  assert.ok(output.sourceId)
+  assert.ok(output.hitCount >= 1)
+  assert.equal(output.firstHitSourceId, output.sourceId)
+  assert.equal(output.matchedStatus, 'selected')
+  assert.ok(output.matchedCitationCount >= 1)
+  assert.match(output.whyMatched || '', /Matched|Retrieved/)
+  assert.equal(output.unmatchedStatus, 'no_match')
+  assert.equal(output.unmatchedCitationCount, 0)
+})
+
+test('archived and superseded sources are excluded by default, restore re-enables search, and restore actions are recorded explicitly', () => {
+  const output = runWithTempDataDir<{
+    archivedDefaultCount: number
+    archivedIncludedCount: number
+    restoredCount: number
+    restoreActionKind: string | null
+    supersededDefaultCount: number
+    supersededIncludedCount: number
+    supersededFinding: boolean
+  }>(`
+    const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
+    const storageMod = await import('./src/lib/server/storage.ts')
+    const knowledge = knowledgeMod.default || knowledgeMod
+    const storage = storageMod.default || storageMod
+
+    const archived = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Orchard Rollback Notes',
+      content: 'orchard sentinel rollback checklist',
+    })
+
+    await knowledge.archiveKnowledgeSource(archived.source.id, { reason: 'manual review' })
+
+    const archivedDefault = await knowledge.searchKnowledgeHits({ query: 'orchard' })
+    const archivedIncluded = await knowledge.searchKnowledgeHits({
+      query: 'orchard',
+      includeArchived: true,
+    })
+
+    await knowledge.restoreKnowledgeSource(archived.source.id)
+    const restored = await knowledge.searchKnowledgeHits({ query: 'orchard' })
+
+    const older = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Legacy API Notes',
+      content: 'legacy endpoint alpha is still enabled',
+      sourceUrl: 'https://example.com/api/reference',
+    })
+    const newer = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Current API Notes',
+      content: 'modern endpoint beta replaced the older route',
+      sourceUrl: 'https://example.com/api/reference',
+    })
+
+    storage.patchKnowledgeSource(older.source.id, (current) => current ? {
+      ...current,
+      lastIndexedAt: 1_000,
+      updatedAt: 1_000,
+    } : null)
+    storage.patchKnowledgeSource(newer.source.id, (current) => current ? {
+      ...current,
+      lastIndexedAt: 2_000,
+      updatedAt: 2_000,
+    } : null)
+
+    await knowledge.runKnowledgeHygieneMaintenance()
+
+    const supersededDefault = await knowledge.searchKnowledgeHits({ query: 'alpha' })
+    const supersededIncluded = await knowledge.searchKnowledgeHits({
+      query: 'alpha',
+      includeArchived: true,
+    })
+    const summary = await knowledge.getKnowledgeHygieneSummary()
+
+    console.log(JSON.stringify({
+      archivedDefaultCount: archivedDefault.length,
+      archivedIncludedCount: archivedIncluded.length,
+      restoredCount: restored.length,
+      restoreActionKind: summary.recentActions.find((action) => action.summary === 'Restored Orchard Rollback Notes')?.kind || null,
+      supersededDefaultCount: supersededDefault.length,
+      supersededIncludedCount: supersededIncluded.length,
+      supersededFinding: summary.findings.some((finding) => finding.kind === 'superseded' && finding.sourceId === older.source.id),
+    }))
+  `, { prefix: 'swarmclaw-knowledge-lifecycle-' })
+
+  assert.equal(output.archivedDefaultCount, 0)
+  assert.equal(output.archivedIncludedCount, 1)
+  assert.equal(output.restoredCount, 1)
+  assert.equal(output.restoreActionKind, 'restore')
+  assert.equal(output.supersededDefaultCount, 0)
+  assert.equal(output.supersededIncludedCount, 1)
+  assert.equal(output.supersededFinding, true)
+})
+
+test('runKnowledgeHygieneMaintenance reindexes stale file sources and archives exact duplicates', () => {
+  const output = runWithTempDataDir<{
+    fileLastAutoSyncAt: number | null
+    fileChunksContainUpdatedText: boolean
+    refreshedHitCount: number
+    archivedDuplicateCount: number
+    recentActionKinds: string[]
+  }>(`
+    const fs = await import('node:fs')
+    const path = await import('node:path')
+    const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
+    const storageMod = await import('./src/lib/server/storage.ts')
+    const knowledge = knowledgeMod.default || knowledgeMod
+    const storage = storageMod.default || storageMod
+
+    const filePath = path.join(process.env.WORKSPACE_DIR, 'ops-runbook.txt')
+    fs.writeFileSync(filePath, 'Initial runbook placeholder.')
+
+    const fileSource = await knowledge.createKnowledgeSource({
+      kind: 'file',
+      title: 'Ops Runbook',
+      sourcePath: filePath,
+    })
+
+    fs.writeFileSync(filePath, 'Updated runbook adds rollback choreography and incident checklist.')
+    storage.patchKnowledgeSource(fileSource.source.id, (current) => current ? {
+      ...current,
+      lastIndexedAt: 1,
+      nextSyncAt: 1,
+      updatedAt: 1,
+    } : null)
+
+    const duplicateA = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Duplicate A',
+      content: 'duplicate payload for archival',
+    })
+    const duplicateB = await knowledge.createKnowledgeSource({
+      kind: 'manual',
+      title: 'Duplicate B',
+      content: 'duplicate payload for archival',
+    })
+
+    const summary = await knowledge.runKnowledgeHygieneMaintenance()
+    const refreshed = await knowledge.getKnowledgeSourceDetail(fileSource.source.id)
+    const duplicateADetail = await knowledge.getKnowledgeSourceDetail(duplicateA.source.id)
+    const duplicateBDetail = await knowledge.getKnowledgeSourceDetail(duplicateB.source.id)
+    const refreshedHits = await knowledge.searchKnowledgeHits({ query: 'choreography' })
+
+    const archivedDuplicateCount = [duplicateADetail, duplicateBDetail]
+      .filter((detail) => !!detail?.source?.archivedAt)
+      .length
+
+    console.log(JSON.stringify({
+      fileLastAutoSyncAt: refreshed?.source?.lastAutoSyncAt || null,
+      fileChunksContainUpdatedText: (refreshed?.chunks || []).some((chunk) => chunk.content.includes('rollback choreography')),
+      refreshedHitCount: refreshedHits.length,
+      archivedDuplicateCount,
+      recentActionKinds: summary.recentActions.map((action) => action.kind),
+    }))
+  `, { prefix: 'swarmclaw-knowledge-maintenance-' })
+
+  assert.ok(typeof output.fileLastAutoSyncAt === 'number' && output.fileLastAutoSyncAt > 0)
+  assert.equal(output.fileChunksContainUpdatedText, true)
+  assert.ok(output.refreshedHitCount >= 1)
+  assert.equal(output.archivedDuplicateCount, 1)
+  assert.ok(output.recentActionKinds.includes('archive'))
+  assert.ok(output.recentActionKinds.includes('reindex') || output.recentActionKinds.includes('sync'))
+})