@swarmclawai/swarmclaw 1.3.4 → 1.3.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +15 -76
  2. package/package.json +1 -1
  3. package/skills/swarmclaw.md +17 -0
  4. package/src/app/api/agents/[id]/dream/route.ts +45 -0
  5. package/src/app/api/knowledge/[id]/route.ts +48 -49
  6. package/src/app/api/knowledge/hygiene/route.ts +13 -0
  7. package/src/app/api/knowledge/route.ts +70 -42
  8. package/src/app/api/knowledge/sources/[id]/archive/route.ts +15 -0
  9. package/src/app/api/knowledge/sources/[id]/restore/route.ts +10 -0
  10. package/src/app/api/knowledge/sources/[id]/route.ts +1 -0
  11. package/src/app/api/knowledge/sources/[id]/supersede/route.ts +26 -0
  12. package/src/app/api/knowledge/sources/[id]/sync/route.ts +17 -0
  13. package/src/app/api/knowledge/sources/route.ts +1 -0
  14. package/src/app/api/knowledge/upload/route.ts +3 -51
  15. package/src/app/api/memory/dream/[id]/route.ts +19 -0
  16. package/src/app/api/memory/dream/route.ts +34 -0
  17. package/src/app/knowledge/layout.tsx +1 -1
  18. package/src/app/knowledge/page.tsx +2 -22
  19. package/src/app/protocols/page.tsx +21 -2
  20. package/src/cli/index.js +16 -0
  21. package/src/cli/spec.js +5 -0
  22. package/src/components/agents/agent-sheet.tsx +65 -0
  23. package/src/components/chat/message-bubble.tsx +10 -0
  24. package/src/components/knowledge/grounding-panel.tsx +99 -0
  25. package/src/components/knowledge/knowledge-detail.tsx +402 -0
  26. package/src/components/knowledge/knowledge-list.tsx +351 -126
  27. package/src/components/knowledge/knowledge-sheet.tsx +208 -119
  28. package/src/components/memory/dream-history.tsx +155 -0
  29. package/src/components/memory/memory-card.tsx +7 -0
  30. package/src/components/memory/memory-detail.tsx +46 -0
  31. package/src/components/runs/run-list.tsx +23 -0
  32. package/src/lib/server/api-routes.test.ts +43 -2
  33. package/src/lib/server/chat-execution/chat-execution-grounding.test.ts +127 -0
  34. package/src/lib/server/chat-execution/chat-execution-types.ts +8 -1
  35. package/src/lib/server/chat-execution/chat-execution.ts +1 -0
  36. package/src/lib/server/chat-execution/chat-turn-finalization.ts +21 -1
  37. package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +6 -1
  38. package/src/lib/server/chat-execution/post-stream-finalization.ts +15 -3
  39. package/src/lib/server/chat-execution/prompt-sections.ts +29 -3
  40. package/src/lib/server/chat-execution/stream-agent-chat.ts +6 -1
  41. package/src/lib/server/execution-engine/task-attempt.ts +8 -2
  42. package/src/lib/server/knowledge-import.ts +159 -0
  43. package/src/lib/server/knowledge-sources.test.ts +215 -0
  44. package/src/lib/server/knowledge-sources.ts +1266 -0
  45. package/src/lib/server/memory/dream-cycles.ts +49 -0
  46. package/src/lib/server/memory/dream-idle-callback.ts +38 -0
  47. package/src/lib/server/memory/dream-service.ts +315 -0
  48. package/src/lib/server/memory/memory-db.ts +37 -2
  49. package/src/lib/server/protocols/protocol-agent-turn.ts +7 -0
  50. package/src/lib/server/protocols/protocol-run-lifecycle.ts +19 -6
  51. package/src/lib/server/protocols/protocol-service.test.ts +99 -0
  52. package/src/lib/server/protocols/protocol-step-helpers.ts +7 -1
  53. package/src/lib/server/protocols/protocol-step-processors.ts +16 -3
  54. package/src/lib/server/protocols/protocol-types.ts +4 -0
  55. package/src/lib/server/runtime/daemon-state/core.ts +6 -1
  56. package/src/lib/server/runtime/run-ledger.test.ts +120 -0
  57. package/src/lib/server/runtime/run-ledger.ts +27 -1
  58. package/src/lib/server/runtime/session-run-manager/drain.ts +5 -0
  59. package/src/lib/server/runtime/session-run-manager/state.ts +19 -2
  60. package/src/lib/server/storage-normalization.ts +5 -0
  61. package/src/lib/server/storage.ts +15 -0
  62. package/src/stores/slices/ui-slice.ts +4 -0
  63. package/src/types/agent.ts +7 -0
  64. package/src/types/dream.ts +45 -0
  65. package/src/types/index.ts +1 -0
  66. package/src/types/message.ts +3 -0
  67. package/src/types/misc.ts +131 -0
  68. package/src/types/protocol.ts +4 -0
  69. package/src/types/run.ts +4 -1
@@ -19,6 +19,8 @@ export function MemoryCard({ entry, active, agentName, agentAvatarSeed, agentAva
19
19
  const [now] = useState(() => Date.now())
20
20
  const scope = deriveMemoryScope(entry)
21
21
  const tier = getMemoryTier(entry)
22
+ const isDreamOrigin = entry.category === 'dream_reflection'
23
+ || (entry.metadata as Record<string, unknown> | undefined)?.origin === 'dream'
22
24
 
23
25
  return (
24
26
  <div
@@ -36,6 +38,11 @@ export function MemoryCard({ entry, active, agentName, agentAvatarSeed, agentAva
36
38
  <span className="shrink-0 text-[9px] font-700 uppercase tracking-wider text-accent-bright/70 bg-accent-soft px-1.5 py-0.5 rounded-[5px]">
37
39
  {entry.category || 'note'}
38
40
  </span>
41
+ {isDreamOrigin && (
42
+ <span className="shrink-0 text-[9px] font-700 uppercase tracking-wider text-violet-300/70 bg-violet-400/10 px-1.5 py-0.5 rounded-[5px]">
43
+ dream
44
+ </span>
45
+ )}
39
46
  {entry.pinned && (
40
47
  <svg width="10" height="10" viewBox="0 0 24 24" fill="currentColor" className="shrink-0 text-amber-400/80">
41
48
  <path d="M16 2l-4 4-4-4-2 2 4 4-5 5v1h1l5-5 4 4 2-2-4-4 4-4z" transform="rotate(45 12 12)" />
@@ -175,6 +175,19 @@ export function MemoryDetail() {
175
175
  const inputClass = "w-full px-4 py-3 rounded-[12px] border border-white/[0.06] bg-white/[0.02] text-text outline-none transition-all duration-200 placeholder:text-text-3/70 focus:border-accent-bright/20 focus:bg-white/[0.03]"
176
176
  const refs = entry.references || []
177
177
  const showRefsCollapse = refs.length > 3
178
+ const entryMeta = entry.metadata && typeof entry.metadata === 'object'
179
+ ? entry.metadata as Record<string, unknown>
180
+ : {}
181
+ const knowledgeSourceId = typeof entryMeta.sourceId === 'string' ? entryMeta.sourceId : null
182
+ const knowledgeSourceTitle = typeof entryMeta.sourceTitle === 'string' ? entryMeta.sourceTitle : null
183
+ const knowledgeSourceKind = typeof entryMeta.sourceKind === 'string' ? entryMeta.sourceKind : null
184
+ const knowledgeSourceLabel = typeof entryMeta.sourceLabel === 'string' ? entryMeta.sourceLabel : null
185
+ const knowledgeSourceUrl = typeof entryMeta.sourceUrl === 'string' ? entryMeta.sourceUrl : null
186
+ const knowledgeChunkIndex = typeof entryMeta.chunkIndex === 'number' ? entryMeta.chunkIndex : null
187
+ const knowledgeChunkCount = typeof entryMeta.chunkCount === 'number' ? entryMeta.chunkCount : null
188
+ const knowledgeSectionLabel = typeof entryMeta.sectionLabel === 'string' ? entryMeta.sectionLabel : null
189
+ const knowledgeCharStart = typeof entryMeta.charStart === 'number' ? entryMeta.charStart : null
190
+ const knowledgeCharEnd = typeof entryMeta.charEnd === 'number' ? entryMeta.charEnd : null
178
191
 
179
192
  return (
180
193
  <div className="flex-1 flex flex-col h-full min-h-0">
@@ -438,6 +451,33 @@ export function MemoryDetail() {
438
451
  {entry.content || '(empty)'}
439
452
  </div>
440
453
 
454
+ {knowledgeSourceId && (
455
+ <div className="rounded-[14px] border border-white/[0.06] bg-white/[0.02] px-4 py-3">
456
+ <label className="block text-[11px] font-600 text-text-3/60 uppercase tracking-[0.06em] mb-2">Source</label>
457
+ <div className="space-y-1.5">
458
+ <p className="text-[13px] text-text-2">
459
+ {knowledgeSourceTitle || entry.title}
460
+ {knowledgeSourceKind ? ` • ${knowledgeSourceKind}` : ''}
461
+ </p>
462
+ {knowledgeSourceLabel && (
463
+ <p className="text-[12px] text-text-3/65">{knowledgeSourceLabel}</p>
464
+ )}
465
+ {knowledgeSourceUrl && (
466
+ <a href={knowledgeSourceUrl} target="_blank" rel="noreferrer" className="text-[12px] text-accent-bright hover:underline break-all">
467
+ {knowledgeSourceUrl}
468
+ </a>
469
+ )}
470
+ <p className="text-[11px] text-text-3/55">
471
+ {knowledgeChunkIndex != null && knowledgeChunkCount != null
472
+ ? `Chunk ${knowledgeChunkIndex + 1} of ${knowledgeChunkCount}`
473
+ : 'Source-backed knowledge'}
474
+ {knowledgeSectionLabel ? ` • ${knowledgeSectionLabel}` : ''}
475
+ {knowledgeCharStart != null && knowledgeCharEnd != null ? ` • chars ${knowledgeCharStart}-${knowledgeCharEnd}` : ''}
476
+ </p>
477
+ </div>
478
+ </div>
479
+ )}
480
+
441
481
  {/* Shared with (read mode) */}
442
482
  {entry.sharedWith && entry.sharedWith.length > 0 && (
443
483
  <div>
@@ -575,6 +615,12 @@ export function MemoryDetail() {
575
615
  <span className="text-text-3/70 block mb-1">Tier</span>
576
616
  <span className="text-text-3/60 font-mono">{tier}</span>
577
617
  </div>
618
+ {knowledgeSourceId && (
619
+ <div>
620
+ <span className="text-text-3/70 block mb-1">Knowledge Source</span>
621
+ <span className="text-text-3/60 font-mono">{knowledgeSourceId}</span>
622
+ </div>
623
+ )}
578
624
  {entry.sessionId && (
579
625
  <div>
580
626
  <span className="text-text-3/70 block mb-1">Chat</span>
@@ -8,6 +8,7 @@ import { BottomSheet } from '@/components/shared/bottom-sheet'
8
8
  import type { RunEventRecord, SessionRunRecord, SessionRunStatus } from '@/types'
9
9
  import { PageLoader } from '@/components/ui/page-loader'
10
10
  import { formatElapsed } from '@/lib/format-display'
11
+ import { GroundingPanel } from '@/components/knowledge/grounding-panel'
11
12
 
12
13
  const STATUS_COLORS: Record<SessionRunStatus, { bg: string; text: string }> = {
13
14
  queued: { bg: 'bg-yellow-500/10', text: 'text-yellow-400' },
@@ -84,6 +85,10 @@ export function RunList() {
84
85
  }, [])
85
86
 
86
87
  const filtered = statusFilter ? runs.filter((r) => r.status === statusFilter) : runs
88
+ const selectedResultGrounding = selectedEvents
89
+ .slice()
90
+ .reverse()
91
+ .find((event) => event.phase === 'status' && ((event.citations?.length || 0) > 0 || event.retrievalTrace?.hits?.length))
87
92
 
88
93
  if (loading) {
89
94
  return <PageLoader label="Loading runs..." />
@@ -240,6 +245,15 @@ export function RunList() {
240
245
  <pre className="text-[11px] text-text-3/80 font-mono whitespace-pre-wrap break-all bg-white/[0.02] rounded-[12px] p-4 max-h-[200px] overflow-auto border border-white/[0.04]">
241
246
  {selected.resultPreview}
242
247
  </pre>
248
+ {selectedResultGrounding && (
249
+ <div className="mt-3">
250
+ <GroundingPanel
251
+ citations={selectedResultGrounding.citations}
252
+ retrievalTrace={selectedResultGrounding.retrievalTrace}
253
+ compact
254
+ />
255
+ </div>
256
+ )}
243
257
  </div>
244
258
  )}
245
259
 
@@ -262,6 +276,15 @@ export function RunList() {
262
276
  <div className="text-[11px] text-text-2 whitespace-pre-wrap break-words">
263
277
  {event.summary || event.event.text || event.event.toolOutput || event.event.toolName || event.event.t}
264
278
  </div>
279
+ {(event.citations?.length || event.retrievalTrace?.hits?.length) ? (
280
+ <div className="mt-2">
281
+ <GroundingPanel
282
+ citations={event.citations}
283
+ retrievalTrace={event.retrievalTrace}
284
+ compact
285
+ />
286
+ </div>
287
+ ) : null}
265
288
  </div>
266
289
  ))}
267
290
  </div>
@@ -78,9 +78,16 @@ function parseKnowledgeQueryParams(url: string) {
78
78
  const q = searchParams.get('q')
79
79
  const tagsParam = searchParams.get('tags')
80
80
  const limitParam = searchParams.get('limit')
81
+ const includeArchivedParam = searchParams.get('includeArchived')
81
82
  const tags = tagsParam ? tagsParam.split(',').map((t) => t.trim()).filter(Boolean) : undefined
82
83
  const limit = limitParam ? Math.max(1, Math.min(500, Number.parseInt(limitParam, 10) || 50)) : undefined
83
- return { q, tags, limit }
84
+ const normalizedIncludeArchived = typeof includeArchivedParam === 'string'
85
+ ? includeArchivedParam.trim().toLowerCase()
86
+ : ''
87
+ const includeArchived = normalizedIncludeArchived === '1'
88
+ || normalizedIncludeArchived === 'true'
89
+ || normalizedIncludeArchived === 'yes'
90
+ return { q, tags, limit, includeArchived }
84
91
  }
85
92
 
86
93
  // ---------------------------------------------------------------------------
@@ -91,7 +98,12 @@ const thisFile = new URL(import.meta.url).pathname
91
98
  const routeDir = path.resolve(path.dirname(thisFile), '../../app/api')
92
99
 
93
100
  function readRoute(...segments: string[]): string {
94
- return fs.readFileSync(path.join(routeDir, ...segments), 'utf-8')
101
+ const direct = path.join(routeDir, ...segments)
102
+ if (fs.existsSync(direct)) {
103
+ return fs.readFileSync(direct, 'utf-8')
104
+ }
105
+ const withTs = direct.endsWith('.ts') ? direct : `${direct}.ts`
106
+ return fs.readFileSync(withTs, 'utf-8')
95
107
  }
96
108
 
97
109
  // ===========================================================================
@@ -225,6 +237,12 @@ describe('Knowledge API contract', () => {
225
237
  it('returns undefined limit when param is absent', () => {
226
238
  assert.equal(parseKnowledgeQueryParams('http://localhost/api/knowledge').limit, undefined)
227
239
  })
240
+
241
+ it('parses includeArchived as a boolean flag', () => {
242
+ assert.equal(parseKnowledgeQueryParams('http://localhost/api/knowledge?includeArchived=true').includeArchived, true)
243
+ assert.equal(parseKnowledgeQueryParams('http://localhost/api/knowledge?includeArchived=1').includeArchived, true)
244
+ assert.equal(parseKnowledgeQueryParams('http://localhost/api/knowledge?includeArchived=no').includeArchived, false)
245
+ })
228
246
  })
229
247
 
230
248
  // --- Route file structure -----------------------------------------------
@@ -241,6 +259,29 @@ describe('Knowledge API contract', () => {
241
259
  assert.match(src, /export\s+async\s+function\s+PUT/)
242
260
  assert.match(src, /export\s+async\s+function\s+DELETE/)
243
261
  })
262
+
263
+ it('knowledge/hygiene/route.ts exports GET and POST', () => {
264
+ const src = readRoute('knowledge', 'hygiene', 'route.ts')
265
+ assert.match(src, /export\s+async\s+function\s+GET/)
266
+ assert.match(src, /export\s+async\s+function\s+POST/)
267
+ })
268
+
269
+ it('knowledge/sources/route.ts re-exports GET and POST', () => {
270
+ const src = readRoute('knowledge', 'sources', 'route.ts')
271
+ assert.match(src, /export\s+\{\s*GET,\s*POST\s*\}/)
272
+ })
273
+
274
+ it('knowledge/sources/[id]/route.ts re-exports GET, PUT, DELETE', () => {
275
+ const src = readRoute('knowledge', 'sources', '[id]', 'route.ts')
276
+ assert.match(src, /export\s+\{\s*GET,\s*PUT,\s*DELETE\s*\}/)
277
+ })
278
+
279
+ it('knowledge source action routes export POST', () => {
280
+ for (const route of ['archive', 'restore', 'supersede', 'sync']) {
281
+ const src = readRoute('knowledge', 'sources', '[id]', route, 'route.ts')
282
+ assert.match(src, /export\s+async\s+function\s+POST/)
283
+ }
284
+ })
244
285
  })
245
286
  })
246
287
 
@@ -0,0 +1,127 @@
1
+ import assert from 'node:assert/strict'
2
+ import fs from 'node:fs'
3
+ import os from 'node:os'
4
+ import path from 'node:path'
5
+ import { spawnSync } from 'node:child_process'
6
+ import test from 'node:test'
7
+
8
+ const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../../..')
9
+
10
+ function runWithTempDataDir(script: string) {
11
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'swarmclaw-chat-grounding-'))
12
+ try {
13
+ const result = spawnSync(process.execPath, ['--import', 'tsx', '--input-type=module', '--eval', script], {
14
+ cwd: repoRoot,
15
+ env: {
16
+ ...process.env,
17
+ DATA_DIR: path.join(tempDir, 'data'),
18
+ WORKSPACE_DIR: path.join(tempDir, 'workspace'),
19
+ BROWSER_PROFILES_DIR: path.join(tempDir, 'browser-profiles'),
20
+ },
21
+ encoding: 'utf-8',
22
+ })
23
+ assert.equal(result.status, 0, result.stderr || result.stdout || 'subprocess failed')
24
+ const lines = (result.stdout || '')
25
+ .trim()
26
+ .split('\n')
27
+ .map((line) => line.trim())
28
+ .filter(Boolean)
29
+ const jsonLine = [...lines].reverse().find((line) => line.startsWith('{'))
30
+ return JSON.parse(jsonLine || '{}')
31
+ } finally {
32
+ fs.rmSync(tempDir, { recursive: true, force: true })
33
+ }
34
+ }
35
+
36
+ test('executeSessionChatTurn persists citations and retrieval traces on grounded assistant messages', () => {
37
+ const output = runWithTempDataDir(`
38
+ const storageMod = await import('@/lib/server/storage')
39
+ const providersMod = await import('@/lib/providers/index')
40
+ const threadMod = await import('@/lib/server/agents/agent-thread-session')
41
+ const execMod = await import('@/lib/server/chat-execution/chat-execution')
42
+ const messageRepoMod = await import('@/lib/server/messages/message-repository')
43
+ const knowledgeMod = await import('@/lib/server/knowledge-sources')
44
+
45
+ const storage = storageMod.default || storageMod['module.exports'] || storageMod
46
+ const ensureAgentThreadSession = threadMod.ensureAgentThreadSession
47
+ || threadMod.default?.ensureAgentThreadSession
48
+ || threadMod['module.exports']?.ensureAgentThreadSession
49
+ const executeSessionChatTurn = execMod.executeSessionChatTurn
50
+ || execMod.default?.executeSessionChatTurn
51
+ || execMod['module.exports']?.executeSessionChatTurn
52
+ const getMessages = messageRepoMod.getMessages
53
+ || messageRepoMod.default?.getMessages
54
+ || messageRepoMod['module.exports']?.getMessages
55
+ const knowledge = knowledgeMod.default || knowledgeMod
56
+ const providers = providersMod.PROVIDERS
57
+ || providersMod.default?.PROVIDERS
58
+ || providersMod['module.exports']?.PROVIDERS
59
+
60
+ providers['test-provider'] = {
61
+ id: 'test-provider',
62
+ name: 'Test Provider',
63
+ models: ['unit'],
64
+ requiresApiKey: false,
65
+ requiresEndpoint: false,
66
+ handler: {
67
+ async streamChat() {
68
+ return 'Use blue green deployment for the gateway migration so rollback stays simple.'
69
+ },
70
+ },
71
+ }
72
+
73
+ const now = Date.now()
74
+ storage.saveAgents({
75
+ molly: {
76
+ id: 'molly',
77
+ name: 'Molly',
78
+ description: 'Grounding test',
79
+ provider: 'test-provider',
80
+ model: 'unit',
81
+ credentialId: null,
82
+ apiEndpoint: null,
83
+ fallbackCredentialIds: [],
84
+ disabled: false,
85
+ proactiveMemory: true,
86
+ extensions: ['memory'],
87
+ createdAt: now,
88
+ updatedAt: now,
89
+ },
90
+ })
91
+
92
+ await knowledge.createKnowledgeSource({
93
+ kind: 'manual',
94
+ title: 'Gateway Migration Runbook',
95
+ content: 'Use blue green deployment for gateway migrations so rollback stays simple and downtime stays low.',
96
+ tags: ['deploy'],
97
+ })
98
+
99
+ const session = ensureAgentThreadSession('molly')
100
+ const result = await executeSessionChatTurn({
101
+ sessionId: session.id,
102
+ message: 'gateway blue green rollback',
103
+ runId: 'run-grounding-chat',
104
+ })
105
+
106
+ const messages = getMessages(session.id)
107
+ const lastMessage = messages[messages.length - 1]
108
+
109
+ console.log(JSON.stringify({
110
+ persisted: result.persisted || false,
111
+ resultCitationCount: Array.isArray(result.citations) ? result.citations.length : 0,
112
+ resultSelectorStatus: result.retrievalTrace?.selectorStatus || null,
113
+ messageCitationCount: Array.isArray(lastMessage?.citations) ? lastMessage.citations.length : 0,
114
+ messageTraceHitCount: Array.isArray(lastMessage?.retrievalTrace?.hits) ? lastMessage.retrievalTrace.hits.length : 0,
115
+ messageSelectorStatus: lastMessage?.retrievalTrace?.selectorStatus || null,
116
+ messageSourceTitle: lastMessage?.citations?.[0]?.sourceTitle || null,
117
+ }))
118
+ `)
119
+
120
+ assert.equal(output.persisted, true)
121
+ assert.equal(output.resultCitationCount >= 1, true)
122
+ assert.equal(output.resultSelectorStatus, 'selected')
123
+ assert.equal(output.messageCitationCount >= 1, true)
124
+ assert.equal(output.messageTraceHitCount >= 1, true)
125
+ assert.equal(output.messageSelectorStatus, 'selected')
126
+ assert.equal(output.messageSourceTitle, 'Gateway Migration Runbook')
127
+ })
@@ -1,4 +1,9 @@
1
- import type { MessageToolEvent, SSEEvent } from '@/types'
1
+ import type {
2
+ KnowledgeCitation,
3
+ KnowledgeRetrievalTrace,
4
+ MessageToolEvent,
5
+ SSEEvent,
6
+ } from '@/types'
2
7
 
3
8
  export interface ExecuteChatTurnInput {
4
9
  sessionId: string
@@ -33,4 +38,6 @@ export interface ExecuteChatTurnResult {
33
38
  inputTokens?: number
34
39
  outputTokens?: number
35
40
  estimatedCost?: number
41
+ citations?: KnowledgeCitation[]
42
+ retrievalTrace?: KnowledgeRetrievalTrace | null
36
43
  }
@@ -112,6 +112,7 @@ export async function executeSessionChatTurn(input: ExecuteChatTurnInput): Promi
112
112
  responseCacheHit: streamResult.responseCacheHit,
113
113
  directUsage: streamResult.directUsage,
114
114
  durationMs: streamResult.durationMs,
115
+ knowledgeRetrievalTrace: streamResult.knowledgeRetrievalTrace || null,
115
116
  emit: partialPersistence.emit,
116
117
  })
117
118
 
@@ -1,4 +1,11 @@
1
- import type { Message, MessageToolEvent, SSEEvent, Session, UsageRecord } from '@/types'
1
+ import type {
2
+ KnowledgeRetrievalTrace,
3
+ Message,
4
+ MessageToolEvent,
5
+ SSEEvent,
6
+ Session,
7
+ UsageRecord,
8
+ } from '@/types'
2
9
  import { sendConnectorMessage } from '../connectors/manager'
3
10
  import { applyExactOutputContract, classifyExactOutputContract, type ExactOutputContract } from '@/lib/server/chat-execution/exact-output-contract'
4
11
  import { stripMainLoopMetaForPersistence } from '@/lib/server/agents/main-agent-loop'
@@ -43,6 +50,7 @@ import {
43
50
  import { appendUsage } from '@/lib/server/usage/usage-repository'
44
51
  import { synchronizeWorkingStateForTurn } from '@/lib/server/working-state/service'
45
52
  import { notify } from '@/lib/server/ws-hub'
53
+ import { selectKnowledgeCitations } from '@/lib/server/knowledge-sources'
46
54
 
47
55
  import type { ExecuteChatTurnInput, ExecuteChatTurnResult } from './chat-execution-types'
48
56
  import type { PartialAssistantPersistence } from '@/lib/server/chat-execution/chat-turn-partial-persistence'
@@ -154,6 +162,7 @@ export async function finalizeChatTurn(params: {
154
162
  received: boolean
155
163
  }
156
164
  durationMs: number
165
+ knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
157
166
  emit: (event: SSEEvent) => void
158
167
  }): Promise<ExecuteChatTurnResult> {
159
168
  const {
@@ -164,6 +173,7 @@ export async function finalizeChatTurn(params: {
164
173
  responseCacheHit,
165
174
  directUsage,
166
175
  durationMs,
176
+ knowledgeRetrievalTrace,
167
177
  emit,
168
178
  } = params
169
179
  let { fullResponse, errorMessage } = params
@@ -313,6 +323,12 @@ export async function finalizeChatTurn(params: {
313
323
  const hiddenControlOnly = shouldSuppressHiddenControlText(rawTextForPersistence)
314
324
  const textForPersistence = stripHiddenControlTokens(rawTextForPersistence)
315
325
  const persistedText = getPersistedAssistantText(textForPersistence, persistedToolEvents)
326
+ const grounding = hiddenControlOnly
327
+ ? { citations: [], retrievalTrace: knowledgeRetrievalTrace || null }
328
+ : selectKnowledgeCitations({
329
+ responseText: persistedText,
330
+ retrievalTrace: knowledgeRetrievalTrace || null,
331
+ })
316
332
  let persistedResponseForHooks = textForPersistence
317
333
 
318
334
  if (isHeartbeatRun && rawTextForPersistence) {
@@ -413,6 +429,8 @@ export async function finalizeChatTurn(params: {
413
429
  thinking: thinkingText || undefined,
414
430
  toolEvents: persistedToolEvents.length ? persistedToolEvents : undefined,
415
431
  kind: persistedKind,
432
+ citations: grounding.citations.length > 0 ? grounding.citations : undefined,
433
+ retrievalTrace: grounding.retrievalTrace || undefined,
416
434
  },
417
435
  enabledIds: extensionsForRun,
418
436
  phase: isHeartbeatRun ? 'heartbeat' : 'assistant_final',
@@ -643,5 +661,7 @@ export async function finalizeChatTurn(params: {
643
661
  inputTokens: accumulatedUsage.inputTokens || undefined,
644
662
  outputTokens: accumulatedUsage.outputTokens || undefined,
645
663
  estimatedCost: accumulatedUsage.estimatedCost || undefined,
664
+ citations: grounding.citations.length > 0 ? grounding.citations : undefined,
665
+ retrievalTrace: grounding.retrievalTrace || undefined,
646
666
  }
647
667
  }
@@ -1,5 +1,5 @@
1
1
  import { CONTEXT_OVERFLOW_RE } from '@/lib/providers/error-classification'
2
- import type { ProviderType } from '@/types'
2
+ import type { KnowledgeRetrievalTrace, ProviderType } from '@/types'
3
3
  import { getEnabledCapabilityIds } from '@/lib/capability-selection'
4
4
  import { isLocalOpenClawEndpoint } from '@/lib/openclaw/openclaw-endpoint'
5
5
  import { streamAgentChat } from '@/lib/server/chat-execution/stream-agent-chat'
@@ -42,6 +42,7 @@ export interface ExecutedPreparedChatTurn {
42
42
  outputTokens: number
43
43
  received: boolean
44
44
  }
45
+ knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
45
46
  }
46
47
 
47
48
  export async function executePreparedChatTurn(params: {
@@ -90,6 +91,7 @@ export async function executePreparedChatTurn(params: {
90
91
  let responseCacheHit = false
91
92
  let responseCacheInput: LlmResponseCacheKeyInput | null = null
92
93
  let durationMs = 0
94
+ let knowledgeRetrievalTrace: KnowledgeRetrievalTrace | null = null
93
95
  const startTs = Date.now()
94
96
  const endLlmPerf = perf.start('chat-execution', 'llm-round-trip', {
95
97
  sessionId,
@@ -111,6 +113,7 @@ export async function executePreparedChatTurn(params: {
111
113
  responseCacheHit,
112
114
  durationMs,
113
115
  directUsage,
116
+ knowledgeRetrievalTrace: null,
114
117
  }
115
118
  }
116
119
 
@@ -157,6 +160,7 @@ export async function executePreparedChatTurn(params: {
157
160
  promptMode,
158
161
  })
159
162
  fullResponse = result.finalResponse || result.fullText
163
+ knowledgeRetrievalTrace = result.knowledgeRetrievalTrace || null
160
164
  } else {
161
165
  let directHistorySnapshot = isAutoRunNoHistory
162
166
  ? (heartbeatLightContext ? [] : getSessionMessages(sessionId).slice(-6))
@@ -298,5 +302,6 @@ export async function executePreparedChatTurn(params: {
298
302
  responseCacheHit,
299
303
  durationMs,
300
304
  directUsage,
305
+ knowledgeRetrievalTrace,
301
306
  }
302
307
  }
@@ -5,7 +5,7 @@
5
5
  * usage recording, forced external service summary, capability hooks,
6
6
  * and OpenClaw sync.
7
7
  */
8
- import type { Session, UsageRecord } from '@/types'
8
+ import type { KnowledgeRetrievalTrace, Session, UsageRecord } from '@/types'
9
9
  import { log } from '@/lib/server/logger'
10
10
  import type { ChatTurnState } from '@/lib/server/chat-execution/chat-turn-state'
11
11
 
@@ -51,6 +51,7 @@ export interface PostStreamResult {
51
51
  fullText: string
52
52
  finalResponse: string
53
53
  toolEvents: import('@/types').MessageToolEvent[]
54
+ knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
54
55
  }
55
56
 
56
57
  export interface FinalizeStreamResultOpts {
@@ -70,6 +71,7 @@ export interface FinalizeStreamResultOpts {
70
71
  cleanup: () => Promise<void>
71
72
  runId: string
72
73
  classification?: MessageClassification | null
74
+ knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
73
75
  }
74
76
 
75
77
  export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Promise<PostStreamResult> {
@@ -138,7 +140,12 @@ export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Prom
138
140
  const finalResponse = await resolveAndSummarize()
139
141
  await emitLlmOutputHook(finalResponse)
140
142
  await cleanup()
141
- return { fullText: state.fullText, finalResponse, toolEvents: state.streamedToolEvents }
143
+ return {
144
+ fullText: state.fullText,
145
+ finalResponse,
146
+ toolEvents: state.streamedToolEvents,
147
+ knowledgeRetrievalTrace: opts.knowledgeRetrievalTrace || null,
148
+ }
142
149
  }
143
150
 
144
151
  // Strip leaked classification JSON from model output (e.g. `{ "isDeliverableTask": true, ... }`)
@@ -212,5 +219,10 @@ export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Prom
212
219
 
213
220
  await cleanup()
214
221
 
215
- return { fullText: state.fullText, finalResponse, toolEvents: state.streamedToolEvents }
222
+ return {
223
+ fullText: state.fullText,
224
+ finalResponse,
225
+ toolEvents: state.streamedToolEvents,
226
+ knowledgeRetrievalTrace: opts.knowledgeRetrievalTrace || null,
227
+ }
216
228
  }
@@ -8,7 +8,7 @@
8
8
 
9
9
  import fs from 'node:fs'
10
10
  import path from 'node:path'
11
- import type { Session, Agent } from '@/types'
11
+ import type { KnowledgeRetrievalTrace, Session, Agent } from '@/types'
12
12
  import type { PromptMode } from '@/lib/server/chat-execution/prompt-mode'
13
13
  import type { MessageClassification } from '@/lib/server/chat-execution/message-classifier'
14
14
  import type { ActiveProjectContext } from '@/lib/server/project-context'
@@ -428,6 +428,7 @@ export function buildSuggestionsSection(
428
428
  export interface ProactiveMemoryResult {
429
429
  section: string | null
430
430
  injectedIds: Record<string, number>
431
+ knowledgeTrace?: KnowledgeRetrievalTrace | null
431
432
  }
432
433
 
433
434
  export async function buildProactiveMemorySection(
@@ -438,22 +439,28 @@ export async function buildProactiveMemorySection(
438
439
  isMinimalPrompt: boolean,
439
440
  currentThreadRecallRequest: boolean,
440
441
  ): Promise<ProactiveMemoryResult> {
441
- const noResult: ProactiveMemoryResult = { section: null, injectedIds: {} }
442
+ const noResult: ProactiveMemoryResult = { section: null, injectedIds: {}, knowledgeTrace: null }
442
443
  if (isMinimalPrompt || !session.agentId || currentThreadRecallRequest || message.length <= 12) return noResult
443
444
  if (!agent?.proactiveMemory) return noResult
444
445
  try {
445
446
  const { getMemoryDb } = await import('@/lib/server/memory/memory-db')
446
447
  const { buildSessionMemoryScopeFilter } = await import('@/lib/server/memory/session-memory-scope')
448
+ const { buildKnowledgeRetrievalTrace } = await import('@/lib/server/knowledge-sources')
447
449
  const memDb = getMemoryDb()
448
450
  const recalled = memDb.search(message, session.agentId, {
449
451
  scope: buildSessionMemoryScopeFilter(session, agent.memoryScopeMode || null, activeProjectRoot),
450
452
  })
453
+ const knowledgeTrace = await buildKnowledgeRetrievalTrace({
454
+ query: message,
455
+ viewerAgentId: session.agentId,
456
+ })
451
457
 
452
458
  // Dedup: skip memories already injected 2+ times in this session
453
459
  const priorCounts = session.injectedMemoryIds || {}
454
460
  const filtered = recalled.filter((entry) => (priorCounts[entry.id] || 0) < 2)
455
461
 
456
462
  const topRecalled = filtered.slice(0, 3)
463
+ const sections: string[] = []
457
464
  if (topRecalled.length > 0) {
458
465
  // Track injection counts
459
466
  const updatedCounts: Record<string, number> = { ...priorCounts }
@@ -464,9 +471,28 @@ export async function buildProactiveMemorySection(
464
471
  const recalledLines = topRecalled.map((entry) =>
465
472
  `- ${entry.abstract || entry.content.slice(0, 300)}`,
466
473
  )
474
+ sections.push(`## Recalled Context\nRelevant memories from previous interactions:\n${recalledLines.join('\n')}`)
475
+ if (knowledgeTrace?.hits.length) {
476
+ const groundingLines = knowledgeTrace.hits.map((hit) =>
477
+ `- [${hit.chunkIndex + 1}/${hit.chunkCount}] ${hit.sourceTitle}: ${hit.snippet}`,
478
+ )
479
+ sections.push(`## Source Grounding\nSource-backed knowledge retrieved for this turn:\n${groundingLines.join('\n')}`)
480
+ }
467
481
  return {
468
- section: `## Recalled Context\nRelevant memories from previous interactions:\n${recalledLines.join('\n')}`,
482
+ section: sections.join('\n\n'),
469
483
  injectedIds: updatedCounts,
484
+ knowledgeTrace,
485
+ }
486
+ }
487
+
488
+ if (knowledgeTrace?.hits.length) {
489
+ const groundingLines = knowledgeTrace.hits.map((hit) =>
490
+ `- [${hit.chunkIndex + 1}/${hit.chunkCount}] ${hit.sourceTitle}: ${hit.snippet}`,
491
+ )
492
+ return {
493
+ section: `## Source Grounding\nSource-backed knowledge retrieved for this turn:\n${groundingLines.join('\n')}`,
494
+ injectedIds: priorCounts,
495
+ knowledgeTrace,
470
496
  }
471
497
  }
472
498
  } catch { /* non-critical */ }
@@ -36,7 +36,7 @@ import { log } from '@/lib/server/logger'
36
36
  import { logExecution } from '@/lib/server/execution-log'
37
37
  import { buildCurrentDateTimePromptContext } from '@/lib/server/prompt-runtime-context'
38
38
  import { expandExtensionIds } from '@/lib/server/tool-aliases'
39
- import type { ExecutionBrief, Session, Message } from '@/types'
39
+ import type { ExecutionBrief, KnowledgeRetrievalTrace, Session, Message } from '@/types'
40
40
  import { getEnabledCapabilityIds } from '@/lib/capability-selection'
41
41
  import { enqueueSystemEvent } from '@/lib/server/runtime/system-events'
42
42
  import { resolveActiveProjectContext } from '@/lib/server/project-context'
@@ -199,6 +199,7 @@ export interface StreamAgentChatResult {
199
199
  finalResponse: string
200
200
  /** Tool events emitted during the streamed run. */
201
201
  toolEvents: import('@/types').MessageToolEvent[]
202
+ knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
202
203
  }
203
204
 
204
205
  type LangChainContentPart =
@@ -267,6 +268,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
267
268
  preferMinimalPrompt: lightweightDirectChat,
268
269
  })
269
270
  const isMinimalPrompt = promptMode === 'minimal'
271
+ let knowledgeRetrievalTrace: KnowledgeRetrievalTrace | null = null
270
272
 
271
273
  // Resolve agent's thinking level for provider-native params
272
274
  let agentThinkingLevel: 'minimal' | 'low' | 'medium' | 'high' | undefined
@@ -309,6 +311,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
309
311
  fullText: requestedToolPreflightResponse,
310
312
  finalResponse: requestedToolPreflightResponse,
311
313
  toolEvents: [],
314
+ knowledgeRetrievalTrace: null,
312
315
  }
313
316
  }
314
317
  const runtime = loadRuntimeSettings()
@@ -490,6 +493,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
490
493
  isMinimalPrompt, currentThreadRecallRequest,
491
494
  )
492
495
  if (memoryResult.section) promptParts.push(memoryResult.section)
496
+ knowledgeRetrievalTrace = memoryResult.knowledgeTrace || null
493
497
  // Persist injection dedup counts so repeated memories are suppressed
494
498
  if (Object.keys(memoryResult.injectedIds).length > 0) {
495
499
  session.injectedMemoryIds = memoryResult.injectedIds
@@ -1269,5 +1273,6 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
1269
1273
  cleanup,
1270
1274
  runId,
1271
1275
  classification,
1276
+ knowledgeRetrievalTrace,
1272
1277
  })
1273
1278
  }