@swarmclawai/swarmclaw 1.3.4 → 1.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +20 -76
  2. package/package.json +3 -2
  3. package/skills/swarmclaw.md +17 -0
  4. package/src/app/api/agents/[id]/dream/route.ts +45 -0
  5. package/src/app/api/knowledge/[id]/route.ts +48 -49
  6. package/src/app/api/knowledge/hygiene/route.ts +13 -0
  7. package/src/app/api/knowledge/route.ts +70 -42
  8. package/src/app/api/knowledge/sources/[id]/archive/route.ts +15 -0
  9. package/src/app/api/knowledge/sources/[id]/restore/route.ts +10 -0
  10. package/src/app/api/knowledge/sources/[id]/route.ts +1 -0
  11. package/src/app/api/knowledge/sources/[id]/supersede/route.ts +26 -0
  12. package/src/app/api/knowledge/sources/[id]/sync/route.ts +17 -0
  13. package/src/app/api/knowledge/sources/route.ts +1 -0
  14. package/src/app/api/knowledge/upload/route.ts +3 -51
  15. package/src/app/api/memory/dream/[id]/route.ts +19 -0
  16. package/src/app/api/memory/dream/route.ts +34 -0
  17. package/src/app/knowledge/layout.tsx +1 -1
  18. package/src/app/knowledge/page.tsx +2 -22
  19. package/src/app/protocols/page.tsx +21 -2
  20. package/src/cli/index.js +16 -0
  21. package/src/cli/spec.js +5 -0
  22. package/src/components/agents/agent-sheet.tsx +65 -0
  23. package/src/components/chat/message-bubble.tsx +10 -0
  24. package/src/components/knowledge/grounding-panel.tsx +99 -0
  25. package/src/components/knowledge/knowledge-detail.tsx +402 -0
  26. package/src/components/knowledge/knowledge-list.tsx +351 -126
  27. package/src/components/knowledge/knowledge-sheet.tsx +208 -119
  28. package/src/components/memory/dream-history.tsx +155 -0
  29. package/src/components/memory/memory-card.tsx +7 -0
  30. package/src/components/memory/memory-detail.tsx +46 -0
  31. package/src/components/runs/run-list.tsx +23 -0
  32. package/src/lib/server/api-routes.test.ts +43 -2
  33. package/src/lib/server/chat-execution/chat-execution-disabled.test.ts +14 -31
  34. package/src/lib/server/chat-execution/chat-execution-eval-history.test.ts +11 -34
  35. package/src/lib/server/chat-execution/chat-execution-grounding.test.ts +108 -0
  36. package/src/lib/server/chat-execution/chat-execution-session-sync.test.ts +35 -36
  37. package/src/lib/server/chat-execution/chat-execution-types.ts +8 -1
  38. package/src/lib/server/chat-execution/chat-execution.ts +1 -0
  39. package/src/lib/server/chat-execution/chat-turn-finalization.ts +21 -1
  40. package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +6 -1
  41. package/src/lib/server/chat-execution/post-stream-finalization.ts +15 -3
  42. package/src/lib/server/chat-execution/prompt-sections.ts +29 -3
  43. package/src/lib/server/chat-execution/stream-agent-chat.ts +6 -1
  44. package/src/lib/server/execution-engine/task-attempt.ts +8 -2
  45. package/src/lib/server/knowledge-import.ts +159 -0
  46. package/src/lib/server/knowledge-sources.test.ts +261 -0
  47. package/src/lib/server/knowledge-sources.ts +1284 -0
  48. package/src/lib/server/memory/dream-cycles.ts +49 -0
  49. package/src/lib/server/memory/dream-idle-callback.ts +38 -0
  50. package/src/lib/server/memory/dream-service.ts +315 -0
  51. package/src/lib/server/memory/memory-db.ts +37 -2
  52. package/src/lib/server/protocols/protocol-agent-turn.ts +7 -0
  53. package/src/lib/server/protocols/protocol-run-lifecycle.ts +19 -6
  54. package/src/lib/server/protocols/protocol-service.test.ts +99 -0
  55. package/src/lib/server/protocols/protocol-step-helpers.ts +7 -1
  56. package/src/lib/server/protocols/protocol-step-processors.ts +16 -3
  57. package/src/lib/server/protocols/protocol-types.ts +4 -0
  58. package/src/lib/server/runtime/daemon-state/core.ts +6 -1
  59. package/src/lib/server/runtime/run-ledger.test.ts +120 -0
  60. package/src/lib/server/runtime/run-ledger.ts +27 -1
  61. package/src/lib/server/runtime/session-run-manager/drain.ts +5 -0
  62. package/src/lib/server/runtime/session-run-manager/state.ts +19 -2
  63. package/src/lib/server/storage-normalization.ts +5 -0
  64. package/src/lib/server/storage.ts +15 -0
  65. package/src/lib/server/test-utils/run-with-temp-data-dir.ts +15 -2
  66. package/src/stores/slices/ui-slice.ts +4 -0
  67. package/src/types/agent.ts +7 -0
  68. package/src/types/dream.ts +45 -0
  69. package/src/types/index.ts +1 -0
  70. package/src/types/message.ts +3 -0
  71. package/src/types/misc.ts +131 -0
  72. package/src/types/protocol.ts +4 -0
  73. package/src/types/run.ts +4 -1
@@ -0,0 +1,49 @@
1
+ import fs from 'fs'
2
+ import path from 'path'
3
+ import { DATA_DIR } from '@/lib/server/data-dir'
4
+ import { hmrSingleton, safeJsonParse } from '@/lib/shared-utils'
5
+ import type { DreamCycle } from '@/types'
6
+
7
+ const DREAM_CYCLES_PATH = path.join(DATA_DIR, 'dream-cycles.json')
8
+
9
+ const state = hmrSingleton('__dream_cycles__', () => ({
10
+ cycles: null as DreamCycle[] | null,
11
+ }))
12
+
13
+ function ensureLoaded(): DreamCycle[] {
14
+ if (state.cycles !== null) return state.cycles
15
+ try {
16
+ const raw = fs.readFileSync(DREAM_CYCLES_PATH, { encoding: 'utf-8' })
17
+ state.cycles = safeJsonParse<DreamCycle[]>(raw, [])
18
+ } catch {
19
+ state.cycles = []
20
+ }
21
+ return state.cycles
22
+ }
23
+
24
+ function persist(): void {
25
+ fs.mkdirSync(path.dirname(DREAM_CYCLES_PATH), { recursive: true })
26
+ fs.writeFileSync(DREAM_CYCLES_PATH, JSON.stringify(ensureLoaded(), null, 2), { encoding: 'utf-8' })
27
+ }
28
+
29
+ export function saveDreamCycle(cycle: DreamCycle): void {
30
+ const cycles = ensureLoaded()
31
+ const idx = cycles.findIndex((c) => c.id === cycle.id)
32
+ if (idx >= 0) {
33
+ cycles[idx] = cycle
34
+ } else {
35
+ cycles.push(cycle)
36
+ }
37
+ persist()
38
+ }
39
+
40
+ export function listDreamCycles(agentId?: string, limit = 50): DreamCycle[] {
41
+ const cycles = ensureLoaded()
42
+ const filtered = agentId ? cycles.filter((c) => c.agentId === agentId) : [...cycles]
43
+ filtered.sort((a, b) => b.startedAt - a.startedAt)
44
+ return filtered.slice(0, limit)
45
+ }
46
+
47
+ export function getDreamCycle(id: string): DreamCycle | null {
48
+ return ensureLoaded().find((c) => c.id === id) ?? null
49
+ }
@@ -0,0 +1,38 @@
1
+ import { onNextIdleWindow } from '@/lib/server/runtime/idle-window'
2
+ import { loadAgents } from '@/lib/server/storage'
3
+ import { executeDreamCycle, resolveDreamConfig } from './dream-service'
4
+ import { log } from '@/lib/server/logger'
5
+ import { errorMessage } from '@/lib/shared-utils'
6
+
7
+ const TAG = 'dream-idle'
8
+ let dreamRegistered = false
9
+
10
+ export function registerDreamIdleCallback(): void {
11
+ if (dreamRegistered) return
12
+ dreamRegistered = true
13
+ onNextIdleWindow(async () => {
14
+ dreamRegistered = false
15
+ await runDreamForEligibleAgents()
16
+ registerDreamIdleCallback()
17
+ })
18
+ }
19
+
20
+ async function runDreamForEligibleAgents(): Promise<void> {
21
+ const agents = loadAgents()
22
+ const now = Date.now()
23
+
24
+ for (const agent of Object.values(agents)) {
25
+ if (agent.trashedAt) continue
26
+ if (!agent.dreamEnabled) continue
27
+
28
+ const config = resolveDreamConfig(agent)
29
+ const lastDream = agent.lastDreamAt ?? 0
30
+ if (lastDream + config.cooldownMinutes * 60_000 >= now) continue
31
+
32
+ try {
33
+ await executeDreamCycle(agent.id, 'idle')
34
+ } catch (err: unknown) {
35
+ log.error(TAG, `Dream cycle failed for agent ${agent.id}: ${errorMessage(err)}`)
36
+ }
37
+ }
38
+ }
@@ -0,0 +1,315 @@
1
+ import crypto from 'crypto'
2
+ import type { DreamCycle, DreamCycleResult, DreamConfig, DreamTrigger, Agent } from '@/types'
3
+ import { DEFAULT_DREAM_CONFIG } from '@/types/dream'
4
+ import { getMemoryDb } from '@/lib/server/memory/memory-db'
5
+ import { saveDreamCycle } from '@/lib/server/memory/dream-cycles'
6
+ import { errorMessage, safeJsonParse } from '@/lib/shared-utils'
7
+ import { log } from '@/lib/server/logger'
8
+
9
+ const TAG = 'dream-service'
10
+
11
+ const EXEMPT_CATEGORIES = new Set(['daily_digest', 'consolidated_insight', 'dream_reflection'])
12
+
13
+ export function resolveDreamConfig(agent: Agent): DreamConfig {
14
+ return { ...DEFAULT_DREAM_CONFIG, ...agent.dreamConfig }
15
+ }
16
+
17
+ export async function runTier1Dream(
18
+ agentId: string,
19
+ config: DreamConfig,
20
+ ): Promise<Partial<DreamCycleResult>> {
21
+ const memDb = getMemoryDb()
22
+ const memories = memDb.getByAgent(agentId, 500)
23
+ const now = Date.now()
24
+ const decayAgeMs = config.decayAgeDays * 86_400_000
25
+ const pruneMs = config.pruneThresholdDays * 86_400_000
26
+
27
+ let decayed = 0
28
+ let pruned = 0
29
+ let promoted = 0
30
+ const errors: string[] = []
31
+
32
+ for (const mem of memories) {
33
+ // Skip exempt categories
34
+ if (EXEMPT_CATEGORIES.has(mem.category)) continue
35
+
36
+ const tier = (mem.metadata as Record<string, unknown> | undefined)?.tier
37
+ // Skip archived entries
38
+ if (tier === 'archive') continue
39
+
40
+ const age = now - (mem.createdAt || mem.updatedAt || 0)
41
+
42
+ // Decay & Prune: old entries with no engagement
43
+ if (age > decayAgeMs && (mem.accessCount || 0) === 0 && (mem.reinforcementCount || 0) === 0) {
44
+ if (tier === 'working' && age > pruneMs) {
45
+ memDb.delete(mem.id)
46
+ pruned++
47
+ } else {
48
+ memDb.update(mem.id, {
49
+ metadata: { ...(mem.metadata as Record<string, unknown>), decayedInDream: true },
50
+ })
51
+ decayed++
52
+ }
53
+ continue
54
+ }
55
+
56
+ // Promote: working-tier entries with high engagement
57
+ if (
58
+ (tier === 'working' || tier === '' || tier === undefined) &&
59
+ (mem.accessCount || 0) >= 3 &&
60
+ (mem.reinforcementCount || 0) >= 2
61
+ ) {
62
+ memDb.update(mem.id, {
63
+ metadata: { ...(mem.metadata as Record<string, unknown>), tier: 'durable' },
64
+ })
65
+ promoted++
66
+ }
67
+ }
68
+
69
+ // Dedup via maintain
70
+ let deduped = 0
71
+ try {
72
+ const maintenance = memDb.maintain({ dedupe: true, pruneWorking: false })
73
+ deduped = maintenance.deduped
74
+ } catch (err: unknown) {
75
+ const msg = `Dedup maintenance failed: ${errorMessage(err)}`
76
+ log.warn(TAG, msg)
77
+ errors.push(msg)
78
+ }
79
+
80
+ return { decayed, pruned, promoted, deduped, errors }
81
+ }
82
+
83
+ interface Tier2Consolidation {
84
+ sourceIds: string[]
85
+ title: string
86
+ content: string
87
+ }
88
+
89
+ interface Tier2Reflection {
90
+ title: string
91
+ content: string
92
+ }
93
+
94
+ interface Tier2Flagged {
95
+ memoryId: string
96
+ reason: string
97
+ }
98
+
99
+ interface Tier2Response {
100
+ consolidations?: Tier2Consolidation[]
101
+ reflections?: Tier2Reflection[]
102
+ flagged?: Tier2Flagged[]
103
+ }
104
+
105
+ export async function runTier2Dream(
106
+ agentId: string,
107
+ config: DreamConfig,
108
+ ): Promise<Partial<DreamCycleResult>> {
109
+ const memDb = getMemoryDb()
110
+ const allMemories = memDb.getByAgent(agentId, config.tier2MaxMemories)
111
+
112
+ // Filter to durable tier and non-exempt entries
113
+ const candidates = allMemories.filter((m) => {
114
+ if (EXEMPT_CATEGORIES.has(m.category)) return false
115
+ const tier = (m.metadata as Record<string, unknown> | undefined)?.tier
116
+ return tier === 'durable' || tier === 'working' || tier === '' || tier === undefined
117
+ })
118
+
119
+ if (candidates.length < 3) {
120
+ return { consolidated: 0, reflections: [], memoriesReviewed: 0, errors: [] }
121
+ }
122
+
123
+ const memoryLines = candidates.map((m) => {
124
+ const content = (m.content || '').slice(0, 300)
125
+ return `[${m.id}] [${m.category}] ${m.title}: ${content}`
126
+ })
127
+
128
+ const prompt = `You are reviewing your memories during a dream cycle to consolidate and reflect.
129
+
130
+ Review the following memories and respond with a JSON object:
131
+ {
132
+ "consolidations": [{ "sourceIds": ["id1", "id2"], "title": "...", "content": "..." }],
133
+ "reflections": [{ "title": "...", "content": "..." }],
134
+ "flagged": [{ "memoryId": "...", "reason": "..." }]
135
+ }
136
+
137
+ Rules:
138
+ - Consolidate 2-5 groups of overlapping memories into single entries
139
+ - Write 1-3 high-level reflections about patterns you notice
140
+ - Flag any outdated or contradictory memories
141
+ - Be concise. Each consolidation and reflection should be 1-3 sentences.
142
+
143
+ MEMORIES:
144
+ ${memoryLines.join('\n')}`
145
+
146
+ const errors: string[] = []
147
+ let consolidated = 0
148
+ const reflectionTitles: string[] = []
149
+
150
+ try {
151
+ const { buildLLM } = await import('@/lib/server/build-llm')
152
+ const { llm } = await buildLLM({ agentId })
153
+ const { HumanMessage } = await import('@langchain/core/messages')
154
+
155
+ const response = await llm.invoke([new HumanMessage(prompt)])
156
+ const text = typeof response.content === 'string'
157
+ ? response.content
158
+ : Array.isArray(response.content)
159
+ ? response.content.map((b) => ('text' in b && typeof b.text === 'string' ? b.text : '')).join('')
160
+ : ''
161
+
162
+ // Extract JSON from response (handle markdown code blocks)
163
+ const jsonMatch = text.match(/\{[\s\S]*\}/)
164
+ const parsed = safeJsonParse<Tier2Response>(jsonMatch?.[0] ?? '', {})
165
+
166
+ // Process consolidations
167
+ if (Array.isArray(parsed.consolidations)) {
168
+ for (const c of parsed.consolidations) {
169
+ if (!c.title || !c.content || !Array.isArray(c.sourceIds)) continue
170
+ const validSourceIds = c.sourceIds.filter((id): id is string => typeof id === 'string' && id.trim().length > 0)
171
+ memDb.add({
172
+ agentId,
173
+ sessionId: null,
174
+ category: 'consolidated_insight',
175
+ title: c.title,
176
+ content: c.content,
177
+ linkedMemoryIds: validSourceIds,
178
+ metadata: { tier: 'durable', origin: 'dream' },
179
+ })
180
+ consolidated++
181
+ }
182
+ }
183
+
184
+ // Process reflections
185
+ if (Array.isArray(parsed.reflections)) {
186
+ for (const r of parsed.reflections) {
187
+ if (!r.title || !r.content) continue
188
+ memDb.add({
189
+ agentId,
190
+ sessionId: null,
191
+ category: 'dream_reflection',
192
+ title: r.title,
193
+ content: r.content,
194
+ metadata: { tier: 'durable', origin: 'dream' },
195
+ })
196
+ reflectionTitles.push(r.title)
197
+ }
198
+ }
199
+
200
+ // Process flagged
201
+ if (Array.isArray(parsed.flagged)) {
202
+ for (const f of parsed.flagged) {
203
+ if (!f.memoryId || !f.reason) continue
204
+ const existing = memDb.get(f.memoryId)
205
+ if (existing) {
206
+ memDb.update(f.memoryId, {
207
+ metadata: {
208
+ ...(existing.metadata as Record<string, unknown>),
209
+ flaggedInDream: true,
210
+ flagReason: f.reason,
211
+ },
212
+ })
213
+ }
214
+ }
215
+ }
216
+ } catch (err: unknown) {
217
+ errors.push(errorMessage(err))
218
+ }
219
+
220
+ return {
221
+ consolidated,
222
+ reflections: reflectionTitles,
223
+ memoriesReviewed: candidates.length,
224
+ errors,
225
+ }
226
+ }
227
+
228
+ export async function executeDreamCycle(
229
+ agentId: string,
230
+ trigger: DreamTrigger,
231
+ ): Promise<DreamCycle> {
232
+ const { loadAgents, patchAgent, logActivity } = await import('@/lib/server/storage')
233
+ const agents = loadAgents()
234
+ const agent = agents[agentId]
235
+ if (!agent) throw new Error(`Agent ${agentId} not found`)
236
+
237
+ const config = resolveDreamConfig(agent)
238
+
239
+ // Check cooldown
240
+ if (agent.lastDreamAt && Date.now() - agent.lastDreamAt < config.cooldownMinutes * 60_000) {
241
+ throw new Error('Dream on cooldown')
242
+ }
243
+
244
+ const cycle: DreamCycle = {
245
+ id: crypto.randomUUID(),
246
+ agentId,
247
+ status: 'running',
248
+ trigger,
249
+ startedAt: Date.now(),
250
+ }
251
+ saveDreamCycle(cycle)
252
+
253
+ try {
254
+ // Tier 1: deterministic server-side operations
255
+ const tier1 = await runTier1Dream(agentId, config)
256
+
257
+ // Tier 2: LLM-driven reflection (optional)
258
+ let tier2: Partial<DreamCycleResult> = {}
259
+ if (config.tier2Enabled) {
260
+ tier2 = await runTier2Dream(agentId, config)
261
+ }
262
+
263
+ const result: DreamCycleResult = {
264
+ decayed: tier1.decayed ?? 0,
265
+ pruned: tier1.pruned ?? 0,
266
+ promoted: tier1.promoted ?? 0,
267
+ deduped: tier1.deduped ?? 0,
268
+ consolidated: tier2.consolidated ?? 0,
269
+ reflections: tier2.reflections ?? [],
270
+ memoriesReviewed: tier2.memoriesReviewed ?? 0,
271
+ durationMs: Date.now() - cycle.startedAt,
272
+ errors: [...(tier1.errors ?? []), ...(tier2.errors ?? [])],
273
+ }
274
+
275
+ cycle.status = 'completed'
276
+ cycle.completedAt = Date.now()
277
+ cycle.result = result
278
+ saveDreamCycle(cycle)
279
+
280
+ // Update agent
281
+ patchAgent(agentId, (current) => {
282
+ if (!current) return null
283
+ return {
284
+ ...current,
285
+ lastDreamAt: Date.now(),
286
+ dreamCycleCount: (current.dreamCycleCount || 0) + 1,
287
+ }
288
+ })
289
+
290
+ logActivity({
291
+ entityType: 'agent',
292
+ entityId: agentId,
293
+ action: 'dream_completed',
294
+ actor: 'system',
295
+ summary: `Dream cycle completed: ${result.decayed} decayed, ${result.pruned} pruned, ${result.consolidated} consolidated`,
296
+ })
297
+
298
+ log.info(TAG, `Dream cycle completed for agent ${agentId}`, {
299
+ trigger,
300
+ decayed: result.decayed,
301
+ pruned: result.pruned,
302
+ promoted: result.promoted,
303
+ consolidated: result.consolidated,
304
+ })
305
+
306
+ return cycle
307
+ } catch (err: unknown) {
308
+ cycle.status = 'failed'
309
+ cycle.completedAt = Date.now()
310
+ cycle.error = errorMessage(err)
311
+ saveDreamCycle(cycle)
312
+ log.error(TAG, `Dream cycle failed for agent ${agentId}: ${errorMessage(err)}`)
313
+ throw err
314
+ }
315
+ }
@@ -146,6 +146,20 @@ function metadataNumber(entry: MemoryEntry, key: string): number | null {
146
146
  return typeof value === 'number' && Number.isFinite(value) ? value : null
147
147
  }
148
148
 
149
+ function knowledgeChunkHashScope(category: string, metadata: unknown): string {
150
+ if (category !== 'knowledge' || !metadata || typeof metadata !== 'object') return category
151
+ const sourceIdValue = (metadata as Record<string, unknown>).sourceId
152
+ const sourceId = typeof sourceIdValue === 'string'
153
+ ? sourceIdValue.trim()
154
+ : ''
155
+ if (!sourceId) return category
156
+ const chunkIndexValue = (metadata as Record<string, unknown>).chunkIndex
157
+ const chunkIndex = typeof chunkIndexValue === 'number'
158
+ ? chunkIndexValue
159
+ : null
160
+ return `${category}:${sourceId}:${chunkIndex ?? 'legacy'}`
161
+ }
162
+
149
163
  function followUpSalienceMultiplier(entry: MemoryEntry, nowTs: number): number {
150
164
  if (entry.category !== 'reflection/open_loop') return 1
151
165
  const resolvedAt = metadataNumber(entry, 'resolvedAt')
@@ -664,7 +678,7 @@ function initDb() {
664
678
  update: db.prepare(`
665
679
  UPDATE memories
666
680
  SET agentId=?, sessionId=?, category=?, title=?, content=?, metadata=?, embedding=?,
667
- "references"=?, filePaths=?, image=?, imagePath=?, linkedMemoryIds=?, pinned=?, sharedWith=?, updatedAt=?
681
+ "references"=?, filePaths=?, image=?, imagePath=?, linkedMemoryIds=?, pinned=?, sharedWith=?, contentHash=?, updatedAt=?
668
682
  WHERE id=?
669
683
  `),
670
684
  delete: db.prepare(`DELETE FROM memories WHERE id=?`),
@@ -677,6 +691,13 @@ function initDb() {
677
691
  listAll: db.prepare(`SELECT * FROM memories ORDER BY updatedAt DESC LIMIT ?`),
678
692
  listByAgent: db.prepare(`SELECT * FROM memories WHERE agentId=? ORDER BY updatedAt DESC LIMIT ?`),
679
693
  listByAgentOrShared: db.prepare(`SELECT * FROM memories WHERE agentId=? OR sharedWith LIKE ? ORDER BY updatedAt DESC LIMIT ?`),
694
+ listByCategoryAll: db.prepare(`SELECT * FROM memories WHERE category=? ORDER BY updatedAt DESC LIMIT ?`),
695
+ listByCategoryAgentOrShared: db.prepare(`SELECT * FROM memories WHERE category=? AND (agentId=? OR sharedWith LIKE ?) ORDER BY updatedAt DESC LIMIT ?`),
696
+ listKnowledgeSourceChunks: db.prepare(`
697
+ SELECT * FROM memories
698
+ WHERE category='knowledge' AND json_extract(metadata, '$.sourceId') = ?
699
+ ORDER BY COALESCE(json_extract(metadata, '$.chunkIndex'), 0) ASC, createdAt ASC
700
+ `),
680
701
  listPinnedByAgent: db.prepare(`SELECT * FROM memories WHERE pinned = 1 AND agentId = ? ORDER BY updatedAt DESC LIMIT ?`),
681
702
  listPinnedAll: db.prepare(`SELECT * FROM memories WHERE pinned = 1 ORDER BY updatedAt DESC LIMIT ?`),
682
703
  search: db.prepare(`
@@ -803,7 +824,7 @@ function initDb() {
803
824
  const category = data.category || 'note'
804
825
  const title = data.title || 'Untitled'
805
826
  const content = data.content || ''
806
- const contentHash = computeContentHash(category, content)
827
+ const contentHash = computeContentHash(knowledgeChunkHashScope(category, data.metadata), content)
807
828
 
808
829
  // Content-hash dedup: if same content already exists for this agent, reinforce instead of duplicating
809
830
  const agentId = data.agentId || null
@@ -896,6 +917,7 @@ function initDb() {
896
917
  const now = Date.now()
897
918
  const pinnedVal = merged.pinned ? 1 : 0
898
919
  const sharedWithVal = Array.isArray(merged.sharedWith) && merged.sharedWith.length ? JSON.stringify(merged.sharedWith) : null
920
+ const nextContentHash = computeContentHash(knowledgeChunkHashScope(merged.category, merged.metadata), merged.content)
899
921
  stmts.update.run(
900
922
  merged.agentId || null, merged.sessionId || null,
901
923
  merged.category, merged.title, merged.content,
@@ -908,6 +930,7 @@ function initDb() {
908
930
  nextLinked.length ? JSON.stringify(nextLinked) : null,
909
931
  pinnedVal,
910
932
  sharedWithVal,
933
+ nextContentHash,
911
934
  now, id,
912
935
  )
913
936
 
@@ -1215,6 +1238,18 @@ function initDb() {
1215
1238
  return rows.map(rowToEntry)
1216
1239
  },
1217
1240
 
1241
+ listByCategory(category: string, agentId?: string, limit = 500): MemoryEntry[] {
1242
+ const safeLimit = Math.max(1, Math.min(10_000, Math.trunc(limit)))
1243
+ const rows = agentId
1244
+ ? stmts.listByCategoryAgentOrShared.all(category, agentId, `%"${agentId}"%`, safeLimit) as Record<string, unknown>[]
1245
+ : stmts.listByCategoryAll.all(category, safeLimit) as Record<string, unknown>[]
1246
+ return rows.map(rowToEntry)
1247
+ },
1248
+
1249
+ listKnowledgeSourceChunks(sourceId: string): MemoryEntry[] {
1250
+ return (stmts.listKnowledgeSourceChunks.all(sourceId) as Record<string, unknown>[]).map(rowToEntry)
1251
+ },
1252
+
1218
1253
  listPinned(agentId?: string, limit = 20): MemoryEntry[] {
1219
1254
  const safeLimit = Math.max(1, Math.min(100, Math.trunc(limit)))
1220
1255
  const rows = agentId
@@ -48,6 +48,7 @@ import { AGENT_TURN_TIMEOUT_MS, cleanText, now } from '@/lib/server/protocols/pr
48
48
  import type { ProtocolAgentTurnResult, ProtocolRunDeps } from '@/lib/server/protocols/protocol-types'
49
49
  import { normalizeProtocolRun } from '@/lib/server/protocols/protocol-normalization'
50
50
  import { persistChatroomInteractionMemory } from '@/lib/server/chatrooms/chatroom-memory-bridge'
51
+ import { selectKnowledgeCitations } from '@/lib/server/knowledge-sources'
51
52
 
52
53
  // ---- Zod schema ----
53
54
 
@@ -262,6 +263,10 @@ export async function defaultExecuteAgentTurn(params: {
262
263
  ])
263
264
  const rawText = result.finalResponse || result.fullText || ''
264
265
  const text = stripHiddenControlTokens(rawText)
266
+ const grounding = selectKnowledgeCitations({
267
+ responseText: text,
268
+ retrievalTrace: result.knowledgeRetrievalTrace || null,
269
+ })
265
270
  if (text.trim() && !shouldSuppressHiddenControlText(rawText)) {
266
271
  appendSyntheticSessionMessage(syntheticSession.id, 'assistant', text)
267
272
  // Persist interaction to agent memory (fire-and-forget)
@@ -278,6 +283,8 @@ export async function defaultExecuteAgentTurn(params: {
278
283
  return {
279
284
  text: cleanText(text, 6_000),
280
285
  toolEvents: result.toolEvents || [],
286
+ citations: grounding.citations,
287
+ retrievalTrace: grounding.retrievalTrace,
281
288
  }
282
289
  } catch (err: unknown) {
283
290
  lastError = err
@@ -47,6 +47,13 @@ const protocolExecutionState = hmrSingleton('__swarmclaw_protocol_engine_executi
47
47
  pendingRunIds: new Set<string>(),
48
48
  }))
49
49
 
50
+ function shouldYieldBetweenProtocolSteps(deps?: ProtocolRunDeps): boolean {
51
+ // Dependency-injected runs are used for deterministic tests and nested orchestration.
52
+ // Keep the production scheduler cooperative, but let injected executions drain
53
+ // their step loop without requiring multiple timer turns.
54
+ return !deps
55
+ }
56
+
50
57
  // ---- Scheduling/Recovery (G10) ----
51
58
 
52
59
  export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDeps): boolean {
@@ -54,7 +61,7 @@ export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDep
54
61
  if (!normalizedId) return false
55
62
  if (protocolExecutionState.pendingRunIds.has(normalizedId)) return false
56
63
  protocolExecutionState.pendingRunIds.add(normalizedId)
57
- setTimeout(() => {
64
+ const invoke = () => {
58
65
  void runProtocolRun(normalizedId, deps)
59
66
  .catch((err: unknown) => {
60
67
  log.warn(TAG, `execution failed for ${normalizedId}: ${errorMessage(err)}`)
@@ -62,7 +69,9 @@ export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDep
62
69
  .finally(() => {
63
70
  protocolExecutionState.pendingRunIds.delete(normalizedId)
64
71
  })
65
- }, 0)
72
+ }
73
+ if (deps) queueMicrotask(invoke)
74
+ else setTimeout(invoke, 0)
66
75
  return true
67
76
  }
68
77
 
@@ -115,8 +124,9 @@ export function wakeProtocolRunFromTaskCompletion(taskId: string, deps?: Protoco
115
124
  }
116
125
 
117
126
  export function ensureProtocolEngineRecovered(deps?: ProtocolRunDeps): void {
118
- if (protocolRecoveryState.completed) return
119
- protocolRecoveryState.completed = true
127
+ const bypassCompletedGuard = Boolean(deps)
128
+ if (!bypassCompletedGuard && protocolRecoveryState.completed) return
129
+ if (!bypassCompletedGuard) protocolRecoveryState.completed = true
120
130
  const runs = Object.values(loadProtocolRuns()).map((entry) => normalizeProtocolRun(entry))
121
131
  for (const run of runs) {
122
132
  if (run.parentRunId) {
@@ -321,8 +331,11 @@ export async function runProtocolRun(runId: string, deps?: ProtocolRunDeps): Pro
321
331
  appendProtocolEvent(run.id, { type: 'failed', summary: `Exceeded maximum step iterations (${MAX_STEP_ITERATIONS}).` }, deps)
322
332
  break
323
333
  }
324
- // Yield between steps so I/O, HTTP responses, and timers can run.
325
- await new Promise(r => setTimeout(r, 0))
334
+ if (shouldYieldBetweenProtocolSteps(deps)) {
335
+ // Yield between steps in the fire-and-forget runtime so I/O, HTTP responses,
336
+ // and timers can run.
337
+ await new Promise(r => setTimeout(r, 0))
338
+ }
326
339
  const latest = loadProtocolRunById(run.id)
327
340
  if (!latest) return null
328
341
  if (latest.status === 'paused' || latest.status === 'cancelled' || latest.status === 'archived' || latest.status === 'completed') {
@@ -96,6 +96,105 @@ test('protocol-service creates a hidden transcript run and completes a structure
96
96
  assert.equal(output.eventTypes.filter((type) => type === 'phase_completed').length, 5)
97
97
  })
98
98
 
99
+ test('protocol-service persists citations on participant and artifact events when grounded responses are provided', () => {
100
+ const output = runWithTempDataDir<{
101
+ participantCitationCounts: number[]
102
+ artifactCitationCounts: number[]
103
+ selectorStatuses: string[]
104
+ }>(`
105
+ const storageMod = await import('./src/lib/server/storage')
106
+ const protocolsMod = await import('./src/lib/server/protocols/protocol-service')
107
+ const storage = storageMod.default || storageMod
108
+ const protocols = protocolsMod.default || protocolsMod
109
+
110
+ storage.upsertStoredItem('agents', 'agentA', {
111
+ id: 'agentA',
112
+ name: 'Agent A',
113
+ provider: 'ollama',
114
+ model: 'test-model',
115
+ systemPrompt: 'test',
116
+ createdAt: 1,
117
+ updatedAt: 1,
118
+ })
119
+
120
+ const groundedCitation = {
121
+ sourceId: 'source-1',
122
+ sourceTitle: 'Gateway Runbook',
123
+ sourceKind: 'manual',
124
+ sourceUrl: null,
125
+ sourceLabel: null,
126
+ chunkId: 'chunk-1',
127
+ chunkIndex: 0,
128
+ chunkCount: 1,
129
+ charStart: 0,
130
+ charEnd: 48,
131
+ sectionLabel: null,
132
+ snippet: 'Use blue green deployment for gateway changes.',
133
+ whyMatched: 'Matched query terms: gateway, deployment',
134
+ score: 0.92,
135
+ }
136
+
137
+ const run = protocols.createProtocolRun({
138
+ title: 'Grounded structured run',
139
+ templateId: 'single_agent_structured_run',
140
+ participantAgentIds: ['agentA'],
141
+ facilitatorAgentId: 'agentA',
142
+ autoStart: false,
143
+ }, { now: () => 1000 })
144
+
145
+ await protocols.runProtocolRun(run.id, {
146
+ now: () => 2000,
147
+ executeAgentTurn: async ({ phase }) => {
148
+ if (phase.kind === 'summarize') {
149
+ return {
150
+ text: 'Blue green deployment keeps the rollback path simple.',
151
+ toolEvents: [],
152
+ citations: [groundedCitation],
153
+ retrievalTrace: {
154
+ query: 'gateway deployment',
155
+ scope: 'source_knowledge',
156
+ hits: [groundedCitation],
157
+ retrievedAt: 2000,
158
+ selectorStatus: 'selected',
159
+ },
160
+ }
161
+ }
162
+ if (phase.kind === 'round_robin') {
163
+ return {
164
+ text: 'Use blue green deployment for the gateway rollout.',
165
+ toolEvents: [],
166
+ citations: [groundedCitation],
167
+ retrievalTrace: {
168
+ query: 'gateway rollout',
169
+ scope: 'source_knowledge',
170
+ hits: [groundedCitation],
171
+ retrievedAt: 2000,
172
+ selectorStatus: 'selected',
173
+ },
174
+ }
175
+ }
176
+ return { text: 'Opened the session.', toolEvents: [] }
177
+ },
178
+ })
179
+
180
+ const detail = protocols.getProtocolRunDetail(run.id)
181
+ const participantEvents = (detail?.events || []).filter((event) => event.type === 'participant_response')
182
+ const artifactEvents = (detail?.events || []).filter((event) => event.type === 'artifact_emitted')
183
+
184
+ console.log(JSON.stringify({
185
+ participantCitationCounts: participantEvents.map((event) => event.citations?.length || 0),
186
+ artifactCitationCounts: artifactEvents.map((event) => event.citations?.length || 0),
187
+ selectorStatuses: participantEvents
188
+ .map((event) => event.retrievalTrace?.selectorStatus)
189
+ .filter((value) => typeof value === 'string'),
190
+ }))
191
+ `, { prefix: 'swarmclaw-protocol-grounding-' })
192
+
193
+ assert.deepEqual(output.participantCitationCounts, [1])
194
+ assert.deepEqual(output.artifactCitationCounts, [1])
195
+ assert.deepEqual(output.selectorStatuses, [])
196
+ })
197
+
99
198
  test('protocol-service supports custom template CRUD and operator actions', () => {
100
199
  const output = runWithTempDataDir<{
101
200
  createdTemplateId: string | null