@swarmclawai/swarmclaw 1.3.4 → 1.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -76
- package/package.json +3 -2
- package/skills/swarmclaw.md +17 -0
- package/src/app/api/agents/[id]/dream/route.ts +45 -0
- package/src/app/api/knowledge/[id]/route.ts +48 -49
- package/src/app/api/knowledge/hygiene/route.ts +13 -0
- package/src/app/api/knowledge/route.ts +70 -42
- package/src/app/api/knowledge/sources/[id]/archive/route.ts +15 -0
- package/src/app/api/knowledge/sources/[id]/restore/route.ts +10 -0
- package/src/app/api/knowledge/sources/[id]/route.ts +1 -0
- package/src/app/api/knowledge/sources/[id]/supersede/route.ts +26 -0
- package/src/app/api/knowledge/sources/[id]/sync/route.ts +17 -0
- package/src/app/api/knowledge/sources/route.ts +1 -0
- package/src/app/api/knowledge/upload/route.ts +3 -51
- package/src/app/api/memory/dream/[id]/route.ts +19 -0
- package/src/app/api/memory/dream/route.ts +34 -0
- package/src/app/knowledge/layout.tsx +1 -1
- package/src/app/knowledge/page.tsx +2 -22
- package/src/app/protocols/page.tsx +21 -2
- package/src/cli/index.js +16 -0
- package/src/cli/spec.js +5 -0
- package/src/components/agents/agent-sheet.tsx +65 -0
- package/src/components/chat/message-bubble.tsx +10 -0
- package/src/components/knowledge/grounding-panel.tsx +99 -0
- package/src/components/knowledge/knowledge-detail.tsx +402 -0
- package/src/components/knowledge/knowledge-list.tsx +351 -126
- package/src/components/knowledge/knowledge-sheet.tsx +208 -119
- package/src/components/memory/dream-history.tsx +155 -0
- package/src/components/memory/memory-card.tsx +7 -0
- package/src/components/memory/memory-detail.tsx +46 -0
- package/src/components/runs/run-list.tsx +23 -0
- package/src/lib/server/api-routes.test.ts +43 -2
- package/src/lib/server/chat-execution/chat-execution-disabled.test.ts +14 -31
- package/src/lib/server/chat-execution/chat-execution-eval-history.test.ts +11 -34
- package/src/lib/server/chat-execution/chat-execution-grounding.test.ts +108 -0
- package/src/lib/server/chat-execution/chat-execution-session-sync.test.ts +35 -36
- package/src/lib/server/chat-execution/chat-execution-types.ts +8 -1
- package/src/lib/server/chat-execution/chat-execution.ts +1 -0
- package/src/lib/server/chat-execution/chat-turn-finalization.ts +21 -1
- package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +6 -1
- package/src/lib/server/chat-execution/post-stream-finalization.ts +15 -3
- package/src/lib/server/chat-execution/prompt-sections.ts +29 -3
- package/src/lib/server/chat-execution/stream-agent-chat.ts +6 -1
- package/src/lib/server/execution-engine/task-attempt.ts +8 -2
- package/src/lib/server/knowledge-import.ts +159 -0
- package/src/lib/server/knowledge-sources.test.ts +261 -0
- package/src/lib/server/knowledge-sources.ts +1284 -0
- package/src/lib/server/memory/dream-cycles.ts +49 -0
- package/src/lib/server/memory/dream-idle-callback.ts +38 -0
- package/src/lib/server/memory/dream-service.ts +315 -0
- package/src/lib/server/memory/memory-db.ts +37 -2
- package/src/lib/server/protocols/protocol-agent-turn.ts +7 -0
- package/src/lib/server/protocols/protocol-run-lifecycle.ts +19 -6
- package/src/lib/server/protocols/protocol-service.test.ts +99 -0
- package/src/lib/server/protocols/protocol-step-helpers.ts +7 -1
- package/src/lib/server/protocols/protocol-step-processors.ts +16 -3
- package/src/lib/server/protocols/protocol-types.ts +4 -0
- package/src/lib/server/runtime/daemon-state/core.ts +6 -1
- package/src/lib/server/runtime/run-ledger.test.ts +120 -0
- package/src/lib/server/runtime/run-ledger.ts +27 -1
- package/src/lib/server/runtime/session-run-manager/drain.ts +5 -0
- package/src/lib/server/runtime/session-run-manager/state.ts +19 -2
- package/src/lib/server/storage-normalization.ts +5 -0
- package/src/lib/server/storage.ts +15 -0
- package/src/lib/server/test-utils/run-with-temp-data-dir.ts +15 -2
- package/src/stores/slices/ui-slice.ts +4 -0
- package/src/types/agent.ts +7 -0
- package/src/types/dream.ts +45 -0
- package/src/types/index.ts +1 -0
- package/src/types/message.ts +3 -0
- package/src/types/misc.ts +131 -0
- package/src/types/protocol.ts +4 -0
- package/src/types/run.ts +4 -1
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import fs from 'fs'
|
|
2
|
+
import path from 'path'
|
|
3
|
+
import { DATA_DIR } from '@/lib/server/data-dir'
|
|
4
|
+
import { hmrSingleton, safeJsonParse } from '@/lib/shared-utils'
|
|
5
|
+
import type { DreamCycle } from '@/types'
|
|
6
|
+
|
|
7
|
+
const DREAM_CYCLES_PATH = path.join(DATA_DIR, 'dream-cycles.json')
|
|
8
|
+
|
|
9
|
+
const state = hmrSingleton('__dream_cycles__', () => ({
|
|
10
|
+
cycles: null as DreamCycle[] | null,
|
|
11
|
+
}))
|
|
12
|
+
|
|
13
|
+
function ensureLoaded(): DreamCycle[] {
|
|
14
|
+
if (state.cycles !== null) return state.cycles
|
|
15
|
+
try {
|
|
16
|
+
const raw = fs.readFileSync(DREAM_CYCLES_PATH, { encoding: 'utf-8' })
|
|
17
|
+
state.cycles = safeJsonParse<DreamCycle[]>(raw, [])
|
|
18
|
+
} catch {
|
|
19
|
+
state.cycles = []
|
|
20
|
+
}
|
|
21
|
+
return state.cycles
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function persist(): void {
|
|
25
|
+
fs.mkdirSync(path.dirname(DREAM_CYCLES_PATH), { recursive: true })
|
|
26
|
+
fs.writeFileSync(DREAM_CYCLES_PATH, JSON.stringify(ensureLoaded(), null, 2), { encoding: 'utf-8' })
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function saveDreamCycle(cycle: DreamCycle): void {
|
|
30
|
+
const cycles = ensureLoaded()
|
|
31
|
+
const idx = cycles.findIndex((c) => c.id === cycle.id)
|
|
32
|
+
if (idx >= 0) {
|
|
33
|
+
cycles[idx] = cycle
|
|
34
|
+
} else {
|
|
35
|
+
cycles.push(cycle)
|
|
36
|
+
}
|
|
37
|
+
persist()
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function listDreamCycles(agentId?: string, limit = 50): DreamCycle[] {
|
|
41
|
+
const cycles = ensureLoaded()
|
|
42
|
+
const filtered = agentId ? cycles.filter((c) => c.agentId === agentId) : [...cycles]
|
|
43
|
+
filtered.sort((a, b) => b.startedAt - a.startedAt)
|
|
44
|
+
return filtered.slice(0, limit)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function getDreamCycle(id: string): DreamCycle | null {
|
|
48
|
+
return ensureLoaded().find((c) => c.id === id) ?? null
|
|
49
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { onNextIdleWindow } from '@/lib/server/runtime/idle-window'
|
|
2
|
+
import { loadAgents } from '@/lib/server/storage'
|
|
3
|
+
import { executeDreamCycle, resolveDreamConfig } from './dream-service'
|
|
4
|
+
import { log } from '@/lib/server/logger'
|
|
5
|
+
import { errorMessage } from '@/lib/shared-utils'
|
|
6
|
+
|
|
7
|
+
const TAG = 'dream-idle'
|
|
8
|
+
let dreamRegistered = false
|
|
9
|
+
|
|
10
|
+
export function registerDreamIdleCallback(): void {
|
|
11
|
+
if (dreamRegistered) return
|
|
12
|
+
dreamRegistered = true
|
|
13
|
+
onNextIdleWindow(async () => {
|
|
14
|
+
dreamRegistered = false
|
|
15
|
+
await runDreamForEligibleAgents()
|
|
16
|
+
registerDreamIdleCallback()
|
|
17
|
+
})
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function runDreamForEligibleAgents(): Promise<void> {
|
|
21
|
+
const agents = loadAgents()
|
|
22
|
+
const now = Date.now()
|
|
23
|
+
|
|
24
|
+
for (const agent of Object.values(agents)) {
|
|
25
|
+
if (agent.trashedAt) continue
|
|
26
|
+
if (!agent.dreamEnabled) continue
|
|
27
|
+
|
|
28
|
+
const config = resolveDreamConfig(agent)
|
|
29
|
+
const lastDream = agent.lastDreamAt ?? 0
|
|
30
|
+
if (lastDream + config.cooldownMinutes * 60_000 >= now) continue
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
await executeDreamCycle(agent.id, 'idle')
|
|
34
|
+
} catch (err: unknown) {
|
|
35
|
+
log.error(TAG, `Dream cycle failed for agent ${agent.id}: ${errorMessage(err)}`)
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
import crypto from 'crypto'
|
|
2
|
+
import type { DreamCycle, DreamCycleResult, DreamConfig, DreamTrigger, Agent } from '@/types'
|
|
3
|
+
import { DEFAULT_DREAM_CONFIG } from '@/types/dream'
|
|
4
|
+
import { getMemoryDb } from '@/lib/server/memory/memory-db'
|
|
5
|
+
import { saveDreamCycle } from '@/lib/server/memory/dream-cycles'
|
|
6
|
+
import { errorMessage, safeJsonParse } from '@/lib/shared-utils'
|
|
7
|
+
import { log } from '@/lib/server/logger'
|
|
8
|
+
|
|
9
|
+
const TAG = 'dream-service'
|
|
10
|
+
|
|
11
|
+
const EXEMPT_CATEGORIES = new Set(['daily_digest', 'consolidated_insight', 'dream_reflection'])
|
|
12
|
+
|
|
13
|
+
export function resolveDreamConfig(agent: Agent): DreamConfig {
|
|
14
|
+
return { ...DEFAULT_DREAM_CONFIG, ...agent.dreamConfig }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function runTier1Dream(
|
|
18
|
+
agentId: string,
|
|
19
|
+
config: DreamConfig,
|
|
20
|
+
): Promise<Partial<DreamCycleResult>> {
|
|
21
|
+
const memDb = getMemoryDb()
|
|
22
|
+
const memories = memDb.getByAgent(agentId, 500)
|
|
23
|
+
const now = Date.now()
|
|
24
|
+
const decayAgeMs = config.decayAgeDays * 86_400_000
|
|
25
|
+
const pruneMs = config.pruneThresholdDays * 86_400_000
|
|
26
|
+
|
|
27
|
+
let decayed = 0
|
|
28
|
+
let pruned = 0
|
|
29
|
+
let promoted = 0
|
|
30
|
+
const errors: string[] = []
|
|
31
|
+
|
|
32
|
+
for (const mem of memories) {
|
|
33
|
+
// Skip exempt categories
|
|
34
|
+
if (EXEMPT_CATEGORIES.has(mem.category)) continue
|
|
35
|
+
|
|
36
|
+
const tier = (mem.metadata as Record<string, unknown> | undefined)?.tier
|
|
37
|
+
// Skip archived entries
|
|
38
|
+
if (tier === 'archive') continue
|
|
39
|
+
|
|
40
|
+
const age = now - (mem.createdAt || mem.updatedAt || 0)
|
|
41
|
+
|
|
42
|
+
// Decay & Prune: old entries with no engagement
|
|
43
|
+
if (age > decayAgeMs && (mem.accessCount || 0) === 0 && (mem.reinforcementCount || 0) === 0) {
|
|
44
|
+
if (tier === 'working' && age > pruneMs) {
|
|
45
|
+
memDb.delete(mem.id)
|
|
46
|
+
pruned++
|
|
47
|
+
} else {
|
|
48
|
+
memDb.update(mem.id, {
|
|
49
|
+
metadata: { ...(mem.metadata as Record<string, unknown>), decayedInDream: true },
|
|
50
|
+
})
|
|
51
|
+
decayed++
|
|
52
|
+
}
|
|
53
|
+
continue
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Promote: working-tier entries with high engagement
|
|
57
|
+
if (
|
|
58
|
+
(tier === 'working' || tier === '' || tier === undefined) &&
|
|
59
|
+
(mem.accessCount || 0) >= 3 &&
|
|
60
|
+
(mem.reinforcementCount || 0) >= 2
|
|
61
|
+
) {
|
|
62
|
+
memDb.update(mem.id, {
|
|
63
|
+
metadata: { ...(mem.metadata as Record<string, unknown>), tier: 'durable' },
|
|
64
|
+
})
|
|
65
|
+
promoted++
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Dedup via maintain
|
|
70
|
+
let deduped = 0
|
|
71
|
+
try {
|
|
72
|
+
const maintenance = memDb.maintain({ dedupe: true, pruneWorking: false })
|
|
73
|
+
deduped = maintenance.deduped
|
|
74
|
+
} catch (err: unknown) {
|
|
75
|
+
const msg = `Dedup maintenance failed: ${errorMessage(err)}`
|
|
76
|
+
log.warn(TAG, msg)
|
|
77
|
+
errors.push(msg)
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return { decayed, pruned, promoted, deduped, errors }
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
interface Tier2Consolidation {
|
|
84
|
+
sourceIds: string[]
|
|
85
|
+
title: string
|
|
86
|
+
content: string
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
interface Tier2Reflection {
|
|
90
|
+
title: string
|
|
91
|
+
content: string
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
interface Tier2Flagged {
|
|
95
|
+
memoryId: string
|
|
96
|
+
reason: string
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
interface Tier2Response {
|
|
100
|
+
consolidations?: Tier2Consolidation[]
|
|
101
|
+
reflections?: Tier2Reflection[]
|
|
102
|
+
flagged?: Tier2Flagged[]
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export async function runTier2Dream(
|
|
106
|
+
agentId: string,
|
|
107
|
+
config: DreamConfig,
|
|
108
|
+
): Promise<Partial<DreamCycleResult>> {
|
|
109
|
+
const memDb = getMemoryDb()
|
|
110
|
+
const allMemories = memDb.getByAgent(agentId, config.tier2MaxMemories)
|
|
111
|
+
|
|
112
|
+
// Filter to durable tier and non-exempt entries
|
|
113
|
+
const candidates = allMemories.filter((m) => {
|
|
114
|
+
if (EXEMPT_CATEGORIES.has(m.category)) return false
|
|
115
|
+
const tier = (m.metadata as Record<string, unknown> | undefined)?.tier
|
|
116
|
+
return tier === 'durable' || tier === 'working' || tier === '' || tier === undefined
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
if (candidates.length < 3) {
|
|
120
|
+
return { consolidated: 0, reflections: [], memoriesReviewed: 0, errors: [] }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
const memoryLines = candidates.map((m) => {
|
|
124
|
+
const content = (m.content || '').slice(0, 300)
|
|
125
|
+
return `[${m.id}] [${m.category}] ${m.title}: ${content}`
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
const prompt = `You are reviewing your memories during a dream cycle to consolidate and reflect.
|
|
129
|
+
|
|
130
|
+
Review the following memories and respond with a JSON object:
|
|
131
|
+
{
|
|
132
|
+
"consolidations": [{ "sourceIds": ["id1", "id2"], "title": "...", "content": "..." }],
|
|
133
|
+
"reflections": [{ "title": "...", "content": "..." }],
|
|
134
|
+
"flagged": [{ "memoryId": "...", "reason": "..." }]
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
Rules:
|
|
138
|
+
- Consolidate 2-5 groups of overlapping memories into single entries
|
|
139
|
+
- Write 1-3 high-level reflections about patterns you notice
|
|
140
|
+
- Flag any outdated or contradictory memories
|
|
141
|
+
- Be concise. Each consolidation and reflection should be 1-3 sentences.
|
|
142
|
+
|
|
143
|
+
MEMORIES:
|
|
144
|
+
${memoryLines.join('\n')}`
|
|
145
|
+
|
|
146
|
+
const errors: string[] = []
|
|
147
|
+
let consolidated = 0
|
|
148
|
+
const reflectionTitles: string[] = []
|
|
149
|
+
|
|
150
|
+
try {
|
|
151
|
+
const { buildLLM } = await import('@/lib/server/build-llm')
|
|
152
|
+
const { llm } = await buildLLM({ agentId })
|
|
153
|
+
const { HumanMessage } = await import('@langchain/core/messages')
|
|
154
|
+
|
|
155
|
+
const response = await llm.invoke([new HumanMessage(prompt)])
|
|
156
|
+
const text = typeof response.content === 'string'
|
|
157
|
+
? response.content
|
|
158
|
+
: Array.isArray(response.content)
|
|
159
|
+
? response.content.map((b) => ('text' in b && typeof b.text === 'string' ? b.text : '')).join('')
|
|
160
|
+
: ''
|
|
161
|
+
|
|
162
|
+
// Extract JSON from response (handle markdown code blocks)
|
|
163
|
+
const jsonMatch = text.match(/\{[\s\S]*\}/)
|
|
164
|
+
const parsed = safeJsonParse<Tier2Response>(jsonMatch?.[0] ?? '', {})
|
|
165
|
+
|
|
166
|
+
// Process consolidations
|
|
167
|
+
if (Array.isArray(parsed.consolidations)) {
|
|
168
|
+
for (const c of parsed.consolidations) {
|
|
169
|
+
if (!c.title || !c.content || !Array.isArray(c.sourceIds)) continue
|
|
170
|
+
const validSourceIds = c.sourceIds.filter((id): id is string => typeof id === 'string' && id.trim().length > 0)
|
|
171
|
+
memDb.add({
|
|
172
|
+
agentId,
|
|
173
|
+
sessionId: null,
|
|
174
|
+
category: 'consolidated_insight',
|
|
175
|
+
title: c.title,
|
|
176
|
+
content: c.content,
|
|
177
|
+
linkedMemoryIds: validSourceIds,
|
|
178
|
+
metadata: { tier: 'durable', origin: 'dream' },
|
|
179
|
+
})
|
|
180
|
+
consolidated++
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Process reflections
|
|
185
|
+
if (Array.isArray(parsed.reflections)) {
|
|
186
|
+
for (const r of parsed.reflections) {
|
|
187
|
+
if (!r.title || !r.content) continue
|
|
188
|
+
memDb.add({
|
|
189
|
+
agentId,
|
|
190
|
+
sessionId: null,
|
|
191
|
+
category: 'dream_reflection',
|
|
192
|
+
title: r.title,
|
|
193
|
+
content: r.content,
|
|
194
|
+
metadata: { tier: 'durable', origin: 'dream' },
|
|
195
|
+
})
|
|
196
|
+
reflectionTitles.push(r.title)
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Process flagged
|
|
201
|
+
if (Array.isArray(parsed.flagged)) {
|
|
202
|
+
for (const f of parsed.flagged) {
|
|
203
|
+
if (!f.memoryId || !f.reason) continue
|
|
204
|
+
const existing = memDb.get(f.memoryId)
|
|
205
|
+
if (existing) {
|
|
206
|
+
memDb.update(f.memoryId, {
|
|
207
|
+
metadata: {
|
|
208
|
+
...(existing.metadata as Record<string, unknown>),
|
|
209
|
+
flaggedInDream: true,
|
|
210
|
+
flagReason: f.reason,
|
|
211
|
+
},
|
|
212
|
+
})
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
} catch (err: unknown) {
|
|
217
|
+
errors.push(errorMessage(err))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
return {
|
|
221
|
+
consolidated,
|
|
222
|
+
reflections: reflectionTitles,
|
|
223
|
+
memoriesReviewed: candidates.length,
|
|
224
|
+
errors,
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export async function executeDreamCycle(
|
|
229
|
+
agentId: string,
|
|
230
|
+
trigger: DreamTrigger,
|
|
231
|
+
): Promise<DreamCycle> {
|
|
232
|
+
const { loadAgents, patchAgent, logActivity } = await import('@/lib/server/storage')
|
|
233
|
+
const agents = loadAgents()
|
|
234
|
+
const agent = agents[agentId]
|
|
235
|
+
if (!agent) throw new Error(`Agent ${agentId} not found`)
|
|
236
|
+
|
|
237
|
+
const config = resolveDreamConfig(agent)
|
|
238
|
+
|
|
239
|
+
// Check cooldown
|
|
240
|
+
if (agent.lastDreamAt && Date.now() - agent.lastDreamAt < config.cooldownMinutes * 60_000) {
|
|
241
|
+
throw new Error('Dream on cooldown')
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const cycle: DreamCycle = {
|
|
245
|
+
id: crypto.randomUUID(),
|
|
246
|
+
agentId,
|
|
247
|
+
status: 'running',
|
|
248
|
+
trigger,
|
|
249
|
+
startedAt: Date.now(),
|
|
250
|
+
}
|
|
251
|
+
saveDreamCycle(cycle)
|
|
252
|
+
|
|
253
|
+
try {
|
|
254
|
+
// Tier 1: deterministic server-side operations
|
|
255
|
+
const tier1 = await runTier1Dream(agentId, config)
|
|
256
|
+
|
|
257
|
+
// Tier 2: LLM-driven reflection (optional)
|
|
258
|
+
let tier2: Partial<DreamCycleResult> = {}
|
|
259
|
+
if (config.tier2Enabled) {
|
|
260
|
+
tier2 = await runTier2Dream(agentId, config)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const result: DreamCycleResult = {
|
|
264
|
+
decayed: tier1.decayed ?? 0,
|
|
265
|
+
pruned: tier1.pruned ?? 0,
|
|
266
|
+
promoted: tier1.promoted ?? 0,
|
|
267
|
+
deduped: tier1.deduped ?? 0,
|
|
268
|
+
consolidated: tier2.consolidated ?? 0,
|
|
269
|
+
reflections: tier2.reflections ?? [],
|
|
270
|
+
memoriesReviewed: tier2.memoriesReviewed ?? 0,
|
|
271
|
+
durationMs: Date.now() - cycle.startedAt,
|
|
272
|
+
errors: [...(tier1.errors ?? []), ...(tier2.errors ?? [])],
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
cycle.status = 'completed'
|
|
276
|
+
cycle.completedAt = Date.now()
|
|
277
|
+
cycle.result = result
|
|
278
|
+
saveDreamCycle(cycle)
|
|
279
|
+
|
|
280
|
+
// Update agent
|
|
281
|
+
patchAgent(agentId, (current) => {
|
|
282
|
+
if (!current) return null
|
|
283
|
+
return {
|
|
284
|
+
...current,
|
|
285
|
+
lastDreamAt: Date.now(),
|
|
286
|
+
dreamCycleCount: (current.dreamCycleCount || 0) + 1,
|
|
287
|
+
}
|
|
288
|
+
})
|
|
289
|
+
|
|
290
|
+
logActivity({
|
|
291
|
+
entityType: 'agent',
|
|
292
|
+
entityId: agentId,
|
|
293
|
+
action: 'dream_completed',
|
|
294
|
+
actor: 'system',
|
|
295
|
+
summary: `Dream cycle completed: ${result.decayed} decayed, ${result.pruned} pruned, ${result.consolidated} consolidated`,
|
|
296
|
+
})
|
|
297
|
+
|
|
298
|
+
log.info(TAG, `Dream cycle completed for agent ${agentId}`, {
|
|
299
|
+
trigger,
|
|
300
|
+
decayed: result.decayed,
|
|
301
|
+
pruned: result.pruned,
|
|
302
|
+
promoted: result.promoted,
|
|
303
|
+
consolidated: result.consolidated,
|
|
304
|
+
})
|
|
305
|
+
|
|
306
|
+
return cycle
|
|
307
|
+
} catch (err: unknown) {
|
|
308
|
+
cycle.status = 'failed'
|
|
309
|
+
cycle.completedAt = Date.now()
|
|
310
|
+
cycle.error = errorMessage(err)
|
|
311
|
+
saveDreamCycle(cycle)
|
|
312
|
+
log.error(TAG, `Dream cycle failed for agent ${agentId}: ${errorMessage(err)}`)
|
|
313
|
+
throw err
|
|
314
|
+
}
|
|
315
|
+
}
|
|
@@ -146,6 +146,20 @@ function metadataNumber(entry: MemoryEntry, key: string): number | null {
|
|
|
146
146
|
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
|
147
147
|
}
|
|
148
148
|
|
|
149
|
+
function knowledgeChunkHashScope(category: string, metadata: unknown): string {
|
|
150
|
+
if (category !== 'knowledge' || !metadata || typeof metadata !== 'object') return category
|
|
151
|
+
const sourceIdValue = (metadata as Record<string, unknown>).sourceId
|
|
152
|
+
const sourceId = typeof sourceIdValue === 'string'
|
|
153
|
+
? sourceIdValue.trim()
|
|
154
|
+
: ''
|
|
155
|
+
if (!sourceId) return category
|
|
156
|
+
const chunkIndexValue = (metadata as Record<string, unknown>).chunkIndex
|
|
157
|
+
const chunkIndex = typeof chunkIndexValue === 'number'
|
|
158
|
+
? chunkIndexValue
|
|
159
|
+
: null
|
|
160
|
+
return `${category}:${sourceId}:${chunkIndex ?? 'legacy'}`
|
|
161
|
+
}
|
|
162
|
+
|
|
149
163
|
function followUpSalienceMultiplier(entry: MemoryEntry, nowTs: number): number {
|
|
150
164
|
if (entry.category !== 'reflection/open_loop') return 1
|
|
151
165
|
const resolvedAt = metadataNumber(entry, 'resolvedAt')
|
|
@@ -664,7 +678,7 @@ function initDb() {
|
|
|
664
678
|
update: db.prepare(`
|
|
665
679
|
UPDATE memories
|
|
666
680
|
SET agentId=?, sessionId=?, category=?, title=?, content=?, metadata=?, embedding=?,
|
|
667
|
-
"references"=?, filePaths=?, image=?, imagePath=?, linkedMemoryIds=?, pinned=?, sharedWith=?, updatedAt=?
|
|
681
|
+
"references"=?, filePaths=?, image=?, imagePath=?, linkedMemoryIds=?, pinned=?, sharedWith=?, contentHash=?, updatedAt=?
|
|
668
682
|
WHERE id=?
|
|
669
683
|
`),
|
|
670
684
|
delete: db.prepare(`DELETE FROM memories WHERE id=?`),
|
|
@@ -677,6 +691,13 @@ function initDb() {
|
|
|
677
691
|
listAll: db.prepare(`SELECT * FROM memories ORDER BY updatedAt DESC LIMIT ?`),
|
|
678
692
|
listByAgent: db.prepare(`SELECT * FROM memories WHERE agentId=? ORDER BY updatedAt DESC LIMIT ?`),
|
|
679
693
|
listByAgentOrShared: db.prepare(`SELECT * FROM memories WHERE agentId=? OR sharedWith LIKE ? ORDER BY updatedAt DESC LIMIT ?`),
|
|
694
|
+
listByCategoryAll: db.prepare(`SELECT * FROM memories WHERE category=? ORDER BY updatedAt DESC LIMIT ?`),
|
|
695
|
+
listByCategoryAgentOrShared: db.prepare(`SELECT * FROM memories WHERE category=? AND (agentId=? OR sharedWith LIKE ?) ORDER BY updatedAt DESC LIMIT ?`),
|
|
696
|
+
listKnowledgeSourceChunks: db.prepare(`
|
|
697
|
+
SELECT * FROM memories
|
|
698
|
+
WHERE category='knowledge' AND json_extract(metadata, '$.sourceId') = ?
|
|
699
|
+
ORDER BY COALESCE(json_extract(metadata, '$.chunkIndex'), 0) ASC, createdAt ASC
|
|
700
|
+
`),
|
|
680
701
|
listPinnedByAgent: db.prepare(`SELECT * FROM memories WHERE pinned = 1 AND agentId = ? ORDER BY updatedAt DESC LIMIT ?`),
|
|
681
702
|
listPinnedAll: db.prepare(`SELECT * FROM memories WHERE pinned = 1 ORDER BY updatedAt DESC LIMIT ?`),
|
|
682
703
|
search: db.prepare(`
|
|
@@ -803,7 +824,7 @@ function initDb() {
|
|
|
803
824
|
const category = data.category || 'note'
|
|
804
825
|
const title = data.title || 'Untitled'
|
|
805
826
|
const content = data.content || ''
|
|
806
|
-
const contentHash = computeContentHash(category, content)
|
|
827
|
+
const contentHash = computeContentHash(knowledgeChunkHashScope(category, data.metadata), content)
|
|
807
828
|
|
|
808
829
|
// Content-hash dedup: if same content already exists for this agent, reinforce instead of duplicating
|
|
809
830
|
const agentId = data.agentId || null
|
|
@@ -896,6 +917,7 @@ function initDb() {
|
|
|
896
917
|
const now = Date.now()
|
|
897
918
|
const pinnedVal = merged.pinned ? 1 : 0
|
|
898
919
|
const sharedWithVal = Array.isArray(merged.sharedWith) && merged.sharedWith.length ? JSON.stringify(merged.sharedWith) : null
|
|
920
|
+
const nextContentHash = computeContentHash(knowledgeChunkHashScope(merged.category, merged.metadata), merged.content)
|
|
899
921
|
stmts.update.run(
|
|
900
922
|
merged.agentId || null, merged.sessionId || null,
|
|
901
923
|
merged.category, merged.title, merged.content,
|
|
@@ -908,6 +930,7 @@ function initDb() {
|
|
|
908
930
|
nextLinked.length ? JSON.stringify(nextLinked) : null,
|
|
909
931
|
pinnedVal,
|
|
910
932
|
sharedWithVal,
|
|
933
|
+
nextContentHash,
|
|
911
934
|
now, id,
|
|
912
935
|
)
|
|
913
936
|
|
|
@@ -1215,6 +1238,18 @@ function initDb() {
|
|
|
1215
1238
|
return rows.map(rowToEntry)
|
|
1216
1239
|
},
|
|
1217
1240
|
|
|
1241
|
+
listByCategory(category: string, agentId?: string, limit = 500): MemoryEntry[] {
|
|
1242
|
+
const safeLimit = Math.max(1, Math.min(10_000, Math.trunc(limit)))
|
|
1243
|
+
const rows = agentId
|
|
1244
|
+
? stmts.listByCategoryAgentOrShared.all(category, agentId, `%"${agentId}"%`, safeLimit) as Record<string, unknown>[]
|
|
1245
|
+
: stmts.listByCategoryAll.all(category, safeLimit) as Record<string, unknown>[]
|
|
1246
|
+
return rows.map(rowToEntry)
|
|
1247
|
+
},
|
|
1248
|
+
|
|
1249
|
+
listKnowledgeSourceChunks(sourceId: string): MemoryEntry[] {
|
|
1250
|
+
return (stmts.listKnowledgeSourceChunks.all(sourceId) as Record<string, unknown>[]).map(rowToEntry)
|
|
1251
|
+
},
|
|
1252
|
+
|
|
1218
1253
|
listPinned(agentId?: string, limit = 20): MemoryEntry[] {
|
|
1219
1254
|
const safeLimit = Math.max(1, Math.min(100, Math.trunc(limit)))
|
|
1220
1255
|
const rows = agentId
|
|
@@ -48,6 +48,7 @@ import { AGENT_TURN_TIMEOUT_MS, cleanText, now } from '@/lib/server/protocols/pr
|
|
|
48
48
|
import type { ProtocolAgentTurnResult, ProtocolRunDeps } from '@/lib/server/protocols/protocol-types'
|
|
49
49
|
import { normalizeProtocolRun } from '@/lib/server/protocols/protocol-normalization'
|
|
50
50
|
import { persistChatroomInteractionMemory } from '@/lib/server/chatrooms/chatroom-memory-bridge'
|
|
51
|
+
import { selectKnowledgeCitations } from '@/lib/server/knowledge-sources'
|
|
51
52
|
|
|
52
53
|
// ---- Zod schema ----
|
|
53
54
|
|
|
@@ -262,6 +263,10 @@ export async function defaultExecuteAgentTurn(params: {
|
|
|
262
263
|
])
|
|
263
264
|
const rawText = result.finalResponse || result.fullText || ''
|
|
264
265
|
const text = stripHiddenControlTokens(rawText)
|
|
266
|
+
const grounding = selectKnowledgeCitations({
|
|
267
|
+
responseText: text,
|
|
268
|
+
retrievalTrace: result.knowledgeRetrievalTrace || null,
|
|
269
|
+
})
|
|
265
270
|
if (text.trim() && !shouldSuppressHiddenControlText(rawText)) {
|
|
266
271
|
appendSyntheticSessionMessage(syntheticSession.id, 'assistant', text)
|
|
267
272
|
// Persist interaction to agent memory (fire-and-forget)
|
|
@@ -278,6 +283,8 @@ export async function defaultExecuteAgentTurn(params: {
|
|
|
278
283
|
return {
|
|
279
284
|
text: cleanText(text, 6_000),
|
|
280
285
|
toolEvents: result.toolEvents || [],
|
|
286
|
+
citations: grounding.citations,
|
|
287
|
+
retrievalTrace: grounding.retrievalTrace,
|
|
281
288
|
}
|
|
282
289
|
} catch (err: unknown) {
|
|
283
290
|
lastError = err
|
|
@@ -47,6 +47,13 @@ const protocolExecutionState = hmrSingleton('__swarmclaw_protocol_engine_executi
|
|
|
47
47
|
pendingRunIds: new Set<string>(),
|
|
48
48
|
}))
|
|
49
49
|
|
|
50
|
+
function shouldYieldBetweenProtocolSteps(deps?: ProtocolRunDeps): boolean {
|
|
51
|
+
// Dependency-injected runs are used for deterministic tests and nested orchestration.
|
|
52
|
+
// Keep the production scheduler cooperative, but let injected executions drain
|
|
53
|
+
// their step loop without requiring multiple timer turns.
|
|
54
|
+
return !deps
|
|
55
|
+
}
|
|
56
|
+
|
|
50
57
|
// ---- Scheduling/Recovery (G10) ----
|
|
51
58
|
|
|
52
59
|
export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDeps): boolean {
|
|
@@ -54,7 +61,7 @@ export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDep
|
|
|
54
61
|
if (!normalizedId) return false
|
|
55
62
|
if (protocolExecutionState.pendingRunIds.has(normalizedId)) return false
|
|
56
63
|
protocolExecutionState.pendingRunIds.add(normalizedId)
|
|
57
|
-
|
|
64
|
+
const invoke = () => {
|
|
58
65
|
void runProtocolRun(normalizedId, deps)
|
|
59
66
|
.catch((err: unknown) => {
|
|
60
67
|
log.warn(TAG, `execution failed for ${normalizedId}: ${errorMessage(err)}`)
|
|
@@ -62,7 +69,9 @@ export function requestProtocolRunExecution(runId: string, deps?: ProtocolRunDep
|
|
|
62
69
|
.finally(() => {
|
|
63
70
|
protocolExecutionState.pendingRunIds.delete(normalizedId)
|
|
64
71
|
})
|
|
65
|
-
}
|
|
72
|
+
}
|
|
73
|
+
if (deps) queueMicrotask(invoke)
|
|
74
|
+
else setTimeout(invoke, 0)
|
|
66
75
|
return true
|
|
67
76
|
}
|
|
68
77
|
|
|
@@ -115,8 +124,9 @@ export function wakeProtocolRunFromTaskCompletion(taskId: string, deps?: Protoco
|
|
|
115
124
|
}
|
|
116
125
|
|
|
117
126
|
export function ensureProtocolEngineRecovered(deps?: ProtocolRunDeps): void {
|
|
118
|
-
|
|
119
|
-
protocolRecoveryState.completed
|
|
127
|
+
const bypassCompletedGuard = Boolean(deps)
|
|
128
|
+
if (!bypassCompletedGuard && protocolRecoveryState.completed) return
|
|
129
|
+
if (!bypassCompletedGuard) protocolRecoveryState.completed = true
|
|
120
130
|
const runs = Object.values(loadProtocolRuns()).map((entry) => normalizeProtocolRun(entry))
|
|
121
131
|
for (const run of runs) {
|
|
122
132
|
if (run.parentRunId) {
|
|
@@ -321,8 +331,11 @@ export async function runProtocolRun(runId: string, deps?: ProtocolRunDeps): Pro
|
|
|
321
331
|
appendProtocolEvent(run.id, { type: 'failed', summary: `Exceeded maximum step iterations (${MAX_STEP_ITERATIONS}).` }, deps)
|
|
322
332
|
break
|
|
323
333
|
}
|
|
324
|
-
|
|
325
|
-
|
|
334
|
+
if (shouldYieldBetweenProtocolSteps(deps)) {
|
|
335
|
+
// Yield between steps in the fire-and-forget runtime so I/O, HTTP responses,
|
|
336
|
+
// and timers can run.
|
|
337
|
+
await new Promise(r => setTimeout(r, 0))
|
|
338
|
+
}
|
|
326
339
|
const latest = loadProtocolRunById(run.id)
|
|
327
340
|
if (!latest) return null
|
|
328
341
|
if (latest.status === 'paused' || latest.status === 'cancelled' || latest.status === 'archived' || latest.status === 'completed') {
|
|
@@ -96,6 +96,105 @@ test('protocol-service creates a hidden transcript run and completes a structure
|
|
|
96
96
|
assert.equal(output.eventTypes.filter((type) => type === 'phase_completed').length, 5)
|
|
97
97
|
})
|
|
98
98
|
|
|
99
|
+
test('protocol-service persists citations on participant and artifact events when grounded responses are provided', () => {
|
|
100
|
+
const output = runWithTempDataDir<{
|
|
101
|
+
participantCitationCounts: number[]
|
|
102
|
+
artifactCitationCounts: number[]
|
|
103
|
+
selectorStatuses: string[]
|
|
104
|
+
}>(`
|
|
105
|
+
const storageMod = await import('./src/lib/server/storage')
|
|
106
|
+
const protocolsMod = await import('./src/lib/server/protocols/protocol-service')
|
|
107
|
+
const storage = storageMod.default || storageMod
|
|
108
|
+
const protocols = protocolsMod.default || protocolsMod
|
|
109
|
+
|
|
110
|
+
storage.upsertStoredItem('agents', 'agentA', {
|
|
111
|
+
id: 'agentA',
|
|
112
|
+
name: 'Agent A',
|
|
113
|
+
provider: 'ollama',
|
|
114
|
+
model: 'test-model',
|
|
115
|
+
systemPrompt: 'test',
|
|
116
|
+
createdAt: 1,
|
|
117
|
+
updatedAt: 1,
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
const groundedCitation = {
|
|
121
|
+
sourceId: 'source-1',
|
|
122
|
+
sourceTitle: 'Gateway Runbook',
|
|
123
|
+
sourceKind: 'manual',
|
|
124
|
+
sourceUrl: null,
|
|
125
|
+
sourceLabel: null,
|
|
126
|
+
chunkId: 'chunk-1',
|
|
127
|
+
chunkIndex: 0,
|
|
128
|
+
chunkCount: 1,
|
|
129
|
+
charStart: 0,
|
|
130
|
+
charEnd: 48,
|
|
131
|
+
sectionLabel: null,
|
|
132
|
+
snippet: 'Use blue green deployment for gateway changes.',
|
|
133
|
+
whyMatched: 'Matched query terms: gateway, deployment',
|
|
134
|
+
score: 0.92,
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const run = protocols.createProtocolRun({
|
|
138
|
+
title: 'Grounded structured run',
|
|
139
|
+
templateId: 'single_agent_structured_run',
|
|
140
|
+
participantAgentIds: ['agentA'],
|
|
141
|
+
facilitatorAgentId: 'agentA',
|
|
142
|
+
autoStart: false,
|
|
143
|
+
}, { now: () => 1000 })
|
|
144
|
+
|
|
145
|
+
await protocols.runProtocolRun(run.id, {
|
|
146
|
+
now: () => 2000,
|
|
147
|
+
executeAgentTurn: async ({ phase }) => {
|
|
148
|
+
if (phase.kind === 'summarize') {
|
|
149
|
+
return {
|
|
150
|
+
text: 'Blue green deployment keeps the rollback path simple.',
|
|
151
|
+
toolEvents: [],
|
|
152
|
+
citations: [groundedCitation],
|
|
153
|
+
retrievalTrace: {
|
|
154
|
+
query: 'gateway deployment',
|
|
155
|
+
scope: 'source_knowledge',
|
|
156
|
+
hits: [groundedCitation],
|
|
157
|
+
retrievedAt: 2000,
|
|
158
|
+
selectorStatus: 'selected',
|
|
159
|
+
},
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
if (phase.kind === 'round_robin') {
|
|
163
|
+
return {
|
|
164
|
+
text: 'Use blue green deployment for the gateway rollout.',
|
|
165
|
+
toolEvents: [],
|
|
166
|
+
citations: [groundedCitation],
|
|
167
|
+
retrievalTrace: {
|
|
168
|
+
query: 'gateway rollout',
|
|
169
|
+
scope: 'source_knowledge',
|
|
170
|
+
hits: [groundedCitation],
|
|
171
|
+
retrievedAt: 2000,
|
|
172
|
+
selectorStatus: 'selected',
|
|
173
|
+
},
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return { text: 'Opened the session.', toolEvents: [] }
|
|
177
|
+
},
|
|
178
|
+
})
|
|
179
|
+
|
|
180
|
+
const detail = protocols.getProtocolRunDetail(run.id)
|
|
181
|
+
const participantEvents = (detail?.events || []).filter((event) => event.type === 'participant_response')
|
|
182
|
+
const artifactEvents = (detail?.events || []).filter((event) => event.type === 'artifact_emitted')
|
|
183
|
+
|
|
184
|
+
console.log(JSON.stringify({
|
|
185
|
+
participantCitationCounts: participantEvents.map((event) => event.citations?.length || 0),
|
|
186
|
+
artifactCitationCounts: artifactEvents.map((event) => event.citations?.length || 0),
|
|
187
|
+
selectorStatuses: participantEvents
|
|
188
|
+
.map((event) => event.retrievalTrace?.selectorStatus)
|
|
189
|
+
.filter((value) => typeof value === 'string'),
|
|
190
|
+
}))
|
|
191
|
+
`, { prefix: 'swarmclaw-protocol-grounding-' })
|
|
192
|
+
|
|
193
|
+
assert.deepEqual(output.participantCitationCounts, [1])
|
|
194
|
+
assert.deepEqual(output.artifactCitationCounts, [1])
|
|
195
|
+
assert.deepEqual(output.selectorStatuses, [])
|
|
196
|
+
})
|
|
197
|
+
|
|
99
198
|
test('protocol-service supports custom template CRUD and operator actions', () => {
|
|
100
199
|
const output = runWithTempDataDir<{
|
|
101
200
|
createdTemplateId: string | null
|