@swarmclawai/swarmclaw 1.3.3 → 1.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -76
- package/package.json +1 -1
- package/skills/swarmclaw.md +17 -0
- package/src/app/api/agents/[id]/dream/route.ts +45 -0
- package/src/app/api/knowledge/[id]/route.ts +48 -49
- package/src/app/api/knowledge/hygiene/route.ts +13 -0
- package/src/app/api/knowledge/route.ts +70 -42
- package/src/app/api/knowledge/sources/[id]/archive/route.ts +15 -0
- package/src/app/api/knowledge/sources/[id]/restore/route.ts +10 -0
- package/src/app/api/knowledge/sources/[id]/route.ts +1 -0
- package/src/app/api/knowledge/sources/[id]/supersede/route.ts +26 -0
- package/src/app/api/knowledge/sources/[id]/sync/route.ts +17 -0
- package/src/app/api/knowledge/sources/route.ts +1 -0
- package/src/app/api/knowledge/upload/route.ts +3 -51
- package/src/app/api/memory/dream/[id]/route.ts +19 -0
- package/src/app/api/memory/dream/route.ts +34 -0
- package/src/app/knowledge/layout.tsx +1 -1
- package/src/app/knowledge/page.tsx +2 -22
- package/src/app/protocols/page.tsx +21 -2
- package/src/cli/index.js +16 -0
- package/src/cli/spec.js +5 -0
- package/src/components/agents/agent-sheet.tsx +65 -0
- package/src/components/chat/message-bubble.tsx +10 -0
- package/src/components/knowledge/grounding-panel.tsx +99 -0
- package/src/components/knowledge/knowledge-detail.tsx +402 -0
- package/src/components/knowledge/knowledge-list.tsx +351 -126
- package/src/components/knowledge/knowledge-sheet.tsx +208 -119
- package/src/components/memory/dream-history.tsx +155 -0
- package/src/components/memory/memory-card.tsx +7 -0
- package/src/components/memory/memory-detail.tsx +46 -0
- package/src/components/runs/run-list.tsx +23 -0
- package/src/lib/providers/cli-utils.ts +3 -4
- package/src/lib/providers/index.ts +12 -22
- package/src/lib/providers/openclaw.ts +1 -2
- package/src/lib/server/agents/subagent-swarm.ts +2 -7
- package/src/lib/server/api-routes.test.ts +43 -2
- package/src/lib/server/chat-execution/chat-execution-grounding.test.ts +127 -0
- package/src/lib/server/chat-execution/chat-execution-types.ts +8 -1
- package/src/lib/server/chat-execution/chat-execution.ts +1 -0
- package/src/lib/server/chat-execution/chat-turn-finalization.ts +23 -6
- package/src/lib/server/chat-execution/chat-turn-stream-execution.ts +6 -1
- package/src/lib/server/chat-execution/post-stream-finalization.ts +15 -3
- package/src/lib/server/chat-execution/prompt-builder.ts +4 -6
- package/src/lib/server/chat-execution/prompt-sections.ts +29 -3
- package/src/lib/server/chat-execution/stream-agent-chat.ts +6 -1
- package/src/lib/server/connectors/openclaw.ts +1 -2
- package/src/lib/server/execution-engine/task-attempt.ts +8 -2
- package/src/lib/server/knowledge-import.ts +159 -0
- package/src/lib/server/knowledge-sources.test.ts +215 -0
- package/src/lib/server/knowledge-sources.ts +1266 -0
- package/src/lib/server/memory/dream-cycles.ts +49 -0
- package/src/lib/server/memory/dream-idle-callback.ts +38 -0
- package/src/lib/server/memory/dream-service.ts +315 -0
- package/src/lib/server/memory/memory-db.ts +37 -2
- package/src/lib/server/protocols/protocol-agent-turn.ts +7 -0
- package/src/lib/server/protocols/protocol-run-lifecycle.ts +19 -6
- package/src/lib/server/protocols/protocol-service.test.ts +99 -0
- package/src/lib/server/protocols/protocol-step-helpers.ts +7 -1
- package/src/lib/server/protocols/protocol-step-processors.ts +16 -3
- package/src/lib/server/protocols/protocol-types.ts +4 -0
- package/src/lib/server/provider-health.ts +2 -7
- package/src/lib/server/runtime/daemon-state/core.ts +6 -1
- package/src/lib/server/runtime/run-ledger.test.ts +120 -0
- package/src/lib/server/runtime/run-ledger.ts +27 -1
- package/src/lib/server/runtime/session-run-manager/drain.ts +5 -0
- package/src/lib/server/runtime/session-run-manager/state.ts +19 -2
- package/src/lib/server/storage-normalization.ts +5 -0
- package/src/lib/server/storage.ts +16 -1
- package/src/stores/slices/ui-slice.ts +4 -0
- package/src/types/agent.ts +7 -0
- package/src/types/dream.ts +45 -0
- package/src/types/index.ts +1 -0
- package/src/types/message.ts +3 -0
- package/src/types/misc.ts +131 -0
- package/src/types/protocol.ts +4 -0
- package/src/types/run.ts +4 -1
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* usage recording, forced external service summary, capability hooks,
|
|
6
6
|
* and OpenClaw sync.
|
|
7
7
|
*/
|
|
8
|
-
import type { Session, UsageRecord } from '@/types'
|
|
8
|
+
import type { KnowledgeRetrievalTrace, Session, UsageRecord } from '@/types'
|
|
9
9
|
import { log } from '@/lib/server/logger'
|
|
10
10
|
import type { ChatTurnState } from '@/lib/server/chat-execution/chat-turn-state'
|
|
11
11
|
|
|
@@ -51,6 +51,7 @@ export interface PostStreamResult {
|
|
|
51
51
|
fullText: string
|
|
52
52
|
finalResponse: string
|
|
53
53
|
toolEvents: import('@/types').MessageToolEvent[]
|
|
54
|
+
knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
export interface FinalizeStreamResultOpts {
|
|
@@ -70,6 +71,7 @@ export interface FinalizeStreamResultOpts {
|
|
|
70
71
|
cleanup: () => Promise<void>
|
|
71
72
|
runId: string
|
|
72
73
|
classification?: MessageClassification | null
|
|
74
|
+
knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
|
|
73
75
|
}
|
|
74
76
|
|
|
75
77
|
export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Promise<PostStreamResult> {
|
|
@@ -138,7 +140,12 @@ export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Prom
|
|
|
138
140
|
const finalResponse = await resolveAndSummarize()
|
|
139
141
|
await emitLlmOutputHook(finalResponse)
|
|
140
142
|
await cleanup()
|
|
141
|
-
return {
|
|
143
|
+
return {
|
|
144
|
+
fullText: state.fullText,
|
|
145
|
+
finalResponse,
|
|
146
|
+
toolEvents: state.streamedToolEvents,
|
|
147
|
+
knowledgeRetrievalTrace: opts.knowledgeRetrievalTrace || null,
|
|
148
|
+
}
|
|
142
149
|
}
|
|
143
150
|
|
|
144
151
|
// Strip leaked classification JSON from model output (e.g. `{ "isDeliverableTask": true, ... }`)
|
|
@@ -212,5 +219,10 @@ export async function finalizeStreamResult(opts: FinalizeStreamResultOpts): Prom
|
|
|
212
219
|
|
|
213
220
|
await cleanup()
|
|
214
221
|
|
|
215
|
-
return {
|
|
222
|
+
return {
|
|
223
|
+
fullText: state.fullText,
|
|
224
|
+
finalResponse,
|
|
225
|
+
toolEvents: state.streamedToolEvents,
|
|
226
|
+
knowledgeRetrievalTrace: opts.knowledgeRetrievalTrace || null,
|
|
227
|
+
}
|
|
216
228
|
}
|
|
@@ -11,6 +11,8 @@ import {
|
|
|
11
11
|
collectCapabilityOperatingGuidance,
|
|
12
12
|
} from '@/lib/server/native-capabilities'
|
|
13
13
|
import { getExtensionManager } from '@/lib/server/extensions'
|
|
14
|
+
import { loadAgents } from '../storage'
|
|
15
|
+
import { resolveTeam } from '../agents/team-resolution'
|
|
14
16
|
import {
|
|
15
17
|
getEnabledToolPlanningView,
|
|
16
18
|
getToolsForCapability,
|
|
@@ -44,12 +46,8 @@ function buildExtensionCapabilityLines(enabledExtensions: string[], opts?: { del
|
|
|
44
46
|
// CLI team hint — if teammates run CLI providers, mention their strengths
|
|
45
47
|
if (opts.agentId) {
|
|
46
48
|
try {
|
|
47
|
-
|
|
48
|
-
const
|
|
49
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
50
|
-
const { resolveTeam } = require('../agents/team-resolution')
|
|
51
|
-
const agents = loadAgents() as Record<string, Record<string, unknown>>
|
|
52
|
-
const team = resolveTeam(opts.agentId, agents)
|
|
49
|
+
const agents = loadAgents()
|
|
50
|
+
const team = resolveTeam(opts.agentId, agents as Record<string, import('@/types').Agent>)
|
|
53
51
|
if (team.mode === 'team') {
|
|
54
52
|
const cliTeammates: string[] = []
|
|
55
53
|
const allMembers = [...(team.coordinator ? [team.coordinator] : []), ...team.peers, ...team.directReports]
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import fs from 'node:fs'
|
|
10
10
|
import path from 'node:path'
|
|
11
|
-
import type { Session, Agent } from '@/types'
|
|
11
|
+
import type { KnowledgeRetrievalTrace, Session, Agent } from '@/types'
|
|
12
12
|
import type { PromptMode } from '@/lib/server/chat-execution/prompt-mode'
|
|
13
13
|
import type { MessageClassification } from '@/lib/server/chat-execution/message-classifier'
|
|
14
14
|
import type { ActiveProjectContext } from '@/lib/server/project-context'
|
|
@@ -428,6 +428,7 @@ export function buildSuggestionsSection(
|
|
|
428
428
|
export interface ProactiveMemoryResult {
|
|
429
429
|
section: string | null
|
|
430
430
|
injectedIds: Record<string, number>
|
|
431
|
+
knowledgeTrace?: KnowledgeRetrievalTrace | null
|
|
431
432
|
}
|
|
432
433
|
|
|
433
434
|
export async function buildProactiveMemorySection(
|
|
@@ -438,22 +439,28 @@ export async function buildProactiveMemorySection(
|
|
|
438
439
|
isMinimalPrompt: boolean,
|
|
439
440
|
currentThreadRecallRequest: boolean,
|
|
440
441
|
): Promise<ProactiveMemoryResult> {
|
|
441
|
-
const noResult: ProactiveMemoryResult = { section: null, injectedIds: {} }
|
|
442
|
+
const noResult: ProactiveMemoryResult = { section: null, injectedIds: {}, knowledgeTrace: null }
|
|
442
443
|
if (isMinimalPrompt || !session.agentId || currentThreadRecallRequest || message.length <= 12) return noResult
|
|
443
444
|
if (!agent?.proactiveMemory) return noResult
|
|
444
445
|
try {
|
|
445
446
|
const { getMemoryDb } = await import('@/lib/server/memory/memory-db')
|
|
446
447
|
const { buildSessionMemoryScopeFilter } = await import('@/lib/server/memory/session-memory-scope')
|
|
448
|
+
const { buildKnowledgeRetrievalTrace } = await import('@/lib/server/knowledge-sources')
|
|
447
449
|
const memDb = getMemoryDb()
|
|
448
450
|
const recalled = memDb.search(message, session.agentId, {
|
|
449
451
|
scope: buildSessionMemoryScopeFilter(session, agent.memoryScopeMode || null, activeProjectRoot),
|
|
450
452
|
})
|
|
453
|
+
const knowledgeTrace = await buildKnowledgeRetrievalTrace({
|
|
454
|
+
query: message,
|
|
455
|
+
viewerAgentId: session.agentId,
|
|
456
|
+
})
|
|
451
457
|
|
|
452
458
|
// Dedup: skip memories already injected 2+ times in this session
|
|
453
459
|
const priorCounts = session.injectedMemoryIds || {}
|
|
454
460
|
const filtered = recalled.filter((entry) => (priorCounts[entry.id] || 0) < 2)
|
|
455
461
|
|
|
456
462
|
const topRecalled = filtered.slice(0, 3)
|
|
463
|
+
const sections: string[] = []
|
|
457
464
|
if (topRecalled.length > 0) {
|
|
458
465
|
// Track injection counts
|
|
459
466
|
const updatedCounts: Record<string, number> = { ...priorCounts }
|
|
@@ -464,9 +471,28 @@ export async function buildProactiveMemorySection(
|
|
|
464
471
|
const recalledLines = topRecalled.map((entry) =>
|
|
465
472
|
`- ${entry.abstract || entry.content.slice(0, 300)}`,
|
|
466
473
|
)
|
|
474
|
+
sections.push(`## Recalled Context\nRelevant memories from previous interactions:\n${recalledLines.join('\n')}`)
|
|
475
|
+
if (knowledgeTrace?.hits.length) {
|
|
476
|
+
const groundingLines = knowledgeTrace.hits.map((hit) =>
|
|
477
|
+
`- [${hit.chunkIndex + 1}/${hit.chunkCount}] ${hit.sourceTitle}: ${hit.snippet}`,
|
|
478
|
+
)
|
|
479
|
+
sections.push(`## Source Grounding\nSource-backed knowledge retrieved for this turn:\n${groundingLines.join('\n')}`)
|
|
480
|
+
}
|
|
467
481
|
return {
|
|
468
|
-
section:
|
|
482
|
+
section: sections.join('\n\n'),
|
|
469
483
|
injectedIds: updatedCounts,
|
|
484
|
+
knowledgeTrace,
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
if (knowledgeTrace?.hits.length) {
|
|
489
|
+
const groundingLines = knowledgeTrace.hits.map((hit) =>
|
|
490
|
+
`- [${hit.chunkIndex + 1}/${hit.chunkCount}] ${hit.sourceTitle}: ${hit.snippet}`,
|
|
491
|
+
)
|
|
492
|
+
return {
|
|
493
|
+
section: `## Source Grounding\nSource-backed knowledge retrieved for this turn:\n${groundingLines.join('\n')}`,
|
|
494
|
+
injectedIds: priorCounts,
|
|
495
|
+
knowledgeTrace,
|
|
470
496
|
}
|
|
471
497
|
}
|
|
472
498
|
} catch { /* non-critical */ }
|
|
@@ -36,7 +36,7 @@ import { log } from '@/lib/server/logger'
|
|
|
36
36
|
import { logExecution } from '@/lib/server/execution-log'
|
|
37
37
|
import { buildCurrentDateTimePromptContext } from '@/lib/server/prompt-runtime-context'
|
|
38
38
|
import { expandExtensionIds } from '@/lib/server/tool-aliases'
|
|
39
|
-
import type { ExecutionBrief, Session, Message } from '@/types'
|
|
39
|
+
import type { ExecutionBrief, KnowledgeRetrievalTrace, Session, Message } from '@/types'
|
|
40
40
|
import { getEnabledCapabilityIds } from '@/lib/capability-selection'
|
|
41
41
|
import { enqueueSystemEvent } from '@/lib/server/runtime/system-events'
|
|
42
42
|
import { resolveActiveProjectContext } from '@/lib/server/project-context'
|
|
@@ -199,6 +199,7 @@ export interface StreamAgentChatResult {
|
|
|
199
199
|
finalResponse: string
|
|
200
200
|
/** Tool events emitted during the streamed run. */
|
|
201
201
|
toolEvents: import('@/types').MessageToolEvent[]
|
|
202
|
+
knowledgeRetrievalTrace?: KnowledgeRetrievalTrace | null
|
|
202
203
|
}
|
|
203
204
|
|
|
204
205
|
type LangChainContentPart =
|
|
@@ -267,6 +268,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
|
|
|
267
268
|
preferMinimalPrompt: lightweightDirectChat,
|
|
268
269
|
})
|
|
269
270
|
const isMinimalPrompt = promptMode === 'minimal'
|
|
271
|
+
let knowledgeRetrievalTrace: KnowledgeRetrievalTrace | null = null
|
|
270
272
|
|
|
271
273
|
// Resolve agent's thinking level for provider-native params
|
|
272
274
|
let agentThinkingLevel: 'minimal' | 'low' | 'medium' | 'high' | undefined
|
|
@@ -309,6 +311,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
|
|
|
309
311
|
fullText: requestedToolPreflightResponse,
|
|
310
312
|
finalResponse: requestedToolPreflightResponse,
|
|
311
313
|
toolEvents: [],
|
|
314
|
+
knowledgeRetrievalTrace: null,
|
|
312
315
|
}
|
|
313
316
|
}
|
|
314
317
|
const runtime = loadRuntimeSettings()
|
|
@@ -490,6 +493,7 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
|
|
|
490
493
|
isMinimalPrompt, currentThreadRecallRequest,
|
|
491
494
|
)
|
|
492
495
|
if (memoryResult.section) promptParts.push(memoryResult.section)
|
|
496
|
+
knowledgeRetrievalTrace = memoryResult.knowledgeTrace || null
|
|
493
497
|
// Persist injection dedup counts so repeated memories are suppressed
|
|
494
498
|
if (Object.keys(memoryResult.injectedIds).length > 0) {
|
|
495
499
|
session.injectedMemoryIds = memoryResult.injectedIds
|
|
@@ -1269,5 +1273,6 @@ async function streamAgentChatCore(opts: StreamAgentChatOpts): Promise<StreamAge
|
|
|
1269
1273
|
cleanup,
|
|
1270
1274
|
runId,
|
|
1271
1275
|
classification,
|
|
1276
|
+
knowledgeRetrievalTrace,
|
|
1272
1277
|
})
|
|
1273
1278
|
}
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type GatewayResponseFrame,
|
|
14
14
|
} from '../gateway/protocol'
|
|
15
15
|
import { log } from '@/lib/server/logger'
|
|
16
|
+
import { setSharedDeviceToken } from '../openclaw/sync'
|
|
16
17
|
|
|
17
18
|
const TAG = 'openclaw'
|
|
18
19
|
|
|
@@ -830,8 +831,6 @@ const openclaw: PlatformConnector = {
|
|
|
830
831
|
// Cross-sync device token for provider identity resolution
|
|
831
832
|
if (normalized) {
|
|
832
833
|
try {
|
|
833
|
-
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
|
834
|
-
const { setSharedDeviceToken } = require('../openclaw/sync')
|
|
835
834
|
setSharedDeviceToken(normalized)
|
|
836
835
|
} catch { /* openclaw-sync not available */ }
|
|
837
836
|
}
|
|
@@ -3,7 +3,7 @@ import { WORKSPACE_DIR } from '@/lib/server/data-dir'
|
|
|
3
3
|
import { log } from '@/lib/server/logger'
|
|
4
4
|
import { loadSettings } from '@/lib/server/settings/settings-repository'
|
|
5
5
|
import { loadSessions } from '@/lib/server/sessions/session-repository'
|
|
6
|
-
import { appendPersistedRunEvent, persistRun } from '@/lib/server/runtime/run-ledger'
|
|
6
|
+
import { appendPersistedRunEvent, buildRetrievalSummary, persistRun } from '@/lib/server/runtime/run-ledger'
|
|
7
7
|
import { notify } from '@/lib/server/ws-hub'
|
|
8
8
|
import { captureGuardianCheckpoint } from '@/lib/server/agents/guardian'
|
|
9
9
|
import {
|
|
@@ -68,6 +68,7 @@ function notifyExecutionState(sessionId: string): void {
|
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Record<string, unknown>): void {
|
|
71
|
+
const { citations, retrievalTrace, ...eventExtra } = extra || {}
|
|
71
72
|
appendPersistedRunEvent({
|
|
72
73
|
runId: run.id,
|
|
73
74
|
sessionId: run.sessionId,
|
|
@@ -78,6 +79,8 @@ function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Rec
|
|
|
78
79
|
phase: 'status',
|
|
79
80
|
status,
|
|
80
81
|
summary: run.resultPreview || run.error || undefined,
|
|
82
|
+
citations: citations as import('@/types').KnowledgeCitation[] | undefined,
|
|
83
|
+
retrievalTrace: (retrievalTrace as import('@/types').KnowledgeRetrievalTrace | undefined) || undefined,
|
|
81
84
|
event: {
|
|
82
85
|
t: 'md',
|
|
83
86
|
text: JSON.stringify({
|
|
@@ -90,7 +93,7 @@ function emitStatus(run: SessionRunRecord, status: SessionRunStatus, extra?: Rec
|
|
|
90
93
|
status,
|
|
91
94
|
source: run.source,
|
|
92
95
|
internal: run.internal,
|
|
93
|
-
...
|
|
96
|
+
...eventExtra,
|
|
94
97
|
},
|
|
95
98
|
}),
|
|
96
99
|
},
|
|
@@ -268,6 +271,7 @@ export function enqueueTaskAttemptExecution(
|
|
|
268
271
|
run.endedAt = Date.now()
|
|
269
272
|
run.error = controller.signal.aborted ? (run.error || 'Cancelled') : result.error
|
|
270
273
|
run.resultPreview = result.text?.slice(0, 280)
|
|
274
|
+
run.retrievalSummary = buildRetrievalSummary(result.citations)
|
|
271
275
|
if (typeof result.inputTokens === 'number') run.totalInputTokens = result.inputTokens
|
|
272
276
|
if (typeof result.outputTokens === 'number') run.totalOutputTokens = result.outputTokens
|
|
273
277
|
if (typeof result.estimatedCost === 'number') run.estimatedCost = result.estimatedCost
|
|
@@ -275,6 +279,8 @@ export function enqueueTaskAttemptExecution(
|
|
|
275
279
|
emitStatus(run, run.status, {
|
|
276
280
|
hasText: !!result.text,
|
|
277
281
|
error: run.error || null,
|
|
282
|
+
citations: result.citations,
|
|
283
|
+
retrievalTrace: result.retrievalTrace,
|
|
278
284
|
})
|
|
279
285
|
return result
|
|
280
286
|
} catch (err: unknown) {
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import fs from 'fs'
|
|
2
|
+
import path from 'path'
|
|
3
|
+
import * as cheerio from 'cheerio'
|
|
4
|
+
|
|
5
|
+
const TEXT_EXTS = new Set([
|
|
6
|
+
'.txt', '.md', '.markdown', '.csv', '.tsv', '.json', '.jsonl',
|
|
7
|
+
'.html', '.htm', '.xml', '.yaml', '.yml', '.toml', '.ini', '.cfg',
|
|
8
|
+
'.js', '.ts', '.tsx', '.jsx', '.py', '.go', '.rs', '.java', '.c', '.cpp', '.h',
|
|
9
|
+
'.rb', '.php', '.sh', '.bash', '.zsh', '.sql', '.r', '.swift', '.kt',
|
|
10
|
+
'.env', '.log', '.conf', '.properties', '.gitignore', '.dockerignore',
|
|
11
|
+
])
|
|
12
|
+
|
|
13
|
+
export const MAX_KNOWLEDGE_IMPORT_BYTES = 10 * 1024 * 1024
|
|
14
|
+
export const MAX_KNOWLEDGE_CONTENT_CHARS = 500_000
|
|
15
|
+
|
|
16
|
+
export function isKnowledgeTextFile(filename: string): boolean {
|
|
17
|
+
const ext = path.extname(filename).toLowerCase()
|
|
18
|
+
return TEXT_EXTS.has(ext) || ext === ''
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function deriveKnowledgeTitle(filename: string): string {
|
|
22
|
+
const name = path.basename(filename, path.extname(filename))
|
|
23
|
+
return name
|
|
24
|
+
.replace(/[-_]+/g, ' ')
|
|
25
|
+
.replace(/([a-z])([A-Z])/g, '$1 $2')
|
|
26
|
+
.replace(/\b\w/g, (char) => char.toUpperCase())
|
|
27
|
+
.trim() || 'Knowledge Source'
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function normalizeKnowledgeContent(content: string): string {
|
|
31
|
+
const normalized = String(content || '')
|
|
32
|
+
.replace(/^\uFEFF/, '')
|
|
33
|
+
.replace(/\r\n/g, '\n')
|
|
34
|
+
.trim()
|
|
35
|
+
|
|
36
|
+
if (normalized.length <= MAX_KNOWLEDGE_CONTENT_CHARS) return normalized
|
|
37
|
+
return `${normalized.slice(0, MAX_KNOWLEDGE_CONTENT_CHARS)}\n\n[... truncated at 500k characters]`
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function extractPdfText(buffer: Buffer, filePathHint?: string): Promise<string> {
|
|
41
|
+
try {
|
|
42
|
+
const pdfParseModule = await import('pdf-parse') as unknown as {
|
|
43
|
+
default?: (input: Buffer) => Promise<{ text?: string }>
|
|
44
|
+
}
|
|
45
|
+
const pdfParse = pdfParseModule.default
|
|
46
|
+
if (typeof pdfParse !== 'function') throw new Error('pdf-parse loader unavailable')
|
|
47
|
+
const result = await pdfParse(buffer)
|
|
48
|
+
return normalizeKnowledgeContent(result.text || '')
|
|
49
|
+
} catch {
|
|
50
|
+
return normalizeKnowledgeContent(
|
|
51
|
+
`[PDF document]\n\nUnable to extract text automatically.${filePathHint ? `\n\nSaved at: ${filePathHint}` : ''}`,
|
|
52
|
+
)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function htmlToReadableText(html: string): { title: string | null; content: string } {
|
|
57
|
+
const $ = cheerio.load(html)
|
|
58
|
+
$('script, style, noscript, svg, nav, footer, header').remove()
|
|
59
|
+
|
|
60
|
+
const title = $('title').first().text().trim() || null
|
|
61
|
+
const root = $('main').first().length
|
|
62
|
+
? $('main').first()
|
|
63
|
+
: $('article').first().length
|
|
64
|
+
? $('article').first()
|
|
65
|
+
: $('body').first().length
|
|
66
|
+
? $('body').first()
|
|
67
|
+
: $('html').first()
|
|
68
|
+
|
|
69
|
+
const text = root
|
|
70
|
+
.text()
|
|
71
|
+
.replace(/\u00a0/g, ' ')
|
|
72
|
+
.split('\n')
|
|
73
|
+
.map((line) => line.trim())
|
|
74
|
+
.filter(Boolean)
|
|
75
|
+
.join('\n\n')
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
title,
|
|
79
|
+
content: normalizeKnowledgeContent(text),
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export async function extractKnowledgeTextFromBuffer(
|
|
84
|
+
buffer: Buffer,
|
|
85
|
+
filename: string,
|
|
86
|
+
filePathHint?: string,
|
|
87
|
+
): Promise<string> {
|
|
88
|
+
if (buffer.length === 0) return ''
|
|
89
|
+
if (buffer.length > MAX_KNOWLEDGE_IMPORT_BYTES) {
|
|
90
|
+
throw new Error('File too large. Maximum 10MB.')
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const ext = path.extname(filename).toLowerCase()
|
|
94
|
+
if (ext === '.pdf') {
|
|
95
|
+
return extractPdfText(buffer, filePathHint)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (isKnowledgeTextFile(filename)) {
|
|
99
|
+
return normalizeKnowledgeContent(buffer.toString('utf-8'))
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return normalizeKnowledgeContent(
|
|
103
|
+
`[Binary file: ${filename}]${filePathHint ? `\n\nSaved at: ${filePathHint}` : ''}`,
|
|
104
|
+
)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export async function extractKnowledgeTextFromFile(filePath: string, filename?: string): Promise<string> {
|
|
108
|
+
const buffer = await fs.promises.readFile(filePath)
|
|
109
|
+
return extractKnowledgeTextFromBuffer(buffer, filename || path.basename(filePath), filePath)
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function extractKnowledgeTextFromUrl(sourceUrl: string): Promise<{
|
|
113
|
+
title: string | null
|
|
114
|
+
content: string
|
|
115
|
+
contentType: string | null
|
|
116
|
+
}> {
|
|
117
|
+
const response = await fetch(sourceUrl, {
|
|
118
|
+
headers: {
|
|
119
|
+
'user-agent': 'SwarmClaw/knowledge-import',
|
|
120
|
+
accept: 'text/html, text/plain, application/json, application/pdf, */*',
|
|
121
|
+
},
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
if (!response.ok) {
|
|
125
|
+
throw new Error(`URL fetch failed (${response.status})`)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const contentType = response.headers.get('content-type')
|
|
129
|
+
const contentLength = Number.parseInt(response.headers.get('content-length') || '', 10)
|
|
130
|
+
if (Number.isFinite(contentLength) && contentLength > MAX_KNOWLEDGE_IMPORT_BYTES) {
|
|
131
|
+
throw new Error('Remote document is too large. Maximum 10MB.')
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
if ((contentType || '').includes('application/pdf') || sourceUrl.toLowerCase().endsWith('.pdf')) {
|
|
135
|
+
const buffer = Buffer.from(await response.arrayBuffer())
|
|
136
|
+
return {
|
|
137
|
+
title: null,
|
|
138
|
+
content: await extractPdfText(buffer, sourceUrl),
|
|
139
|
+
contentType,
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const text = await response.text()
|
|
144
|
+
const looksLikeHtml = (contentType || '').includes('text/html') || /<html[\s>]|<body[\s>]/i.test(text)
|
|
145
|
+
if (looksLikeHtml) {
|
|
146
|
+
const parsed = htmlToReadableText(text)
|
|
147
|
+
return {
|
|
148
|
+
title: parsed.title,
|
|
149
|
+
content: parsed.content,
|
|
150
|
+
contentType,
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return {
|
|
155
|
+
title: null,
|
|
156
|
+
content: normalizeKnowledgeContent(text),
|
|
157
|
+
contentType,
|
|
158
|
+
}
|
|
159
|
+
}
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import assert from 'node:assert/strict'
|
|
2
|
+
import { test } from 'node:test'
|
|
3
|
+
import { runWithTempDataDir } from '@/lib/server/test-utils/run-with-temp-data-dir'
|
|
4
|
+
|
|
5
|
+
test('buildKnowledgeRetrievalTrace returns active hits and selectKnowledgeCitations marks empty replies as no_match', () => {
|
|
6
|
+
const output = runWithTempDataDir<{
|
|
7
|
+
sourceId: string | null
|
|
8
|
+
hitCount: number
|
|
9
|
+
firstHitSourceId: string | null
|
|
10
|
+
matchedStatus: string | null
|
|
11
|
+
matchedCitationCount: number
|
|
12
|
+
whyMatched: string | null
|
|
13
|
+
unmatchedStatus: string | null
|
|
14
|
+
unmatchedCitationCount: number
|
|
15
|
+
}>(`
|
|
16
|
+
const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
|
|
17
|
+
const knowledge = knowledgeMod.default || knowledgeMod
|
|
18
|
+
|
|
19
|
+
const detail = await knowledge.createKnowledgeSource({
|
|
20
|
+
kind: 'manual',
|
|
21
|
+
title: 'Gateway Migration Runbook',
|
|
22
|
+
content: 'Use blue green deployment for gateway migrations so rollback stays simple and downtime stays low.',
|
|
23
|
+
tags: ['deploy'],
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
const trace = await knowledge.buildKnowledgeRetrievalTrace({
|
|
27
|
+
query: 'gateway blue green rollback',
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
const matched = knowledge.selectKnowledgeCitations({
|
|
31
|
+
responseText: 'Use blue green deployment for the gateway migration so rollback stays simple.',
|
|
32
|
+
retrievalTrace: trace,
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
const unmatched = knowledge.selectKnowledgeCitations({
|
|
36
|
+
responseText: '',
|
|
37
|
+
retrievalTrace: trace,
|
|
38
|
+
})
|
|
39
|
+
|
|
40
|
+
console.log(JSON.stringify({
|
|
41
|
+
sourceId: detail?.source?.id || null,
|
|
42
|
+
hitCount: trace?.hits?.length || 0,
|
|
43
|
+
firstHitSourceId: trace?.hits?.[0]?.sourceId || null,
|
|
44
|
+
matchedStatus: matched.retrievalTrace?.selectorStatus || null,
|
|
45
|
+
matchedCitationCount: matched.citations.length,
|
|
46
|
+
whyMatched: matched.citations[0]?.whyMatched || null,
|
|
47
|
+
unmatchedStatus: unmatched.retrievalTrace?.selectorStatus || null,
|
|
48
|
+
unmatchedCitationCount: unmatched.citations.length,
|
|
49
|
+
}))
|
|
50
|
+
`, { prefix: 'swarmclaw-knowledge-trace-' })
|
|
51
|
+
|
|
52
|
+
assert.ok(output.sourceId)
|
|
53
|
+
assert.ok(output.hitCount >= 1)
|
|
54
|
+
assert.equal(output.firstHitSourceId, output.sourceId)
|
|
55
|
+
assert.equal(output.matchedStatus, 'selected')
|
|
56
|
+
assert.ok(output.matchedCitationCount >= 1)
|
|
57
|
+
assert.match(output.whyMatched || '', /Matched|Retrieved/)
|
|
58
|
+
assert.equal(output.unmatchedStatus, 'no_match')
|
|
59
|
+
assert.equal(output.unmatchedCitationCount, 0)
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
test('archived and superseded sources are excluded by default, restore re-enables search, and restore actions are recorded explicitly', () => {
|
|
63
|
+
const output = runWithTempDataDir<{
|
|
64
|
+
archivedDefaultCount: number
|
|
65
|
+
archivedIncludedCount: number
|
|
66
|
+
restoredCount: number
|
|
67
|
+
restoreActionKind: string | null
|
|
68
|
+
supersededDefaultCount: number
|
|
69
|
+
supersededIncludedCount: number
|
|
70
|
+
supersededFinding: boolean
|
|
71
|
+
}>(`
|
|
72
|
+
const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
|
|
73
|
+
const storageMod = await import('./src/lib/server/storage.ts')
|
|
74
|
+
const knowledge = knowledgeMod.default || knowledgeMod
|
|
75
|
+
const storage = storageMod.default || storageMod
|
|
76
|
+
|
|
77
|
+
const archived = await knowledge.createKnowledgeSource({
|
|
78
|
+
kind: 'manual',
|
|
79
|
+
title: 'Orchard Rollback Notes',
|
|
80
|
+
content: 'orchard sentinel rollback checklist',
|
|
81
|
+
})
|
|
82
|
+
|
|
83
|
+
await knowledge.archiveKnowledgeSource(archived.source.id, { reason: 'manual review' })
|
|
84
|
+
|
|
85
|
+
const archivedDefault = await knowledge.searchKnowledgeHits({ query: 'orchard' })
|
|
86
|
+
const archivedIncluded = await knowledge.searchKnowledgeHits({
|
|
87
|
+
query: 'orchard',
|
|
88
|
+
includeArchived: true,
|
|
89
|
+
})
|
|
90
|
+
|
|
91
|
+
await knowledge.restoreKnowledgeSource(archived.source.id)
|
|
92
|
+
const restored = await knowledge.searchKnowledgeHits({ query: 'orchard' })
|
|
93
|
+
|
|
94
|
+
const older = await knowledge.createKnowledgeSource({
|
|
95
|
+
kind: 'manual',
|
|
96
|
+
title: 'Legacy API Notes',
|
|
97
|
+
content: 'legacy endpoint alpha is still enabled',
|
|
98
|
+
sourceUrl: 'https://example.com/api/reference',
|
|
99
|
+
})
|
|
100
|
+
const newer = await knowledge.createKnowledgeSource({
|
|
101
|
+
kind: 'manual',
|
|
102
|
+
title: 'Current API Notes',
|
|
103
|
+
content: 'modern endpoint beta replaced the older route',
|
|
104
|
+
sourceUrl: 'https://example.com/api/reference',
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
storage.patchKnowledgeSource(older.source.id, (current) => current ? {
|
|
108
|
+
...current,
|
|
109
|
+
lastIndexedAt: 1_000,
|
|
110
|
+
updatedAt: 1_000,
|
|
111
|
+
} : null)
|
|
112
|
+
storage.patchKnowledgeSource(newer.source.id, (current) => current ? {
|
|
113
|
+
...current,
|
|
114
|
+
lastIndexedAt: 2_000,
|
|
115
|
+
updatedAt: 2_000,
|
|
116
|
+
} : null)
|
|
117
|
+
|
|
118
|
+
await knowledge.runKnowledgeHygieneMaintenance()
|
|
119
|
+
|
|
120
|
+
const supersededDefault = await knowledge.searchKnowledgeHits({ query: 'alpha' })
|
|
121
|
+
const supersededIncluded = await knowledge.searchKnowledgeHits({
|
|
122
|
+
query: 'alpha',
|
|
123
|
+
includeArchived: true,
|
|
124
|
+
})
|
|
125
|
+
const summary = await knowledge.getKnowledgeHygieneSummary()
|
|
126
|
+
|
|
127
|
+
console.log(JSON.stringify({
|
|
128
|
+
archivedDefaultCount: archivedDefault.length,
|
|
129
|
+
archivedIncludedCount: archivedIncluded.length,
|
|
130
|
+
restoredCount: restored.length,
|
|
131
|
+
restoreActionKind: summary.recentActions.find((action) => action.summary === 'Restored Orchard Rollback Notes')?.kind || null,
|
|
132
|
+
supersededDefaultCount: supersededDefault.length,
|
|
133
|
+
supersededIncludedCount: supersededIncluded.length,
|
|
134
|
+
supersededFinding: summary.findings.some((finding) => finding.kind === 'superseded' && finding.sourceId === older.source.id),
|
|
135
|
+
}))
|
|
136
|
+
`, { prefix: 'swarmclaw-knowledge-lifecycle-' })
|
|
137
|
+
|
|
138
|
+
assert.equal(output.archivedDefaultCount, 0)
|
|
139
|
+
assert.equal(output.archivedIncludedCount, 1)
|
|
140
|
+
assert.equal(output.restoredCount, 1)
|
|
141
|
+
assert.equal(output.restoreActionKind, 'restore')
|
|
142
|
+
assert.equal(output.supersededDefaultCount, 0)
|
|
143
|
+
assert.equal(output.supersededIncludedCount, 1)
|
|
144
|
+
assert.equal(output.supersededFinding, true)
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
test('runKnowledgeHygieneMaintenance reindexes stale file sources and archives exact duplicates', () => {
|
|
148
|
+
const output = runWithTempDataDir<{
|
|
149
|
+
fileLastAutoSyncAt: number | null
|
|
150
|
+
fileChunksContainUpdatedText: boolean
|
|
151
|
+
refreshedHitCount: number
|
|
152
|
+
archivedDuplicateCount: number
|
|
153
|
+
recentActionKinds: string[]
|
|
154
|
+
}>(`
|
|
155
|
+
const fs = await import('node:fs')
|
|
156
|
+
const path = await import('node:path')
|
|
157
|
+
const knowledgeMod = await import('./src/lib/server/knowledge-sources.ts')
|
|
158
|
+
const storageMod = await import('./src/lib/server/storage.ts')
|
|
159
|
+
const knowledge = knowledgeMod.default || knowledgeMod
|
|
160
|
+
const storage = storageMod.default || storageMod
|
|
161
|
+
|
|
162
|
+
const filePath = path.join(process.env.WORKSPACE_DIR, 'ops-runbook.txt')
|
|
163
|
+
fs.writeFileSync(filePath, 'Initial runbook placeholder.')
|
|
164
|
+
|
|
165
|
+
const fileSource = await knowledge.createKnowledgeSource({
|
|
166
|
+
kind: 'file',
|
|
167
|
+
title: 'Ops Runbook',
|
|
168
|
+
sourcePath: filePath,
|
|
169
|
+
})
|
|
170
|
+
|
|
171
|
+
fs.writeFileSync(filePath, 'Updated runbook adds rollback choreography and incident checklist.')
|
|
172
|
+
storage.patchKnowledgeSource(fileSource.source.id, (current) => current ? {
|
|
173
|
+
...current,
|
|
174
|
+
lastIndexedAt: 1,
|
|
175
|
+
nextSyncAt: 1,
|
|
176
|
+
updatedAt: 1,
|
|
177
|
+
} : null)
|
|
178
|
+
|
|
179
|
+
const duplicateA = await knowledge.createKnowledgeSource({
|
|
180
|
+
kind: 'manual',
|
|
181
|
+
title: 'Duplicate A',
|
|
182
|
+
content: 'duplicate payload for archival',
|
|
183
|
+
})
|
|
184
|
+
const duplicateB = await knowledge.createKnowledgeSource({
|
|
185
|
+
kind: 'manual',
|
|
186
|
+
title: 'Duplicate B',
|
|
187
|
+
content: 'duplicate payload for archival',
|
|
188
|
+
})
|
|
189
|
+
|
|
190
|
+
const summary = await knowledge.runKnowledgeHygieneMaintenance()
|
|
191
|
+
const refreshed = await knowledge.getKnowledgeSourceDetail(fileSource.source.id)
|
|
192
|
+
const duplicateADetail = await knowledge.getKnowledgeSourceDetail(duplicateA.source.id)
|
|
193
|
+
const duplicateBDetail = await knowledge.getKnowledgeSourceDetail(duplicateB.source.id)
|
|
194
|
+
const refreshedHits = await knowledge.searchKnowledgeHits({ query: 'choreography' })
|
|
195
|
+
|
|
196
|
+
const archivedDuplicateCount = [duplicateADetail, duplicateBDetail]
|
|
197
|
+
.filter((detail) => !!detail?.source?.archivedAt)
|
|
198
|
+
.length
|
|
199
|
+
|
|
200
|
+
console.log(JSON.stringify({
|
|
201
|
+
fileLastAutoSyncAt: refreshed?.source?.lastAutoSyncAt || null,
|
|
202
|
+
fileChunksContainUpdatedText: (refreshed?.chunks || []).some((chunk) => chunk.content.includes('rollback choreography')),
|
|
203
|
+
refreshedHitCount: refreshedHits.length,
|
|
204
|
+
archivedDuplicateCount,
|
|
205
|
+
recentActionKinds: summary.recentActions.map((action) => action.kind),
|
|
206
|
+
}))
|
|
207
|
+
`, { prefix: 'swarmclaw-knowledge-maintenance-' })
|
|
208
|
+
|
|
209
|
+
assert.ok(typeof output.fileLastAutoSyncAt === 'number' && output.fileLastAutoSyncAt > 0)
|
|
210
|
+
assert.equal(output.fileChunksContainUpdatedText, true)
|
|
211
|
+
assert.ok(output.refreshedHitCount >= 1)
|
|
212
|
+
assert.equal(output.archivedDuplicateCount, 1)
|
|
213
|
+
assert.ok(output.recentActionKinds.includes('archive'))
|
|
214
|
+
assert.ok(output.recentActionKinds.includes('reindex') || output.recentActionKinds.includes('sync'))
|
|
215
|
+
})
|