@desplega.ai/agent-swarm 1.92.0 → 1.92.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/openapi.json +276 -3
- package/package.json +6 -6
- package/plugin/skills/pages/SKILL.md +5 -2
- package/src/be/db.ts +416 -20
- package/src/be/memory/boot-reembed.ts +85 -0
- package/src/be/memory/constants.ts +44 -2
- package/src/be/memory/providers/openai-embedding.ts +15 -5
- package/src/be/memory/providers/sqlite-store.ts +325 -76
- package/src/be/memory/reranker.ts +35 -17
- package/src/be/memory/types.ts +43 -0
- package/src/be/migrations/084_script_run_journal_duration.sql +5 -0
- package/src/be/migrations/085_script_runs_kind.sql +9 -0
- package/src/be/migrations/086_pages_default_authed.sql +64 -0
- package/src/be/migrations/087_skill_files.sql +19 -0
- package/src/be/modelsdev-cache.json +5622 -2543
- package/src/be/seed-scripts/catalog/boot-triage.ts +221 -0
- package/src/be/seed-scripts/catalog/catalog-report.ts +457 -0
- package/src/be/seed-scripts/catalog/compound-insights.ts +465 -0
- package/src/be/seed-scripts/catalog/gh-pr-snapshot.ts +1 -1
- package/src/be/seed-scripts/catalog/memory-eval.ts +1059 -0
- package/src/be/seed-scripts/catalog/ops-catalog-audit.ts +34 -439
- package/src/be/seed-scripts/catalog/schedule-health.ts +78 -2
- package/src/be/seed-scripts/catalog/task-failure-audit.ts +48 -1
- package/src/be/seed-scripts/index.ts +32 -4
- package/src/be/seed-skills/index.ts +0 -7
- package/src/be/skill-sync.ts +91 -7
- package/src/commands/runner.ts +6 -2
- package/src/heartbeat/templates.ts +20 -16
- package/src/http/index.ts +50 -7
- package/src/http/mcp-user.ts +23 -0
- package/src/http/mcp.ts +58 -0
- package/src/http/memory.ts +62 -0
- package/src/http/pages.ts +1 -1
- package/src/http/script-runs.ts +2 -0
- package/src/http/scripts.ts +39 -2
- package/src/http/skills.ts +225 -0
- package/src/providers/claude-adapter.ts +56 -24
- package/src/script-workflows/workflow-ctx.ts +7 -3
- package/src/scripts-runtime/sdk-allowlist.ts +1 -0
- package/src/scripts-runtime/swarm-sdk.ts +13 -0
- package/src/scripts-runtime/types/stdlib.d.ts +1 -0
- package/src/scripts-runtime/types/swarm-sdk.d.ts +1 -0
- package/src/server.ts +2 -0
- package/src/tasks/worker-follow-up.ts +12 -0
- package/src/tests/claude-adapter-binary.test.ts +135 -81
- package/src/tests/create-page-tool.test.ts +19 -2
- package/src/tests/heartbeat-checklist.test.ts +36 -0
- package/src/tests/mcp-transport-gc.test.ts +58 -0
- package/src/tests/memory-e2e.test.ts +6 -6
- package/src/tests/memory-health-endpoint.test.ts +78 -0
- package/src/tests/memory-rater-e2e.test.ts +4 -5
- package/src/tests/memory-reranker.test.ts +135 -124
- package/src/tests/memory-store.test.ts +221 -1
- package/src/tests/memory.test.ts +13 -12
- package/src/tests/pages-http.test.ts +20 -2
- package/src/tests/pages-storage.test.ts +26 -0
- package/src/tests/scripts-mcp-e2e.test.ts +53 -0
- package/src/tests/seed-scripts.test.ts +328 -3
- package/src/tests/skill-files-http.test.ts +171 -0
- package/src/tests/skill-files.test.ts +162 -0
- package/src/tests/skill-get-file-tool.test.ts +110 -0
- package/src/tests/skill-sync.test.ts +125 -6
- package/src/tests/task-cascade-fail.test.ts +304 -0
- package/src/tools/create-page.ts +2 -2
- package/src/tools/skills/index.ts +1 -0
- package/src/tools/skills/skill-get-file.ts +80 -0
- package/src/tools/tool-config.ts +2 -1
- package/src/types.ts +20 -0
- package/src/utils/internal-ai/complete-structured.ts +2 -2
- package/templates/schedules/daily-blocker-digest/content.md +68 -54
- package/templates/schedules/daily-compounding-reflection/content.md +4 -4
- package/templates/schedules/daily-hn-briefing/content.md +5 -5
- package/templates/schedules/daily-workflow-health-audit/content.md +6 -6
- package/templates/schedules/gtm-weekly-review/content.md +9 -9
- package/templates/schedules/weekly-dependabot-triage/content.md +24 -20
- package/templates/skills/agentmail-sending/content.md +6 -7
- package/templates/skills/desloppify/content.md +8 -9
- package/templates/skills/jira-interaction/content.md +25 -33
- package/templates/skills/kapso-whatsapp/content.md +29 -30
- package/templates/skills/linear-interaction/content.md +8 -9
- package/templates/skills/profile-corruption-escalation/content.md +44 -85
- package/templates/skills/sprite-cli/content.md +4 -5
- package/templates/skills/turso-interaction/content.md +14 -17
- package/templates/skills/workflow-iterate/content.md +38 -391
- package/templates/skills/x-api-interactions/content.md +4 -6
- package/templates/workflows/llm-safe-release-context/config.json +13 -0
- package/templates/workflows/llm-safe-release-context/content.md +69 -0
- package/templates/skills/scheduled-task-resilience/config.json +0 -14
- package/templates/skills/scheduled-task-resilience/content.md +0 -95
|
@@ -15,12 +15,54 @@ export const TTL_DEFAULTS: Record<AgentMemorySource, number | null> = {
|
|
|
15
15
|
manual: null,
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
+
// Per-source recency decay half-life (in days).
|
|
19
|
+
// manual = Infinity (no decay — curated knowledge stays relevant forever).
|
|
20
|
+
// A global MEMORY_RECENCY_HALF_LIFE_DAYS override forces ALL sources to the same value.
|
|
21
|
+
const GLOBAL_HALF_LIFE_OVERRIDE = process.env.MEMORY_RECENCY_HALF_LIFE_DAYS;
|
|
22
|
+
const GLOBAL_HALF_LIFE =
|
|
23
|
+
GLOBAL_HALF_LIFE_OVERRIDE != null && GLOBAL_HALF_LIFE_OVERRIDE !== ""
|
|
24
|
+
? Number(GLOBAL_HALF_LIFE_OVERRIDE)
|
|
25
|
+
: null;
|
|
26
|
+
|
|
27
|
+
export const RECENCY_DECAY_HALF_LIFE: Record<AgentMemorySource, number> =
|
|
28
|
+
GLOBAL_HALF_LIFE != null && Number.isFinite(GLOBAL_HALF_LIFE)
|
|
29
|
+
? {
|
|
30
|
+
manual: GLOBAL_HALF_LIFE,
|
|
31
|
+
file_index: GLOBAL_HALF_LIFE,
|
|
32
|
+
task_completion: GLOBAL_HALF_LIFE,
|
|
33
|
+
session_summary: GLOBAL_HALF_LIFE,
|
|
34
|
+
}
|
|
35
|
+
: {
|
|
36
|
+
manual: Number.POSITIVE_INFINITY,
|
|
37
|
+
file_index: 180,
|
|
38
|
+
task_completion: 14,
|
|
39
|
+
session_summary: 7,
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
// Legacy export — callers that don't have a source fall back to task_completion's value.
|
|
43
|
+
export const RECENCY_DECAY_HALF_LIFE_DAYS = RECENCY_DECAY_HALF_LIFE.task_completion;
|
|
44
|
+
|
|
45
|
+
// Source-quality multiplier for reranking.
|
|
46
|
+
// Curated manual memories rank higher; ephemeral session summaries rank lower.
|
|
47
|
+
export const SOURCE_QUALITY_MULTIPLIER: Record<AgentMemorySource, number> = {
|
|
48
|
+
manual: 1.5,
|
|
49
|
+
file_index: 1.0,
|
|
50
|
+
task_completion: 0.7,
|
|
51
|
+
session_summary: 0.5,
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
// Minimum raw cosine similarity to keep a candidate. Below this, the result is noise.
|
|
55
|
+
export const MIN_SIMILARITY = numEnv("MEMORY_MIN_SIMILARITY", 0.1);
|
|
56
|
+
|
|
18
57
|
// Reranking parameters
|
|
19
|
-
export const RECENCY_DECAY_HALF_LIFE_DAYS = numEnv("MEMORY_RECENCY_HALF_LIFE_DAYS", 14);
|
|
20
58
|
export const ACCESS_BOOST_MAX_MULTIPLIER = numEnv("MEMORY_ACCESS_BOOST_MAX", 1.5);
|
|
21
59
|
export const ACCESS_BOOST_RECENCY_WINDOW_HOURS = numEnv("MEMORY_ACCESS_RECENCY_HOURS", 48);
|
|
22
60
|
export const CANDIDATE_SET_MULTIPLIER = numEnv("MEMORY_CANDIDATE_MULTIPLIER", 3);
|
|
23
61
|
|
|
24
62
|
// Embedding defaults
|
|
25
|
-
export const
|
|
63
|
+
export const EMBEDDING_DIMENSIONS = numEnv("EMBEDDING_DIMENSIONS", 512);
|
|
64
|
+
export const DEFAULT_EMBEDDING_DIMENSIONS = EMBEDDING_DIMENSIONS;
|
|
26
65
|
export const DEFAULT_EMBEDDING_MODEL = "openai/text-embedding-3-small";
|
|
66
|
+
|
|
67
|
+
// Manual memories must NEVER be deleted by automated processes (curator, GC, etc.)
|
|
68
|
+
export const PROTECTED_SOURCES: ReadonlySet<AgentMemorySource> = new Set(["manual"]);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import OpenAI from "openai";
|
|
2
|
-
import {
|
|
2
|
+
import { DEFAULT_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS } from "../constants";
|
|
3
3
|
import type { EmbeddingProvider } from "../types";
|
|
4
4
|
|
|
5
5
|
interface OpenAIEmbeddingConfig {
|
|
@@ -21,10 +21,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
|
|
21
21
|
|
|
22
22
|
this.model = config?.model ?? process.env.EMBEDDING_MODEL ?? "text-embedding-3-small";
|
|
23
23
|
|
|
24
|
-
this.dimensions =
|
|
25
|
-
config?.dimensions ??
|
|
26
|
-
Number(process.env.EMBEDDING_DIMENSIONS) ??
|
|
27
|
-
DEFAULT_EMBEDDING_DIMENSIONS;
|
|
24
|
+
this.dimensions = config?.dimensions ?? EMBEDDING_DIMENSIONS;
|
|
28
25
|
|
|
29
26
|
this.name = config?.model
|
|
30
27
|
? `openai/${config.model}`
|
|
@@ -58,6 +55,13 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
|
|
58
55
|
const values = response.data[0]?.embedding;
|
|
59
56
|
if (!values) return null;
|
|
60
57
|
|
|
58
|
+
if (values.length !== this.dimensions) {
|
|
59
|
+
console.error(
|
|
60
|
+
`[memory] Embedding dimension mismatch: expected=${this.dimensions} got=${values.length}. Provider may not support the 'dimensions' parameter.`,
|
|
61
|
+
);
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
61
65
|
return new Float32Array(values);
|
|
62
66
|
} catch (err) {
|
|
63
67
|
console.error("[memory] Embedding failed:", (err as Error).message);
|
|
@@ -93,6 +97,12 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
|
|
93
97
|
for (const item of response.data) {
|
|
94
98
|
const originalIndex = nonEmptyIndices[item.index];
|
|
95
99
|
if (originalIndex !== undefined && item.embedding) {
|
|
100
|
+
if (item.embedding.length !== this.dimensions) {
|
|
101
|
+
console.error(
|
|
102
|
+
`[memory] Batch embedding dimension mismatch: expected=${this.dimensions} got=${item.embedding.length}. Provider may not support the 'dimensions' parameter.`,
|
|
103
|
+
);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
96
106
|
results[originalIndex] = new Float32Array(item.embedding);
|
|
97
107
|
}
|
|
98
108
|
}
|
|
@@ -1,16 +1,25 @@
|
|
|
1
1
|
import { getDb, isSqliteVecAvailable } from "@/be/db";
|
|
2
2
|
import { cosineSimilarity, deserializeEmbedding, serializeEmbedding } from "@/be/embedding";
|
|
3
3
|
import type { AgentMemory, AgentMemoryScope, AgentMemorySource } from "@/types";
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
EMBEDDING_DIMENSIONS,
|
|
6
|
+
MIN_SIMILARITY,
|
|
7
|
+
PROTECTED_SOURCES,
|
|
8
|
+
TTL_DEFAULTS,
|
|
9
|
+
} from "../constants";
|
|
5
10
|
import type {
|
|
6
11
|
MemoryCandidate,
|
|
12
|
+
MemoryHealth,
|
|
7
13
|
MemoryInput,
|
|
8
14
|
MemoryListOptions,
|
|
9
15
|
MemorySearchOptions,
|
|
10
16
|
MemoryStats,
|
|
11
17
|
MemoryStore,
|
|
18
|
+
MemoryVecPopulateStats,
|
|
12
19
|
} from "../types";
|
|
13
20
|
|
|
21
|
+
const VECTOR_BYTES = EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
|
|
22
|
+
|
|
14
23
|
type AgentMemoryRow = {
|
|
15
24
|
id: string;
|
|
16
25
|
agentId: string | null;
|
|
@@ -76,54 +85,152 @@ function computeExpiresAt(source: AgentMemorySource): string | null {
|
|
|
76
85
|
|
|
77
86
|
export class SqliteMemoryStore implements MemoryStore {
|
|
78
87
|
private vecInitialized = false;
|
|
88
|
+
private lastPopulate: MemoryVecPopulateStats | null = null;
|
|
79
89
|
|
|
80
90
|
constructor() {
|
|
81
91
|
this.ensureVecTable();
|
|
82
92
|
}
|
|
83
93
|
|
|
84
94
|
private ensureVecTable(): void {
|
|
85
|
-
if (this.vecInitialized
|
|
95
|
+
if (this.vecInitialized) return;
|
|
96
|
+
|
|
97
|
+
if (!isSqliteVecAvailable()) {
|
|
98
|
+
console.warn("[memory-vec] sqlite-vec extension_loaded=false; retrieval_mode=fallback");
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
86
101
|
|
|
87
102
|
const db = getDb();
|
|
88
|
-
// Create the virtual table if it doesn't exist
|
|
89
103
|
try {
|
|
104
|
+
console.log(
|
|
105
|
+
`[memory-vec] sqlite-vec extension_loaded=true vector_dimensions=${EMBEDDING_DIMENSIONS}`,
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
const existingSchema = this.getVecTableSchema();
|
|
109
|
+
if (existingSchema && !existingSchema.includes("distance_metric=cosine")) {
|
|
110
|
+
console.warn(
|
|
111
|
+
"[memory-vec] Existing memory_vec table is missing cosine distance metric; rebuilding from agent_memory",
|
|
112
|
+
);
|
|
113
|
+
db.run("DROP TABLE memory_vec");
|
|
114
|
+
}
|
|
115
|
+
|
|
90
116
|
db.run(`
|
|
91
117
|
CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
|
|
92
118
|
memory_id TEXT PRIMARY KEY,
|
|
93
|
-
embedding float[
|
|
119
|
+
embedding float[${EMBEDDING_DIMENSIONS}] distance_metric=cosine
|
|
94
120
|
)
|
|
95
121
|
`);
|
|
96
122
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
.
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
if (existing.length > 0) {
|
|
105
|
-
const vecCount = db
|
|
106
|
-
.prepare<{ count: number }, []>("SELECT COUNT(*) as count FROM memory_vec")
|
|
107
|
-
.get();
|
|
108
|
-
|
|
109
|
-
if ((vecCount?.count ?? 0) < existing.length) {
|
|
110
|
-
const insert = db.prepare(
|
|
111
|
-
"INSERT OR IGNORE INTO memory_vec(memory_id, embedding) VALUES (?, ?)",
|
|
112
|
-
);
|
|
113
|
-
const tx = db.transaction(() => {
|
|
114
|
-
for (const row of existing) {
|
|
115
|
-
insert.run(row.id, row.embedding);
|
|
116
|
-
}
|
|
117
|
-
});
|
|
118
|
-
tx();
|
|
119
|
-
console.log(`[memory] Synced ${existing.length} embeddings to memory_vec`);
|
|
120
|
-
}
|
|
123
|
+
const healthBefore = this.getHealthCounts();
|
|
124
|
+
if (healthBefore.missingFromVec > 0 || healthBefore.extraInVec > 0) {
|
|
125
|
+
this.populateVecTable(healthBefore.memoryVec);
|
|
126
|
+
} else {
|
|
127
|
+
console.log(
|
|
128
|
+
`[memory-vec] populate skipped attempted=0 inserted=0 memory_vec=${healthBefore.memoryVec} valid_embedding=${healthBefore.validEmbedding}`,
|
|
129
|
+
);
|
|
121
130
|
}
|
|
122
131
|
|
|
123
132
|
this.vecInitialized = true;
|
|
124
133
|
} catch (err) {
|
|
125
|
-
|
|
134
|
+
this.vecInitialized = false;
|
|
135
|
+
console.error("[memory-vec] Failed to initialize memory_vec:", (err as Error).message);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private getVecTableSchema(): string | null {
|
|
140
|
+
try {
|
|
141
|
+
return (
|
|
142
|
+
getDb()
|
|
143
|
+
.prepare<{ sql: string | null }, []>(
|
|
144
|
+
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_vec'",
|
|
145
|
+
)
|
|
146
|
+
.get()?.sql ?? null
|
|
147
|
+
);
|
|
148
|
+
} catch {
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private getVecCount(): number {
|
|
154
|
+
if (!this.getVecTableSchema()) return 0;
|
|
155
|
+
return (
|
|
156
|
+
getDb().prepare<{ count: number }, []>("SELECT COUNT(*) as count FROM memory_vec").get()
|
|
157
|
+
?.count ?? 0
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
private populateVecTable(beforeCount: number): void {
|
|
162
|
+
const db = getDb();
|
|
163
|
+
const deletedExtra = db
|
|
164
|
+
.prepare(
|
|
165
|
+
`DELETE FROM memory_vec
|
|
166
|
+
WHERE memory_id NOT IN (SELECT id FROM agent_memory)`,
|
|
167
|
+
)
|
|
168
|
+
.run();
|
|
169
|
+
if (deletedExtra.changes > 0) {
|
|
170
|
+
console.warn(`[memory-vec] removed_extra_rows count=${deletedExtra.changes}`);
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
const rows = db
|
|
174
|
+
.prepare<{ id: string; embedding: Buffer }, []>(
|
|
175
|
+
"SELECT id, embedding FROM agent_memory WHERE embedding IS NOT NULL",
|
|
176
|
+
)
|
|
177
|
+
.all();
|
|
178
|
+
const deleteVec = db.prepare("DELETE FROM memory_vec WHERE memory_id = ?");
|
|
179
|
+
const insertVec = db.prepare("INSERT INTO memory_vec(memory_id, embedding) VALUES (?, ?)");
|
|
180
|
+
|
|
181
|
+
let attempted = 0;
|
|
182
|
+
let inserted = 0;
|
|
183
|
+
let skippedInvalidDimensions = 0;
|
|
184
|
+
let failed = 0;
|
|
185
|
+
|
|
186
|
+
for (const row of rows) {
|
|
187
|
+
const embeddingBuffer = this.toVecBuffer(row.embedding);
|
|
188
|
+
if (!embeddingBuffer) {
|
|
189
|
+
skippedInvalidDimensions++;
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
attempted++;
|
|
194
|
+
try {
|
|
195
|
+
deleteVec.run(row.id);
|
|
196
|
+
insertVec.run(row.id, embeddingBuffer);
|
|
197
|
+
inserted++;
|
|
198
|
+
} catch (err) {
|
|
199
|
+
failed++;
|
|
200
|
+
console.error(
|
|
201
|
+
`[memory-vec] populate failed memory_id=${row.id}: ${(err as Error).message}`,
|
|
202
|
+
);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const afterCount = this.getVecCount();
|
|
207
|
+
this.lastPopulate = {
|
|
208
|
+
attempted,
|
|
209
|
+
inserted,
|
|
210
|
+
skippedInvalidDimensions,
|
|
211
|
+
failed,
|
|
212
|
+
beforeCount,
|
|
213
|
+
afterCount,
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
console.log(
|
|
217
|
+
`[memory-vec] populate attempted=${attempted} inserted=${inserted} skipped_invalid_dimensions=${skippedInvalidDimensions} failed=${failed} before_count=${beforeCount} after_count=${afterCount}`,
|
|
218
|
+
);
|
|
219
|
+
|
|
220
|
+
if (failed > 0 || afterCount < attempted) {
|
|
221
|
+
console.error(
|
|
222
|
+
`[memory-vec] populate incomplete attempted=${attempted} after_count=${afterCount} failed=${failed}`,
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
private toVecBuffer(embedding: Buffer | Float32Array): Buffer | null {
|
|
228
|
+
if (embedding instanceof Float32Array) {
|
|
229
|
+
if (embedding.length !== EMBEDDING_DIMENSIONS) return null;
|
|
230
|
+
return serializeEmbedding(embedding);
|
|
126
231
|
}
|
|
232
|
+
if (embedding.length !== VECTOR_BYTES) return null;
|
|
233
|
+
return embedding;
|
|
127
234
|
}
|
|
128
235
|
|
|
129
236
|
store(input: MemoryInput): AgentMemory {
|
|
@@ -223,7 +330,11 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
223
330
|
): MemoryCandidate[] {
|
|
224
331
|
const { scope = "all", limit = 10, source, isLead = false, includeExpired = false } = options;
|
|
225
332
|
|
|
226
|
-
|
|
333
|
+
const health = this.getHealth();
|
|
334
|
+
if (health.retrievalMode === "vec" && embedding.length === EMBEDDING_DIMENSIONS) {
|
|
335
|
+
console.log(
|
|
336
|
+
`[memory-search] retrieval_path=vec scope=${scope} limit=${limit} vec_rows=${health.counts.memoryVec} searchable=${health.counts.searchable}`,
|
|
337
|
+
);
|
|
227
338
|
return this.searchWithVec(embedding, agentId, {
|
|
228
339
|
scope,
|
|
229
340
|
limit,
|
|
@@ -232,6 +343,10 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
232
343
|
includeExpired,
|
|
233
344
|
});
|
|
234
345
|
}
|
|
346
|
+
|
|
347
|
+
console.log(
|
|
348
|
+
`[memory-search] retrieval_path=fallback scope=${scope} limit=${limit} reason=${embedding.length !== EMBEDDING_DIMENSIONS ? "query_dimension_mismatch" : health.reasons.join("|") || "vec_unavailable"}`,
|
|
349
|
+
);
|
|
235
350
|
return this.searchBruteForce(embedding, agentId, {
|
|
236
351
|
scope,
|
|
237
352
|
limit,
|
|
@@ -255,58 +370,46 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
255
370
|
const db = getDb();
|
|
256
371
|
const { scope, limit, source, isLead, includeExpired } = options;
|
|
257
372
|
|
|
258
|
-
// KNN query — fetch more candidates than needed for post-filtering
|
|
259
|
-
const knnLimit = limit * 5; // over-fetch to account for scope/expiry filters
|
|
260
373
|
const embeddingBuffer = serializeEmbedding(queryEmbedding);
|
|
374
|
+
// sqlite-vec hard ceiling is 4096 for knn queries
|
|
375
|
+
const knnLimit = Math.min(Math.max(limit, this.getVecCount()), 4096);
|
|
261
376
|
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
"SELECT memory_id, distance FROM memory_vec WHERE embedding MATCH ? AND k = ?",
|
|
265
|
-
)
|
|
266
|
-
.all(embeddingBuffer, knnLimit);
|
|
267
|
-
|
|
268
|
-
if (vecRows.length === 0) return [];
|
|
269
|
-
|
|
270
|
-
// Build ID list and distance map
|
|
271
|
-
const distanceMap = new Map<string, number>();
|
|
272
|
-
const ids: string[] = [];
|
|
273
|
-
for (const vr of vecRows) {
|
|
274
|
-
distanceMap.set(vr.memory_id, vr.distance);
|
|
275
|
-
ids.push(vr.memory_id);
|
|
276
|
-
}
|
|
377
|
+
const conditions: string[] = ["v.embedding MATCH ?"];
|
|
378
|
+
const params: (Buffer | string | number | null)[] = [embeddingBuffer];
|
|
277
379
|
|
|
278
|
-
|
|
279
|
-
const placeholders = ids.map(() => "?").join(",");
|
|
280
|
-
const conditions: string[] = [`id IN (${placeholders})`];
|
|
281
|
-
const params: (string | null)[] = [...ids];
|
|
282
|
-
|
|
283
|
-
this.addScopeConditions(conditions, params, agentId, scope, isLead);
|
|
380
|
+
this.addScopeConditions(conditions, params, agentId, scope, isLead, "m");
|
|
284
381
|
|
|
285
382
|
if (source) {
|
|
286
|
-
conditions.push("source = ?");
|
|
383
|
+
conditions.push("m.source = ?");
|
|
287
384
|
params.push(source);
|
|
288
385
|
}
|
|
289
386
|
|
|
290
387
|
if (!includeExpired) {
|
|
291
|
-
conditions.push("(expiresAt IS NULL OR expiresAt > datetime('now'))");
|
|
388
|
+
conditions.push("(m.expiresAt IS NULL OR m.expiresAt > datetime('now'))");
|
|
292
389
|
}
|
|
293
390
|
|
|
391
|
+
conditions.push("v.k = ?");
|
|
392
|
+
params.push(knnLimit);
|
|
393
|
+
|
|
294
394
|
const rows = db
|
|
295
|
-
.prepare<AgentMemoryRow, (string | null)[]>(
|
|
296
|
-
`SELECT
|
|
395
|
+
.prepare<AgentMemoryRow & { distance: number }, (Buffer | string | number | null)[]>(
|
|
396
|
+
`SELECT m.*, v.distance
|
|
397
|
+
FROM memory_vec v
|
|
398
|
+
JOIN agent_memory m ON m.id = v.memory_id
|
|
399
|
+
WHERE ${conditions.join(" AND ")}
|
|
400
|
+
ORDER BY v.distance
|
|
401
|
+
LIMIT ?`,
|
|
297
402
|
)
|
|
298
|
-
.all(...params);
|
|
403
|
+
.all(...params, limit);
|
|
299
404
|
|
|
300
|
-
// Map to candidates with similarity scores
|
|
301
405
|
const candidates: MemoryCandidate[] = [];
|
|
302
406
|
for (const row of rows) {
|
|
303
|
-
const
|
|
304
|
-
|
|
407
|
+
const similarity = 1 - row.distance;
|
|
408
|
+
if (similarity < MIN_SIMILARITY) continue;
|
|
305
409
|
candidates.push(rowToCandidate(row, similarity));
|
|
306
410
|
}
|
|
307
411
|
|
|
308
|
-
candidates
|
|
309
|
-
return candidates.slice(0, limit);
|
|
412
|
+
return candidates;
|
|
310
413
|
}
|
|
311
414
|
|
|
312
415
|
private searchBruteForce(
|
|
@@ -349,6 +452,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
349
452
|
const emb = deserializeEmbedding(row.embedding);
|
|
350
453
|
if (emb.length !== queryEmbedding.length) continue;
|
|
351
454
|
const similarity = cosineSimilarity(queryEmbedding, emb);
|
|
455
|
+
if (similarity < MIN_SIMILARITY) continue;
|
|
352
456
|
candidates.push(rowToCandidate(row, similarity));
|
|
353
457
|
}
|
|
354
458
|
|
|
@@ -358,26 +462,28 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
358
462
|
|
|
359
463
|
private addScopeConditions(
|
|
360
464
|
conditions: string[],
|
|
361
|
-
params: (string | null)[],
|
|
465
|
+
params: (Buffer | string | number | null)[],
|
|
362
466
|
agentId: string,
|
|
363
467
|
scope: string,
|
|
364
468
|
isLead: boolean,
|
|
469
|
+
tableAlias = "",
|
|
365
470
|
): void {
|
|
471
|
+
const col = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name);
|
|
366
472
|
if (!isLead) {
|
|
367
473
|
if (scope === "agent") {
|
|
368
|
-
conditions.push("agentId = ? AND scope = 'agent'
|
|
474
|
+
conditions.push(`${col("agentId")} = ? AND ${col("scope")} = 'agent'`);
|
|
369
475
|
params.push(agentId);
|
|
370
476
|
} else if (scope === "swarm") {
|
|
371
|
-
conditions.push("scope = 'swarm'
|
|
477
|
+
conditions.push(`${col("scope")} = 'swarm'`);
|
|
372
478
|
} else {
|
|
373
|
-
conditions.push("
|
|
479
|
+
conditions.push(`(${col("agentId")} = ? OR ${col("scope")} = 'swarm')`);
|
|
374
480
|
params.push(agentId);
|
|
375
481
|
}
|
|
376
482
|
} else {
|
|
377
483
|
if (scope === "agent") {
|
|
378
|
-
conditions.push("scope = 'agent'
|
|
484
|
+
conditions.push(`${col("scope")} = 'agent'`);
|
|
379
485
|
} else if (scope === "swarm") {
|
|
380
|
-
conditions.push("scope = 'swarm'
|
|
486
|
+
conditions.push(`${col("scope")} = 'swarm'`);
|
|
381
487
|
}
|
|
382
488
|
}
|
|
383
489
|
}
|
|
@@ -424,6 +530,31 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
424
530
|
return rows.map(rowToAgentMemory);
|
|
425
531
|
}
|
|
426
532
|
|
|
533
|
+
isSourceProtected(source: AgentMemorySource): boolean {
|
|
534
|
+
return PROTECTED_SOURCES.has(source);
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
listForCuration(
|
|
538
|
+
agentId?: string,
|
|
539
|
+
): { id: string; source: string; name: string; createdAt: string }[] {
|
|
540
|
+
const db = getDb();
|
|
541
|
+
const protectedList = [...PROTECTED_SOURCES].map((s) => `'${s}'`).join(",");
|
|
542
|
+
if (agentId) {
|
|
543
|
+
return db
|
|
544
|
+
.prepare<{ id: string; source: string; name: string; createdAt: string }, [string]>(
|
|
545
|
+
`SELECT id, source, name, createdAt FROM agent_memory
|
|
546
|
+
WHERE agentId = ? AND source NOT IN (${protectedList})`,
|
|
547
|
+
)
|
|
548
|
+
.all(agentId);
|
|
549
|
+
}
|
|
550
|
+
return db
|
|
551
|
+
.prepare<{ id: string; source: string; name: string; createdAt: string }, []>(
|
|
552
|
+
`SELECT id, source, name, createdAt FROM agent_memory
|
|
553
|
+
WHERE source NOT IN (${protectedList})`,
|
|
554
|
+
)
|
|
555
|
+
.all();
|
|
556
|
+
}
|
|
557
|
+
|
|
427
558
|
listForReembedding(options?: { agentId?: string }): { id: string; content: string }[] {
|
|
428
559
|
const db = getDb();
|
|
429
560
|
if (options?.agentId) {
|
|
@@ -440,7 +571,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
440
571
|
|
|
441
572
|
delete(id: string): boolean {
|
|
442
573
|
const db = getDb();
|
|
443
|
-
if (
|
|
574
|
+
if (this.vecInitialized && this.getVecTableSchema()) {
|
|
444
575
|
db.prepare("DELETE FROM memory_vec WHERE memory_id = ?").run(id);
|
|
445
576
|
}
|
|
446
577
|
const result = db.prepare("DELETE FROM agent_memory WHERE id = ?").run(id);
|
|
@@ -450,7 +581,7 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
450
581
|
deleteBySourcePath(sourcePath: string, agentId: string): number {
|
|
451
582
|
const db = getDb();
|
|
452
583
|
|
|
453
|
-
if (
|
|
584
|
+
if (this.vecInitialized && this.getVecTableSchema()) {
|
|
454
585
|
// Get IDs first for vec table cleanup
|
|
455
586
|
const ids = db
|
|
456
587
|
.prepare<{ id: string }, [string, string]>(
|
|
@@ -472,6 +603,40 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
472
603
|
return result.changes;
|
|
473
604
|
}
|
|
474
605
|
|
|
606
|
+
purgeExpired(): number {
|
|
607
|
+
const db = getDb();
|
|
608
|
+
|
|
609
|
+
const expiredIds = db
|
|
610
|
+
.prepare<{ id: string }, []>(
|
|
611
|
+
"SELECT id FROM agent_memory WHERE expiresAt IS NOT NULL AND expiresAt <= datetime('now')",
|
|
612
|
+
)
|
|
613
|
+
.all();
|
|
614
|
+
|
|
615
|
+
if (expiredIds.length === 0) return 0;
|
|
616
|
+
|
|
617
|
+
if (this.vecInitialized && this.getVecTableSchema()) {
|
|
618
|
+
const batchSize = 500;
|
|
619
|
+
for (let i = 0; i < expiredIds.length; i += batchSize) {
|
|
620
|
+
const batch = expiredIds.slice(i, i + batchSize);
|
|
621
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
622
|
+
db.prepare(`DELETE FROM memory_vec WHERE memory_id IN (${placeholders})`).run(
|
|
623
|
+
...batch.map((r) => r.id),
|
|
624
|
+
);
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
const result = db
|
|
629
|
+
.prepare(
|
|
630
|
+
"DELETE FROM agent_memory WHERE expiresAt IS NOT NULL AND expiresAt <= datetime('now')",
|
|
631
|
+
)
|
|
632
|
+
.run();
|
|
633
|
+
|
|
634
|
+
console.log(
|
|
635
|
+
`[memory] Purged ${result.changes} expired memory row(s) (vec cleanup: ${expiredIds.length} id(s))`,
|
|
636
|
+
);
|
|
637
|
+
return result.changes;
|
|
638
|
+
}
|
|
639
|
+
|
|
475
640
|
updateEmbedding(id: string, embedding: Float32Array, model: string): void {
|
|
476
641
|
const db = getDb();
|
|
477
642
|
const buffer = serializeEmbedding(embedding);
|
|
@@ -481,11 +646,20 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
481
646
|
id,
|
|
482
647
|
);
|
|
483
648
|
|
|
484
|
-
if (
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
649
|
+
if (this.vecInitialized && this.getVecTableSchema()) {
|
|
650
|
+
const vecBuffer = this.toVecBuffer(embedding);
|
|
651
|
+
if (!vecBuffer) {
|
|
652
|
+
console.warn(
|
|
653
|
+
`[memory-vec] update skipped memory_id=${id} reason=invalid_dimensions dimensions=${embedding.length} expected=${EMBEDDING_DIMENSIONS}`,
|
|
654
|
+
);
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
try {
|
|
658
|
+
db.prepare("DELETE FROM memory_vec WHERE memory_id = ?").run(id);
|
|
659
|
+
db.prepare("INSERT INTO memory_vec(memory_id, embedding) VALUES (?, ?)").run(id, vecBuffer);
|
|
660
|
+
} catch (err) {
|
|
661
|
+
console.error(`[memory-vec] update failed memory_id=${id}: ${(err as Error).message}`);
|
|
662
|
+
}
|
|
489
663
|
}
|
|
490
664
|
}
|
|
491
665
|
|
|
@@ -536,4 +710,79 @@ export class SqliteMemoryStore implements MemoryStore {
|
|
|
536
710
|
expired: expired?.count ?? 0,
|
|
537
711
|
};
|
|
538
712
|
}
|
|
713
|
+
|
|
714
|
+
private getHealthCounts(): MemoryHealth["counts"] {
|
|
715
|
+
const db = getDb();
|
|
716
|
+
const tableExists = this.getVecTableSchema() !== null;
|
|
717
|
+
const tableUsable = tableExists && isSqliteVecAvailable();
|
|
718
|
+
const count = (sql: string) => db.prepare<{ count: number }, []>(sql).get()?.count ?? 0;
|
|
719
|
+
|
|
720
|
+
return {
|
|
721
|
+
total: count("SELECT COUNT(*) as count FROM agent_memory"),
|
|
722
|
+
withEmbedding: count(
|
|
723
|
+
"SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL",
|
|
724
|
+
),
|
|
725
|
+
validEmbedding: count(
|
|
726
|
+
`SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) = ${VECTOR_BYTES}`,
|
|
727
|
+
),
|
|
728
|
+
invalidEmbedding: count(
|
|
729
|
+
`SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}`,
|
|
730
|
+
),
|
|
731
|
+
searchable: count(
|
|
732
|
+
`SELECT COUNT(*) as count FROM agent_memory
|
|
733
|
+
WHERE embedding IS NOT NULL
|
|
734
|
+
AND length(embedding) = ${VECTOR_BYTES}
|
|
735
|
+
AND (expiresAt IS NULL OR expiresAt > datetime('now'))`,
|
|
736
|
+
),
|
|
737
|
+
memoryVec: tableUsable ? count("SELECT COUNT(*) as count FROM memory_vec") : 0,
|
|
738
|
+
missingFromVec: tableUsable
|
|
739
|
+
? count(
|
|
740
|
+
`SELECT COUNT(*) as count
|
|
741
|
+
FROM agent_memory m
|
|
742
|
+
LEFT JOIN memory_vec v ON v.memory_id = m.id
|
|
743
|
+
WHERE m.embedding IS NOT NULL
|
|
744
|
+
AND length(m.embedding) = ${VECTOR_BYTES}
|
|
745
|
+
AND v.memory_id IS NULL`,
|
|
746
|
+
)
|
|
747
|
+
: count(
|
|
748
|
+
`SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) = ${VECTOR_BYTES}`,
|
|
749
|
+
),
|
|
750
|
+
extraInVec: tableUsable
|
|
751
|
+
? count(
|
|
752
|
+
`SELECT COUNT(*) as count
|
|
753
|
+
FROM memory_vec v
|
|
754
|
+
LEFT JOIN agent_memory m ON m.id = v.memory_id
|
|
755
|
+
WHERE m.id IS NULL`,
|
|
756
|
+
)
|
|
757
|
+
: 0,
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
getHealth(): MemoryHealth {
|
|
762
|
+
const schema = this.getVecTableSchema();
|
|
763
|
+
const counts = this.getHealthCounts();
|
|
764
|
+
const reasons: string[] = [];
|
|
765
|
+
|
|
766
|
+
if (!isSqliteVecAvailable()) reasons.push("sqlite_vec_extension_unavailable");
|
|
767
|
+
if (!schema) reasons.push("memory_vec_table_missing");
|
|
768
|
+
if (!this.vecInitialized) reasons.push("memory_vec_not_initialized");
|
|
769
|
+
if (counts.memoryVec === 0) reasons.push("memory_vec_empty");
|
|
770
|
+
if (counts.missingFromVec > 0) reasons.push("memory_vec_missing_embeddings");
|
|
771
|
+
if (counts.extraInVec > 0) reasons.push("memory_vec_extra_rows");
|
|
772
|
+
|
|
773
|
+
return {
|
|
774
|
+
sqliteVec: {
|
|
775
|
+
extensionLoaded: isSqliteVecAvailable(),
|
|
776
|
+
tableExists: schema !== null,
|
|
777
|
+
initialized: this.vecInitialized,
|
|
778
|
+
vectorDimensions: EMBEDDING_DIMENSIONS,
|
|
779
|
+
distanceMetric: "cosine",
|
|
780
|
+
schema,
|
|
781
|
+
lastPopulate: this.lastPopulate,
|
|
782
|
+
},
|
|
783
|
+
counts,
|
|
784
|
+
retrievalMode: reasons.length === 0 ? "vec" : "fallback",
|
|
785
|
+
reasons,
|
|
786
|
+
};
|
|
787
|
+
}
|
|
539
788
|
}
|