@desplega.ai/agent-swarm 1.92.0 → 1.92.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +1 -1
  2. package/openapi.json +276 -3
  3. package/package.json +6 -6
  4. package/plugin/skills/pages/SKILL.md +5 -2
  5. package/src/be/db.ts +416 -20
  6. package/src/be/memory/boot-reembed.ts +85 -0
  7. package/src/be/memory/constants.ts +44 -2
  8. package/src/be/memory/providers/openai-embedding.ts +15 -5
  9. package/src/be/memory/providers/sqlite-store.ts +325 -76
  10. package/src/be/memory/reranker.ts +35 -17
  11. package/src/be/memory/types.ts +43 -0
  12. package/src/be/migrations/084_script_run_journal_duration.sql +5 -0
  13. package/src/be/migrations/085_script_runs_kind.sql +9 -0
  14. package/src/be/migrations/086_pages_default_authed.sql +64 -0
  15. package/src/be/migrations/087_skill_files.sql +19 -0
  16. package/src/be/modelsdev-cache.json +5622 -2543
  17. package/src/be/seed-scripts/catalog/boot-triage.ts +221 -0
  18. package/src/be/seed-scripts/catalog/catalog-report.ts +457 -0
  19. package/src/be/seed-scripts/catalog/compound-insights.ts +465 -0
  20. package/src/be/seed-scripts/catalog/gh-pr-snapshot.ts +1 -1
  21. package/src/be/seed-scripts/catalog/memory-eval.ts +1059 -0
  22. package/src/be/seed-scripts/catalog/ops-catalog-audit.ts +34 -439
  23. package/src/be/seed-scripts/catalog/schedule-health.ts +78 -2
  24. package/src/be/seed-scripts/catalog/task-failure-audit.ts +48 -1
  25. package/src/be/seed-scripts/index.ts +32 -4
  26. package/src/be/seed-skills/index.ts +0 -7
  27. package/src/be/skill-sync.ts +91 -7
  28. package/src/commands/runner.ts +6 -2
  29. package/src/heartbeat/templates.ts +20 -16
  30. package/src/http/index.ts +50 -7
  31. package/src/http/mcp-user.ts +23 -0
  32. package/src/http/mcp.ts +58 -0
  33. package/src/http/memory.ts +62 -0
  34. package/src/http/pages.ts +1 -1
  35. package/src/http/script-runs.ts +2 -0
  36. package/src/http/scripts.ts +39 -2
  37. package/src/http/skills.ts +225 -0
  38. package/src/providers/claude-adapter.ts +56 -24
  39. package/src/script-workflows/workflow-ctx.ts +7 -3
  40. package/src/scripts-runtime/sdk-allowlist.ts +1 -0
  41. package/src/scripts-runtime/swarm-sdk.ts +13 -0
  42. package/src/scripts-runtime/types/stdlib.d.ts +1 -0
  43. package/src/scripts-runtime/types/swarm-sdk.d.ts +1 -0
  44. package/src/server.ts +2 -0
  45. package/src/tasks/worker-follow-up.ts +12 -0
  46. package/src/tests/claude-adapter-binary.test.ts +135 -81
  47. package/src/tests/create-page-tool.test.ts +19 -2
  48. package/src/tests/heartbeat-checklist.test.ts +36 -0
  49. package/src/tests/mcp-transport-gc.test.ts +58 -0
  50. package/src/tests/memory-e2e.test.ts +6 -6
  51. package/src/tests/memory-health-endpoint.test.ts +78 -0
  52. package/src/tests/memory-rater-e2e.test.ts +4 -5
  53. package/src/tests/memory-reranker.test.ts +135 -124
  54. package/src/tests/memory-store.test.ts +221 -1
  55. package/src/tests/memory.test.ts +13 -12
  56. package/src/tests/pages-http.test.ts +20 -2
  57. package/src/tests/pages-storage.test.ts +26 -0
  58. package/src/tests/scripts-mcp-e2e.test.ts +53 -0
  59. package/src/tests/seed-scripts.test.ts +328 -3
  60. package/src/tests/skill-files-http.test.ts +171 -0
  61. package/src/tests/skill-files.test.ts +162 -0
  62. package/src/tests/skill-get-file-tool.test.ts +110 -0
  63. package/src/tests/skill-sync.test.ts +125 -6
  64. package/src/tests/task-cascade-fail.test.ts +304 -0
  65. package/src/tools/create-page.ts +2 -2
  66. package/src/tools/skills/index.ts +1 -0
  67. package/src/tools/skills/skill-get-file.ts +80 -0
  68. package/src/tools/tool-config.ts +2 -1
  69. package/src/types.ts +20 -0
  70. package/src/utils/internal-ai/complete-structured.ts +2 -2
  71. package/templates/schedules/daily-blocker-digest/content.md +68 -54
  72. package/templates/schedules/daily-compounding-reflection/content.md +4 -4
  73. package/templates/schedules/daily-hn-briefing/content.md +5 -5
  74. package/templates/schedules/daily-workflow-health-audit/content.md +6 -6
  75. package/templates/schedules/gtm-weekly-review/content.md +9 -9
  76. package/templates/schedules/weekly-dependabot-triage/content.md +24 -20
  77. package/templates/skills/agentmail-sending/content.md +6 -7
  78. package/templates/skills/desloppify/content.md +8 -9
  79. package/templates/skills/jira-interaction/content.md +25 -33
  80. package/templates/skills/kapso-whatsapp/content.md +29 -30
  81. package/templates/skills/linear-interaction/content.md +8 -9
  82. package/templates/skills/profile-corruption-escalation/content.md +44 -85
  83. package/templates/skills/sprite-cli/content.md +4 -5
  84. package/templates/skills/turso-interaction/content.md +14 -17
  85. package/templates/skills/workflow-iterate/content.md +38 -391
  86. package/templates/skills/x-api-interactions/content.md +4 -6
  87. package/templates/workflows/llm-safe-release-context/config.json +13 -0
  88. package/templates/workflows/llm-safe-release-context/content.md +69 -0
  89. package/templates/skills/scheduled-task-resilience/config.json +0 -14
  90. package/templates/skills/scheduled-task-resilience/content.md +0 -95
@@ -15,12 +15,54 @@ export const TTL_DEFAULTS: Record<AgentMemorySource, number | null> = {
15
15
  manual: null,
16
16
  };
17
17
 
18
+ // Per-source recency decay half-life (in days).
19
+ // manual = Infinity (no decay — curated knowledge stays relevant forever).
20
+ // A global MEMORY_RECENCY_HALF_LIFE_DAYS override forces ALL sources to the same value.
21
+ const GLOBAL_HALF_LIFE_OVERRIDE = process.env.MEMORY_RECENCY_HALF_LIFE_DAYS;
22
+ const GLOBAL_HALF_LIFE =
23
+ GLOBAL_HALF_LIFE_OVERRIDE != null && GLOBAL_HALF_LIFE_OVERRIDE !== ""
24
+ ? Number(GLOBAL_HALF_LIFE_OVERRIDE)
25
+ : null;
26
+
27
+ export const RECENCY_DECAY_HALF_LIFE: Record<AgentMemorySource, number> =
28
+ GLOBAL_HALF_LIFE != null && Number.isFinite(GLOBAL_HALF_LIFE)
29
+ ? {
30
+ manual: GLOBAL_HALF_LIFE,
31
+ file_index: GLOBAL_HALF_LIFE,
32
+ task_completion: GLOBAL_HALF_LIFE,
33
+ session_summary: GLOBAL_HALF_LIFE,
34
+ }
35
+ : {
36
+ manual: Number.POSITIVE_INFINITY,
37
+ file_index: 180,
38
+ task_completion: 14,
39
+ session_summary: 7,
40
+ };
41
+
42
+ // Legacy export — callers that don't have a source fall back to task_completion's value.
43
+ export const RECENCY_DECAY_HALF_LIFE_DAYS = RECENCY_DECAY_HALF_LIFE.task_completion;
44
+
45
+ // Source-quality multiplier for reranking.
46
+ // Curated manual memories rank higher; ephemeral session summaries rank lower.
47
+ export const SOURCE_QUALITY_MULTIPLIER: Record<AgentMemorySource, number> = {
48
+ manual: 1.5,
49
+ file_index: 1.0,
50
+ task_completion: 0.7,
51
+ session_summary: 0.5,
52
+ };
53
+
54
+ // Minimum raw cosine similarity to keep a candidate. Below this, the result is noise.
55
+ export const MIN_SIMILARITY = numEnv("MEMORY_MIN_SIMILARITY", 0.1);
56
+
18
57
  // Reranking parameters
19
- export const RECENCY_DECAY_HALF_LIFE_DAYS = numEnv("MEMORY_RECENCY_HALF_LIFE_DAYS", 14);
20
58
  export const ACCESS_BOOST_MAX_MULTIPLIER = numEnv("MEMORY_ACCESS_BOOST_MAX", 1.5);
21
59
  export const ACCESS_BOOST_RECENCY_WINDOW_HOURS = numEnv("MEMORY_ACCESS_RECENCY_HOURS", 48);
22
60
  export const CANDIDATE_SET_MULTIPLIER = numEnv("MEMORY_CANDIDATE_MULTIPLIER", 3);
23
61
 
24
62
  // Embedding defaults
25
- export const DEFAULT_EMBEDDING_DIMENSIONS = 512;
63
+ export const EMBEDDING_DIMENSIONS = numEnv("EMBEDDING_DIMENSIONS", 512);
64
+ export const DEFAULT_EMBEDDING_DIMENSIONS = EMBEDDING_DIMENSIONS;
26
65
  export const DEFAULT_EMBEDDING_MODEL = "openai/text-embedding-3-small";
66
+
67
+ // Manual memories must NEVER be deleted by automated processes (curator, GC, etc.)
68
+ export const PROTECTED_SOURCES: ReadonlySet<AgentMemorySource> = new Set(["manual"]);
@@ -1,5 +1,5 @@
1
1
  import OpenAI from "openai";
2
- import { DEFAULT_EMBEDDING_DIMENSIONS, DEFAULT_EMBEDDING_MODEL } from "../constants";
2
+ import { DEFAULT_EMBEDDING_MODEL, EMBEDDING_DIMENSIONS } from "../constants";
3
3
  import type { EmbeddingProvider } from "../types";
4
4
 
5
5
  interface OpenAIEmbeddingConfig {
@@ -21,10 +21,7 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
21
21
 
22
22
  this.model = config?.model ?? process.env.EMBEDDING_MODEL ?? "text-embedding-3-small";
23
23
 
24
- this.dimensions =
25
- config?.dimensions ??
26
- Number(process.env.EMBEDDING_DIMENSIONS) ??
27
- DEFAULT_EMBEDDING_DIMENSIONS;
24
+ this.dimensions = config?.dimensions ?? EMBEDDING_DIMENSIONS;
28
25
 
29
26
  this.name = config?.model
30
27
  ? `openai/${config.model}`
@@ -58,6 +55,13 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
58
55
  const values = response.data[0]?.embedding;
59
56
  if (!values) return null;
60
57
 
58
+ if (values.length !== this.dimensions) {
59
+ console.error(
60
+ `[memory] Embedding dimension mismatch: expected=${this.dimensions} got=${values.length}. Provider may not support the 'dimensions' parameter.`,
61
+ );
62
+ return null;
63
+ }
64
+
61
65
  return new Float32Array(values);
62
66
  } catch (err) {
63
67
  console.error("[memory] Embedding failed:", (err as Error).message);
@@ -93,6 +97,12 @@ export class OpenAIEmbeddingProvider implements EmbeddingProvider {
93
97
  for (const item of response.data) {
94
98
  const originalIndex = nonEmptyIndices[item.index];
95
99
  if (originalIndex !== undefined && item.embedding) {
100
+ if (item.embedding.length !== this.dimensions) {
101
+ console.error(
102
+ `[memory] Batch embedding dimension mismatch: expected=${this.dimensions} got=${item.embedding.length}. Provider may not support the 'dimensions' parameter.`,
103
+ );
104
+ continue;
105
+ }
96
106
  results[originalIndex] = new Float32Array(item.embedding);
97
107
  }
98
108
  }
@@ -1,16 +1,25 @@
1
1
  import { getDb, isSqliteVecAvailable } from "@/be/db";
2
2
  import { cosineSimilarity, deserializeEmbedding, serializeEmbedding } from "@/be/embedding";
3
3
  import type { AgentMemory, AgentMemoryScope, AgentMemorySource } from "@/types";
4
- import { TTL_DEFAULTS } from "../constants";
4
+ import {
5
+ EMBEDDING_DIMENSIONS,
6
+ MIN_SIMILARITY,
7
+ PROTECTED_SOURCES,
8
+ TTL_DEFAULTS,
9
+ } from "../constants";
5
10
  import type {
6
11
  MemoryCandidate,
12
+ MemoryHealth,
7
13
  MemoryInput,
8
14
  MemoryListOptions,
9
15
  MemorySearchOptions,
10
16
  MemoryStats,
11
17
  MemoryStore,
18
+ MemoryVecPopulateStats,
12
19
  } from "../types";
13
20
 
21
+ const VECTOR_BYTES = EMBEDDING_DIMENSIONS * Float32Array.BYTES_PER_ELEMENT;
22
+
14
23
  type AgentMemoryRow = {
15
24
  id: string;
16
25
  agentId: string | null;
@@ -76,54 +85,152 @@ function computeExpiresAt(source: AgentMemorySource): string | null {
76
85
 
77
86
  export class SqliteMemoryStore implements MemoryStore {
78
87
  private vecInitialized = false;
88
+ private lastPopulate: MemoryVecPopulateStats | null = null;
79
89
 
80
90
  constructor() {
81
91
  this.ensureVecTable();
82
92
  }
83
93
 
84
94
  private ensureVecTable(): void {
85
- if (this.vecInitialized || !isSqliteVecAvailable()) return;
95
+ if (this.vecInitialized) return;
96
+
97
+ if (!isSqliteVecAvailable()) {
98
+ console.warn("[memory-vec] sqlite-vec extension_loaded=false; retrieval_mode=fallback");
99
+ return;
100
+ }
86
101
 
87
102
  const db = getDb();
88
- // Create the virtual table if it doesn't exist
89
103
  try {
104
+ console.log(
105
+ `[memory-vec] sqlite-vec extension_loaded=true vector_dimensions=${EMBEDDING_DIMENSIONS}`,
106
+ );
107
+
108
+ const existingSchema = this.getVecTableSchema();
109
+ if (existingSchema && !existingSchema.includes("distance_metric=cosine")) {
110
+ console.warn(
111
+ "[memory-vec] Existing memory_vec table is missing cosine distance metric; rebuilding from agent_memory",
112
+ );
113
+ db.run("DROP TABLE memory_vec");
114
+ }
115
+
90
116
  db.run(`
91
117
  CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
92
118
  memory_id TEXT PRIMARY KEY,
93
- embedding float[512]
119
+ embedding float[${EMBEDDING_DIMENSIONS}] distance_metric=cosine
94
120
  )
95
121
  `);
96
122
 
97
- // Populate from existing embeddings that aren't yet in the vec table
98
- const existing = db
99
- .prepare<{ id: string; embedding: Buffer }, []>(
100
- "SELECT id, embedding FROM agent_memory WHERE embedding IS NOT NULL",
101
- )
102
- .all();
103
-
104
- if (existing.length > 0) {
105
- const vecCount = db
106
- .prepare<{ count: number }, []>("SELECT COUNT(*) as count FROM memory_vec")
107
- .get();
108
-
109
- if ((vecCount?.count ?? 0) < existing.length) {
110
- const insert = db.prepare(
111
- "INSERT OR IGNORE INTO memory_vec(memory_id, embedding) VALUES (?, ?)",
112
- );
113
- const tx = db.transaction(() => {
114
- for (const row of existing) {
115
- insert.run(row.id, row.embedding);
116
- }
117
- });
118
- tx();
119
- console.log(`[memory] Synced ${existing.length} embeddings to memory_vec`);
120
- }
123
+ const healthBefore = this.getHealthCounts();
124
+ if (healthBefore.missingFromVec > 0 || healthBefore.extraInVec > 0) {
125
+ this.populateVecTable(healthBefore.memoryVec);
126
+ } else {
127
+ console.log(
128
+ `[memory-vec] populate skipped attempted=0 inserted=0 memory_vec=${healthBefore.memoryVec} valid_embedding=${healthBefore.validEmbedding}`,
129
+ );
121
130
  }
122
131
 
123
132
  this.vecInitialized = true;
124
133
  } catch (err) {
125
- console.warn("[memory] Failed to initialize memory_vec:", (err as Error).message);
134
+ this.vecInitialized = false;
135
+ console.error("[memory-vec] Failed to initialize memory_vec:", (err as Error).message);
136
+ }
137
+ }
138
+
139
+ private getVecTableSchema(): string | null {
140
+ try {
141
+ return (
142
+ getDb()
143
+ .prepare<{ sql: string | null }, []>(
144
+ "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'memory_vec'",
145
+ )
146
+ .get()?.sql ?? null
147
+ );
148
+ } catch {
149
+ return null;
150
+ }
151
+ }
152
+
153
+ private getVecCount(): number {
154
+ if (!this.getVecTableSchema()) return 0;
155
+ return (
156
+ getDb().prepare<{ count: number }, []>("SELECT COUNT(*) as count FROM memory_vec").get()
157
+ ?.count ?? 0
158
+ );
159
+ }
160
+
161
+ private populateVecTable(beforeCount: number): void {
162
+ const db = getDb();
163
+ const deletedExtra = db
164
+ .prepare(
165
+ `DELETE FROM memory_vec
166
+ WHERE memory_id NOT IN (SELECT id FROM agent_memory)`,
167
+ )
168
+ .run();
169
+ if (deletedExtra.changes > 0) {
170
+ console.warn(`[memory-vec] removed_extra_rows count=${deletedExtra.changes}`);
171
+ }
172
+
173
+ const rows = db
174
+ .prepare<{ id: string; embedding: Buffer }, []>(
175
+ "SELECT id, embedding FROM agent_memory WHERE embedding IS NOT NULL",
176
+ )
177
+ .all();
178
+ const deleteVec = db.prepare("DELETE FROM memory_vec WHERE memory_id = ?");
179
+ const insertVec = db.prepare("INSERT INTO memory_vec(memory_id, embedding) VALUES (?, ?)");
180
+
181
+ let attempted = 0;
182
+ let inserted = 0;
183
+ let skippedInvalidDimensions = 0;
184
+ let failed = 0;
185
+
186
+ for (const row of rows) {
187
+ const embeddingBuffer = this.toVecBuffer(row.embedding);
188
+ if (!embeddingBuffer) {
189
+ skippedInvalidDimensions++;
190
+ continue;
191
+ }
192
+
193
+ attempted++;
194
+ try {
195
+ deleteVec.run(row.id);
196
+ insertVec.run(row.id, embeddingBuffer);
197
+ inserted++;
198
+ } catch (err) {
199
+ failed++;
200
+ console.error(
201
+ `[memory-vec] populate failed memory_id=${row.id}: ${(err as Error).message}`,
202
+ );
203
+ }
204
+ }
205
+
206
+ const afterCount = this.getVecCount();
207
+ this.lastPopulate = {
208
+ attempted,
209
+ inserted,
210
+ skippedInvalidDimensions,
211
+ failed,
212
+ beforeCount,
213
+ afterCount,
214
+ };
215
+
216
+ console.log(
217
+ `[memory-vec] populate attempted=${attempted} inserted=${inserted} skipped_invalid_dimensions=${skippedInvalidDimensions} failed=${failed} before_count=${beforeCount} after_count=${afterCount}`,
218
+ );
219
+
220
+ if (failed > 0 || afterCount < attempted) {
221
+ console.error(
222
+ `[memory-vec] populate incomplete attempted=${attempted} after_count=${afterCount} failed=${failed}`,
223
+ );
224
+ }
225
+ }
226
+
227
+ private toVecBuffer(embedding: Buffer | Float32Array): Buffer | null {
228
+ if (embedding instanceof Float32Array) {
229
+ if (embedding.length !== EMBEDDING_DIMENSIONS) return null;
230
+ return serializeEmbedding(embedding);
126
231
  }
232
+ if (embedding.length !== VECTOR_BYTES) return null;
233
+ return embedding;
127
234
  }
128
235
 
129
236
  store(input: MemoryInput): AgentMemory {
@@ -223,7 +330,11 @@ export class SqliteMemoryStore implements MemoryStore {
223
330
  ): MemoryCandidate[] {
224
331
  const { scope = "all", limit = 10, source, isLead = false, includeExpired = false } = options;
225
332
 
226
- if (isSqliteVecAvailable() && this.vecInitialized) {
333
+ const health = this.getHealth();
334
+ if (health.retrievalMode === "vec" && embedding.length === EMBEDDING_DIMENSIONS) {
335
+ console.log(
336
+ `[memory-search] retrieval_path=vec scope=${scope} limit=${limit} vec_rows=${health.counts.memoryVec} searchable=${health.counts.searchable}`,
337
+ );
227
338
  return this.searchWithVec(embedding, agentId, {
228
339
  scope,
229
340
  limit,
@@ -232,6 +343,10 @@ export class SqliteMemoryStore implements MemoryStore {
232
343
  includeExpired,
233
344
  });
234
345
  }
346
+
347
+ console.log(
348
+ `[memory-search] retrieval_path=fallback scope=${scope} limit=${limit} reason=${embedding.length !== EMBEDDING_DIMENSIONS ? "query_dimension_mismatch" : health.reasons.join("|") || "vec_unavailable"}`,
349
+ );
235
350
  return this.searchBruteForce(embedding, agentId, {
236
351
  scope,
237
352
  limit,
@@ -255,58 +370,46 @@ export class SqliteMemoryStore implements MemoryStore {
255
370
  const db = getDb();
256
371
  const { scope, limit, source, isLead, includeExpired } = options;
257
372
 
258
- // KNN query — fetch more candidates than needed for post-filtering
259
- const knnLimit = limit * 5; // over-fetch to account for scope/expiry filters
260
373
  const embeddingBuffer = serializeEmbedding(queryEmbedding);
374
+ // sqlite-vec hard ceiling is 4096 for knn queries
375
+ const knnLimit = Math.min(Math.max(limit, this.getVecCount()), 4096);
261
376
 
262
- const vecRows = db
263
- .prepare<{ memory_id: string; distance: number }, [Buffer, number]>(
264
- "SELECT memory_id, distance FROM memory_vec WHERE embedding MATCH ? AND k = ?",
265
- )
266
- .all(embeddingBuffer, knnLimit);
267
-
268
- if (vecRows.length === 0) return [];
269
-
270
- // Build ID list and distance map
271
- const distanceMap = new Map<string, number>();
272
- const ids: string[] = [];
273
- for (const vr of vecRows) {
274
- distanceMap.set(vr.memory_id, vr.distance);
275
- ids.push(vr.memory_id);
276
- }
377
+ const conditions: string[] = ["v.embedding MATCH ?"];
378
+ const params: (Buffer | string | number | null)[] = [embeddingBuffer];
277
379
 
278
- // Hydrate from agent_memory with filters
279
- const placeholders = ids.map(() => "?").join(",");
280
- const conditions: string[] = [`id IN (${placeholders})`];
281
- const params: (string | null)[] = [...ids];
282
-
283
- this.addScopeConditions(conditions, params, agentId, scope, isLead);
380
+ this.addScopeConditions(conditions, params, agentId, scope, isLead, "m");
284
381
 
285
382
  if (source) {
286
- conditions.push("source = ?");
383
+ conditions.push("m.source = ?");
287
384
  params.push(source);
288
385
  }
289
386
 
290
387
  if (!includeExpired) {
291
- conditions.push("(expiresAt IS NULL OR expiresAt > datetime('now'))");
388
+ conditions.push("(m.expiresAt IS NULL OR m.expiresAt > datetime('now'))");
292
389
  }
293
390
 
391
+ conditions.push("v.k = ?");
392
+ params.push(knnLimit);
393
+
294
394
  const rows = db
295
- .prepare<AgentMemoryRow, (string | null)[]>(
296
- `SELECT * FROM agent_memory WHERE ${conditions.join(" AND ")}`,
395
+ .prepare<AgentMemoryRow & { distance: number }, (Buffer | string | number | null)[]>(
396
+ `SELECT m.*, v.distance
397
+ FROM memory_vec v
398
+ JOIN agent_memory m ON m.id = v.memory_id
399
+ WHERE ${conditions.join(" AND ")}
400
+ ORDER BY v.distance
401
+ LIMIT ?`,
297
402
  )
298
- .all(...params);
403
+ .all(...params, limit);
299
404
 
300
- // Map to candidates with similarity scores
301
405
  const candidates: MemoryCandidate[] = [];
302
406
  for (const row of rows) {
303
- const distance = distanceMap.get(row.id) ?? 1;
304
- const similarity = 1 - distance; // cosine distance to similarity
407
+ const similarity = 1 - row.distance;
408
+ if (similarity < MIN_SIMILARITY) continue;
305
409
  candidates.push(rowToCandidate(row, similarity));
306
410
  }
307
411
 
308
- candidates.sort((a, b) => b.similarity - a.similarity);
309
- return candidates.slice(0, limit);
412
+ return candidates;
310
413
  }
311
414
 
312
415
  private searchBruteForce(
@@ -349,6 +452,7 @@ export class SqliteMemoryStore implements MemoryStore {
349
452
  const emb = deserializeEmbedding(row.embedding);
350
453
  if (emb.length !== queryEmbedding.length) continue;
351
454
  const similarity = cosineSimilarity(queryEmbedding, emb);
455
+ if (similarity < MIN_SIMILARITY) continue;
352
456
  candidates.push(rowToCandidate(row, similarity));
353
457
  }
354
458
 
@@ -358,26 +462,28 @@ export class SqliteMemoryStore implements MemoryStore {
358
462
 
359
463
  private addScopeConditions(
360
464
  conditions: string[],
361
- params: (string | null)[],
465
+ params: (Buffer | string | number | null)[],
362
466
  agentId: string,
363
467
  scope: string,
364
468
  isLead: boolean,
469
+ tableAlias = "",
365
470
  ): void {
471
+ const col = (name: string) => (tableAlias ? `${tableAlias}.${name}` : name);
366
472
  if (!isLead) {
367
473
  if (scope === "agent") {
368
- conditions.push("agentId = ? AND scope = 'agent'");
474
+ conditions.push(`${col("agentId")} = ? AND ${col("scope")} = 'agent'`);
369
475
  params.push(agentId);
370
476
  } else if (scope === "swarm") {
371
- conditions.push("scope = 'swarm'");
477
+ conditions.push(`${col("scope")} = 'swarm'`);
372
478
  } else {
373
- conditions.push("(agentId = ? OR scope = 'swarm')");
479
+ conditions.push(`(${col("agentId")} = ? OR ${col("scope")} = 'swarm')`);
374
480
  params.push(agentId);
375
481
  }
376
482
  } else {
377
483
  if (scope === "agent") {
378
- conditions.push("scope = 'agent'");
484
+ conditions.push(`${col("scope")} = 'agent'`);
379
485
  } else if (scope === "swarm") {
380
- conditions.push("scope = 'swarm'");
486
+ conditions.push(`${col("scope")} = 'swarm'`);
381
487
  }
382
488
  }
383
489
  }
@@ -424,6 +530,31 @@ export class SqliteMemoryStore implements MemoryStore {
424
530
  return rows.map(rowToAgentMemory);
425
531
  }
426
532
 
533
+ isSourceProtected(source: AgentMemorySource): boolean {
534
+ return PROTECTED_SOURCES.has(source);
535
+ }
536
+
537
+ listForCuration(
538
+ agentId?: string,
539
+ ): { id: string; source: string; name: string; createdAt: string }[] {
540
+ const db = getDb();
541
+ const protectedList = [...PROTECTED_SOURCES].map((s) => `'${s}'`).join(",");
542
+ if (agentId) {
543
+ return db
544
+ .prepare<{ id: string; source: string; name: string; createdAt: string }, [string]>(
545
+ `SELECT id, source, name, createdAt FROM agent_memory
546
+ WHERE agentId = ? AND source NOT IN (${protectedList})`,
547
+ )
548
+ .all(agentId);
549
+ }
550
+ return db
551
+ .prepare<{ id: string; source: string; name: string; createdAt: string }, []>(
552
+ `SELECT id, source, name, createdAt FROM agent_memory
553
+ WHERE source NOT IN (${protectedList})`,
554
+ )
555
+ .all();
556
+ }
557
+
427
558
  listForReembedding(options?: { agentId?: string }): { id: string; content: string }[] {
428
559
  const db = getDb();
429
560
  if (options?.agentId) {
@@ -440,7 +571,7 @@ export class SqliteMemoryStore implements MemoryStore {
440
571
 
441
572
  delete(id: string): boolean {
442
573
  const db = getDb();
443
- if (isSqliteVecAvailable() && this.vecInitialized) {
574
+ if (this.vecInitialized && this.getVecTableSchema()) {
444
575
  db.prepare("DELETE FROM memory_vec WHERE memory_id = ?").run(id);
445
576
  }
446
577
  const result = db.prepare("DELETE FROM agent_memory WHERE id = ?").run(id);
@@ -450,7 +581,7 @@ export class SqliteMemoryStore implements MemoryStore {
450
581
  deleteBySourcePath(sourcePath: string, agentId: string): number {
451
582
  const db = getDb();
452
583
 
453
- if (isSqliteVecAvailable() && this.vecInitialized) {
584
+ if (this.vecInitialized && this.getVecTableSchema()) {
454
585
  // Get IDs first for vec table cleanup
455
586
  const ids = db
456
587
  .prepare<{ id: string }, [string, string]>(
@@ -472,6 +603,40 @@ export class SqliteMemoryStore implements MemoryStore {
472
603
  return result.changes;
473
604
  }
474
605
 
606
+ purgeExpired(): number {
607
+ const db = getDb();
608
+
609
+ const expiredIds = db
610
+ .prepare<{ id: string }, []>(
611
+ "SELECT id FROM agent_memory WHERE expiresAt IS NOT NULL AND expiresAt <= datetime('now')",
612
+ )
613
+ .all();
614
+
615
+ if (expiredIds.length === 0) return 0;
616
+
617
+ if (this.vecInitialized && this.getVecTableSchema()) {
618
+ const batchSize = 500;
619
+ for (let i = 0; i < expiredIds.length; i += batchSize) {
620
+ const batch = expiredIds.slice(i, i + batchSize);
621
+ const placeholders = batch.map(() => "?").join(",");
622
+ db.prepare(`DELETE FROM memory_vec WHERE memory_id IN (${placeholders})`).run(
623
+ ...batch.map((r) => r.id),
624
+ );
625
+ }
626
+ }
627
+
628
+ const result = db
629
+ .prepare(
630
+ "DELETE FROM agent_memory WHERE expiresAt IS NOT NULL AND expiresAt <= datetime('now')",
631
+ )
632
+ .run();
633
+
634
+ console.log(
635
+ `[memory] Purged ${result.changes} expired memory row(s) (vec cleanup: ${expiredIds.length} id(s))`,
636
+ );
637
+ return result.changes;
638
+ }
639
+
475
640
  updateEmbedding(id: string, embedding: Float32Array, model: string): void {
476
641
  const db = getDb();
477
642
  const buffer = serializeEmbedding(embedding);
@@ -481,11 +646,20 @@ export class SqliteMemoryStore implements MemoryStore {
481
646
  id,
482
647
  );
483
648
 
484
- if (isSqliteVecAvailable() && this.vecInitialized) {
485
- db.prepare("INSERT OR REPLACE INTO memory_vec(memory_id, embedding) VALUES (?, ?)").run(
486
- id,
487
- buffer,
488
- );
649
+ if (this.vecInitialized && this.getVecTableSchema()) {
650
+ const vecBuffer = this.toVecBuffer(embedding);
651
+ if (!vecBuffer) {
652
+ console.warn(
653
+ `[memory-vec] update skipped memory_id=${id} reason=invalid_dimensions dimensions=${embedding.length} expected=${EMBEDDING_DIMENSIONS}`,
654
+ );
655
+ return;
656
+ }
657
+ try {
658
+ db.prepare("DELETE FROM memory_vec WHERE memory_id = ?").run(id);
659
+ db.prepare("INSERT INTO memory_vec(memory_id, embedding) VALUES (?, ?)").run(id, vecBuffer);
660
+ } catch (err) {
661
+ console.error(`[memory-vec] update failed memory_id=${id}: ${(err as Error).message}`);
662
+ }
489
663
  }
490
664
  }
491
665
 
@@ -536,4 +710,79 @@ export class SqliteMemoryStore implements MemoryStore {
536
710
  expired: expired?.count ?? 0,
537
711
  };
538
712
  }
713
+
714
+ private getHealthCounts(): MemoryHealth["counts"] {
715
+ const db = getDb();
716
+ const tableExists = this.getVecTableSchema() !== null;
717
+ const tableUsable = tableExists && isSqliteVecAvailable();
718
+ const count = (sql: string) => db.prepare<{ count: number }, []>(sql).get()?.count ?? 0;
719
+
720
+ return {
721
+ total: count("SELECT COUNT(*) as count FROM agent_memory"),
722
+ withEmbedding: count(
723
+ "SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL",
724
+ ),
725
+ validEmbedding: count(
726
+ `SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) = ${VECTOR_BYTES}`,
727
+ ),
728
+ invalidEmbedding: count(
729
+ `SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) != ${VECTOR_BYTES}`,
730
+ ),
731
+ searchable: count(
732
+ `SELECT COUNT(*) as count FROM agent_memory
733
+ WHERE embedding IS NOT NULL
734
+ AND length(embedding) = ${VECTOR_BYTES}
735
+ AND (expiresAt IS NULL OR expiresAt > datetime('now'))`,
736
+ ),
737
+ memoryVec: tableUsable ? count("SELECT COUNT(*) as count FROM memory_vec") : 0,
738
+ missingFromVec: tableUsable
739
+ ? count(
740
+ `SELECT COUNT(*) as count
741
+ FROM agent_memory m
742
+ LEFT JOIN memory_vec v ON v.memory_id = m.id
743
+ WHERE m.embedding IS NOT NULL
744
+ AND length(m.embedding) = ${VECTOR_BYTES}
745
+ AND v.memory_id IS NULL`,
746
+ )
747
+ : count(
748
+ `SELECT COUNT(*) as count FROM agent_memory WHERE embedding IS NOT NULL AND length(embedding) = ${VECTOR_BYTES}`,
749
+ ),
750
+ extraInVec: tableUsable
751
+ ? count(
752
+ `SELECT COUNT(*) as count
753
+ FROM memory_vec v
754
+ LEFT JOIN agent_memory m ON m.id = v.memory_id
755
+ WHERE m.id IS NULL`,
756
+ )
757
+ : 0,
758
+ };
759
+ }
760
+
761
+ getHealth(): MemoryHealth {
762
+ const schema = this.getVecTableSchema();
763
+ const counts = this.getHealthCounts();
764
+ const reasons: string[] = [];
765
+
766
+ if (!isSqliteVecAvailable()) reasons.push("sqlite_vec_extension_unavailable");
767
+ if (!schema) reasons.push("memory_vec_table_missing");
768
+ if (!this.vecInitialized) reasons.push("memory_vec_not_initialized");
769
+ if (counts.memoryVec === 0) reasons.push("memory_vec_empty");
770
+ if (counts.missingFromVec > 0) reasons.push("memory_vec_missing_embeddings");
771
+ if (counts.extraInVec > 0) reasons.push("memory_vec_extra_rows");
772
+
773
+ return {
774
+ sqliteVec: {
775
+ extensionLoaded: isSqliteVecAvailable(),
776
+ tableExists: schema !== null,
777
+ initialized: this.vecInitialized,
778
+ vectorDimensions: EMBEDDING_DIMENSIONS,
779
+ distanceMetric: "cosine",
780
+ schema,
781
+ lastPopulate: this.lastPopulate,
782
+ },
783
+ counts,
784
+ retrievalMode: reasons.length === 0 ? "vec" : "fallback",
785
+ reasons,
786
+ };
787
+ }
539
788
  }