nemoris 0.1.0

package/src/memory/memory-store.js

@@ -0,0 +1,648 @@
+import path from "node:path";
+import { formatRuntimeError, RuntimeError } from "../utils/errors.js";
+import { appendJsonLine, ensureDir, listFiles, readJson, readJsonLines, removePath, writeJson } from "../utils/fs.js";
+import { DEFAULT_RETRIEVAL_BLEND, computeLexicalScore, scoreMemory } from "./scoring.js";
+import { evaluateDurableWrite } from "./write-policy.js";
+import { SqliteActiveStore, buildDuplicateSignature } from "./sqlite-active-store.js";
+import { buildResultSignature } from "./memory-signature.js";
+import { createRuntimeId } from "../utils/ids.js";
+import { computeSimhash, simhashToHex } from "./simhash.js";
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+// --- Secret write guard ---
+const SECRET_PATTERN = /^(sk|key|token|secret|password)[_-][a-zA-Z0-9]{20,}/i;
+const KNOWN_PREFIXES = [
+  /\bsk-ant-[a-zA-Z0-9_-]{10,}/,
+  /\bsk-[a-zA-Z0-9]{20,}/,
+  /\bkey-[a-zA-Z0-9]{20,}/
+];
+const ENV_REF_PATTERN = /^\$\{?\w+\}?$/;
+
+export function containsPlaintextSecret(value) {
+  if (value == null) return false;
+  if (typeof value === "object") {
+    return Object.values(value).some((v) => containsPlaintextSecret(v));
+  }
+  if (typeof value !== "string") return false;
+  const trimmed = value.trim();
+  if (ENV_REF_PATTERN.test(trimmed)) return false;
+  if (SECRET_PATTERN.test(trimmed)) return true;
+  for (const re of KNOWN_PREFIXES) {
+    if (re.test(trimmed)) return true;
+  }
+  return false;
+}
+
+function guardAgainstSecrets(record) {
+  if (containsPlaintextSecret(record)) {
+    const err = new RuntimeError(
+      "Rejected: plaintext secret detected in memory write. Use env-backed SecretRef pattern instead.",
+      { category: "security", context: { guard: "secret_write" }, recoverable: false }
+    );
+    throw new Error(formatRuntimeError(err));
+  }
+}
+
+// --- Cross-agent memory isolation ---
+function guardAgentAccess(storeAgentId, requestedAgentId, options = {}) {
+  if (storeAgentId == null) return; // no owner configured — unrestricted
+  if (requestedAgentId === storeAgentId) return;
+  if (options.allowCrossAgentRead) return;
+  const err = new RuntimeError(
+    `Rejected: cross-agent memory access denied. Agent '${requestedAgentId}' cannot access memory of agent '${storeAgentId}'.`,
+    { category: "memory", context: { storeAgentId, requestedAgentId }, recoverable: false }
+  );
+  throw new Error(formatRuntimeError(err));
+}
+
+// --- Protected path guard (inspired by Codex security model) ---
+const PROTECTED_PATH_PATTERNS = [
+  /\bconfig\//,
+  /\bTRUST_POLICY\.md\b/i,
+  /\bruntime\.toml\b/i,
+  /(?:^|[\s/])\.github\//,
+  /\bpackage\.json\b/i,
+  /\bpackage-lock\.json\b/i,
+  /(?:^|[\s/])\.env\b/
+];
+
+export function isProtectedPath(value) {
+  if (value == null) return false;
+  const str = String(value);
+  return PROTECTED_PATH_PATTERNS.some((re) => re.test(str));
+}
+
+const PATH_LIKE_FIELDS = new Set(["title", "filePath", "path", "target", "ref"]);
+
+function guardProtectedPaths(record) {
+  if (record == null) return;
+  if (typeof record === "string") {
+    if (isProtectedPath(record)) throwProtectedPathError(record);
+    return;
+  }
+  if (typeof record !== "object") return;
+  for (const [key, v] of Object.entries(record)) {
+    if (typeof v !== "string") continue;
+    // Only check path-like fields and short values that look like file references
+    if (PATH_LIKE_FIELDS.has(key) || v.length < 120) {
+      if (isProtectedPath(v)) throwProtectedPathError(v);
+    }
+  }
+}
+
+function throwProtectedPathError(path) {
+  const err = new RuntimeError(
+    `Rejected: memory write references protected path "${path}". Config, trust policy, and environment files are immutable from agent operations.`,
+    { category: "security", context: { guard: "protected_path", path }, recoverable: false }
+  );
+  throw new Error(formatRuntimeError(err));
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function basePaths(rootDir, agentId) {
+  const agentRoot = path.join(rootDir, agentId);
+  return {
+    agentRoot,
+    eventsFile: path.join(agentRoot, "events.jsonl"),
+    factsFile: path.join(agentRoot, "facts.jsonl"),
+    summariesFile: path.join(agentRoot, "summaries.jsonl"),
+    scratchpadsDir: path.join(agentRoot, "scratchpads"),
+    checkpointsDir: path.join(agentRoot, "checkpoints"),
+    countersFile: path.join(agentRoot, "counters.json"),
+    sqliteFile: path.join(agentRoot, "memory.sqlite"),
+    lockDir: path.join(agentRoot, ".lock")
+  };
+}
+
+export class MemoryStore {
+  constructor({ rootDir, agentId = null, lockConfig = {}, embeddingIndex = null, embeddingConfig = null }) {
+    this.rootDir = rootDir;
+    this.agentId = agentId;
+    this.sqliteStores = new Map();
+    this.embeddingIndex = embeddingIndex;
+    this.embeddingConfig = embeddingConfig || {};
+    this.lockConfig = {
+      ttlMs: Number(lockConfig.ttlMs ?? 15000),
+      retryDelayMs: Number(lockConfig.retryDelayMs ?? 25),
+      maxRetries: Number(lockConfig.maxRetries ?? 40)
+    };
+  }
+
+  /**
+   * Fire-and-forget: embed newly written entries if index_on_write is enabled.
+   * Never blocks the write path. Logs warnings on failure.
+   */
+  _maybeIndexOnWrite(agentId, records) {
+    if (!this.embeddingIndex) return;
+    if (!this.embeddingConfig.indexOnWrite) return;
+    const items = Array.isArray(records) ? records : [records];
+    const valid = items.filter((r) => r && r.type && !r.skipped);
+    if (valid.length === 0) return;
+
+    // Fire-and-forget — do not await
+    this.embeddingIndex.rebuild(agentId, valid).catch((err) => {
+      console.warn(`[nemoris] index-on-write failed for ${agentId}: ${err?.message || err}`);
+    });
+  }
+
+  async initAgent(agentId) {
+    const paths = basePaths(this.rootDir, agentId);
+    await ensureDir(paths.agentRoot);
+    await ensureDir(paths.scratchpadsDir);
+    await ensureDir(paths.checkpointsDir);
+    const counters = await readJson(paths.countersFile, null);
+    if (!counters) {
+      await writeJson(paths.countersFile, { durableWrites: 0, reads: 0 });
+    }
+    return paths;
+  }
+
+  getSqliteStore(paths) {
+    if (!this.sqliteStores.has(paths.sqliteFile)) {
+      this.sqliteStores.set(paths.sqliteFile, new SqliteActiveStore({ dbPath: paths.sqliteFile }));
+    }
+    return this.sqliteStores.get(paths.sqliteFile);
+  }
+
+  async rebuildSqliteStore(paths) {
+    const [facts, summaries, events] = await Promise.all([
+      readJsonLines(paths.factsFile),
+      readJsonLines(paths.summariesFile),
+      readJsonLines(paths.eventsFile)
+    ]);
+    const scratchpads = await this.readScratchpadsFromFiles(paths);
+    const sqlite = this.getSqliteStore(paths);
+    sqlite.clearAll();
+    const seenFactSignatures = new Set();
+    const seenSummarySignatures = new Set();
+    const seenEventDedupeKeys = new Set();
+
+    for (const record of facts) {
+      const duplicateSignature = buildDuplicateSignature(record, "fact");
+      if (duplicateSignature && seenFactSignatures.has(duplicateSignature)) continue;
+      sqlite.insert(record, {
+        duplicateSignature
+      });
+      if (duplicateSignature) seenFactSignatures.add(duplicateSignature);
+    }
+    for (const record of summaries) {
+      const duplicateSignature = buildDuplicateSignature(record, "summary");
+      if (duplicateSignature && seenSummarySignatures.has(duplicateSignature)) continue;
+      sqlite.insert(record, {
+        duplicateSignature
+      });
+      if (duplicateSignature) seenSummarySignatures.add(duplicateSignature);
+    }
+    for (const record of events) {
+      if (record.dedupeKey) {
+        if (seenEventDedupeKeys.has(record.dedupeKey)) continue;
+        seenEventDedupeKeys.add(record.dedupeKey);
+      }
+      sqlite.insert(record);
+    }
+    for (const { scratchpadId, record } of scratchpads) {
+      sqlite.upsertScratchpad(scratchpadId, record);
+    }
+
+    return sqlite;
+  }
+
+  async ensureSqliteStore(paths) {
+    const sqlite = this.getSqliteStore(paths);
+    if (sqlite.count() === 0) {
+      await this.rebuildSqliteStore(paths);
+    }
+    return sqlite;
+  }
+
+  async readScratchpadsFromFiles(paths) {
+    const files = await listFiles(paths.scratchpadsDir);
+    return Promise.all(
+      files.map(async (fileName) => ({
+        scratchpadId: path.basename(fileName, ".json"),
+        record: await readJson(path.join(paths.scratchpadsDir, fileName), null),
+        filePath: path.join(paths.scratchpadsDir, fileName)
+      }))
+    );
+  }
+
+  async appendEvent(agentId, event, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    return this.withAgentLock(paths, "memory-write", async () => {
+      const sqlite = await this.ensureSqliteStore(paths);
+      const record = {
+        type: "event",
+        timestamp: nowIso(),
+        salience: 0.35,
+        ...event
+      };
+      if (event.dedupeKey && sqlite.hasEventDedupe(event.dedupeKey)) {
+        return { skipped: true, record: null };
+      }
+      await appendJsonLine(paths.eventsFile, record);
+      sqlite.insert(record);
+      return record;
+    });
+  }
+
+  async writeSummary(agentId, summary, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    return this.withAgentLock(paths, "memory-write", async () => {
+      const sqlite = await this.ensureSqliteStore(paths);
+      guardAgainstSecrets(summary);
+      guardProtectedPaths(summary);
+      const record = {
+        type: "summary",
+        timestamp: nowIso(),
+        salience: 0.7,
+        accessCount: 0,
+        ...summary
+      };
+      const duplicateSignature = buildDuplicateSignature(record, "summary");
+      if (sqlite.hasDuplicateSignature("summary", duplicateSignature)) {
+        return { skipped: true, record: null };
+      }
+      // Simhash near-duplicate check
+      const hash = computeSimhash((summary.content || "") + " " + (summary.title || ""));
+      const simhashHex = simhashToHex(hash);
+      const nearDupes = sqlite.findNearDuplicates("summary", simhashHex, 3);
+      if (nearDupes.length > 0) {
+        const best = nearDupes[0];
+        sqlite.boostSalience(best.entryId, 0.15);
+        return { accepted: true, skipped: true, nearDuplicate: true, boostedEntryId: best.entryId, record: null };
+      }
+      record.simhash = simhashHex;
+      await appendJsonLine(paths.summariesFile, record);
+      sqlite.insert(record, { duplicateSignature });
+      this._maybeIndexOnWrite(agentId, record);
+      return record;
+    });
+  }
+
+  async writeFact(agentId, fact, policy, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const writesThisRun = options.writesThisRun ?? 0;
+    return this.withAgentLock(paths, "memory-write", async () => {
+      guardAgainstSecrets(fact);
+      guardProtectedPaths(fact);
+      const counters = await readJson(paths.countersFile, { durableWrites: 0, reads: 0 });
+      const sqlite = await this.ensureSqliteStore(paths);
+      const evaluation = evaluateDurableWrite(fact, policy, writesThisRun);
+      if (!evaluation.accepted) {
+        return {
+          accepted: false,
+          reasons: evaluation.reasons
+        };
+      }
+
+      const record = {
+        type: "fact",
+        timestamp: nowIso(),
+        salience: 0.8,
+        accessCount: 0,
+        ...fact
+      };
+      const duplicateSignature = buildDuplicateSignature(record, "fact");
+      if (sqlite.hasDuplicateSignature("fact", duplicateSignature)) {
+        return {
+          accepted: true,
+          skipped: true,
+          record: null
+        };
+      }
+      // Simhash near-duplicate check
+      const hash = computeSimhash((fact.content || "") + " " + (fact.title || ""));
+      const simhashHex = simhashToHex(hash);
+      const nearDupes = sqlite.findNearDuplicates("fact", simhashHex, 3);
+      if (nearDupes.length > 0) {
+        const best = nearDupes[0];
+        sqlite.boostSalience(best.entryId, 0.15);
+        return {
+          accepted: true,
+          skipped: true,
+          nearDuplicate: true,
+          boostedEntryId: best.entryId,
+          record: null
+        };
+      }
+      record.simhash = simhashHex;
+      await appendJsonLine(paths.factsFile, record);
+      sqlite.insert(record, { duplicateSignature });
+      await writeJson(paths.countersFile, {
+        ...counters,
+        durableWrites: counters.durableWrites + 1
+      });
+      this._maybeIndexOnWrite(agentId, record);
+      return {
+        accepted: true,
+        record
+      };
+    });
+  }
+
+  async putScratchpad(agentId, scratchpadId, scratchpad, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    return this.withAgentLock(paths, "memory-write", async () => {
+      const sqlite = await this.ensureSqliteStore(paths);
+      const record = {
+        type: "scratchpad",
+        timestamp: nowIso(),
+        expiresAt: scratchpad.expiresAt,
+        salience: scratchpad.salience ?? 0.5,
+        ...scratchpad
+      };
+      const filePath = path.join(paths.scratchpadsDir, `${scratchpadId}.json`);
+      await writeJson(filePath, record);
+      sqlite.upsertScratchpad(scratchpadId, record);
+      return record;
+    });
+  }
+
+  async getScratchpads(agentId, now = Date.now(), options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const sqlite = await this.ensureSqliteStore(paths);
+    const nowIsoValue = new Date(now).toISOString();
+    const expiredEntryIds = sqlite.listExpiredScratchpads(nowIsoValue);
+
+    if (expiredEntryIds.length) {
+      await this.withAgentLock(paths, "memory-write", async () => {
+        for (const entryId of expiredEntryIds) {
+          const scratchpadId = entryId.replace(/^scratchpad:/, "");
+          const filePath = path.join(paths.scratchpadsDir, `${scratchpadId}.json`);
+          sqlite.deleteByEntryId(entryId);
+          await removePath(filePath);
+        }
+      });
+    }
+    return sqlite
+      .listAll(nowIsoValue)
+      .filter((entry) => entry.type === "scratchpad");
+  }
+
+  async saveCheckpoint(agentId, checkpointId, checkpoint, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const record = {
+      timestamp: nowIso(),
+      ...checkpoint
+    };
+    await writeJson(path.join(paths.checkpointsDir, `${checkpointId}.json`), record);
+    return record;
+  }
+
+  async loadCheckpoint(agentId, checkpointId = "latest", options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    return readJson(path.join(paths.checkpointsDir, `${checkpointId}.json`), null);
+  }
+
+  async listAll(agentId, now = Date.now(), options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    await this.getScratchpads(agentId, now, { allowCrossAgentRead: true });
+    const sqlite = await this.ensureSqliteStore(paths);
+    return sqlite.listAll(new Date(now).toISOString());
+  }
+
+  async sqliteStatus(agentId, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const sqlite = await this.ensureSqliteStore(paths);
+    return sqlite.getWalStatus(options.mode || "passive");
+  }
+
+  async manageSqlite(agentId, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const sqlite = await this.ensureSqliteStore(paths);
+    return sqlite.manageWal(options);
+  }
+
+  async getEmbeddingHealth(agentId, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const paths = await this.initAgent(agentId);
+    const sqlite = await this.ensureSqliteStore(paths);
+    let probe = null;
+
+    if (options.probe && options.embeddingIndex) {
+      try {
+        const embedded = await options.embeddingIndex.embedQuery(options.probeText || "nemoris embedding health probe");
+        if (embedded?.config?.provider) sqlite.setRuntimeMeta("embedding_last_provider", embedded.config.provider);
+        if (embedded?.config?.model) sqlite.setRuntimeMeta("embedding_last_model", embedded.config.model);
+        sqlite.setRuntimeMeta("embedding_last_success_at", nowIso());
+        sqlite.setRuntimeMeta("embedding_last_query_mode", embedded?.vector ? "embedding_probe" : "lexical_only");
+        sqlite.clearRuntimeMeta("embedding_last_error");
+        sqlite.clearRuntimeMeta("embedding_last_error_at");
+        probe = {
+          ok: true,
+          dimensions: Array.isArray(embedded?.vector) ? embedded.vector.length : 0,
+          config: embedded?.config || null
+        };
+      } catch (error) {
+        const message = error?.message || String(error);
+        sqlite.setRuntimeMeta("embedding_last_error", message);
+        sqlite.setRuntimeMeta("embedding_last_error_at", nowIso());
+        sqlite.setRuntimeMeta("embedding_last_query_mode", "embedding_probe_failed");
+        probe = {
+          ok: false,
+          error: message
+        };
+      }
+    }
+
+    return {
+      agentId,
+      totalEntries: sqlite.count(),
+      embeddingHealth: sqlite.getEmbeddingHealth(),
+      probe
+    };
+  }
+
+  async rebuildEmbeddings(agentId, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    if (!options.embeddingIndex) {
+      throw new Error("No embedding index configured for rebuild");
+    }
+
+    const paths = await this.initAgent(agentId);
+    const sqlite = await this.ensureSqliteStore(paths);
+    const items = await this.listAll(agentId);
+
+    try {
+      const result = await options.embeddingIndex.rebuild(agentId, items);
+      if (result?.config?.provider) sqlite.setRuntimeMeta("embedding_last_provider", result.config.provider);
+      if (result?.config?.model) sqlite.setRuntimeMeta("embedding_last_model", result.config.model);
+      sqlite.setRuntimeMeta("embedding_last_success_at", nowIso());
+      sqlite.setRuntimeMeta("embedding_last_query_mode", "embedding_rebuild");
+      sqlite.clearRuntimeMeta("embedding_last_error");
+      sqlite.clearRuntimeMeta("embedding_last_error_at");
+
+      return {
+        agentId,
+        itemCount: items.length,
+        updatedAt: result.updatedAt,
+        config: result.config,
+        embeddingHealth: sqlite.getEmbeddingHealth()
+      };
+    } catch (error) {
+      const message = error?.message || String(error);
+      sqlite.setRuntimeMeta("embedding_last_error", message);
+      sqlite.setRuntimeMeta("embedding_last_error_at", nowIso());
+      sqlite.setRuntimeMeta("embedding_last_query_mode", "embedding_rebuild_failed");
+      throw error;
+    }
+  }
+
+  async query(agentId, query, options = {}) {
+    guardAgentAccess(this.agentId, agentId, options);
+    const limit = options.limit ?? 8;
+    const now = options.now ?? Date.now();
+    const retrievalBlend = {
+      ...DEFAULT_RETRIEVAL_BLEND,
+      ...(options.retrievalBlend || {})
+    };
+    const paths = await this.initAgent(agentId);
+    await this.getScratchpads(agentId, now, { allowCrossAgentRead: true });
+    const sqlite = await this.ensureSqliteStore(paths);
+    let queryVector = null;
+    let embeddingConfig = null;
+    let embeddingError = null;
+    let embeddingQueryMode = "lexical_only";
+    if (options.embeddingIndex) {
+      try {
+        const embedded = await options.embeddingIndex.embedQuery(query);
+        queryVector = embedded.vector;
+        embeddingConfig = embedded.config || null;
+        embeddingQueryMode = queryVector ? "embedding_query" : "lexical_only";
+        if (embeddingConfig?.provider) sqlite.setRuntimeMeta("embedding_last_provider", embeddingConfig.provider);
+        if (embeddingConfig?.model) sqlite.setRuntimeMeta("embedding_last_model", embeddingConfig.model);
+        sqlite.setRuntimeMeta("embedding_last_success_at", nowIso());
+        sqlite.setRuntimeMeta("embedding_last_query_mode", embeddingQueryMode);
+        sqlite.clearRuntimeMeta("embedding_last_error");
+        sqlite.clearRuntimeMeta("embedding_last_error_at");
+      } catch (error) {
+        queryVector = null;
+        embeddingConfig = null;
+        embeddingError = error?.message || String(error);
+        embeddingQueryMode = "lexical_fallback";
+        sqlite.setRuntimeMeta("embedding_last_error", embeddingError);
+        sqlite.setRuntimeMeta("embedding_last_error_at", nowIso());
+        sqlite.setRuntimeMeta("embedding_last_query_mode", embeddingQueryMode);
+      }
+    }
+    const retrievalCandidates = sqlite.queryRetrievalCandidates(query, {
+      nowIso: new Date(now).toISOString(),
+      limit,
+      queryVector
+    });
+
+    const ranked = retrievalCandidates
+      .map((candidate) => ({
+        ...candidate.item,
+        ...scoreMemory(
+          {
+            ...candidate.item,
+            lexicalScore: computeLexicalScore(
+              query,
+              [candidate.item.title, candidate.item.content, candidate.item.summary, candidate.item.reason, candidate.item.category]
+                .filter(Boolean)
+                .join(" ")
+            ),
+            embeddingSimilarity: candidate.embeddingSimilarity,
+            embeddingFreshness: candidate.embeddingFreshness
+          },
+          query,
+          now,
+          retrievalBlend
+        ),
+        retrievalSources: candidate.retrievalSources,
+        embeddingFreshness: candidate.embeddingFreshness,
+        embeddingProvider: candidate.embeddingProvider,
+        embeddingModel: candidate.embeddingModel,
+        candidateSource: candidate.retrievalSources.includes("semantic") && candidate.retrievalSources.includes("lexical")
+          ? "indexed+semantic"
+          : candidate.retrievalSources.includes("semantic")
+            ? "semantic"
+            : "indexed"
+      }))
+      .sort((a, b) => b.score - a.score);
+
+    const deduped = [];
+    const seen = new Set();
+    for (const item of ranked) {
+      const key = buildResultSignature(item);
+      if (seen.has(key)) continue;
+      seen.add(key);
+      deduped.push(item);
+      if (deduped.length >= limit) break;
+    }
+
+    return {
+      query,
+      totalCandidates: sqlite.count(),
+      indexedCandidates: retrievalCandidates.filter((item) => item.lexicalMatch).length,
+      semanticCandidates: retrievalCandidates.filter((item) => item.embeddingSimilarity > 0).length,
+      retrieval: {
+        blend: retrievalBlend,
+        embeddingQueryAvailable: Boolean(queryVector),
+        embeddingConfig,
+        embeddingQueryMode,
+        embeddingError,
+        embeddingHealth: sqlite.getEmbeddingHealth(),
+        candidates: retrievalCandidates.map((candidate) => ({
+          entryId: candidate.item.entryId,
+          title: candidate.item.title,
+          retrievalSources: candidate.retrievalSources,
+          lexicalMatch: candidate.lexicalMatch,
+          lexicalRank: candidate.lexicalRank,
+          embeddingSimilarity: Number(candidate.embeddingSimilarity.toFixed(4)),
+          embeddingFreshness: candidate.embeddingFreshness
+        }))
+      },
+      items: deduped
+    };
+  }
+
+  async withAgentLock(paths, lockKey, fn) {
+    const sqlite = this.getSqliteStore(paths);
+    const ownerId = createRuntimeId("memory-lock");
+    const ttlMs = this.lockConfig.ttlMs;
+    const retryDelayMs = this.lockConfig.retryDelayMs;
+    const maxRetries = this.lockConfig.maxRetries;
+
+    let acquired = false;
+    for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+      const acquiredAt = new Date();
+      const expiresAt = new Date(acquiredAt.getTime() + ttlMs);
+      acquired = sqlite.tryAcquireLock(lockKey, {
+        ownerId,
+        acquiredAt: acquiredAt.toISOString(),
+        expiresAt: expiresAt.toISOString()
+      });
+      if (acquired) break;
+      if (attempt === maxRetries) {
+        throw new Error(`Timed out waiting for SQLite memory lock ${lockKey}`);
+      }
+      await sleep(retryDelayMs);
+    }
+
+    try {
+      return await fn();
+    } finally {
+      if (acquired) {
+        sqlite.releaseLock(lockKey, ownerId);
+      }
+    }
+  }
+}