nemoris 0.1.0

package/src/memory/backends/qmd-backend.js

@@ -0,0 +1,219 @@
+import path from "node:path";
+import { listFilesRecursive, readText } from "../../utils/fs.js";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+
+function parseQmdCollections(yaml) {
+  const lines = String(yaml || "").split("\n");
+  const collections = [];
+  let current = null;
+
+  for (const rawLine of lines) {
+    const line = rawLine.trimEnd();
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const collectionMatch = /^([a-zA-Z0-9_-]+):\s*$/.exec(trimmed);
+    if (collectionMatch && !trimmed.startsWith("collections:")) {
+      current = {
+        id: collectionMatch[1]
+      };
+      collections.push(current);
+      continue;
+    }
+    if (!current) continue;
+    const pathMatch = /^path:\s*(.+)$/.exec(trimmed);
+    if (pathMatch) {
+      current.path = pathMatch[1];
+      continue;
+    }
+    const patternMatch = /^pattern:\s*(.+)$/.exec(trimmed);
+    if (patternMatch) {
+      current.pattern = patternMatch[1];
+    }
+  }
+
+  return collections;
+}
+
+export class QmdMemoryBackend {
+  constructor({ liveRoot }) {
+    this.kind = "qmd";
+    this.liveRoot = liveRoot;
+  }
+
+  async inspect(agentId, workspaceRoot = null) {
+    if (!this.liveRoot) {
+      return {
+        kind: this.kind,
+        available: false,
+        agentId: null,
+        collections: [],
+        indexSqlite: null
+      };
+    }
+    const direct = await this.inspectDirect(agentId);
+    if (direct.available) return direct;
+
+    if (workspaceRoot) {
+      const byWorkspace = await this.inspectByWorkspace(workspaceRoot);
+      if (byWorkspace.available) {
+        return {
+          ...byWorkspace,
+          mappedFromWorkspace: true
+        };
+      }
+    }
+
+    return direct;
+  }
+
+  async inspectDirect(agentId) {
+    const qmdRoot = path.join(this.liveRoot, "agents", agentId, "qmd");
+    return this.inspectRoot(qmdRoot, agentId);
+  }
+
+  async inspectByWorkspace(workspaceRoot) {
+    const qmdConfigs = (await listFilesRecursive(path.join(this.liveRoot, "agents"))).filter((filePath) =>
+      filePath.endsWith("/qmd/xdg-config/index.yml")
+    );
+
+    for (const configPath of qmdConfigs) {
+      const configText = await readText(configPath, "");
+      const collections = parseQmdCollections(configText);
+      if (collections.some((collection) => matchesWorkspaceCollection(workspaceRoot, collection.path))) {
+        const agentId = configPath.split(path.sep).slice(-4, -3)[0];
+        const qmdRoot = path.join(this.liveRoot, "agents", agentId, "qmd");
+        return this.inspectRoot(qmdRoot, agentId);
+      }
+    }
+
+    return {
+      kind: this.kind,
+      available: false,
+      agentId: null,
+      collections: [],
+      indexSqlite: null
+    };
+  }
+
+  async inspectRoot(qmdRoot, agentId) {
+    const configPath = path.join(qmdRoot, "xdg-config", "index.yml");
+    const indexSqlite = path.join(qmdRoot, "xdg-cache", "qmd", "index.sqlite");
+    const configText = await readText(configPath, null);
+    const indexExists = await readText(indexSqlite, null).then(() => true, () => false);
+    const stats = indexExists ? await this.readStats(indexSqlite) : null;
+
+    return {
+      kind: this.kind,
+      available: Boolean(configText),
+      agentId,
+      configPath,
+      indexSqlite: indexExists ? indexSqlite : null,
+      collections: parseQmdCollections(configText || ""),
+      stats
+    };
+  }
+
+  async query(agentId, query, options = {}) {
+    const inspected = await this.inspect(agentId, options.workspaceRoot || null);
+    if (!inspected.available || !inspected.indexSqlite) {
+      return {
+        backend: this.kind,
+        available: false,
+        items: []
+      };
+    }
+
+    const normalized = normalizeFtsQuery(query);
+    const limit = options.limit ?? 5;
+    const sql =
+      "select documents_fts.filepath, documents_fts.title, documents.path as document_path, documents.collection, " +
+      "snippet(documents_fts, 2, '[', ']', ' … ', 12) as snippet " +
+      "from documents_fts join documents on documents.id = documents_fts.rowid " +
+      `where documents_fts match '${escapeSql(normalized)}' limit ${Number(limit) * 3 || 15}`;
+
+    try {
+      const { stdout } = await execFileAsync("sqlite3", ["-json", inspected.indexSqlite, sql]);
+      return {
+        backend: this.kind,
+        available: true,
+        items: dedupeQmdItems(JSON.parse(stdout || "[]")).slice(0, limit),
+        stats: inspected.stats || null
+      };
+    } catch {
+      return {
+        backend: this.kind,
+        available: true,
+        items: [],
+        stats: inspected.stats || null
+      };
+    }
+  }
+
+  async readStats(indexSqlite) {
+    try {
+      const sql =
+        "select " +
+        "(select count(*) from documents where active=1) as docs, " +
+        "(select count(*) from content_vectors) as vectors;";
+      const { stdout } = await execFileAsync("sqlite3", ["-json", indexSqlite, sql]);
+      return JSON.parse(stdout || "[]")[0] || null;
+    } catch {
+      return null;
+    }
+  }
+}
+
+function normalizeFtsQuery(query) {
+  return String(query || "")
+    .split(/\s+/)
+    .filter(Boolean)
+    .map((token) => token.replace(/"/g, ""))
+    .join(" OR ");
+}
+
+function escapeSql(value) {
+  return String(value).replace(/'/g, "''");
+}
+
+function dedupeQmdItems(items) {
+  const seen = new Set();
+  const deduped = [];
+
+  for (const item of items) {
+    const key = buildDedupKey(item);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(item);
+  }
+
+  return deduped;
+}
+
+function buildDedupKey(item) {
+  const pathKey = normalizePath(item.document_path || item.filepath || "");
+  const titleKey = String(item.title || "").trim().toLowerCase();
+  const snippetKey = String(item.snippet || "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .toLowerCase();
+
+  return [pathKey, titleKey, snippetKey].join("::");
+}
+
+function normalizePath(value) {
+  return String(value || "")
+    .replace(/\\/g, "/")
+    .trim()
+    .toLowerCase();
+}
+
+function matchesWorkspaceCollection(workspaceRoot, collectionPath) {
+  const normalizedWorkspace = normalizePath(workspaceRoot);
+  const normalizedCollection = normalizePath(collectionPath);
+  if (!normalizedWorkspace || !normalizedCollection) return false;
+  if (normalizedCollection === normalizedWorkspace) return true;
+  return normalizedCollection === `${normalizedWorkspace}/memory`;
+}

package/src/memory/embedding-guards.js

@@ -0,0 +1,24 @@
+function readFlag(name) {
+  const raw = process.env[name];
+  return raw === "1" || raw === "true";
+}
+
+export function getEmbeddingPolicy() {
+  return {
+    allowEmbeddings: readFlag("NEMORIS_ALLOW_EMBEDDINGS"),
+    allowRemoteEmbeddings: readFlag("NEMORIS_ALLOW_REMOTE_EMBEDDINGS"),
+    requireHealthyProvider: true,
+    allowedProvider: "ollama"
+  };
+}
+
+export function assertEmbeddingsAllowed(providerId) {
+  const policy = getEmbeddingPolicy();
+  if (!policy.allowEmbeddings) {
+    throw new Error("Embeddings disabled. Set NEMORIS_ALLOW_EMBEDDINGS=1 to enable embedding indexing.");
+  }
+  if (providerId !== policy.allowedProvider && !policy.allowRemoteEmbeddings) {
+    throw new Error(`Embedding provider ${providerId} is blocked by policy.`);
+  }
+  return policy;
+}

package/src/memory/embedding-index.js

@@ -0,0 +1,118 @@
+import path from "node:path";
+import { ensureDir } from "../utils/fs.js";
+import { buildResultSignature } from "./memory-signature.js";
+import { SqliteActiveStore } from "./sqlite-active-store.js";
+
+export class EmbeddingIndex {
+  constructor({ rootDir, embeddingService = null } = {}) {
+    this.rootDir = rootDir;
+    this.embeddingService = embeddingService;
+    this.stores = new Map();
+  }
+
+  dbPath(agentId) {
+    return path.join(this.rootDir, agentId, "memory.sqlite");
+  }
+
+  async getStore(agentId) {
+    const dbPath = this.dbPath(agentId);
+    await ensureDir(path.dirname(dbPath));
+    if (!this.stores.has(dbPath)) {
+      this.stores.set(dbPath, new SqliteActiveStore({ dbPath }));
+    }
+    return this.stores.get(dbPath);
+  }
+
+  async load(agentId) {
+    const store = await this.getStore(agentId);
+    return {
+      agentId,
+      ...store.getEmbeddingIndex()
+    };
+  }
+
+  async save(agentId, payload) {
+    const store = await this.getStore(agentId);
+    return {
+      agentId,
+      ...store.replaceEmbeddings(payload)
+    };
+  }
+
+  async rebuild(agentId, memoryItems) {
+    if (!this.embeddingService) {
+      throw new Error("No embedding service configured");
+    }
+
+    const texts = memoryItems.map((item) =>
+      [item.title, item.summary, item.content, item.reason, item.category].filter(Boolean).join("\n")
+    );
+    const embedded = await this.embeddingService.embedTexts(texts);
+    const items = memoryItems.map((item, index) => ({
+      entryId: item.entryId || `synthetic:${index}:${buildResultSignature(item)}`,
+      signature: buildResultSignature(item),
+      item,
+      vector: embedded.vectors[index] || []
+    }));
+
+    return this.save(agentId, {
+      agentId,
+      updatedAt: new Date().toISOString(),
+      config: embedded.config,
+      items
+    });
+  }
+
+  async query(agentId, queryText, limit = 8) {
+    const store = await this.getStore(agentId);
+    const index = store.getEmbeddingIndex();
+    if (!this.embeddingService || !index.items?.length) {
+      return [];
+    }
+
+    const embedded = await this.embedQuery(queryText);
+    const queryVector = embedded.vector;
+    if (!Array.isArray(queryVector)) return [];
+
+    const sqliteResults = store.queryEmbeddingSimilarities(queryVector, limit);
+    if (sqliteResults.length) {
+      return sqliteResults;
+    }
+
+    return index.items
+      .map((entry) => ({
+        entryId: entry.entryId || null,
+        signature: entry.signature,
+        similarity: cosineSimilarity(queryVector, entry.vector || []),
+        freshnessStatus: entry.freshnessStatus || "fresh",
+        item: entry.item
+      }))
+      .sort((a, b) => b.similarity - a.similarity)
+      .slice(0, limit);
+  }
+
+  async embedQuery(queryText) {
+    if (!this.embeddingService) {
+      throw new Error("No embedding service configured");
+    }
+    const embedded = await this.embeddingService.embedTexts([queryText]);
+    return {
+      vector: embedded.vectors[0],
+      config: embedded.config || null
+    };
+  }
+}
+
+function cosineSimilarity(a = [], b = []) {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const length = Math.min(a.length, b.length);
+  for (let i = 0; i < length; i += 1) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}

package/src/memory/embedding-service.js

@@ -0,0 +1,179 @@
+import path from "node:path";
+import { ConfigLoader } from "../config/loader.js";
+import { ProviderRegistry } from "../providers/registry.js";
+import { assertEmbeddingsAllowed, getEmbeddingPolicy } from "./embedding-guards.js";
+
+function modelToProviderId(modelId) {
+  return String(modelId || "").split("/")[0] || null;
+}
+
+function normalizeOllamaModelName(modelId) {
+  const normalized = String(modelId || "").replace(/^ollama\//, "");
+  return normalized;
+}
+
+function matchesAvailableModel(targetModel, availableModels) {
+  const target = normalizeOllamaModelName(targetModel);
+  if (!target || !Array.isArray(availableModels)) return false;
+  return availableModels.some((model) => {
+    const candidate = normalizeOllamaModelName(model);
+    return candidate === target || candidate === `${target}:latest`;
+  });
+}
+
+export class EmbeddingService {
+  constructor({ projectRoot, fetchImpl } = {}) {
+    this.projectRoot = projectRoot;
+    this.loader = new ConfigLoader({ rootDir: path.join(projectRoot, "config") });
+    this.registry = new ProviderRegistry({ fetchImpl });
+  }
+
+  async loadConfig() {
+    const raw = await this.loader.loadEmbeddings();
+    return {
+      enabled: raw.enabled ?? false,
+      provider: raw.provider || modelToProviderId(raw.model),
+      model: raw.model || null,
+      dimensions: raw.dimensions ?? 128,
+      indexOnWrite: raw.indexOnWrite ?? false
+    };
+  }
+
+  async embedTexts(texts) {
+    const config = await this.loadConfig();
+    const providerId = config.provider || modelToProviderId(config.model);
+    const policy = assertEmbeddingsAllowed(providerId);
+    const providers = await this.loader.loadProviders();
+    const providerConfig = providers[providerId];
+    if (!providerConfig) {
+      throw new Error(`No provider config for embedding provider ${providerId}`);
+    }
+
+    const adapter = this.registry.create(providerConfig);
+    if (policy.requireHealthyProvider) {
+      const health = await adapter.healthCheck();
+      if (!health.ok) {
+        throw new Error(`Embedding provider health check failed for ${providerId} with status ${health.status}`);
+      }
+    }
+    if (typeof adapter.embed !== "function") {
+      throw new Error(`Provider ${providerId} does not implement embeddings`);
+    }
+
+    const result = await adapter.embed({
+      model: config.model,
+      input: texts
+    });
+
+    return {
+      config,
+      policy: getEmbeddingPolicy(),
+      vectors: result.embeddings || []
+    };
+  }
+
+  async getReadiness() {
+    const config = await this.loadConfig();
+    const providerId = config.provider || modelToProviderId(config.model);
+    const policy = getEmbeddingPolicy();
+
+    if (!policy.allowEmbeddings) {
+      return {
+        ready: false,
+        config,
+        policy,
+        providerId,
+        reason: "Embeddings disabled. Set NEMORIS_ALLOW_EMBEDDINGS=1 to enable embedding indexing."
+      };
+    }
+
+    if (providerId !== policy.allowedProvider && !policy.allowRemoteEmbeddings) {
+      return {
+        ready: false,
+        config,
+        policy,
+        providerId,
+        reason: `Embedding provider ${providerId} is blocked by policy.`
+      };
+    }
+
+    const providers = await this.loader.loadProviders();
+    const providerConfig = providers[providerId];
+    if (!providerConfig) {
+      return {
+        ready: false,
+        config,
+        policy,
+        providerId,
+        reason: `No provider config for embedding provider ${providerId}`
+      };
+    }
+
+    const adapter = this.registry.create(providerConfig);
+    const health = await adapter.healthCheck();
+    if (!health.ok) {
+      return {
+        ready: false,
+        config,
+        policy,
+        providerId,
+        providerHealth: health,
+        reason: `Embedding provider health check failed for ${providerId} with status ${health.status}`
+      };
+    }
+
+    if (typeof adapter.embed !== "function") {
+      return {
+        ready: false,
+        config,
+        policy,
+        providerId,
+        providerHealth: health,
+        reason: `Provider ${providerId} does not implement embeddings`
+      };
+    }
+
+    let availableModels = null;
+    let modelAvailable = null;
+    if (typeof adapter.listModels === "function") {
+      try {
+        availableModels = await adapter.listModels();
+        if (Array.isArray(availableModels) && config.model) {
+          const normalizedTarget = normalizeOllamaModelName(config.model);
+          modelAvailable = matchesAvailableModel(config.model, availableModels);
+          if (!modelAvailable) {
+            return {
+              ready: false,
+              config,
+              policy,
+              providerId,
+              providerHealth: health,
+              availableModels,
+              modelAvailable,
+              reason: `Embedding model ${normalizedTarget} is not available in ${providerId}`
+            };
+          }
+        }
+      } catch (error) {
+        return {
+          ready: false,
+          config,
+          policy,
+          providerId,
+          providerHealth: health,
+          reason: error?.message || String(error)
+        };
+      }
+    }
+
+    return {
+      ready: true,
+      config,
+      policy,
+      providerId,
+      providerHealth: health,
+      availableModels,
+      modelAvailable
+    };
+  }
+}