memorix 1.2.0 → 1.2.2

package/src/dashboard/server.ts

@@ -22,6 +22,9 @@ import { loadDotenv } from '../config/dotenv-loader.js';
 import { resetDotenv } from '../config/dotenv-loader.js';
 import { initProjectRoot } from '../config/yaml-loader.js';
 import { clearProjectRoot } from '../config/yaml-loader.js';
+import { scopeKnowledgeGraphToProject } from '../memory/graph-scope.js';
+import { canManageObservation, filterReadableObservations } from '../memory/visibility.js';
+import type { Observation } from '../types.js';
 
 // MIME types for static file serving
 const MIME_TYPES: Record<string, string> = {
@@ -58,6 +61,15 @@ function filterByProject<T extends { projectId?: string }>(items: T[], projectId
     return items.filter(item => item.projectId === projectId);
 }
 
+/**
+ * The dashboard has no per-agent authentication. Treat it as an unbound reader:
+ * project facts remain inspectable, while personal and team-scoped records stay
+ * in their owning MCP session.
+ */
+function filterDashboardObservations(observations: Observation[], projectId: string): Observation[] {
+    return filterReadableObservations(observations, { projectId });
+}
+
 function isActiveStatus(status?: string): boolean {
     return (status ?? 'active') === 'active';
 }
@@ -92,22 +104,15 @@ export function prepareDashboardConfig(projectRoot: string | null): void {
 
 /**
  * Compute project-scoped graph counts from observations.
- * Only entities referenced by this project's active observations are counted.
+ * Active observation entities and their explicit cross-references are counted.
  */
 function computeProjectGraphCounts(
     allEntities: Array<{ name: string }>,
     allRelations: Array<{ from: string; to: string }>,
-    projectObs: Array<{ entityName?: string; status?: string }>,
+    projectObs: Array<{ entityName?: string; relatedEntities?: string[]; status?: string }>,
 ): { entities: number; relations: number; entityNames: Set<string> } {
-    const entityNames = new Set(
-        projectObs
-            .filter(o => (o.status ?? 'active') === 'active' && o.entityName)
-            .map(o => o.entityName!),
-    );
-    const entities = allEntities.filter(e => entityNames.has(e.name));
-    const entityNameSet = new Set(entities.map(e => e.name));
-    const relations = allRelations.filter(r => entityNameSet.has(r.from) && entityNameSet.has(r.to));
-    return { entities: entities.length, relations: relations.length, entityNames };
+    const scoped = scopeKnowledgeGraphToProject({ entities: allEntities, relations: allRelations }, projectObs);
+    return { entities: scoped.entities.length, relations: scoped.relations.length, entityNames: scoped.entityNames };
 }
 
 /**
@@ -153,7 +158,10 @@ async function handleApi(
                 // List all unique project IDs from observations data (flat storage)
                 // Deduplicate using alias registry – aliased IDs are merged under canonical
                 try {
-                    const allObs = await getObservationStore().loadAll() as Array<{ projectId?: string; status?: string }>;
+                    const allObs = filterReadableObservations(
+                        await getObservationStore().loadAll(),
+                        {},
+                    ) as Array<{ projectId?: string; status?: string }>;
                     const projectSet = new Map<string, number>();
                     for (const obs of allObs) {
                         if (!isActiveStatus(obs.status)) continue;
@@ -210,22 +218,20 @@ async function handleApi(
                 await initGraphStore(effectiveDataDir);
                 const gStore = getGraphStore();
                 const graph = { entities: gStore.loadEntities(), relations: gStore.loadRelations() };
-                // Project-scope the graph: only include entities that have observations in this project
-                const graphObs = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }) as Array<{ entityName?: string }>;
-                const projectEntityNames = new Set(
-                    graphObs
-                        .filter(o => o.entityName)
-                        .map(o => o.entityName!),
+                const graphObs = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
                 );
-                const entities = graph.entities.filter((e: any) => projectEntityNames.has(e.name));
-                const entityNameSet = new Set(entities.map((e: any) => e.name));
-                const relations = graph.relations.filter((r: any) => entityNameSet.has(r.from) && entityNameSet.has(r.to));
-                sendJson(res, { entities, relations });
+                const scoped = scopeKnowledgeGraphToProject(graph, graphObs);
+                sendJson(res, { entities: scoped.entities, relations: scoped.relations });
                 break;
             }
 
             case '/observations': {
-                const observations = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' });
+                const observations = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
+                );
                 sendJson(res, observations);
                 break;
             }
@@ -240,11 +246,14 @@ async function handleApi(
             case '/stats': {
                 await initGraphStore(effectiveDataDir);
                 const graph = { entities: getGraphStore().loadEntities(), relations: getGraphStore().loadRelations() };
-                const observations = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }) as Array<{ type?: string; id?: number; createdAt?: string; title?: string; entityName?: string; status?: string }>;
+                const observations = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
+                ) as Array<{ type?: string; id?: number; createdAt?: string; title?: string; entityName?: string; relatedEntities?: string[]; status?: string }>;
                 const nextId = await getObservationStore().loadIdCounter();
 
                 // Project-scoped graph counts (must match /api/graph and /api/export)
-                const projectGraphCounts = computeProjectGraphCounts(graph.entities, graph.relations, observations as Array<{ entityName?: string; status?: string }>);
+                const projectGraphCounts = computeProjectGraphCounts(graph.entities, graph.relations, observations);
 
                 // Type counts (exclude probe -- operational heartbeats, not durable knowledge)
                 const typeCounts: Record<string, number> = {};
@@ -376,7 +385,10 @@ async function handleApi(
             }
 
             case '/retention': {
-                const observations = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }) as Array<{
+                const observations = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
+                ) as Array<{
                     id?: number;
                     title?: string;
                     type?: string;
@@ -437,7 +449,10 @@ async function handleApi(
 
                 await initMiniSkillStore(effectiveDataDir);
 
-                const allObs = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' });
+                const allObs = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
+                );
                 const skills = await getMiniSkillStore().loadByProject(effectiveProjectId);
 
                 const overview = generateKnowledgeBase({
@@ -458,27 +473,21 @@ async function handleApi(
                 await initMiniSkillStore(effectiveDataDir);
                 await initGraphStore(effectiveDataDir);
 
-                const allObs = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' });
+                const allObs = filterDashboardObservations(
+                    await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                    effectiveProjectId,
+                );
                 const skills = await getMiniSkillStore().loadByProject(effectiveProjectId);
 
-                // Project-scope graph store (same logic as /api/graph)
                 const fullGraph = { entities: getGraphStore().loadEntities(), relations: getGraphStore().loadRelations() };
-                const graphObs = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }) as Array<{ entityName?: string }>;
-                const projectEntityNames = new Set(
-                    graphObs
-                        .filter(o => o.entityName)
-                        .map(o => o.entityName!),
-                );
-                const scopedEntities = fullGraph.entities.filter((e: any) => projectEntityNames.has(e.name));
-                const scopedEntityNameSet = new Set(scopedEntities.map((e: any) => e.name));
-                const scopedRelations = fullGraph.relations.filter((r: any) => scopedEntityNameSet.has(r.from) && scopedEntityNameSet.has(r.to));
+                const scoped = scopeKnowledgeGraphToProject(fullGraph, allObs);
 
                 const graph = generateKnowledgeGraph({
                     projectId: effectiveProjectId,
                     observations: allObs,
                     miniSkills: skills,
-                    graphEntities: scopedEntities,
-                    graphRelations: scopedRelations,
+                    graphEntities: scoped.entities,
+                    graphRelations: scoped.relations,
                 });
 
                 sendJson(res, graph);
@@ -620,7 +629,7 @@ async function handleApi(
 
             case '/identity': {
                 // Project identity health — with classification layering (matches control-plane contract)
-                const allObs = await getObservationStore().loadAll() as Array<{ projectId?: string }>;
+                const allObs = filterReadableObservations(await getObservationStore().loadAll(), {}) as Array<{ projectId?: string }>;
                 const allProjectIds = [...new Set(allObs.map(o => o.projectId).filter(Boolean))] as string[];
 
                 // Classify every known ID (real / temporary / placeholder) + dirty axis
@@ -721,6 +730,8 @@ async function handleApi(
                     } else if (matchObs.projectId !== effectiveProjectId) {
                         // Cross-project deletion guard: reject if obs belongs to a different project
                         sendError(res, `Observation #${obsId} belongs to project "${matchObs.projectId}", not "${effectiveProjectId}"`, 403);
+                    } else if (!canManageObservation(matchObs, { projectId: effectiveProjectId })) {
+                        sendError(res, `Observation #${obsId} is not manageable from the unbound dashboard.`, 403);
                     } else {
                         await obsStore.remove(obsId);
 
@@ -745,21 +756,16 @@ async function handleApi(
                 if (apiPath === '/export') {
                     await initGraphStore(effectiveDataDir);
                     const fullGraph = { entities: getGraphStore().loadEntities(), relations: getGraphStore().loadRelations() };
-                    const observations = await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' });
-                    const nextId = await getObservationStore().loadIdCounter();
-                    // Project-scope the graph: only entities referenced by this project's observations
-                    const exportEntityNames = new Set(
-                        observations
-                            .filter(o => (o.status ?? 'active') === 'active' && o.entityName)
-                            .map(o => o.entityName!),
+                    const observations = filterDashboardObservations(
+                        await getObservationStore().loadByProject(effectiveProjectId, { status: 'active' }),
+                        effectiveProjectId,
                     );
-                    const exportEntities = fullGraph.entities.filter((e: any) => exportEntityNames.has(e.name));
-                    const exportEntitySet = new Set(exportEntities.map((e: any) => e.name));
-                    const exportRelations = fullGraph.relations.filter((r: any) => exportEntitySet.has(r.from) && exportEntitySet.has(r.to));
+                    const nextId = await getObservationStore().loadIdCounter();
+                    const scoped = scopeKnowledgeGraphToProject(fullGraph, observations);
                     const exportData = {
                         project: { id: effectiveProjectId, name: effectiveProjectName },
                         exportedAt: new Date().toISOString(),
-                        graph: { entities: exportEntities, relations: exportRelations },
+                        graph: { entities: scoped.entities, relations: scoped.relations },
                         observations,
                         nextId,
                     };

package/src/embedding/fastembed-provider.ts

@@ -1,142 +1,142 @@
-/**
- * FastEmbed Provider
- *
- * Local ONNX-based embedding using fastembed (Qdrant).
- * Model: BAAI/bge-small-en-v1.5 (384 dimensions, ~30MB)
- *
- * This is an optional dependency — if fastembed is not installed,
- * the provider module gracefully falls back to fulltext-only search.
- *
- * Persistent disk cache: embeddings are saved to ~/.memorix/data/.embedding-cache.json
- * so server restarts don't need to regenerate them (saves minutes of CPU on 500+ obs).
- */
-
-import { createHash } from 'node:crypto';
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { homedir } from 'node:os';
-import type { EmbeddingProvider } from './provider.js';
-
-const CACHE_DIR = process.env.MEMORIX_DATA_DIR || join(homedir(), '.memorix', 'data');
-const CACHE_FILE = join(CACHE_DIR, '.embedding-cache.json');
-
-// In-memory cache keyed by text hash → embedding
-const cache = new Map<string, number[]>();
-const MAX_CACHE_SIZE = 5000;
-let diskCacheDirty = false;
-
-function textHash(text: string): string {
-  return createHash('sha256').update(text).digest('hex').slice(0, 16);
-}
-
-async function loadDiskCache(): Promise<void> {
-  try {
-    const raw = await readFile(CACHE_FILE, 'utf-8');
-    const entries: [string, number[]][] = JSON.parse(raw);
-    for (const [k, v] of entries) cache.set(k, v);
-    console.error(`[memorix] Loaded ${entries.length} cached embeddings from disk`);
-  } catch {
-    // No cache file or corrupt — start fresh
-  }
-}
-
-async function saveDiskCache(): Promise<void> {
-  if (!diskCacheDirty) return;
-  try {
-    await mkdir(CACHE_DIR, { recursive: true });
-    const entries = Array.from(cache.entries());
-    await writeFile(CACHE_FILE, JSON.stringify(entries));
-    diskCacheDirty = false;
-  } catch {
-    // Ignore write errors — cache is an optimization, not critical
-  }
-}
-
-export class FastEmbedProvider implements EmbeddingProvider {
-  readonly name = 'fastembed-bge-small';
-  readonly dimensions = 384;
-
-  private model: { embed: (docs: string[], batchSize?: number) => AsyncGenerator<number[][]>; queryEmbed: (query: string) => Promise<number[]> };
-
-  private constructor(model: FastEmbedProvider['model']) {
-    this.model = model;
-  }
-
-  /**
-   * Initialize the FastEmbed provider.
-   * Downloads model on first use (~30MB), cached locally after.
-   * Loads persistent embedding cache from disk.
-   */
-  static async create(): Promise<FastEmbedProvider> {
-    // Dynamic import — throws if fastembed is not installed
-    const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
-    const model = await FlagEmbedding.init({
-      model: EmbeddingModel.BGESmallENV15,
-    });
-    // Load disk cache before returning — subsequent embedBatch calls will hit cache
-    await loadDiskCache();
-    return new FastEmbedProvider(model);
-  }
-
-  async embed(text: string): Promise<number[]> {
-    const hash = textHash(text);
-    const cached = cache.get(hash);
-    if (cached) return cached;
-
-    const raw = await this.model.queryEmbed(text);
-    // Ensure plain number[] (fastembed may return Float32Array)
-    const result = Array.from(raw) as number[];
-    if (result.length !== this.dimensions) {
-      throw new Error(`Expected ${this.dimensions}d embedding, got ${result.length}d`);
-    }
-    this.cacheSet(hash, result);
-    return result;
-  }
-
-  async embedBatch(texts: string[]): Promise<number[][]> {
-    const results: number[][] = new Array(texts.length);
-    const uncachedIndices: number[] = [];
-    const uncachedTexts: string[] = [];
-
-    // Check cache for each text (by hash)
-    for (let i = 0; i < texts.length; i++) {
-      const hash = textHash(texts[i]);
-      const cached = cache.get(hash);
-      if (cached) {
-        results[i] = cached;
-      } else {
-        uncachedIndices.push(i);
-        uncachedTexts.push(texts[i]);
-      }
-    }
-
-    // Batch embed uncached texts
-    if (uncachedTexts.length > 0) {
-      console.error(`[memorix] Embedding ${uncachedTexts.length}/${texts.length} uncached texts (${texts.length - uncachedTexts.length} from cache)`);
-      let batchIdx = 0;
-      for await (const batch of this.model.embed(uncachedTexts, 64)) {
-        for (const vec of batch) {
-          const originalIdx = uncachedIndices[batchIdx];
-          const plain = Array.from(vec) as number[];
-          results[originalIdx] = plain;
-          this.cacheSet(textHash(uncachedTexts[batchIdx]), plain);
-          batchIdx++;
-        }
-      }
-      // Persist cache to disk after batch operations
-      await saveDiskCache();
-    }
-
-    return results;
-  }
-
-  private cacheSet(hash: string, value: number[]): void {
-    // Evict oldest entries if cache is full
-    if (cache.size >= MAX_CACHE_SIZE) {
-      const firstKey = cache.keys().next().value;
-      if (firstKey !== undefined) cache.delete(firstKey);
-    }
-    cache.set(hash, value);
-    diskCacheDirty = true;
-  }
-}
+/**
+ * FastEmbed Provider
+ *
+ * Local ONNX-based embedding using fastembed (Qdrant).
+ * Model: BAAI/bge-small-en-v1.5 (384 dimensions, ~30MB)
+ *
+ * This is an optional dependency — if fastembed is not installed,
+ * the provider module gracefully falls back to fulltext-only search.
+ *
+ * Persistent disk cache: embeddings are saved to ~/.memorix/data/.embedding-cache.json
+ * so server restarts don't need to regenerate them (saves minutes of CPU on 500+ obs).
+ */
+
+import { createHash } from 'node:crypto';
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import type { EmbeddingProvider } from './provider.js';
+
+const CACHE_DIR = process.env.MEMORIX_DATA_DIR || join(homedir(), '.memorix', 'data');
+const CACHE_FILE = join(CACHE_DIR, '.embedding-cache.json');
+
+// In-memory cache keyed by text hash → embedding
+const cache = new Map<string, number[]>();
+const MAX_CACHE_SIZE = 5000;
+let diskCacheDirty = false;
+
+function textHash(text: string): string {
+  return createHash('sha256').update(text).digest('hex').slice(0, 16);
+}
+
+async function loadDiskCache(): Promise<void> {
+  try {
+    const raw = await readFile(CACHE_FILE, 'utf-8');
+    const entries: [string, number[]][] = JSON.parse(raw);
+    for (const [k, v] of entries) cache.set(k, v);
+    console.error(`[memorix] Loaded ${entries.length} cached embeddings from disk`);
+  } catch {
+    // No cache file or corrupt — start fresh
+  }
+}
+
+async function saveDiskCache(): Promise<void> {
+  if (!diskCacheDirty) return;
+  try {
+    await mkdir(CACHE_DIR, { recursive: true });
+    const entries = Array.from(cache.entries());
+    await writeFile(CACHE_FILE, JSON.stringify(entries));
+    diskCacheDirty = false;
+  } catch {
+    // Ignore write errors — cache is an optimization, not critical
+  }
+}
+
+export class FastEmbedProvider implements EmbeddingProvider {
+  readonly name = 'fastembed-bge-small';
+  readonly dimensions = 384;
+
+  private model: { embed: (docs: string[], batchSize?: number) => AsyncGenerator<number[][]>; queryEmbed: (query: string) => Promise<number[]> };
+
+  private constructor(model: FastEmbedProvider['model']) {
+    this.model = model;
+  }
+
+  /**
+   * Initialize the FastEmbed provider.
+   * Downloads model on first use (~30MB), cached locally after.
+   * Loads persistent embedding cache from disk.
+   */
+  static async create(): Promise<FastEmbedProvider> {
+    // Dynamic import — throws if fastembed is not installed
+    const { EmbeddingModel, FlagEmbedding } = await import('fastembed');
+    const model = await FlagEmbedding.init({
+      model: EmbeddingModel.BGESmallENV15,
+    });
+    // Load disk cache before returning — subsequent embedBatch calls will hit cache
+    await loadDiskCache();
+    return new FastEmbedProvider(model);
+  }
+
+  async embed(text: string): Promise<number[]> {
+    const hash = textHash(text);
+    const cached = cache.get(hash);
+    if (cached) return cached;
+
+    const raw = await this.model.queryEmbed(text);
+    // Ensure plain number[] (fastembed may return Float32Array)
+    const result = Array.from(raw) as number[];
+    if (result.length !== this.dimensions) {
+      throw new Error(`Expected ${this.dimensions}d embedding, got ${result.length}d`);
+    }
+    this.cacheSet(hash, result);
+    return result;
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    const results: number[][] = new Array(texts.length);
+    const uncachedIndices: number[] = [];
+    const uncachedTexts: string[] = [];
+
+    // Check cache for each text (by hash)
+    for (let i = 0; i < texts.length; i++) {
+      const hash = textHash(texts[i]);
+      const cached = cache.get(hash);
+      if (cached) {
+        results[i] = cached;
+      } else {
+        uncachedIndices.push(i);
+        uncachedTexts.push(texts[i]);
+      }
+    }
+
+    // Batch embed uncached texts
+    if (uncachedTexts.length > 0) {
+      console.error(`[memorix] Embedding ${uncachedTexts.length}/${texts.length} uncached texts (${texts.length - uncachedTexts.length} from cache)`);
+      let batchIdx = 0;
+      for await (const batch of this.model.embed(uncachedTexts, 64)) {
+        for (const vec of batch) {
+          const originalIdx = uncachedIndices[batchIdx];
+          const plain = Array.from(vec) as number[];
+          results[originalIdx] = plain;
+          this.cacheSet(textHash(uncachedTexts[batchIdx]), plain);
+          batchIdx++;
+        }
+      }
+      // Persist cache to disk after batch operations
+      await saveDiskCache();
+    }
+
+    return results;
+  }
+
+  private cacheSet(hash: string, value: number[]): void {
+    // Evict oldest entries if cache is full
+    if (cache.size >= MAX_CACHE_SIZE) {
+      const firstKey = cache.keys().next().value;
+      if (firstKey !== undefined) cache.delete(firstKey);
+    }
+    cache.set(hash, value);
+    diskCacheDirty = true;
+  }
+}