memorix 1.2.0 → 1.2.2

package/src/store/orama-store.ts

@@ -9,9 +9,10 @@
  */
 
 import { create, insert, search, remove, update, count, getByID, type AnyOrama } from '@orama/orama';
-import type { MemorixDocument, SearchOptions, IndexEntry, KnowledgeLayer } from '../types.js';
+import type { MemorixDocument, SearchOptions, IndexEntry, KnowledgeLayer, ObservationReader } from '../types.js';
 import { OBSERVATION_ICONS, type ObservationType } from '../types.js';
 import { resolveKnowledgeLayer } from '../skills/mini-skills.js';
+import { canReadObservation } from '../memory/visibility.js';
 import { getEmbeddingProvider, type EmbeddingProvider } from '../embedding/provider.js';
 import { calculateProjectAffinity, extractProjectKeywords, type AffinityContext, type MemoryContent } from './project-affinity.js';
 import { detectQueryIntent, applyIntentBoost } from '../search/intent-detector.js';
@@ -175,6 +176,11 @@ async function initializeDb(
     source: 'string' as const,
     sourceDetail: 'string' as const,
     valueCategory: 'string' as const,
+    admissionState: 'string' as const,
+    admissionReason: 'string' as const,
+    visibility: 'string' as const,
+    createdByAgentId: 'string' as const,
+    sharedWithAgentIds: 'string' as const,
     documentType: 'string' as const,
     knowledgeLayer: 'string' as const,
   };
@@ -390,6 +396,13 @@ export async function hydrateIndex(
         lastAccessedAt: obs.lastAccessedAt || '',
         status: obs.status ?? 'active',
         source: obs.source || 'agent',
+        sourceDetail: obs.sourceDetail ?? '',
+        valueCategory: obs.valueCategory ?? '',
+        admissionState: obs.admissionState ?? '',
+        admissionReason: obs.admissionReason ?? '',
+        visibility: obs.visibility ?? 'project',
+        createdByAgentId: obs.createdByAgentId ?? '',
+        sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
         documentType: 'observation',
         knowledgeLayer: resolveKnowledgeLayer('observation', obs.sourceDetail, obs.source),
         ...(compatibleVector ? { embedding: compatibleVector } : {}),
@@ -443,6 +456,26 @@ export async function insertObservation(doc: MemorixDocument): Promise<void> {
   rememberObservationDoc(doc);
 }
 
+/**
+ * Update retrieval metadata without rebuilding an embedding. Admission is
+ * deliberately metadata-only: promotion must not create another embedding
+ * request or delay a background qualification job.
+ */
+export async function updateObservationMetadata(
+  projectId: string,
+  observationId: number,
+  patch: Partial<Pick<MemorixDocument, 'admissionState' | 'admissionReason' | 'visibility' | 'createdByAgentId' | 'sharedWithAgentIds'>>,
+): Promise<boolean> {
+  const database = await getDb();
+  const id = makeOramaObservationId(projectId, observationId);
+  const existing = getByID(database, id) as MemorixDocument | undefined;
+  if (!existing) return false;
+  const next = { ...existing, ...patch };
+  await update(database, id, next);
+  rememberObservationDoc(next);
+  return true;
+}
+
 /**
  * Remove an observation document by its Orama internal ID.
  */
@@ -643,6 +676,9 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
       const doc = hit.document as unknown as MemorixDocument;
       return projectIds.includes(doc.projectId);
     })
+    // Visibility is enforced before scoring/ranking so private records cannot
+    // affect result order, fallback behavior, or downstream detail reads.
+    .filter((hit) => canReadObservation(hit.document as unknown as MemorixDocument, options.reader))
     // Post-filter by status (active/resolved/archived)
     .filter((hit) => {
       if (statusFilter === 'all') return true;
@@ -682,6 +718,8 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
         source: (doc.source || 'agent') as 'agent' | 'git' | 'manual',
         sourceDetail: (doc.sourceDetail || undefined) as 'explicit' | 'hook' | 'git-ingest' | undefined,
         valueCategory: (doc.valueCategory || undefined) as 'core' | 'contextual' | 'ephemeral' | undefined,
+        admissionState: (doc.admissionState || undefined) as IndexEntry['admissionState'],
+        visibility: (doc.visibility || undefined) as IndexEntry['visibility'],
         entityName: doc.entityName || undefined,
         documentType: (doc.documentType || 'observation') as 'observation' | 'mini-skill',
         knowledgeLayer: (doc.knowledgeLayer || 'project-truth') as KnowledgeLayer,
@@ -752,6 +790,16 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
     }));
   }
 
+  // Automatic capture is useful as an audit trail, but unqualified evidence
+  // should not crowd out durable project knowledge in ordinary retrieval.
+  // Keep it available as a fallback when it is all the project has.
+  if (hasQuery) {
+    const qualifiedEntries = intermediate.filter(
+      (entry) => entry.admissionState !== 'candidate' && entry.admissionState !== 'ephemeral',
+    );
+    if (qualifiedEntries.length > 0) intermediate = qualifiedEntries;
+  }
+
   // Re-sort: chronological for WHEN queries, relevance for others
   if (intentResult?.preferChronological) {
     intermediate.sort((a, b) => new Date(b.rawTime).getTime() - new Date(a.rawTime).getTime());
@@ -960,7 +1008,8 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
   let entries: IndexEntry[] = intermediate.map(({ rawTime: _, _isCommandLog: _c, ...rest }: any) => rest);
 
   for (const hit of results.hits) {
-    rememberObservationDoc(hit.document as unknown as MemorixDocument);
+    const doc = hit.document as unknown as MemorixDocument;
+    if (canReadObservation(doc, options.reader)) rememberObservationDoc(doc);
   }
 
   // Explainable recall: annotate entries with match reasons (O(1) lookup via Map)
@@ -1013,7 +1062,10 @@ export async function searchObservations(options: SearchOptions): Promise<IndexE
 
   // Record access for returned results (fire-and-forget, non-blocking).
   if (options.trackAccess !== false) {
-    const hitDocs = results.hits.map((h) => ({ id: h.id, doc: h.document as unknown as MemorixDocument }));
+    const returnedKeys = new Set(entries.map((entry) => makeEntryKey(entry.projectId, entry.id)));
+    const hitDocs = results.hits
+      .map((hit) => ({ id: hit.id, doc: hit.document as unknown as MemorixDocument }))
+      .filter(({ doc }) => returnedKeys.has(makeEntryKey(doc.projectId, doc.observationId)));
     recordAccessBatch(hitDocs).catch(() => {});
   }
 
@@ -1071,6 +1123,7 @@ export async function getTimeline(
   projectId?: string,
   depthBefore = 3,
   depthAfter = 3,
+  reader?: ObservationReader,
 ): Promise<{ before: IndexEntry[]; anchor: IndexEntry | null; after: IndexEntry[] }> {
   // Use in-memory observations for reliable lookup
   // (Orama search with empty term is unreliable — same fix as compactDetail)
@@ -1079,9 +1132,9 @@ export async function getTimeline(
   const rawObs = await withFreshIndex(() => getAllObservations());
 
   // Filter by project if specified — prevents cross-project context leaking
-  const allObs = projectId
+  const allObs = (projectId
     ? rawObs.filter((o) => o.projectId === projectId)
-    : rawObs;
+    : rawObs).filter((observation) => canReadObservation(observation, reader));
 
   // Sort by creation time
   const sorted = allObs.sort((a, b) => a.createdAt.localeCompare(b.createdAt));
@@ -1093,7 +1146,7 @@ export async function getTimeline(
 
   const toIndexEntry = (obs: {
     id: number; type: string; title: string; tokens: number; createdAt: string;
-    source?: string; sourceDetail?: string; valueCategory?: string;
+    source?: string; sourceDetail?: string; valueCategory?: string; admissionState?: string; visibility?: string;
   }): IndexEntry => {
     const obsType = obs.type as ObservationType;
     return {
@@ -1106,6 +1159,8 @@ export async function getTimeline(
       source: (obs.source as IndexEntry['source']) || undefined,
       sourceDetail: (obs.sourceDetail as IndexEntry['sourceDetail']) || undefined,
       valueCategory: (obs.valueCategory as IndexEntry['valueCategory']) || undefined,
+      admissionState: (obs.admissionState as IndexEntry['admissionState']) || undefined,
+      visibility: (obs.visibility as IndexEntry['visibility']) || undefined,
     };
   };
 

package/src/store/persistence-json.ts

@@ -1,212 +1,212 @@
-/**
- * JSON Persistence Helpers — Migration / Export / Debug Only
- *
- * These functions read/write JSON/JSONL files for one-time migration
- * from legacy storage formats into SQLite, or for export/import and debug.
- *
- * NOT used as runtime canonical store — SQLite is the sole canonical backend.
- */
-
-import { promises as fs } from 'node:fs';
-import path from 'node:path';
-import { atomicWriteFile } from './file-lock.js';
-
-/**
- * Get the file path for the knowledge graph JSONL file.
- * (MCP-compatible format, same as official Memory Server)
- */
-export function getGraphFilePath(projectDir: string): string {
-  return path.join(projectDir, 'graph.jsonl');
-}
-
-/**
- * Save the knowledge graph in JSONL format (MCP-compatible).
- * Each line is a JSON object with type: "entity" or "relation".
- *
- * Format adopted from MCP Official Memory Server.
- */
-export async function saveGraphJsonl(
-  projectDir: string,
-  entities: Array<{ name: string; entityType: string; observations: string[] }>,
-  relations: Array<{ from: string; to: string; relationType: string }>,
-): Promise<void> {
-  const lines = [
-    ...entities.map((e) =>
-      JSON.stringify({ type: 'entity', name: e.name, entityType: e.entityType, observations: e.observations }),
-    ),
-    ...relations.map((r) =>
-      JSON.stringify({ type: 'relation', from: r.from, to: r.to, relationType: r.relationType }),
-    ),
-  ];
-  await atomicWriteFile(getGraphFilePath(projectDir), lines.join('\n'));
-}
-
-/**
- * Load the knowledge graph from JSONL format.
- */
-export async function loadGraphJsonl(
-  projectDir: string,
-): Promise<{
-  entities: Array<{ name: string; entityType: string; observations: string[] }>;
-  relations: Array<{ from: string; to: string; relationType: string }>;
-}> {
-  const filePath = getGraphFilePath(projectDir);
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    const lines = data.split('\n').filter((line) => line.trim() !== '');
-    return lines.reduce(
-      (graph, line) => {
-        const item = JSON.parse(line);
-        if (item.type === 'entity') {
-          graph.entities.push({
-            name: item.name,
-            entityType: item.entityType,
-            observations: item.observations,
-          });
-        }
-        if (item.type === 'relation') {
-          graph.relations.push({
-            from: item.from,
-            to: item.to,
-            relationType: item.relationType,
-          });
-        }
-        return graph;
-      },
-      {
-        entities: [] as Array<{ name: string; entityType: string; observations: string[] }>,
-        relations: [] as Array<{ from: string; to: string; relationType: string }>
-      },
-    );
-  } catch (error) {
-    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return { entities: [], relations: [] };
-    }
-    throw error;
-  }
-}
-
-/**
- * Save observation data as JSON (for Orama restore / export).
- */
-export async function saveObservationsJson(
-  projectDir: string,
-  observations: unknown[],
-): Promise<void> {
-  const filePath = path.join(projectDir, 'observations.json');
-  await atomicWriteFile(filePath, JSON.stringify(observations, null, 2));
-}
-
-/**
- * Load observation data from JSON.
- */
-export async function loadObservationsJson(projectDir: string): Promise<unknown[]> {
-  const filePath = path.join(projectDir, 'observations.json');
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    return JSON.parse(data);
-  } catch (error) {
-    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return [];
-    }
-    throw error;
-  }
-}
-
-/**
- * Save the next observation ID counter (legacy JSON format).
- */
-export async function saveIdCounter(projectDir: string, nextId: number): Promise<void> {
-  const filePath = path.join(projectDir, 'counter.json');
-  await atomicWriteFile(filePath, JSON.stringify({ nextId }));
-}
-
-/**
- * Load the next observation ID counter (legacy JSON format).
- * For runtime use, prefer the SQLite meta table via SqliteBackend.
- */
-export async function loadIdCounter(projectDir: string): Promise<number> {
-  const filePath = path.join(projectDir, 'counter.json');
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    return JSON.parse(data).nextId ?? 1;
-  } catch {
-    return 1;
-  }
-}
-
-/**
- * Save mini-skills data as JSON (migration source only).
- */
-export async function saveMiniSkillsJson(
-  projectDir: string,
-  skills: unknown[],
-): Promise<void> {
-  const filePath = path.join(projectDir, 'mini-skills.json');
-  await atomicWriteFile(filePath, JSON.stringify(skills, null, 2));
-}
-
-/**
- * Load mini-skills data from JSON (migration source only).
- */
-export async function loadMiniSkillsJson(projectDir: string): Promise<unknown[]> {
-  const filePath = path.join(projectDir, 'mini-skills.json');
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    const parsed = JSON.parse(data);
-    return Array.isArray(parsed) ? parsed : [];
-  } catch (error) {
-    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return [];
-    }
-    throw error;
-  }
-}
-
-/**
- * Load the mini-skills ID counter (legacy JSON format).
- */
-export async function loadMiniSkillsCounter(projectDir: string): Promise<number> {
-  const filePath = path.join(projectDir, 'mini-skills-counter.json');
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    return JSON.parse(data).nextId ?? 1;
-  } catch {
-    return 1;
-  }
-}
-
-/**
- * Save the mini-skills ID counter (legacy JSON format).
- */
-export async function saveMiniSkillsCounter(projectDir: string, nextId: number): Promise<void> {
-  const filePath = path.join(projectDir, 'mini-skills-counter.json');
-  await atomicWriteFile(filePath, JSON.stringify({ nextId }));
-}
-
-/**
- * Save sessions data as JSON (migration source only).
- */
-export async function saveSessionsJson(
-  projectDir: string,
-  sessions: unknown[],
-): Promise<void> {
-  const filePath = path.join(projectDir, 'sessions.json');
-  await atomicWriteFile(filePath, JSON.stringify(sessions, null, 2));
-}
-
-/**
- * Load sessions data from JSON (migration source only).
- */
-export async function loadSessionsJson(projectDir: string): Promise<unknown[]> {
-  const filePath = path.join(projectDir, 'sessions.json');
-  try {
-    const data = await fs.readFile(filePath, 'utf-8');
-    return JSON.parse(data);
-  } catch (error) {
-    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return [];
-    }
-    throw error;
-  }
-}
+/**
+ * JSON Persistence Helpers — Migration / Export / Debug Only
+ *
+ * These functions read/write JSON/JSONL files for one-time migration
+ * from legacy storage formats into SQLite, or for export/import and debug.
+ *
+ * NOT used as runtime canonical store — SQLite is the sole canonical backend.
+ */
+
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { atomicWriteFile } from './file-lock.js';
+
+/**
+ * Get the file path for the knowledge graph JSONL file.
+ * (MCP-compatible format, same as official Memory Server)
+ */
+export function getGraphFilePath(projectDir: string): string {
+  return path.join(projectDir, 'graph.jsonl');
+}
+
+/**
+ * Save the knowledge graph in JSONL format (MCP-compatible).
+ * Each line is a JSON object with type: "entity" or "relation".
+ *
+ * Format adopted from MCP Official Memory Server.
+ */
+export async function saveGraphJsonl(
+  projectDir: string,
+  entities: Array<{ name: string; entityType: string; observations: string[] }>,
+  relations: Array<{ from: string; to: string; relationType: string }>,
+): Promise<void> {
+  const lines = [
+    ...entities.map((e) =>
+      JSON.stringify({ type: 'entity', name: e.name, entityType: e.entityType, observations: e.observations }),
+    ),
+    ...relations.map((r) =>
+      JSON.stringify({ type: 'relation', from: r.from, to: r.to, relationType: r.relationType }),
+    ),
+  ];
+  await atomicWriteFile(getGraphFilePath(projectDir), lines.join('\n'));
+}
+
+/**
+ * Load the knowledge graph from JSONL format.
+ */
+export async function loadGraphJsonl(
+  projectDir: string,
+): Promise<{
+  entities: Array<{ name: string; entityType: string; observations: string[] }>;
+  relations: Array<{ from: string; to: string; relationType: string }>;
+}> {
+  const filePath = getGraphFilePath(projectDir);
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    const lines = data.split('\n').filter((line) => line.trim() !== '');
+    return lines.reduce(
+      (graph, line) => {
+        const item = JSON.parse(line);
+        if (item.type === 'entity') {
+          graph.entities.push({
+            name: item.name,
+            entityType: item.entityType,
+            observations: item.observations,
+          });
+        }
+        if (item.type === 'relation') {
+          graph.relations.push({
+            from: item.from,
+            to: item.to,
+            relationType: item.relationType,
+          });
+        }
+        return graph;
+      },
+      {
+        entities: [] as Array<{ name: string; entityType: string; observations: string[] }>,
+        relations: [] as Array<{ from: string; to: string; relationType: string }>
+      },
+    );
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return { entities: [], relations: [] };
+    }
+    throw error;
+  }
+}
+
+/**
+ * Save observation data as JSON (for Orama restore / export).
+ */
+export async function saveObservationsJson(
+  projectDir: string,
+  observations: unknown[],
+): Promise<void> {
+  const filePath = path.join(projectDir, 'observations.json');
+  await atomicWriteFile(filePath, JSON.stringify(observations, null, 2));
+}
+
+/**
+ * Load observation data from JSON.
+ */
+export async function loadObservationsJson(projectDir: string): Promise<unknown[]> {
+  const filePath = path.join(projectDir, 'observations.json');
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    return JSON.parse(data);
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return [];
+    }
+    throw error;
+  }
+}
+
+/**
+ * Save the next observation ID counter (legacy JSON format).
+ */
+export async function saveIdCounter(projectDir: string, nextId: number): Promise<void> {
+  const filePath = path.join(projectDir, 'counter.json');
+  await atomicWriteFile(filePath, JSON.stringify({ nextId }));
+}
+
+/**
+ * Load the next observation ID counter (legacy JSON format).
+ * For runtime use, prefer the SQLite meta table via SqliteBackend.
+ */
+export async function loadIdCounter(projectDir: string): Promise<number> {
+  const filePath = path.join(projectDir, 'counter.json');
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    return JSON.parse(data).nextId ?? 1;
+  } catch {
+    return 1;
+  }
+}
+
+/**
+ * Save mini-skills data as JSON (migration source only).
+ */
+export async function saveMiniSkillsJson(
+  projectDir: string,
+  skills: unknown[],
+): Promise<void> {
+  const filePath = path.join(projectDir, 'mini-skills.json');
+  await atomicWriteFile(filePath, JSON.stringify(skills, null, 2));
+}
+
+/**
+ * Load mini-skills data from JSON (migration source only).
+ */
+export async function loadMiniSkillsJson(projectDir: string): Promise<unknown[]> {
+  const filePath = path.join(projectDir, 'mini-skills.json');
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    const parsed = JSON.parse(data);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return [];
+    }
+    throw error;
+  }
+}
+
+/**
+ * Load the mini-skills ID counter (legacy JSON format).
+ */
+export async function loadMiniSkillsCounter(projectDir: string): Promise<number> {
+  const filePath = path.join(projectDir, 'mini-skills-counter.json');
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    return JSON.parse(data).nextId ?? 1;
+  } catch {
+    return 1;
+  }
+}
+
+/**
+ * Save the mini-skills ID counter (legacy JSON format).
+ */
+export async function saveMiniSkillsCounter(projectDir: string, nextId: number): Promise<void> {
+  const filePath = path.join(projectDir, 'mini-skills-counter.json');
+  await atomicWriteFile(filePath, JSON.stringify({ nextId }));
+}
+
+/**
+ * Save sessions data as JSON (migration source only).
+ */
+export async function saveSessionsJson(
+  projectDir: string,
+  sessions: unknown[],
+): Promise<void> {
+  const filePath = path.join(projectDir, 'sessions.json');
+  await atomicWriteFile(filePath, JSON.stringify(sessions, null, 2));
+}
+
+/**
+ * Load sessions data from JSON (migration source only).
+ */
+export async function loadSessionsJson(projectDir: string): Promise<unknown[]> {
+  const filePath = path.join(projectDir, 'sessions.json');
+  try {
+    const data = await fs.readFile(filePath, 'utf-8');
+    return JSON.parse(data);
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
+      return [];
+    }
+    throw error;
+  }
+}