memorix 1.2.0 → 1.2.2

package/src/memory/observations.ts

@@ -8,7 +8,16 @@
  * and in the Orama search index (for full-text + vector search).
  */
 
-import type { Observation, ObservationType, ObservationStatus, MemorixDocument, ProgressInfo } from '../types.js';
+import type {
+  Observation,
+  ObservationAdmissionState,
+  ObservationReader,
+  ObservationVisibility,
+  ObservationType,
+  ObservationStatus,
+  MemorixDocument,
+  ProgressInfo,
+} from '../types.js';
 import { TOPIC_KEY_FAMILIES } from '../types.js';
 import {
   insertObservation,
@@ -25,13 +34,15 @@ import {
   makeOramaObservationId,
   getLastSearchMode,
   searchObservations,
+  updateObservationMetadata,
 } from '../store/orama-store.js';
 import { getObservationStore, initObservationStore } from '../store/obs-store.js';
 import { countTextTokens } from '../compact/token-budget.js';
 import { extractEntities, enrichConcepts } from './entity-extractor.js';
 import { getEmbeddingProvider, isEmbeddingExplicitlyDisabled } from '../embedding/provider.js';
 import { sanitizeCredentials } from './secret-filter.js';
-import { enqueueClaimDerivation } from '../runtime/lifecycle.js';
+import { enqueueClaimDerivation, enqueueObservationQualification } from '../runtime/lifecycle.js';
+import { canManageObservation, resolveObservationVisibility } from './visibility.js';
 
 /** In-memory observation list (loaded from persistence on init) */
 let observations: Observation[] = [];
@@ -137,6 +148,20 @@ function queueClaimDerivation(observation: Observation): void {
   }
 }
 
+function queueObservationQualification(observation: Observation): void {
+  const dataDir = projectDir;
+  if (!dataDir || observation.admissionState !== 'candidate') return;
+  try {
+    enqueueObservationQualification({
+      dataDir,
+      projectId: observation.projectId,
+      source: 'automatic-capture:' + observation.id,
+    });
+  } catch {
+    // Candidate evidence remains available for a later qualification scan.
+  }
+}
+
 function isVectorCompatibleWithCurrentIndex(embedding: number[] | null): boolean {
   if (!embedding) return false;
   const vectorDimensions = getVectorDimensions();
@@ -253,7 +278,13 @@ export async function storeObservation(input: {
   relatedEntities?: string[];
   sourceDetail?: 'explicit' | 'hook' | 'git-ingest';
   valueCategory?: 'core' | 'contextual' | 'ephemeral';
+  admissionState?: ObservationAdmissionState;
+  admissionReason?: string;
+  visibility?: ObservationVisibility;
+  sharedWithAgentIds?: string[];
   createdByAgentId?: string;
+  /** Optional caller-bound write scope for topic-key upserts. */
+  visibilityReader?: ObservationReader;
 }): Promise<{ observation: Observation; upserted: boolean }> {
   const now = new Date().toISOString();
 
@@ -273,6 +304,9 @@ export async function storeObservation(input: {
       o => o.topicKey === input.topicKey && o.projectId === input.projectId,
     );
     if (existing) {
+      if (input.visibilityReader && !canManageObservation(existing, input.visibilityReader)) {
+        throw new Error('Cannot update a memory outside this session\'s write scope.');
+      }
       return { observation: await upsertObservation(existing, input, now), upserted: true };
     }
   }
@@ -320,6 +354,9 @@ export async function storeObservation(input: {
         if (input.topicKey) {
           const diskExisting = await tx.findByTopicKey(input.projectId, input.topicKey);
           if (diskExisting) {
+            if (input.visibilityReader && !canManageObservation(diskExisting, input.visibilityReader)) {
+              throw new Error('Cannot update a memory outside this session\'s write scope.');
+            }
             // Switch to upsert path — update the existing observation after this
             // short transaction has released its lock.
             upsertedInsideLock = true;
@@ -355,6 +392,10 @@ export async function storeObservation(input: {
           relatedEntities: input.relatedEntities,
           sourceDetail: input.sourceDetail,
           valueCategory: input.valueCategory,
+          admissionState: input.admissionState,
+          admissionReason: input.admissionReason ? sanitizeCredentials(input.admissionReason) : undefined,
+          visibility: input.visibility ?? 'project',
+          sharedWithAgentIds: input.sharedWithAgentIds,
           createdByAgentId: input.createdByAgentId,
           // Predict the generation that atomic() will commit after this callback.
           // bumpGeneration() runs after fn(tx) returns, incrementing by 1.
@@ -409,6 +450,10 @@ export async function storeObservation(input: {
         relatedEntities: input.relatedEntities,
         sourceDetail: input.sourceDetail,
         valueCategory: input.valueCategory,
+        admissionState: input.admissionState,
+        admissionReason: input.admissionReason ? sanitizeCredentials(input.admissionReason) : undefined,
+        visibility: input.visibility ?? 'project',
+        sharedWithAgentIds: input.sharedWithAgentIds,
         createdByAgentId: input.createdByAgentId,
         writeGeneration: 0,
       };
@@ -435,6 +480,11 @@ export async function storeObservation(input: {
       source: input.source ?? 'agent',
       sourceDetail: input.sourceDetail ?? '',
       valueCategory: input.valueCategory ?? '',
+      admissionState: input.admissionState ?? '',
+      admissionReason: input.admissionReason ? sanitizeCredentials(input.admissionReason) : '',
+      visibility: input.visibility ?? 'project',
+      createdByAgentId: input.createdByAgentId ?? '',
+      sharedWithAgentIds: JSON.stringify(input.sharedWithAgentIds ?? []),
     };
 
     await insertObservation(doc);
@@ -448,7 +498,10 @@ export async function storeObservation(input: {
   }
 
   await bindObservationCodeRefsBestEffort(observation);
-  queueClaimDerivation(observation);
+  queueObservationQualification(observation);
+  if (resolveObservationVisibility(observation) === 'project') {
+    queueClaimDerivation(observation);
+  }
 
   // Generate embedding async (fire-and-forget) — never blocks MCP response
   // Track in vectorMissingIds until embedding is successfully written.
@@ -518,6 +571,10 @@ async function upsertObservation(
     progress?: ProgressInfo;
     sourceDetail?: 'explicit' | 'hook' | 'git-ingest';
     valueCategory?: 'core' | 'contextual' | 'ephemeral';
+    admissionState?: ObservationAdmissionState;
+    admissionReason?: string;
+    visibility?: ObservationVisibility;
+    sharedWithAgentIds?: string[];
   },
   now: string,
 ): Promise<Observation> {
@@ -556,6 +613,10 @@ async function upsertObservation(
   if (input.progress) existing.progress = input.progress;
   if (input.sourceDetail !== undefined) existing.sourceDetail = input.sourceDetail;
   if (input.valueCategory !== undefined) existing.valueCategory = input.valueCategory;
+  if (input.admissionState !== undefined) existing.admissionState = input.admissionState;
+  if (input.admissionReason !== undefined) existing.admissionReason = sanitizeCredentials(input.admissionReason);
+  if (input.visibility !== undefined) existing.visibility = input.visibility;
+  if (input.sharedWithAgentIds !== undefined) existing.sharedWithAgentIds = input.sharedWithAgentIds;
 
   // Re-index in Orama WITHOUT embedding first (non-blocking)
   const doc: MemorixDocument = {
@@ -577,6 +638,11 @@ async function upsertObservation(
     source: existing.source ?? 'agent',
     sourceDetail: existing.sourceDetail ?? '',
     valueCategory: existing.valueCategory ?? '',
+    admissionState: existing.admissionState ?? '',
+    admissionReason: existing.admissionReason ?? '',
+    visibility: existing.visibility ?? 'project',
+    createdByAgentId: existing.createdByAgentId ?? '',
+    sharedWithAgentIds: JSON.stringify(existing.sharedWithAgentIds ?? []),
   };
 
   // Remove old doc and insert updated one (with retry for concurrent upsert race)
@@ -603,7 +669,10 @@ async function upsertObservation(
   }
 
   await bindObservationCodeRefsBestEffort(existing);
-  queueClaimDerivation(existing);
+  queueObservationQualification(existing);
+  if (resolveObservationVisibility(existing) === 'project') {
+    queueClaimDerivation(existing);
+  }
 
   // Generate embedding async (fire-and-forget) — never blocks MCP response
   const searchableText = [input.title, input.narrative, ...(input.facts ?? [])].join(' ');
@@ -651,6 +720,79 @@ export function getObservation(id: number, projectId?: string): Observation | un
   return observations.find((o) => o.id === id && (projectId ? o.projectId === projectId : true));
 }
 
+/**
+ * Promote or demote an automatic observation without changing its content.
+ * The expected-state guard keeps concurrent maintenance workers from
+ * resurrecting an observation that another writer has already changed.
+ */
+export async function updateObservationAdmission(input: {
+  observationId: number;
+  projectId?: string;
+  expectedState?: ObservationAdmissionState;
+  admissionState: ObservationAdmissionState;
+  admissionReason: string;
+  visibility?: ObservationVisibility;
+}): Promise<Observation | undefined> {
+  await ensureFreshObservations();
+  const cached = observations.find((observation) =>
+    observation.id === input.observationId &&
+    (!input.projectId || observation.projectId === input.projectId),
+  );
+  if (!cached) return undefined;
+
+  const now = new Date().toISOString();
+  const admissionReason = sanitizeCredentials(input.admissionReason).slice(0, 240);
+  const apply = (current: Observation): Observation | undefined => {
+    if (input.projectId && current.projectId !== input.projectId) return undefined;
+    if (input.expectedState && current.admissionState !== input.expectedState) return undefined;
+    return {
+      ...current,
+      admissionState: input.admissionState,
+      admissionReason,
+      ...(input.visibility !== undefined ? { visibility: input.visibility } : {}),
+      updatedAt: now,
+    };
+  };
+
+  let updated: Observation | undefined;
+  if (projectDir) {
+    const store = getObservationStore();
+    await store.atomic(async (tx) => {
+      const current = await tx.getById(input.observationId);
+      if (!current) return;
+      const next = apply(current);
+      if (!next) return;
+      await tx.update(next);
+      updated = next;
+    });
+  } else {
+    updated = apply(cached);
+  }
+
+  if (!updated) return undefined;
+  observations = observations.map((observation) =>
+    observation.id === updated!.id && observation.projectId === updated!.projectId
+      ? updated!
+      : observation,
+  );
+
+  try {
+    await updateObservationMetadata(updated.projectId, updated.id, {
+      admissionState: updated.admissionState,
+      admissionReason: updated.admissionReason,
+      ...(updated.visibility !== undefined ? { visibility: updated.visibility } : {}),
+    });
+  } catch {
+    // SQLite is canonical. The next index hydration repairs a missed update.
+  }
+
+  if (updated.admissionState === 'qualified' && resolveObservationVisibility(updated) === 'project') {
+    queueClaimDerivation(updated);
+  }
+
+  return updated;
+}
+
 /**
  * Resolve observations — mark them as resolved (completed/no longer active).
  * This prevents resolved memories from appearing in default search results.
@@ -701,6 +843,11 @@ export async function resolveObservations(
         source: obs.source ?? 'agent',
         sourceDetail: obs.sourceDetail ?? '',
         valueCategory: obs.valueCategory ?? '',
+        admissionState: obs.admissionState ?? '',
+        admissionReason: obs.admissionReason ?? '',
+        visibility: obs.visibility ?? 'project',
+        createdByAgentId: obs.createdByAgentId ?? '',
+        sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
       };
       await insertObservation(doc);
       // Async embedding update (fire-and-forget)
@@ -904,6 +1051,11 @@ export async function reindexObservations(): Promise<number> {
         source: obs.source ?? 'agent',
         sourceDetail: obs.sourceDetail ?? '',
         valueCategory: obs.valueCategory ?? '',
+        admissionState: obs.admissionState ?? '',
+        admissionReason: obs.admissionReason ?? '',
+        visibility: obs.visibility ?? 'project',
+        createdByAgentId: obs.createdByAgentId ?? '',
+        sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
         ...(compatibleEmbedding ? { embedding: compatibleEmbedding } : {}),
       };
       await insertObservation(doc);
@@ -1028,6 +1180,7 @@ export async function probeSearchIndex(projectId: string): Promise<string> {
   await searchObservations({
     query: 'semantic memory retrieval status',
     projectId,
+    reader: { projectId },
     limit: 1,
     status: 'all',
     trackAccess: false,
@@ -1114,6 +1267,11 @@ export async function backfillVectorEmbeddings(options: {
             source: obs.source ?? 'agent',
             sourceDetail: obs.sourceDetail ?? '',
             valueCategory: obs.valueCategory ?? '',
+            admissionState: obs.admissionState ?? '',
+            admissionReason: obs.admissionReason ?? '',
+            visibility: obs.visibility ?? 'project',
+            createdByAgentId: obs.createdByAgentId ?? '',
+            sharedWithAgentIds: JSON.stringify(obs.sharedWithAgentIds ?? []),
             embedding,
           };
           await insertObservation(doc);

package/src/memory/quality-audit.ts

@@ -137,6 +137,8 @@ export function auditMemoryQuality(
       source: obs.source ?? 'agent',
       sourceDetail: obs.sourceDetail ?? '',
       valueCategory: obs.valueCategory ?? '',
+      admissionState: obs.admissionState ?? '',
+      admissionReason: obs.admissionReason ?? '',
     }, options.referenceTime) !== 'active')
     .map((obs) => makeEntry(obs, 'Outside active retention zone'));
 

package/src/memory/refs.ts

@@ -1,94 +1,94 @@
-/**
- * Typed Memory Reference Protocol (Phase 3a)
- *
- * Provides a formal, unambiguous way to reference memory objects
- * (observations and mini-skills) across internal code and the MCP API.
- *
- * String format:
- *   obs:42          — observation #42
- *   skill:3         — mini-skill #3
- *   obs:42@org/proj — observation #42 in project org/proj
- *
- * Legacy support:
- *   42   (bare number)  → obs:42
- *   "42" (bare string)  → obs:42
- *
- * Display short forms (presentation only):
- *   #42  — observation
- *   S3   — mini-skill
- */
-
-import type { MemoryRef } from '../types.js';
-
-// ── Parsing ──────────────────────────────────────────────────────
-
-const TYPED_REF_RE = /^(obs|skill):(\d+)(?:@(.+))?$/;
-
-/**
- * Parse a typed memory reference from a string or number.
- *
- * Accepts:
- *   - "obs:42", "skill:3", "obs:42@org/proj"
- *   - 42 (bare number → obs:42)
- *   - "42" (bare numeric string → obs:42)
- *
- * Throws on invalid input.
- */
-export function parseMemoryRef(input: string | number): MemoryRef {
-  // Bare number → legacy observation ref
-  if (typeof input === 'number') {
-    if (!Number.isInteger(input) || input < 0) {
-      throw new Error(`Invalid memory ref: ${input} (must be a non-negative integer)`);
-    }
-    return { kind: 'obs', id: input };
-  }
-
-  const trimmed = input.trim();
-
-  // Bare numeric string → legacy observation ref
-  if (/^\d+$/.test(trimmed)) {
-    return { kind: 'obs', id: parseInt(trimmed, 10) };
-  }
-
-  // Typed ref: obs:42 or skill:3 or obs:42@org/proj
-  const match = trimmed.match(TYPED_REF_RE);
-  if (!match) {
-    throw new Error(
-      `Invalid memory ref: "${input}". Expected format: obs:<id>, skill:<id>, obs:<id>@<projectId>, or a bare number.`,
-    );
-  }
-
-  const kind = match[1] as 'obs' | 'skill';
-  const id = parseInt(match[2], 10);
-  const projectId = match[3] || undefined;
-
-  return { kind, id, projectId };
-}
-
-// ── Serialization ────────────────────────────────────────────────
-
-/**
- * Serialize a MemoryRef to its canonical string form.
- *
- * Examples:
- *   { kind: 'obs', id: 42 }                    → "obs:42"
- *   { kind: 'skill', id: 3 }                   → "skill:3"
- *   { kind: 'obs', id: 42, projectId: 'o/p' }  → "obs:42@o/p"
- */
-export function serializeMemoryRef(ref: MemoryRef): string {
-  const base = `${ref.kind}:${ref.id}`;
-  return ref.projectId ? `${base}@${ref.projectId}` : base;
-}
-
-// ── Display ──────────────────────────────────────────────────────
-
-/**
- * Format a MemoryRef for human-readable display.
- *
- * Short forms:
- *   obs:42  → "#42"
- *   skill:3 → "S3"
- */
-export function displayRef(ref: MemoryRef): string {
-  return ref.kind === 'obs' ? `#${ref.id}` : `S${ref.id}`;
-}
+/**
+ * Typed Memory Reference Protocol (Phase 3a)
+ *
+ * Provides a formal, unambiguous way to reference memory objects
+ * (observations and mini-skills) across internal code and the MCP API.
+ *
+ * String format:
+ *   obs:42          — observation #42
+ *   skill:3         — mini-skill #3
+ *   obs:42@org/proj — observation #42 in project org/proj
+ *
+ * Legacy support:
+ *   42   (bare number)  → obs:42
+ *   "42" (bare string)  → obs:42
+ *
+ * Display short forms (presentation only):
+ *   #42  — observation
+ *   S3   — mini-skill
+ */
+
+import type { MemoryRef } from '../types.js';
+
+// ── Parsing ──────────────────────────────────────────────────────
+
+const TYPED_REF_RE = /^(obs|skill):(\d+)(?:@(.+))?$/;
+
+/**
+ * Parse a typed memory reference from a string or number.
+ *
+ * Accepts:
+ *   - "obs:42", "skill:3", "obs:42@org/proj"
+ *   - 42 (bare number → obs:42)
+ *   - "42" (bare numeric string → obs:42)
+ *
+ * Throws on invalid input.
+ */
+export function parseMemoryRef(input: string | number): MemoryRef {
+  // Bare number → legacy observation ref
+  if (typeof input === 'number') {
+    if (!Number.isInteger(input) || input < 0) {
+      throw new Error(`Invalid memory ref: ${input} (must be a non-negative integer)`);
+    }
+    return { kind: 'obs', id: input };
+  }
+
+  const trimmed = input.trim();
+
+  // Bare numeric string → legacy observation ref
+  if (/^\d+$/.test(trimmed)) {
+    return { kind: 'obs', id: parseInt(trimmed, 10) };
+  }
+
+  // Typed ref: obs:42 or skill:3 or obs:42@org/proj
+  const match = trimmed.match(TYPED_REF_RE);
+  if (!match) {
+    throw new Error(
+      `Invalid memory ref: "${input}". Expected format: obs:<id>, skill:<id>, obs:<id>@<projectId>, or a bare number.`,
+    );
+  }
+
+  const kind = match[1] as 'obs' | 'skill';
+  const id = parseInt(match[2], 10);
+  const projectId = match[3] || undefined;
+
+  return { kind, id, projectId };
+}
+
+// ── Serialization ────────────────────────────────────────────────
+
+/**
+ * Serialize a MemoryRef to its canonical string form.
+ *
+ * Examples:
+ *   { kind: 'obs', id: 42 }                    → "obs:42"
+ *   { kind: 'skill', id: 3 }                   → "skill:3"
+ *   { kind: 'obs', id: 42, projectId: 'o/p' }  → "obs:42@o/p"
+ */
+export function serializeMemoryRef(ref: MemoryRef): string {
+  const base = `${ref.kind}:${ref.id}`;
+  return ref.projectId ? `${base}@${ref.projectId}` : base;
+}
+
+// ── Display ──────────────────────────────────────────────────────
+
+/**
+ * Format a MemoryRef for human-readable display.
+ *
+ * Short forms:
+ *   obs:42  → "#42"
+ *   skill:3 → "S3"
+ */
+export function displayRef(ref: MemoryRef): string {
+  return ref.kind === 'obs' ? `#${ref.id}` : `S${ref.id}`;
+}

package/src/memory/retention.ts

@@ -15,8 +15,9 @@
  * retention period but are no longer permanently immune — they decay normally.
  */
 
-import type { MemorixDocument, Observation } from '../types.js';
+import type { MemorixDocument, Observation, ObservationReader } from '../types.js';
 import { getObservationStore } from '../store/obs-store.js';
+import { canManageObservation } from './visibility.js';
 
 // ── Importance → Retention Period mapping ────────────────────────────
 
@@ -113,6 +114,10 @@ export function isImmune(doc: MemorixDocument): boolean {
   // Probe observations are operational heartbeats -- never immune, regardless of valueCategory or access.
   if (doc.type === 'probe') return false;
 
+  // A candidate has not earned durable status. Never let an automatic core
+  // classification turn unqualified capture into permanently retained memory.
+  if (doc.admissionState === 'candidate' || doc.admissionState === 'ephemeral') return false;
+
   // formation-classified core memories are immune regardless of type
   if (doc.valueCategory === 'core') return true;
 
@@ -135,6 +140,11 @@ export function getImmunityReason(doc: MemorixDocument): string | null {
   // Probe observations are never immune
   if (doc.type === 'probe') return null;
 
+  // Automatic traces and candidates have not earned durable status. Keep the
+  // explanation aligned with isImmune() even when an earlier classifier gave
+  // a candidate the core value category.
+  if (doc.admissionState === 'candidate' || doc.admissionState === 'ephemeral') return null;
+
   if (doc.valueCategory === 'core') return 'core valueCategory (formation-classified)';
   const importance = getImportanceLevel(doc);
   if (importance === 'critical') return 'critical importance';
@@ -369,6 +379,8 @@ export interface ArchiveExpiredBatchOptions {
   limit?: number;
   referenceTime?: Date;
   accessMap?: Map<number, { accessCount: number; lastAccessedAt: string }>;
+  /** Omit only for trusted background maintenance. */
+  reader?: ObservationReader;
 }
 
 export interface ArchiveExpiredBatchResult {
@@ -401,6 +413,8 @@ function toRetentionDocument(
     source: obs.source ?? 'agent',
     sourceDetail: obs.sourceDetail ?? '',
     valueCategory: obs.valueCategory ?? '',
+    admissionState: obs.admissionState ?? '',
+    admissionReason: obs.admissionReason ?? '',
   };
 }
 
@@ -423,6 +437,7 @@ export async function archiveExpiredBatch(
   const hasMore = page.length > limit;
   const scanned = hasMore ? page.slice(0, limit) : page;
   const candidateIds = scanned
+    .filter((observation) => !options.reader || canManageObservation(observation, options.reader))
     .filter((observation) => getRetentionZone(
       toRetentionDocument(observation, options.accessMap),
       options.referenceTime,
@@ -461,6 +476,7 @@ export async function archiveExpired(
   referenceTime?: Date,
   accessMap?: Map<number, { accessCount: number; lastAccessedAt: string }>,
   projectId?: string,
+  reader?: ObservationReader,
 ): Promise<{ archived: number; remaining: number }> {
   const store = getObservationStore();
   if (projectId) {
@@ -472,12 +488,16 @@ export async function archiveExpired(
         afterId,
         referenceTime,
         accessMap,
+        reader,
       });
       archived += batch.archived;
       afterId = batch.nextCursor;
     } while (afterId !== undefined);
 
-    const remaining = await store.countByProject(projectId, { status: 'active' });
+    const remainingObservations = await store.loadByProject(projectId, { status: 'active' });
+    const remaining = reader
+      ? remainingObservations.filter((observation) => canManageObservation(observation, reader)).length
+      : remainingObservations.length;
     return { archived, remaining };
   }