memorix 1.6.1 → 1.6.3

package/src/memory/observations.ts

@@ -37,7 +37,7 @@ import {
   searchObservations,
   updateObservationMetadata,
 } from '../store/orama-store.js';
-import { getObservationStore, initObservationStore } from '../store/obs-store.js';
+import { getObservationStore, initObservationStore, type ObservationStore } from '../store/obs-store.js';
 import { countTextTokens } from '../compact/token-budget.js';
 import { extractEntities, enrichConcepts } from './entity-extractor.js';
 import { getEmbeddingProvider, isEmbeddingExplicitlyDisabled, validateEmbeddingInput } from '../embedding/provider.js';
@@ -57,10 +57,32 @@ export interface ObservationRuntimeOptions {
   embeddingWriteMode?: ObservationEmbeddingWriteMode;
   /** Git root used by the detached vector worker. */
   projectRoot?: string;
+  /**
+   * Open SQLite and the ID counter without materializing the full corpus.
+   * Hook writes use disk atomic() + findByTopicKey; they must not loadAll()
+   * tens of thousands of rows on every PostToolUse.
+   */
+  skipCorpusLoad?: boolean;
 }
 
 let embeddingWriteMode: ObservationEmbeddingWriteMode = 'background';
 let embeddingWorkerProjectRoot: string | undefined;
+let corpusLoaded = false;
+let loadedObservationStore: ObservationStore | null = null;
+
+/** @internal Reset in-process observation state between tests. */
+export function resetObservationRuntime(): void {
+  observations = [];
+  nextId = 1;
+  projectDir = null;
+  searchIndexPrepared = false;
+  corpusLoaded = false;
+  loadedObservationStore = null;
+  embeddingWriteMode = 'background';
+  embeddingWorkerProjectRoot = undefined;
+  vectorMissingIds.clear();
+  vectorSchemaUpgradePromise = null;
+}
 
 // ── Vector-missing tracking ──────────────────────────────────────
 // Tracks observation IDs whose async embedding write failed or was skipped.
@@ -233,14 +255,26 @@ export async function initObservations(
 ): Promise<void> {
   embeddingWriteMode = options.embeddingWriteMode ?? 'background';
   embeddingWorkerProjectRoot = options.projectRoot;
-  if (projectDir === dir) return;
   await initObservationStore(dir);
   const store = getObservationStore();
-  observations = await store.loadAll();
+
+  if (projectDir === dir && loadedObservationStore === store) {
+    if (options.skipCorpusLoad || corpusLoaded) return;
+  }
+
   nextId = await store.loadIdCounter();
   projectDir = dir;
   searchIndexPrepared = false;
   vectorSchemaUpgradePromise = null;
+  if (options.skipCorpusLoad) {
+    observations = [];
+    corpusLoaded = false;
+    loadedObservationStore = store;
+    return;
+  }
+  observations = await store.loadAll();
+  corpusLoaded = true;
+  loadedObservationStore = store;
 }
 
 function scheduleObservationEmbedding(input: {
@@ -252,14 +286,18 @@ function scheduleObservationEmbedding(input: {
   const { observationId, projectId, searchableText, doc } = input;
   vectorMissingIds.add(observationId);
 
+  // Ordinary store must not pay embedding-provider or cache I/O when vectors
+  // are disabled. Check before generateEmbedding() so MEMORIX_EMBEDDING=off
+  // stays on the lexical write path.
+  if (isEmbeddingExplicitlyDisabled()) {
+    vectorMissingIds.delete(observationId);
+    return;
+  }
+
   // A network fetch keeps Node's event loop alive even when its Promise is not
   // awaited. One-shot CLI commands therefore persist first and let a detached
   // worker consume the same durable queue.
   if (embeddingWriteMode === 'deferred') {
-    if (isEmbeddingExplicitlyDisabled()) {
-      vectorMissingIds.delete(observationId);
-      return;
-    }
     queueVectorBackfill(projectId, {
       detachedWorker: true,
       projectRoot: embeddingWorkerProjectRoot,
@@ -322,7 +360,7 @@ function scheduleObservationEmbedding(input: {
  * For DegradedBackend this is a no-op (always returns false).
  */
 export async function ensureFreshObservations(): Promise<boolean> {
-  if (!projectDir) return false;
+  if (!projectDir || !corpusLoaded) return false;
   try {
     const store = getObservationStore();
     const wasStale = await store.ensureFresh();
@@ -424,12 +462,13 @@ export async function storeObservation(input: {
 
   // Sync the local cache before using it as the topicKey fast path. This costs a
   // generation read in the normal case and only reloads when another process wrote.
+  // Write-only / hook processes skip the corpus cache; disk findByTopicKey is authoritative.
   await ensureFreshObservations();
 
   // Topic key upsert: fast-path check in-memory (optimistic, may be stale).
   // A second authoritative check happens inside the file lock to prevent TOCTOU races
   // where two concurrent calls with the same topicKey both miss this check.
-  if (input.topicKey) {
+  if (input.topicKey && corpusLoaded) {
     const existing = observations.find(
       o => o.topicKey === input.topicKey && o.projectId === input.projectId,
     );
@@ -546,14 +585,18 @@ export async function storeObservation(input: {
       observation.writeGeneration = store.getGeneration();
 
       if (upsertedInsideLock || reloadCacheAfterCommit) {
-        observations = await store.loadAll();
-        nextId = await store.loadIdCounter();
-        if (upsertedInsideLock) {
-          observation = observations.find((candidate) => candidate.id === observation.id)
-            ?? observation;
+        if (corpusLoaded) {
+          observations = await store.loadAll();
+          nextId = await store.loadIdCounter();
+          if (upsertedInsideLock) {
+            observation = observations.find((candidate) => candidate.id === observation.id)
+              ?? observation;
+          }
+        } else {
+          nextId = await store.loadIdCounter();
         }
       } else {
-        observations.push(observation);
+        if (corpusLoaded) observations.push(observation);
         nextId = observation.id + 1;
       }
 
@@ -627,7 +670,12 @@ export async function storeObservation(input: {
       sharedWithAgentIds: JSON.stringify(input.sharedWithAgentIds ?? []),
     };
 
-    await insertObservation(doc);
+    // Hook / write-only processes persist SQLite only. The control plane
+    // picks the row up via ensureFresh. Creating Orama here would load the
+    // embedding provider and the on-disk vector cache in a 10s hook budget.
+    if (corpusLoaded) {
+      await insertObservation(doc);
+    }
   };
 
   await assignAndPersist();
@@ -1163,10 +1211,12 @@ export async function reindexObservations(): Promise<number> {
  * coupled to embedding provider throughput. Missing vectors are queued for the
  * existing background backfill cycle.
  */
-export async function prepareSearchIndex(): Promise<number> {
+export async function prepareSearchIndex(options: { skipCachedVectors?: boolean } = {}): Promise<number> {
   if (searchIndexPrepared) return 0;
 
-  const count = await hydrateIndexForStartup(observations as unknown as any[]);
+  const count = await hydrateIndexForStartup(observations as unknown as any[], {
+    skipCachedVectors: options.skipCachedVectors,
+  });
   if (count === 0) {
     searchIndexPrepared = true;
     return 0;

package/src/memory/refs.ts

@@ -1,94 +1,94 @@
-/**
- * Typed Memory Reference Protocol (Phase 3a)
- *
- * Provides a formal, unambiguous way to reference memory objects
- * (observations and mini-skills) across internal code and the MCP API.
- *
- * String format:
- *   obs:42          — observation #42
- *   skill:3         — mini-skill #3
- *   obs:42@org/proj — observation #42 in project org/proj
- *
- * Legacy support:
- *   42   (bare number)  → obs:42
- *   "42" (bare string)  → obs:42
- *
- * Display short forms (presentation only):
- *   #42  — observation
- *   S3   — mini-skill
- */
-
-import type { MemoryRef } from '../types.js';
-
-// ── Parsing ──────────────────────────────────────────────────────
-
-const TYPED_REF_RE = /^(obs|skill):(\d+)(?:@(.+))?$/;
-
-/**
- * Parse a typed memory reference from a string or number.
- *
- * Accepts:
- *   - "obs:42", "skill:3", "obs:42@org/proj"
- *   - 42 (bare number → obs:42)
- *   - "42" (bare numeric string → obs:42)
- *
- * Throws on invalid input.
- */
-export function parseMemoryRef(input: string | number): MemoryRef {
-  // Bare number → legacy observation ref
-  if (typeof input === 'number') {
-    if (!Number.isInteger(input) || input < 0) {
-      throw new Error(`Invalid memory ref: ${input} (must be a non-negative integer)`);
-    }
-    return { kind: 'obs', id: input };
-  }
-
-  const trimmed = input.trim();
-
-  // Bare numeric string → legacy observation ref
-  if (/^\d+$/.test(trimmed)) {
-    return { kind: 'obs', id: parseInt(trimmed, 10) };
-  }
-
-  // Typed ref: obs:42 or skill:3 or obs:42@org/proj
-  const match = trimmed.match(TYPED_REF_RE);
-  if (!match) {
-    throw new Error(
-      `Invalid memory ref: "${input}". Expected format: obs:<id>, skill:<id>, obs:<id>@<projectId>, or a bare number.`,
-    );
-  }
-
-  const kind = match[1] as 'obs' | 'skill';
-  const id = parseInt(match[2], 10);
-  const projectId = match[3] || undefined;
-
-  return { kind, id, projectId };
-}
-
-// ── Serialization ────────────────────────────────────────────────
-
-/**
- * Serialize a MemoryRef to its canonical string form.
- *
- * Examples:
- *   { kind: 'obs', id: 42 }                    → "obs:42"
- *   { kind: 'skill', id: 3 }                   → "skill:3"
- *   { kind: 'obs', id: 42, projectId: 'o/p' }  → "obs:42@o/p"
- */
-export function serializeMemoryRef(ref: MemoryRef): string {
-  const base = `${ref.kind}:${ref.id}`;
-  return ref.projectId ? `${base}@${ref.projectId}` : base;
-}
-
-// ── Display ──────────────────────────────────────────────────────
-
-/**
- * Format a MemoryRef for human-readable display.
- *
- * Short forms:
- *   obs:42  → "#42"
- *   skill:3 → "S3"
- */
-export function displayRef(ref: MemoryRef): string {
-  return ref.kind === 'obs' ? `#${ref.id}` : `S${ref.id}`;
-}
+/**
+ * Typed Memory Reference Protocol (Phase 3a)
+ *
+ * Provides a formal, unambiguous way to reference memory objects
+ * (observations and mini-skills) across internal code and the MCP API.
+ *
+ * String format:
+ *   obs:42          — observation #42
+ *   skill:3         — mini-skill #3
+ *   obs:42@org/proj — observation #42 in project org/proj
+ *
+ * Legacy support:
+ *   42   (bare number)  → obs:42
+ *   "42" (bare string)  → obs:42
+ *
+ * Display short forms (presentation only):
+ *   #42  — observation
+ *   S3   — mini-skill
+ */
+
+import type { MemoryRef } from '../types.js';
+
+// ── Parsing ──────────────────────────────────────────────────────
+
+const TYPED_REF_RE = /^(obs|skill):(\d+)(?:@(.+))?$/;
+
+/**
+ * Parse a typed memory reference from a string or number.
+ *
+ * Accepts:
+ *   - "obs:42", "skill:3", "obs:42@org/proj"
+ *   - 42 (bare number → obs:42)
+ *   - "42" (bare numeric string → obs:42)
+ *
+ * Throws on invalid input.
+ */
+export function parseMemoryRef(input: string | number): MemoryRef {
+  // Bare number → legacy observation ref
+  if (typeof input === 'number') {
+    if (!Number.isInteger(input) || input < 0) {
+      throw new Error(`Invalid memory ref: ${input} (must be a non-negative integer)`);
+    }
+    return { kind: 'obs', id: input };
+  }
+
+  const trimmed = input.trim();
+
+  // Bare numeric string → legacy observation ref
+  if (/^\d+$/.test(trimmed)) {
+    return { kind: 'obs', id: parseInt(trimmed, 10) };
+  }
+
+  // Typed ref: obs:42 or skill:3 or obs:42@org/proj
+  const match = trimmed.match(TYPED_REF_RE);
+  if (!match) {
+    throw new Error(
+      `Invalid memory ref: "${input}". Expected format: obs:<id>, skill:<id>, obs:<id>@<projectId>, or a bare number.`,
+    );
+  }
+
+  const kind = match[1] as 'obs' | 'skill';
+  const id = parseInt(match[2], 10);
+  const projectId = match[3] || undefined;
+
+  return { kind, id, projectId };
+}
+
+// ── Serialization ────────────────────────────────────────────────
+
+/**
+ * Serialize a MemoryRef to its canonical string form.
+ *
+ * Examples:
+ *   { kind: 'obs', id: 42 }                    → "obs:42"
+ *   { kind: 'skill', id: 3 }                   → "skill:3"
+ *   { kind: 'obs', id: 42, projectId: 'o/p' }  → "obs:42@o/p"
+ */
+export function serializeMemoryRef(ref: MemoryRef): string {
+  const base = `${ref.kind}:${ref.id}`;
+  return ref.projectId ? `${base}@${ref.projectId}` : base;
+}
+
+// ── Display ──────────────────────────────────────────────────────
+
+/**
+ * Format a MemoryRef for human-readable display.
+ *
+ * Short forms:
+ *   obs:42  → "#42"
+ *   skill:3 → "S3"
+ */
+export function displayRef(ref: MemoryRef): string {
+  return ref.kind === 'obs' ? `#${ref.id}` : `S${ref.id}`;
+}

package/src/memory/secret-filter.ts

@@ -1,79 +1,79 @@
-/**
- * Secret Filter
- *
- * Conservative credential detection and redaction for Memorix memory content.
- * Designed for low false-positive risk: only matches explicit credential
- * assignments (key=value / key: value), not generic discussion of auth concepts.
- *
- *   sanitizeCredentials()  — store-time: called inside storeObservation() before write
- *   redactCredentials()    — retrieval-time: called in format/output paths for legacy safety
- *   containsCredential()   — predicate used for testing and optional logging
- *
- * Both sanitize and redact share the same pattern logic. They are kept as
- * separate named exports so call-site semantics remain clear.
- */
-
-/**
- * Replaces the VALUE portion of matched credential patterns with [REDACTED].
- * The credential key name is preserved so context is not lost.
- *
- * Pattern categories:
- *   1. key=value / key: value forms — password, token, api_key, secret, etc.
- *      Requires a real value (6+ non-whitespace chars) to avoid matching
- *      discussion text like "password must be 8 chars".
- *   2. Bearer authorization header tokens (20+ chars)
- *   3. Structured token prefixes: GitHub (ghp_/ghs_/gho_), OpenAI (sk-), JWT (eyJ)
- */
-function applyRedaction(text: string): string {
-  if (!text) return text;
-  let out = text;
-
-  // key=value / key: "value" / key='value' — preserve key, redact value
-  out = out.replace(
-    /((?:password|passwd|pwd|secret|token|api[_-]?key|auth[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret)\s*[:=]\s*["']?)([^\s"',;\n]{6,})/gi,
-    '$1[REDACTED]',
-  );
-
-  // Bearer <token> — preserve "Bearer ", redact token
-  out = out.replace(
-    /(Bearer\s+)([A-Za-z0-9._\-/+]{20,})/g,
-    '$1[REDACTED]',
-  );
-
-  // GitHub tokens (ghp_, ghs_, gho_, github_pat_)
-  out = out.replace(/\b(?:ghp|ghs|gho|github_pat)_[A-Za-z0-9]{36,}\b/g, '[REDACTED]');
-
-  // OpenAI / similar keys (sk-...)
-  out = out.replace(/\bsk-[A-Za-z0-9T-]{20,}\b/g, '[REDACTED]');
-
-  // JWT tokens (base64url header eyJ...)
-  out = out.replace(/\beyJ[A-Za-z0-9._-]{40,}\b/g, '[REDACTED]');
-
-  return out;
-}
-
-/**
- * Store-time sanitization: strips credential values before any durable write.
- * Called inside storeObservation() / upsertObservation() so that every write
- * path (hooks, git-ingest, CLI, reasoning, compact-on-write) is covered.
- */
-export function sanitizeCredentials(text: string): string {
-  return applyRedaction(text);
-}
-
-/**
- * Retrieval-time redaction: masks credential values in display output.
- * Applied in all output formatters (detail, index table, timeline, session context)
- * as a safety net for legacy observations stored before sanitization was in place.
- */
-export function redactCredentials(text: string): string {
-  return applyRedaction(text);
-}
-
-/**
- * Returns true if the text contains an obvious credential pattern.
- * Used for testing and optional diagnostic logging.
- */
-export function containsCredential(text: string): boolean {
-  return applyRedaction(text) !== text;
-}
+/**
+ * Secret Filter
+ *
+ * Conservative credential detection and redaction for Memorix memory content.
+ * Designed for low false-positive risk: only matches explicit credential
+ * assignments (key=value / key: value), not generic discussion of auth concepts.
+ *
+ *   sanitizeCredentials()  — store-time: called inside storeObservation() before write
+ *   redactCredentials()    — retrieval-time: called in format/output paths for legacy safety
+ *   containsCredential()   — predicate used for testing and optional logging
+ *
+ * Both sanitize and redact share the same pattern logic. They are kept as
+ * separate named exports so call-site semantics remain clear.
+ */
+
+/**
+ * Replaces the VALUE portion of matched credential patterns with [REDACTED].
+ * The credential key name is preserved so context is not lost.
+ *
+ * Pattern categories:
+ *   1. key=value / key: value forms — password, token, api_key, secret, etc.
+ *      Requires a real value (6+ non-whitespace chars) to avoid matching
+ *      discussion text like "password must be 8 chars".
+ *   2. Bearer authorization header tokens (20+ chars)
+ *   3. Structured token prefixes: GitHub (ghp_/ghs_/gho_), OpenAI (sk-), JWT (eyJ)
+ */
+function applyRedaction(text: string): string {
+  if (!text) return text;
+  let out = text;
+
+  // key=value / key: "value" / key='value' — preserve key, redact value
+  out = out.replace(
+    /((?:password|passwd|pwd|secret|token|api[_-]?key|auth[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret)\s*[:=]\s*["']?)([^\s"',;\n]{6,})/gi,
+    '$1[REDACTED]',
+  );
+
+  // Bearer <token> — preserve "Bearer ", redact token
+  out = out.replace(
+    /(Bearer\s+)([A-Za-z0-9._\-/+]{20,})/g,
+    '$1[REDACTED]',
+  );
+
+  // GitHub tokens (ghp_, ghs_, gho_, github_pat_)
+  out = out.replace(/\b(?:ghp|ghs|gho|github_pat)_[A-Za-z0-9]{36,}\b/g, '[REDACTED]');
+
+  // OpenAI / similar keys (sk-...)
+  out = out.replace(/\bsk-[A-Za-z0-9T-]{20,}\b/g, '[REDACTED]');
+
+  // JWT tokens (base64url header eyJ...)
+  out = out.replace(/\beyJ[A-Za-z0-9._-]{40,}\b/g, '[REDACTED]');
+
+  return out;
+}
+
+/**
+ * Store-time sanitization: strips credential values before any durable write.
+ * Called inside storeObservation() / upsertObservation() so that every write
+ * path (hooks, git-ingest, CLI, reasoning, compact-on-write) is covered.
+ */
+export function sanitizeCredentials(text: string): string {
+  return applyRedaction(text);
+}
+
+/**
+ * Retrieval-time redaction: masks credential values in display output.
+ * Applied in all output formatters (detail, index table, timeline, session context)
+ * as a safety net for legacy observations stored before sanitization was in place.
+ */
+export function redactCredentials(text: string): string {
+  return applyRedaction(text);
+}
+
+/**
+ * Returns true if the text contains an obvious credential pattern.
+ * Used for testing and optional diagnostic logging.
+ */
+export function containsCredential(text: string): boolean {
+  return applyRedaction(text) !== text;
+}

package/src/memory/session.ts

@@ -140,10 +140,20 @@ async function loadAliasActiveSessions(projectIds: Set<string>): Promise<Session
   return groups.flat();
 }
 
-async function loadAliasActiveObservations(projectIds: Set<string>): Promise<Observation[]> {
+const SESSION_RESUME_OBSERVATION_LIMIT = 64;
+const SESSION_CONTEXT_OBSERVATION_LIMIT = 128;
+
+async function loadAliasActiveObservations(
+  projectIds: Set<string>,
+  limit: number = SESSION_RESUME_OBSERVATION_LIMIT,
+): Promise<Observation[]> {
   const store = getObservationStore();
   const groups = await Promise.all(
-    [...projectIds].map((projectId) => store.loadByProject(projectId, { status: 'active' })),
+    [...projectIds].map((projectId) => store.loadByProject(projectId, {
+      status: 'active',
+      limit,
+      newestFirst: true,
+    })),
   );
   return groups.flat();
 }
@@ -522,7 +532,7 @@ export async function getSessionContext(
   const aliasSet = await resolveProjectIds(projectId);
   const [sessions, allObs] = await Promise.all([
     loadAliasSessions(aliasSet),
-    loadAliasActiveObservations(aliasSet),
+    loadAliasActiveObservations(aliasSet, SESSION_CONTEXT_OBSERVATION_LIMIT),
   ]);
   const readableObs = readableAliasObservations(allObs, aliasSet, reader);
   /** Check if a session summary contains noise/system-self content */