@remnic/core 9.3.761 → 9.3.763

package/src/memory-projection-store.ts

@@ -17,7 +17,7 @@ import {
   type BetterSqlite3Database,
 } from "./runtime/better-sqlite.js";
 
-export const MEMORY_PROJECTION_SCHEMA_VERSION = 2;
+export const MEMORY_PROJECTION_SCHEMA_VERSION = 3;
 
 export interface ProjectedMemoryBrowseOptions {
   query?: string;
@@ -153,11 +153,13 @@ function migrateProjectionSchemaIfNeeded(memoryDir: string): void {
 export function memoryCurrentSelectExpressions(db: BetterSqlite3Database): {
   tagsJson: string;
   previewText: string;
+  hasPathValid: boolean;
 } {
   const columns = listTableColumns(db, "memory_current");
   return {
     tagsJson: columns.has("tags_json") ? "tags_json" : `'[]' AS tags_json`,
     previewText: columns.has("preview_text") ? "preview_text" : `'' AS preview_text`,
+    hasPathValid: columns.has("path_valid"),
   };
 }
 
@@ -174,6 +176,7 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
       status TEXT NOT NULL,
       lifecycle_state TEXT,
       path_rel TEXT NOT NULL,
+      path_valid INTEGER NOT NULL DEFAULT 0 CHECK (path_valid IN (0, 1)),
       created_at TEXT NOT NULL,
       updated_at TEXT NOT NULL,
       archived_at TEXT,
@@ -195,8 +198,6 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
     CREATE INDEX IF NOT EXISTS idx_memory_current_category
       ON memory_current(category);
 
-    CREATE INDEX IF NOT EXISTS idx_memory_current_updated
-      ON memory_current(updated_at DESC);
 
     CREATE TABLE IF NOT EXISTS memory_timeline (
       event_id TEXT PRIMARY KEY,
@@ -297,6 +298,18 @@ export function initializeMemoryProjectionDb(db: BetterSqlite3Database): void {
   `);
 
   migrateMemoryCurrentTable(db);
+  if (listTableColumns(db, "memory_current").has("path_valid")) {
+    db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_memory_current_browse_updated_desc
+        ON memory_current(path_valid, updated_at DESC, created_at DESC, memory_id ASC);
+      CREATE INDEX IF NOT EXISTS idx_memory_current_browse_updated_asc
+        ON memory_current(path_valid, updated_at ASC, created_at ASC, memory_id ASC);
+      CREATE INDEX IF NOT EXISTS idx_memory_current_browse_created_desc
+        ON memory_current(path_valid, created_at DESC, updated_at DESC, memory_id ASC);
+      CREATE INDEX IF NOT EXISTS idx_memory_current_browse_created_asc
+        ON memory_current(path_valid, created_at ASC, updated_at ASC, memory_id ASC);
+    `);
+  }
   db.prepare("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)")
     .run("schemaVersion", String(MEMORY_PROJECTION_SCHEMA_VERSION));
 }
@@ -309,6 +322,42 @@ function openProjectionReadonly(memoryDir: string): BetterSqlite3Database | null
     return null;
   }
 }
+function updateProjectionBestEffort(
+  memoryDir: string,
+  sql: string,
+  params: unknown[],
+): void {
+  let db: BetterSqlite3Database | null = null;
+  try {
+    db = openBetterSqlite3(getMemoryProjectionPath(memoryDir), { fileMustExist: true });
+    db.prepare(sql).run(...params);
+  } catch {
+    // Projection updates must never block the canonical filesystem mutation.
+  } finally {
+    db?.close();
+  }
+}
+
+export function markProjectedMemoryPathInvalid(memoryDir: string, memoryId: string): void {
+  updateProjectionBestEffort(
+    memoryDir,
+    "UPDATE memory_current SET path_valid = 0 WHERE memory_id = ?",
+    [memoryId],
+  );
+}
+
+export function updateProjectedMemoryPath(
+  memoryDir: string,
+  memoryId: string,
+  pathRel: string,
+): void {
+  const pathValid = isProjectedMemoryPathValid(memoryDir, pathRel) ? 1 : 0;
+  updateProjectionBestEffort(
+    memoryDir,
+    "UPDATE memory_current SET path_rel = ?, path_valid = ? WHERE memory_id = ?",
+    [pathRel, pathValid, memoryId],
+  );
+}
 
 function withProjectionReadonly<T>(
   memoryDir: string,
@@ -363,7 +412,11 @@ function isPathInsideRoot(candidatePath: string, rootPath: string): boolean {
   return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
 }
 
-function resolveProjectedMemoryPath(memoryDir: string, pathRel: string): string | null {
+function resolveProjectedMemoryPath(
+  memoryDir: string,
+  pathRel: string,
+  requireExisting = false,
+): string | null {
   if (path.isAbsolute(pathRel)) return null;
   const root = resolveMemoryDirRoot(memoryDir);
   const candidate = path.resolve(root, pathRel);
@@ -372,14 +425,21 @@ function resolveProjectedMemoryPath(memoryDir: string, pathRel: string): string
     const realCandidate = fs.realpathSync(candidate);
     if (!isPathInsideRoot(realCandidate, root)) return null;
   } catch {
+    if (requireExisting) return null;
     // Missing files are handled by callers. Still keep the lexical guard above.
   }
   return candidate;
 }
 
+export function isProjectedMemoryPathValid(memoryDir: string, pathRel: string): boolean {
+  return resolveProjectedMemoryPath(memoryDir, pathRel) !== null;
+}
+
 function projectedBrowseRowFromCurrentRow(
   memoryDir: string,
   row: Record<string, unknown>,
+  resolvedFilePath?: string,
+  requireExisting = false,
 ): ProjectedMemoryBrowseRow | null {
   if (
     typeof row.memory_id !== "string" ||
@@ -389,7 +449,7 @@ function projectedBrowseRowFromCurrentRow(
   ) {
     return null;
   }
-  const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
+  const filePath = resolvedFilePath ?? resolveProjectedMemoryPath(memoryDir, row.path_rel, requireExisting);
   if (!filePath) return null;
   return {
     id: row.memory_id,
@@ -613,8 +673,15 @@ export function readProjectedMemoryBrowse(
     const whereSql = whereClauses.length > 0 ? `WHERE ${whereClauses.join(" AND ")}` : "";
 
     if (normalizedQuery) {
-      // Query-based browse: fetch all matching rows, filter by full file content, then paginate in JS
-      const allRows = db
+      // Full-content search still examines every eligible row so totals and
+      // matches stay exact, but iteration keeps memory use proportional to the page.
+      const queryWhereClauses = currentSelect.hasPathValid
+        ? [...whereClauses, ...projectedBrowsePathSqlClauses(), "path_valid = 1"]
+        : whereClauses;
+      const queryWhereSql = queryWhereClauses.length > 0
+        ? `WHERE ${queryWhereClauses.join(" AND ")}`
+        : "";
+      const rows = db
         .prepare(`
           SELECT
             memory_id,
@@ -627,47 +694,88 @@ export function readProjectedMemoryBrowse(
             ${currentSelect.tagsJson},
             ${currentSelect.previewText}
           FROM memory_current
-          ${whereSql}
+          ${queryWhereSql}
           ORDER BY ${orderBySql}
         `)
-        .all(...params) as Array<Record<string, unknown>>;
+        .iterate(...params) as Iterable<Record<string, unknown>>;
+      const pageRows: ProjectedMemoryBrowseRow[] = [];
+      let total = 0;
 
-      const filtered = allRows.filter((row) => {
-        if (typeof row.memory_id !== "string" || typeof row.path_rel !== "string") return false;
-        const filePath = resolveProjectedMemoryPath(memoryDir, row.path_rel);
-        if (!filePath) return false;
-        // Check preview, category, entity_ref, tags first (fast)
+      for (const row of rows) {
+        if (typeof row.memory_id !== "string" || typeof row.path_rel !== "string") continue;
+        const filePath = resolveProjectedMemoryPath(
+          memoryDir,
+          row.path_rel,
+          currentSelect.hasPathValid,
+        );
+        if (!filePath) continue;
         const preview = typeof row.preview_text === "string" ? row.preview_text.toLowerCase() : "";
         const category = typeof row.category === "string" ? row.category.toLowerCase() : "";
         const entityRef = typeof row.entity_ref === "string" ? row.entity_ref.toLowerCase() : "";
         const tags = typeof row.tags_json === "string" ? row.tags_json.toLowerCase() : "";
-        if (preview.includes(normalizedQuery) || category.includes(normalizedQuery) ||
-            entityRef.includes(normalizedQuery) || tags.includes(normalizedQuery)) {
-          return true;
+        let matches = preview.includes(normalizedQuery) || category.includes(normalizedQuery) ||
+          entityRef.includes(normalizedQuery) || tags.includes(normalizedQuery);
+        if (!matches) {
+          try {
+            matches = fs.readFileSync(filePath, "utf-8").toLowerCase().includes(normalizedQuery);
+          } catch {
+            matches = false;
+          }
         }
-        // Fall back to reading full file content from disk
-        try {
-          const content = fs.readFileSync(filePath, "utf-8").toLowerCase();
-          return content.includes(normalizedQuery);
-        } catch {
-          return false;
+        if (!matches) continue;
+        if (total >= options.offset && pageRows.length < options.limit) {
+          const browseRow = projectedBrowseRowFromCurrentRow(memoryDir, row, filePath);
+          if (browseRow) pageRows.push(browseRow);
         }
-      });
-
-      const pageRows = filtered.slice(options.offset, options.offset + options.limit);
-      return {
-        total: filtered.length,
-        memories: pageRows
-          .map((row) => projectedBrowseRowFromCurrentRow(memoryDir, row))
-          .filter((row): row is ProjectedMemoryBrowseRow => row !== null),
-      };
+        total += 1;
+      }
+
+      return { total, memories: pageRows };
     }
 
-    // No query: push lexical path safety into SQL, then count through the same
-    // realpath-aware row parser used for returned rows so symlink escapes do not
-    // inflate totals.
     const browseWhereClauses = [...whereClauses, ...projectedBrowsePathSqlClauses()];
+    if (currentSelect.hasPathValid) {
+      browseWhereClauses.push("path_valid = 1");
+    }
     const browseWhereSql = `WHERE ${browseWhereClauses.join(" AND ")}`;
+
+    if (currentSelect.hasPathValid) {
+      const rows = db
+        .prepare(`
+          SELECT
+            memory_id,
+            path_rel,
+            category,
+            status,
+            created_at,
+            updated_at,
+            entity_ref,
+            ${currentSelect.tagsJson},
+            ${currentSelect.previewText}
+          FROM memory_current
+          ${browseWhereSql}
+          ORDER BY ${orderBySql}
+        `)
+        .iterate(...params) as Iterable<Record<string, unknown>>;
+      const pageRows: ProjectedMemoryBrowseRow[] = [];
+      let validRowsSeen = 0;
+      for (const row of rows) {
+        const browseRow = projectedBrowseRowFromCurrentRow(memoryDir, row, undefined, true);
+        if (!browseRow) continue;
+        if (validRowsSeen >= options.offset) {
+          pageRows.push(browseRow);
+          if (pageRows.length >= options.limit) break;
+        }
+        validRowsSeen += 1;
+      }
+      const totalRow = db
+        .prepare(`SELECT COUNT(*) AS total FROM memory_current ${browseWhereSql}`)
+        .get(...params) as { total: number };
+      return { total: totalRow.total, memories: pageRows };
+    }
+
+    // Legacy projections have no materialized path validity. Preserve the
+    // realpath-aware full scan rather than trusting rows from an older writer.
     const pageRows: ProjectedMemoryBrowseRow[] = [];
     const fetchSize = Math.max(options.limit * 2, 50);
     let validRowsSeen = 0;

package/src/storage-guards.ts

@@ -0,0 +1,30 @@
+import { log } from "./logger.js";
+
+const PROJECTION_FALLBACK_WARN_INTERVAL_MS = 5 * 60_000;
+const projectionFallbackWarnedAt = new Map<string, number>();
+
+export function assertMemoryFrontmatterId(
+  fm: { id: string; category?: string; created?: string; entityRef?: string },
+): void {
+  if (typeof fm.id !== "string" || fm.id.trim().length === 0) {
+    const context = [
+      fm.category ? `category=${fm.category}` : null,
+      fm.created ? `created=${fm.created}` : null,
+      fm.entityRef ? `entityRef=${fm.entityRef}` : null,
+    ].filter(Boolean).join(" ");
+    throw new Error(
+      `memory frontmatter id must not be blank${context ? ` (${context})` : ""}`,
+    );
+  }
+}
+
+export function warnProjectionFallback(memoryDir: string, consumer: string): null {
+  const key = `${memoryDir}\0${consumer}`;
+  const now = Date.now();
+  const warnedAt = projectionFallbackWarnedAt.get(key) ?? 0;
+  if (now - warnedAt >= PROJECTION_FALLBACK_WARN_INTERVAL_MS) {
+    projectionFallbackWarnedAt.set(key, now);
+    log.warn(`storage.${consumer}: memory projection absent or empty; falling back to full corpus`);
+  }
+  return null;
+}

package/src/storage.ts

@@ -4,6 +4,7 @@ import { createHash } from "node:crypto"
 import { normalizeContent, computeContentHash } from "./content-hash.js";;
 import path from "node:path";
 import { log } from "./logger.js";
+import { assertMemoryFrontmatterId, warnProjectionFallback } from "./storage-guards.js";
 import { MemoryReadStore } from "./storage/memory-read-store.js";
 import {
   readMaybeEncryptedLines,
@@ -102,11 +103,11 @@ import type {
 import { confidenceTier, SPECULATIVE_TTL_DAYS } from "./types.js";
 import {
   type ProjectedMemoryBrowseOptions,
-  type ProjectedMemoryBrowsePage,
+  type ProjectedMemoryBrowsePage, markProjectedMemoryPathInvalid,
   readProjectedMemoryState,
   readProjectedMemoryBrowse,
   readProjectedGovernanceRecord,
-  readProjectedMemoryTimeline,
+  readProjectedMemoryTimeline, updateProjectedMemoryPath,
 } from "./memory-projection-store.js";
 import {
   inferMemoryStatus,
@@ -308,6 +309,7 @@ function stripCitationMarkersForHashRemoval(value: string, template: string): st
 }
 
 function serializeFrontmatter(fm: MemoryFrontmatter): string {
+  assertMemoryFrontmatterId(fm);
   const lines = [
     "---",
     `id: ${fm.id}`,
@@ -4564,14 +4566,12 @@ export class StorageManager {
     const dir = categoryDirName(memory.frontmatter.category);
     return path.join(root, dir, this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
   }
-
   private async writeMemoryFileAtomic(targetPath: string, memory: MemoryFile): Promise<void> {
     const fileContent = `${serializeFrontmatter(memory.frontmatter)}\n\n${memory.content}\n`;
     await writeMaybeEncryptedFile(targetPath, fileContent, this.resolveWriteKey(), {}, this.baseDir);
     this.invalidateAllMemoriesCache();
     this.notifyCatalogWrite();
   }
-
   async moveMemoryToPath(memory: MemoryFile, targetPath: string): Promise<void> {
     await this.writeMemoryFileAtomic(targetPath, memory);
     const sourcePath = path.resolve(memory.path);
@@ -4589,6 +4589,7 @@ export class StorageManager {
       // invalidated, but a concurrent readAllMemories() may have re-populated
       // the cache between the write and the unlink.
       this.invalidateAllMemoriesCache();
+      updateProjectedMemoryPath(this.baseDir, memory.frontmatter.id, toMemoryPathRel(this.baseDir, targetPath));
     }
   }
 
@@ -4613,6 +4614,7 @@ export class StorageManager {
           throw err;
         }
       }
+      updateProjectedMemoryPath(this.baseDir, memory.frontmatter.id, toMemoryPathRel(this.baseDir, targetPath));
       this.bumpMemoryStatusVersion();
       return { changed: false, targetPath };
     }
@@ -4660,7 +4662,7 @@ export class StorageManager {
 
       // Write to archive location first (encrypted if applicable), then remove original.
       await this.writeStorageSecureFile(destPath, fileContent);
-      await unlink(memory.path);
+      await unlink(memory.path); markProjectedMemoryPathInvalid(this.baseDir, memory.frontmatter.id);
       this.invalidateAllMemoriesCache();
       await this.appendGeneratedMemoryLifecycleEventFailOpen(
         "storage.archiveMemory",
@@ -4765,7 +4767,7 @@ export class StorageManager {
     if (!memory) return false;
 
     try {
-      await unlink(memory.path);
+      await unlink(memory.path); markProjectedMemoryPathInvalid(this.baseDir, id);
       this.invalidateAllMemoriesCache();
       this.bumpMemoryStatusVersion();
       log.debug(`invalidated memory ${id}`);
@@ -4904,7 +4906,7 @@ export class StorageManager {
       const expiresAt = new Date(m.frontmatter.expiresAt).getTime();
       if (expiresAt < now) {
         try {
-          await unlink(m.path);
+          await unlink(m.path); markProjectedMemoryPathInvalid(this.baseDir, m.frontmatter.id);
           deleted.push(m);
           log.debug(`cleaned expired memory ${m.frontmatter.id} (TTL expired)`);
         } catch {
@@ -5979,7 +5981,7 @@ export class StorageManager {
       if (updatedAt < cutoff) {
         // Remove the file
         try {
-          await unlink(m.path);
+          await unlink(m.path); markProjectedMemoryPathInvalid(this.baseDir, m.frontmatter.id);
           deleted.push(m);
           log.debug(`cleaned expired commitment ${m.frontmatter.id}`);
         } catch {
@@ -6020,8 +6022,8 @@ export class StorageManager {
         lastAccessed: entry.lastAccessed,
       };
 
-      const fileContent = `${serializeFrontmatter(newFm)}\n\n${memory.content}\n`;
       try {
+        const fileContent = `${serializeFrontmatter(newFm)}\n\n${memory.content}\n`;
         await this.writeStorageSecureFile(memory.path, fileContent);
         updated++;
       } catch (err) {
@@ -6070,8 +6072,7 @@ export class StorageManager {
 
   async getProjectedMemoryState(id: string): Promise<MemoryProjectionCurrentState | null> {
     const projected = readProjectedMemoryState(this.baseDir, id);
-    if (projected) return projected;
-
+    if (projected) return projected; warnProjectionFallback(this.baseDir, "getProjectedMemoryState");
     const active = await this.getMemoryById(id);
     if (active) return this.toProjectedCurrentState(active, "active");
 
@@ -6084,7 +6085,8 @@ export class StorageManager {
   async browseProjectedMemories(
     options: ProjectedMemoryBrowseOptions,
   ): Promise<ProjectedMemoryBrowsePage | null> {
-    return readProjectedMemoryBrowse(this.baseDir, options);
+    return readProjectedMemoryBrowse(this.baseDir, options)
+      ?? warnProjectionFallback(this.baseDir, "browseProjectedMemories");
   }
 
   async getProjectedGovernanceRecord(): Promise<ReturnType<typeof readProjectedGovernanceRecord>> {
@@ -6124,8 +6126,7 @@ export class StorageManager {
     if (cappedLimit === 0) return [];
 
     const projected = readProjectedMemoryTimeline(this.baseDir, memoryId, cappedLimit);
-    if (projected && projected.length > 0) return projected;
-
+    if (projected && projected.length > 0) return projected; warnProjectionFallback(this.baseDir, "getMemoryTimeline");
     const events = await this.readAllMemoryLifecycleEvents();
     return events.filter((event) => event.memoryId === memoryId).slice(-cappedLimit);
   }
@@ -6382,9 +6383,8 @@ export class StorageManager {
         updated: now,
       };
 
-      const fileContent = `${serializeFrontmatter(updatedFm)}\n\n${memory.content}\n`;
-
       try {
+        const fileContent = `${serializeFrontmatter(updatedFm)}\n\n${memory.content}\n`;
         await this.writeStorageSecureFile(memory.path, fileContent);
         await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.archiveMemories", {
           memoryId: id,