@remnic/core 9.3.644 → 9.3.646

package/src/transcript.ts

@@ -3,13 +3,8 @@ import path from "node:path";
 import { log } from "./logger.js";
 import type { TranscriptEntry, Checkpoint, PluginConfig } from "./types.js";
 import { analyzeSessionIntegrity, type SessionIntegrityReport } from "./session-integrity.js";
-import {
-  encodeStoragePathSegment,
-  encodeStoragePathSegmentWithHash,
-  isSafeLegacyPathSegment,
-  resolveSafeStoragePath,
-  storagePathHash,
-} from "./storage-paths.js";
+import { resolveSafeStoragePath } from "./storage-paths.js";
+import { sessionStoragePaths } from "./session-identity.js";
 
 type DirectorySessionStatus = "missing" | "empty" | "matches" | "occupied";
 type DirectoryOwnershipCacheEntry = {
@@ -17,16 +12,25 @@ type DirectoryOwnershipCacheEntry = {
   fileSizes: Map<string, number>;
 };
 
-function legacyTranscriptDirFor(
-  channelType: string,
-  channelId: string,
-  encodedDir: string,
-): string | undefined {
-  if (!isSafeLegacyPathSegment(channelType) || !isSafeLegacyPathSegment(channelId)) {
-    return undefined;
-  }
-  const legacyDir = path.join(channelType, channelId);
-  return legacyDir === encodedDir ? undefined : legacyDir;
+/**
+ * De-duplicate JSONL lines collected across multiple read-back candidate
+ * directories. A partially-applied migration (issue #1496) can leave the SAME
+ * raw row in both the primary `session/<hash>` tree and a legacy read-back dir
+ * (`other/default` or an old `parts.length >= 3` directory) when the source was
+ * copied but not yet trimmed. Without this guard, `readRecent`, `readToolUse`,
+ * footprint estimation, and the summarizer fetch would return that row twice
+ * (cursor review on PR #1504). The exact raw JSONL line is the stable identity:
+ * the migration preserves byte content per session, so a copied-but-not-trimmed
+ * row is byte-identical in both locations. Returns `true` the first time a line
+ * is seen, `false` for every subsequent duplicate.
+ */
+function makeRawLineDeduper(): (rawLine: string) => boolean {
+  const seen = new Set<string>();
+  return (rawLine: string): boolean => {
+    if (seen.has(rawLine)) return false;
+    seen.add(rawLine);
+    return true;
+  };
 }
 
 /**
@@ -64,13 +68,17 @@ export class TranscriptManager {
   }
 
   /**
-   * Parse a sessionKey to extract channel type and ID.
+   * Resolve the storage path pieces for a sessionKey via the shared
+   * {@link sessionStoragePaths} helper (issue #1496, rule #22).
    *
-   * SessionKey patterns:
-   *   - agent:<agent-id>:main → type="main", id="default"
-   *   - agent:<agent-id>:discord:channel:<channel-id> → type="discord", id="<channel-id>"
-   *   - agent:<agent-id>:cron:<job-id> → type="cron", id="<job-id>"
-   *   - agent:<agent-id>:slack:channel:<channel-id> → type="slack", id="<channel-id>"
+   * Legacy `agent:<id>:...` shapes keep their existing readable channel paths.
+   * Arbitrary / non-legacy keys (e.g. `pi-geek:abc123`) route to
+   * `transcripts/session/<hash>/YYYY-MM-DD.jsonl` from the FIRST write — they
+   * never start life in `other/default`. For those keys a list of
+   * read-back-only `readbackDirs` is returned so data written by older builds
+   * (under `other/default`, or under an old `parts.length >= 3` directory such
+   * as `baz/default`) stays discoverable until migrated. New writes never
+   * target read-back dirs. See `session-identity.ts`.
    *
    * @returns Object with raw channel identifiers and encoded storage path pieces.
    */
@@ -81,49 +89,21 @@ export class TranscriptManager {
     channelId: string;
     alternateDir: string;
     legacyDir?: string;
+    readbackDirs: string[];
   } {
-    const parts = sessionKey.split(":");
-
-    // Default fallback
-    let channelType = "other";
-    let channelId = "default";
-
-    if (parts.length >= 3) {
-      // parts[0] = "agent", parts[1] = agent name, parts[2] = channel type
-      channelType = parts[2];
-
-      // Extract channel ID based on pattern
-      if (channelType === "main") {
-        channelId = "default";
-      } else if (channelType === "discord" && parts.length >= 5 && parts[3] === "channel") {
-        channelId = parts[4];
-      } else if (channelType === "slack" && parts.length >= 5 && parts[3] === "channel") {
-        channelId = parts[4];
-      } else if (channelType === "cron" && parts.length >= 4) {
-        channelId = parts[3];
-      } else if (parts.length >= 4) {
-        // For other types, use the 4th part as ID if available
-        channelId = parts[3];
-      }
-    }
+    const paths = sessionStoragePaths(sessionKey);
 
     // Daily rotation: transcripts/{channelType}/{channelId}/YYYY-MM-DD.jsonl
     const today = new Date().toISOString().slice(0, 10);
-    const dir = path.join(
-      encodeStoragePathSegment(channelType),
-      encodeStoragePathSegment(channelId),
-    );
-    const alternateDir = path.join(
-      encodeStoragePathSegmentWithHash(channelType),
-      `${encodeStoragePathSegmentWithHash(channelId)}--session-${storagePathHash(sessionKey)}`,
-    );
+
     return {
-      dir,
+      dir: paths.dir,
       file: `${today}.jsonl`,
-      channelType,
-      channelId,
-      alternateDir,
-      legacyDir: legacyTranscriptDirFor(channelType, channelId, dir),
+      channelType: paths.channelType,
+      channelId: paths.channelId,
+      alternateDir: paths.alternateDir,
+      legacyDir: paths.legacyDir,
+      readbackDirs: paths.readbackDirs,
     };
   }
 
@@ -183,9 +163,16 @@ export class TranscriptManager {
     file: string;
     alternateDir: string;
     legacyDir?: string;
+    readbackDirs: string[];
   } {
     const p = this.getTranscriptPath(sessionKey);
-    return { dir: p.dir, file: p.file, alternateDir: p.alternateDir, legacyDir: p.legacyDir };
+    return {
+      dir: p.dir,
+      file: p.file,
+      alternateDir: p.alternateDir,
+      legacyDir: p.legacyDir,
+      readbackDirs: p.readbackDirs,
+    };
   }
 
   private async selectStorageDirForWrite(
@@ -344,11 +331,12 @@ export class TranscriptManager {
     dir: string,
     legacyDir?: string,
     alternateDir?: string,
+    readbackDirs: string[] = [],
   ): Promise<Array<{ cacheKey: string; name: string; path: string }>> {
     const files: Array<{ cacheKey: string; name: string; path: string }> = [];
     const seenDirs = new Set<string>();
 
-    for (const candidateDir of [dir, alternateDir, legacyDir]) {
+    for (const candidateDir of [dir, alternateDir, legacyDir, ...readbackDirs]) {
       if (!candidateDir || seenDirs.has(candidateDir)) continue;
       seenDirs.add(candidateDir);
 
@@ -400,10 +388,19 @@ export class TranscriptManager {
     startTime: Date,
     endTime: Date,
   ): Promise<Array<{ timestamp: string; sessionKey: string; tool: string }>> {
-    const { dir, alternateDir, legacyDir } = this.getToolUsagePath(sessionKey);
+    const { dir, alternateDir, legacyDir, readbackDirs } = this.getToolUsagePath(sessionKey);
     try {
-      const files = await this.getSessionStorageFiles(this.toolUsageDir, dir, legacyDir, alternateDir);
+      const files = await this.getSessionStorageFiles(
+        this.toolUsageDir,
+        dir,
+        legacyDir,
+        alternateDir,
+        readbackDirs,
+      );
       const out: Array<{ timestamp: string; sessionKey: string; tool: string }> = [];
+      // Dedup identical raw rows that a partially-applied migration may have left
+      // in both the primary dir and a read-back dir (issue #1496, cursor review).
+      const keepRawLine = makeRawLineDeduper();
       for (const file of files) {
         const raw = await readFile(file.path, "utf-8");
         for (const line of raw.split("\n")) {
@@ -414,7 +411,7 @@ export class TranscriptManager {
             if (!Number.isFinite(ts)) continue;
             if (ts >= startTime.getTime() && ts < endTime.getTime()) {
               if (typeof obj.tool === "string" && typeof obj.sessionKey === "string") {
-                if (obj.sessionKey === sessionKey) {
+                if (obj.sessionKey === sessionKey && keepRawLine(line)) {
                   out.push({ timestamp: obj.timestamp, sessionKey: obj.sessionKey, tool: obj.tool });
                 }
               }
@@ -431,11 +428,24 @@ export class TranscriptManager {
   }
 
   async estimateSessionFootprint(sessionKey: string): Promise<{ bytes: number; tokens: number }> {
-    const { dir, alternateDir, legacyDir } = this.getTranscriptPath(sessionKey);
+    const { dir, alternateDir, legacyDir, readbackDirs } = this.getTranscriptPath(sessionKey);
     let bytes = 0;
 
+    // NOTE: this is a best-effort byte ESTIMATE for compaction sizing, not an
+    // exact row count, and is maintained via an incremental per-file cache. A
+    // copied-but-not-trimmed migration window (issue #1496) can transiently
+    // over-estimate by counting a duplicated row in both the primary and a
+    // read-back dir; that self-heals once the migration trims the source. The
+    // exact-once guarantee that matters for callers lives in readRecent /
+    // readToolUse / the summarizer fetch, which dedup by raw line.
     try {
-      const files = await this.getSessionStorageFiles(this.transcriptsDir, dir, legacyDir, alternateDir);
+      const files = await this.getSessionStorageFiles(
+        this.transcriptsDir,
+        dir,
+        legacyDir,
+        alternateDir,
+        readbackDirs,
+      );
       const cached = this.sessionFootprintCache.get(sessionKey);
       if (!cached) {
         const fileBytes = new Map<string, number>();
@@ -642,6 +652,16 @@ export class TranscriptManager {
     const end = new Date(endTime);
     const entries: TranscriptEntry[] = [];
 
+    // When a sessionKey is given, a partially-applied #1496 migration can leave
+    // the SAME raw row in both the primary `session/<hash>` dir and a read-back
+    // dir (`other/default` or an old `parts.length >= 3` dir). `readRange` scans
+    // EVERY transcript file, so it would append that row once per directory.
+    // Dedup by exact raw line to give the same exact-once guarantee `readRecent`
+    // / `readToolUse` / the summarizer fetch already provide (cursor review on
+    // PR #1504). Only applied for the session-scoped path so unfiltered range
+    // scans (each file already enumerated once) keep their existing behavior.
+    const keepRawLine = sessionKey ? makeRawLineDeduper() : undefined;
+
     try {
       // Get all transcript files from the hierarchical structure
       const transcriptFiles = await this.getAllTranscriptFiles();
@@ -662,6 +682,7 @@ export class TranscriptManager {
               if (entryTime >= start && entryTime < end) {
                 // Filter by sessionKey if provided
                 if (!sessionKey || entry.sessionKey === sessionKey) {
+                  if (keepRawLine && !keepRawLine(line)) continue;
                   entries.push(entry);
                 }
               }
@@ -711,7 +732,7 @@ export class TranscriptManager {
     end: Date,
     sessionKey: string,
   ): Promise<TranscriptEntry[]> {
-    const { dir, alternateDir, legacyDir } = this.getTranscriptPath(sessionKey);
+    const { dir, alternateDir, legacyDir, readbackDirs } = this.getTranscriptPath(sessionKey);
 
     // Build set of date strings that overlap with [start, end].
     // Always include end's date to handle midnight-crossing lookbacks
@@ -725,8 +746,17 @@ export class TranscriptManager {
     dateStrings.add(end.toISOString().slice(0, 10));
 
     const entries: TranscriptEntry[] = [];
-    const files = await this.getSessionStorageFiles(this.transcriptsDir, dir, legacyDir, alternateDir);
+    const files = await this.getSessionStorageFiles(
+      this.transcriptsDir,
+      dir,
+      legacyDir,
+      alternateDir,
+      readbackDirs,
+    );
 
+    // Dedup identical raw rows that a partially-applied migration may have left
+    // in both the primary dir and a read-back dir (issue #1496, cursor review).
+    const keepRawLine = makeRawLineDeduper();
     for (const file of files) {
       // Only read files whose date is within the window
       const dateStr = file.name.slice(0, 10);
@@ -739,7 +769,7 @@ export class TranscriptManager {
           try {
             const entry = JSON.parse(line) as TranscriptEntry;
             const ts = new Date(entry.timestamp);
-            if (ts >= start && ts < end && entry.sessionKey === sessionKey) {
+            if (ts >= start && ts < end && entry.sessionKey === sessionKey && keepRawLine(line)) {
               entries.push(entry);
             }
           } catch {

package/src/utils/category-dir.ts

@@ -28,6 +28,32 @@ export const ALL_CATEGORY_DIRS: string[] = [
   ...Object.values(CATEGORY_DIR_MAP),
 ];
 
+/**
+ * Category directories whose files are NOT part of the recall memory corpus.
+ *
+ * `questions/` holds operational question-QUEUE items written by
+ * `StorageManager.writeQuestion()` (frontmatter `{ id, created, priority,
+ * resolved }`, body `<question>\n\n**Context:** ...`). They are read only via
+ * `readQuestions()` and surfaced through the dedicated, disabled-by-default
+ * `injectQuestions` recall-pipeline stage (`{"id":"questions","enabled":false}`)
+ * — never as standard frontmatter-backed recall memories. The QMD primary
+ * recall path treats them the same way (they are not part of the `memories`
+ * corpus), so the filesystem fallback must not pull them into recall either.
+ */
+export const RECALL_NON_MEMORY_DIRS: ReadonlySet<string> = new Set(["questions"]);
+
+/**
+ * Category directories the filesystem-fallback recall scan should read.
+ *
+ * This is `ALL_CATEGORY_DIRS` minus the non-memory queue dirs in
+ * `RECALL_NON_MEMORY_DIRS`. It is the single source of truth for the corpus the
+ * QMD-unavailable fallback (`StorageManager.collectActiveMemoryPaths()`) treats
+ * as recall memories — kept in parity with the QMD primary recall corpus.
+ */
+export const RECALL_FALLBACK_DIRS: string[] = ALL_CATEGORY_DIRS.filter(
+  (dir) => !RECALL_NON_MEMORY_DIRS.has(dir),
+);
+
 /** All category keys (singular form) — used when iterating categories and calling getCategoryDir. */
 export const ALL_CATEGORY_KEYS: string[] = [
   "fact",