@selesai/code 0.13.15 → 0.13.18

package/dist/extensions/pi-hermes-memory/src/store/fts-query.ts

@@ -2,6 +2,30 @@ const FTS5_OPERATOR_PATTERN = /\b(OR|AND|NOT|NEAR)\b/;
 const FTS5_TOKEN_PATTERN = /"([^"]*)"|(\S+)/g;
 const NATURAL_LANGUAGE_CONNECTORS = new Set(['and', 'or', 'not', 'near']);
 
+// Common English stop words that add noise to FTS5 searches. They are silently
+// dropped from natural-language queries so common filler words cannot dominate
+// the FTS5 ranking or produce misleading "AND"-style misses. The list is
+// deliberately conservative: only high-frequency function words are dropped.
+// Words that frequently carry the actual search intent (can, need, off, like,
+// get) stay, so "can the app start" still searches for "can".
+const STOP_WORDS = new Set([
+  'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+  'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should',
+  'may', 'might', 'shall', 'dare', 'ought', 'used',
+  'i', 'me', 'my', 'mine', 'we', 'our', 'ours', 'you', 'your', 'yours',
+  'he', 'him', 'his', 'she', 'her', 'hers', 'it', 'its', 'they', 'them',
+  'their', 'theirs', 'that', 'this', 'these', 'those', 'what', 'which',
+  'who', 'whom', 'whose', 'when', 'where', 'why', 'how', 'all', 'each',
+  'every', 'both', 'few', 'more', 'most', 'other', 'some', 'such', 'no',
+  'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's',
+  't', 'just', 'don', 'now', 'about', 'above', 'after', 'again', 'against',
+  'am', 'at', 'before', 'behind', 'below', 'between', 'by', 'during',
+  'from', 'further', 'into', 'over', 'per', 'through',
+  'throughout', 'till', 'to', 'under', 'until', 'up', 'upon', 'via', 'with',
+  'within', 'without', 'as', 'if', 'or', 'but', 'and', 'out', 'of',
+  'for', 'in', 'on', 'any', 'got', 'here', 'there',
+]);
+
 export function hasExplicitFts5Operator(query: string): boolean {
   return FTS5_OPERATOR_PATTERN.test(query.trim());
 }
@@ -15,6 +39,10 @@ function collectNaturalLanguageTerms(query: string): string[] {
     if (phrase === undefined && term && NATURAL_LANGUAGE_CONNECTORS.has(term.toLowerCase())) {
       continue;
     }
+    // Skip high-frequency stop words so they cannot dominate FTS5 ranking.
+    if (phrase === undefined && term && STOP_WORDS.has(term.toLowerCase())) {
+      continue;
+    }
 
     const rawValue = phrase ?? term ?? '';
     if (rawValue.length > 0) terms.push(rawValue);
@@ -67,6 +95,30 @@ function quoteTerms(terms: string[], separator: string): string {
   return terms.map((term) => `"${term.replace(/"/g, '""')}"`).join(separator);
 }
 
+/**
+ * Collect the raw terms of a query for literal (LIKE) fallback searches.
+ * Unlike normalizeFts5Query this does NOT drop stop words — a literal
+ * substring fallback is the last resort, and the original terms are what the
+ * user typed. Natural-language connectors are skipped because they never
+ * carry search intent.
+ */
+export function collectLikeTerms(query: string): string[] {
+  const terms: string[] = [];
+
+  for (const match of query.matchAll(FTS5_TOKEN_PATTERN)) {
+    const phrase = match[1];
+    const term = match[2];
+    if (phrase === undefined && term && NATURAL_LANGUAGE_CONNECTORS.has(term.toLowerCase())) {
+      continue;
+    }
+
+    const rawValue = phrase ?? term ?? '';
+    if (rawValue.length > 0) terms.push(rawValue);
+  }
+
+  return terms;
+}
+
 /**
  * Normalize a query as natural language even when it contains uppercase
  * operator words. Queries like "DO NOT USE FIND /" are passed through as raw
@@ -97,5 +149,10 @@ export function buildNaturalLanguageFallbackQuery(query: string): string | null
 export function isFts5QueryError(err: unknown): boolean {
   if (!(err instanceof Error)) return false;
   const msg = err.message.toLowerCase();
-  return msg.includes('fts5') || msg.includes('unterminated string');
+  // NOTE: "malformed" and "sql logic error" are intentionally NOT matched here.
+  // They are corruption/recovery signals used by DatabaseManager (#186); swallowing
+  // them would suppress database quarantine/rebuild. Only genuine FTS5 query/index
+  // errors are considered recoverable search-path failures.
+  return msg.includes('fts5') ||
+    msg.includes('unterminated string');
 }

package/dist/extensions/pi-hermes-memory/src/store/memory-store.ts

@@ -40,6 +40,7 @@ import { canonicalMarkdownIdentity, withMarkdownMutationLock } from "./markdown-
 
 const MAX_EXTERNAL_WRITE_RETRIES = 2;
 const RECOVERY_ACTIVE_GRACE_MS = 7 * 24 * 60 * 60 * 1000;
+const RECOVERY_SNAPSHOT_MIN_INTERVAL_MS = 60 * 60 * 1000;
 const RECOVERY_MAX_COUNT = 32;
 const RECOVERY_MAX_BYTES = 64 * 1024 * 1024;
 const RETIRED_RECOVERY_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -119,6 +120,9 @@ export class MemoryStore {
     if (target === "failure") return this.config.memoryCharLimit * 2; // Failures get more space
     return target === "user" ? this.config.userCharLimit : this.config.memoryCharLimit;
   }
+  private get capEnforced(): boolean {
+    return this.config.memoryMode !== "policy-only";
+  }
 
   private charCount(target: "memory" | "user" | "failure"): number {
     const entries = this.entriesFor(target);
@@ -190,10 +194,11 @@ export class MemoryStore {
     toolState?: string;
     correctedTo?: string;
     project?: string;
+    signal?: AbortSignal;
   }): Promise<MemoryResult> {
     const failureText = this.buildFailureMemoryText(content, options);
     return this.addWithConsolidation(
-      "failure", failureText, undefined, 1, "Failure memory saved: " + options.category, options.project,
+      "failure", failureText, options.signal, 1, "Failure memory saved: " + options.category, options.project,
     );
   }
 
@@ -244,7 +249,7 @@ export class MemoryStore {
     const encoded = this.encodeEntry(content, today, today, project);
 
     const newTotal = [...entries, encoded].join(ENTRY_DELIMITER).length;
-    if (newTotal > limit) {
+    if (this.capEnforced && newTotal > limit) {
       this.overflowSince[target] ??= Date.now();
       const strategy = this.memoryOverflowStrategy();
 
@@ -276,6 +281,7 @@ export class MemoryStore {
     const result = await this.runTargetMutation(
       target,
       (markMutation) => this._add(target, content, signal, addedMessage, project, markMutation),
+      signal,
     );
     if (
       result.success
@@ -374,7 +380,7 @@ export class MemoryStore {
   async applyMutationPlan(
     target: "memory" | "user" | "failure",
     operations: MemoryMutationOperation[],
-    options: { requireShrink?: boolean } = {},
+    options: { requireShrink?: boolean; signal?: AbortSignal } = {},
   ): Promise<MemoryResult> {
     return this.runTargetMutation(target, async (markMutation) => {
       await this.syncTargetFromDiskIfChanged(target);
@@ -440,7 +446,7 @@ export class MemoryStore {
 
       const originalTotal = originalEntries.join(ENTRY_DELIMITER).length;
       const plannedTotal = plannedEntries.join(ENTRY_DELIMITER).length;
-      if (plannedTotal > this.charLimit(target)) {
+      if (this.capEnforced && plannedTotal > this.charLimit(target)) {
         return {
           success: false,
           error: `Memory mutation plan would put memory at ${plannedTotal}/${this.charLimit(target)} chars.`,
@@ -457,13 +463,14 @@ export class MemoryStore {
       await this.saveToDisk(target);
       markMutation();
       return this.successResponse(target, `Applied ${operations.length} memory operations atomically.`);
-    });
+    }, options.signal);
   }
 
-  async replace(target: "memory" | "user" | "failure", oldText: string, newContent: string): Promise<MemoryResult> {
+  async replace(target: "memory" | "user" | "failure", oldText: string, newContent: string, signal?: AbortSignal): Promise<MemoryResult> {
     return this.runTargetMutation(
       target,
       (markMutation) => this.replaceUnlocked(target, oldText, newContent, markMutation),
+      signal,
     );
   }
 
@@ -506,7 +513,7 @@ export class MemoryStore {
 
     const newTotal = testEntries.join(ENTRY_DELIMITER).length;
 
-    if (newTotal > this.charLimit(target)) {
+    if (this.capEnforced && newTotal > this.charLimit(target)) {
       return {
         success: false,
         error: `Replacement would put memory at ${newTotal}/${this.charLimit(target)} chars. Shorten or remove other entries first.`,
@@ -520,10 +527,11 @@ export class MemoryStore {
     return this.successResponse(target, "Entry replaced.");
   }
 
-  async remove(target: "memory" | "user" | "failure", oldText: string): Promise<MemoryResult> {
+  async remove(target: "memory" | "user" | "failure", oldText: string, signal?: AbortSignal): Promise<MemoryResult> {
     return this.runTargetMutation(
       target,
       (markMutation) => this.removeUnlocked(target, oldText, markMutation),
+      signal,
     );
   }
 
@@ -839,10 +847,14 @@ export class MemoryStore {
   private async runTargetMutation(
     target: "memory" | "user" | "failure",
     mutation: (markMutation: () => void) => Promise<MemoryResult>,
+    signal?: AbortSignal,
   ): Promise<MemoryResult> {
     const storagePath = await this.resolveStoragePath(target);
     return withMarkdownMutationLock(storagePath, async () => {
       for (let attempt = 0; ; attempt++) {
+        if (signal?.aborted) {
+          return { success: false, error: "Memory mutation cancelled." };
+        }
         let mutated = false;
         try {
           const result = await mutation(() => {
@@ -914,6 +926,15 @@ export class MemoryStore {
           }
           throw error;
         }
+      } else if (await this.shouldReuseRecoverySnapshot(filePath)) {
+        try {
+          await fs.rename(tmpPath, filePath);
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+            throw new ExternalMemoryWriteConflict();
+          }
+          throw error;
+        }
       } else {
         const recoveryPath = this.recoveryPathFor(filePath);
         const publishedIdentity = await this.fileIdentity(tmpPath);
@@ -1048,6 +1069,37 @@ export class MemoryStore {
     );
   }
 
+  private async shouldReuseRecoverySnapshot(filePath: string): Promise<boolean> {
+    const directory = path.dirname(filePath);
+    const escapedName = path.basename(filePath).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    const uuidPattern = "[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}";
+    const recoveryPattern = new RegExp(`^\\.${escapedName}\\.recovery-\\d+-${uuidPattern}$`, "i");
+    try {
+      const names = await fs.readdir(directory);
+      const mtimes = await Promise.all(
+        names
+          .filter((name) => recoveryPattern.test(name))
+          .map(async (name) => {
+            try {
+              const state = await fs.lstat(path.join(directory, name));
+              return state.isFile() ? state.mtimeMs : null;
+            } catch {
+              return null;
+            }
+          }),
+      );
+      let newestMtimeMs: number | null = null;
+      for (const mtimeMs of mtimes) {
+        if (mtimeMs !== null && (newestMtimeMs === null || mtimeMs > newestMtimeMs)) {
+          newestMtimeMs = mtimeMs;
+        }
+      }
+      return newestMtimeMs !== null && Date.now() - newestMtimeMs < RECOVERY_SNAPSHOT_MIN_INTERVAL_MS;
+    } catch {
+      return false;
+    }
+  }
+
   private retiredRecoveryPathFor(filePath: string): string {
     return path.join(
       path.dirname(filePath),

package/dist/extensions/pi-hermes-memory/src/store/session-indexer.ts

@@ -25,6 +25,8 @@ export interface BulkIndexResult {
   messagesIndexed: number;
   errors: string[];
   reachedLimit?: boolean;
+  /** Session files skipped because their mtime is outside the retention window. */
+  expiredSkipped?: number;
 }
 
 interface SessionFileMetadata {
@@ -36,6 +38,25 @@ interface SessionFileMetadata {
 export interface IncrementalIndexOptions {
   projectDir?: string;
   maxFilesToIndex?: number;
+  /**
+   * Optional retention cutoff (ms epoch). JSONL session files whose mtime is
+   * strictly older than this cutoff are considered outside the retained window
+   * and are skipped entirely (not queued for indexing or counted as changed).
+   * This keeps incremental backfill aligned with session retention pruning:
+   * sessions pruned by pruneOldSessions() are never re-indexed on a later
+   * startup. Omit/0 to index every file (backwards-compatible default).
+   */
+  retentionCutoffMs?: number;
+}
+
+/**
+ * True when a session JSONL file's last-modified time falls within the
+ * retention window (mtime >= cutoff). Files older than the cutoff are treated
+ * as expired and are ineligible for backfill/indexing so that pruned sessions
+ * are not re-surfaced.
+ */
+function isWithinRetention(mtimeMs: number, retentionCutoffMs: number | undefined): boolean {
+  return !retentionCutoffMs || mtimeMs >= retentionCutoffMs;
 }
 
 export function truncateMessageContent(
@@ -339,21 +360,39 @@ function indexSessionFile(dbManager: DatabaseManager, file: string, result: Bulk
 
 /**
  * Index all sessions from disk.
+ * With retentionCutoffMs > 0, files whose mtime falls outside the window are
+ * skipped (counted in expiredSkipped) so a manual reindex honors the same
+ * retention policy the auto pruning enforces, instead of re-adding expired
+ * sessions that pruning just deleted.
  *
  * @param dbManager — Database manager instance
  * @param sessionsDir — Path to ~/.selesai/agent/sessions/
  * @param projectDir — Optional: specific project directory to index
+ * @param retentionCutoffMs — Optional: epoch ms; files modified before it are skipped
  * @returns Bulk index result
  */
 export function indexAllSessions(
   dbManager: DatabaseManager,
   sessionsDir: string,
-  projectDir?: string
+  projectDir?: string,
+  retentionCutoffMs = 0,
 ): BulkIndexResult {
   const files = getSessionFiles(sessionsDir, projectDir);
   const result = emptyBulkIndexResult();
+  let expiredSkipped = 0;
 
   for (const file of files) {
+    if (retentionCutoffMs > 0) {
+      try {
+        if (!isWithinRetention(getSessionFileMetadata(file).mtimeMs, retentionCutoffMs)) {
+          expiredSkipped++;
+          continue;
+        }
+      } catch {
+        // Unreadable metadata: let indexSessionFile report the real error.
+      }
+    }
+
     try {
       indexSessionFile(dbManager, file, result);
     } catch (err) {
@@ -361,6 +400,10 @@ export function indexAllSessions(
     }
   }
 
+  if (expiredSkipped > 0) {
+    result.expiredSkipped = expiredSkipped;
+  }
+
   return result;
 }
 
@@ -390,6 +433,12 @@ export function indexChangedSessions(
   for (const file of files) {
     try {
       const metadata = getSessionFileMetadata(file);
+      if (!isWithinRetention(metadata.mtimeMs, options.retentionCutoffMs)) {
+        // Outside the retained window (e.g. pruned by pruneOldSessions):
+        // never re-queue it for indexing, so a pruned session does not come
+        // back on the next startup backfill.
+        continue;
+      }
       if (storedSessionFileMatches(dbManager, metadata)) {
         result.sessionsSkipped++;
         continue;
@@ -443,25 +492,58 @@ function isRecentBackfillTimestamp(value: string | null, nowMs: number): boolean
  * The check stays cheap: it compares file counts and stored file size/mtime
  * metadata. Full JSONL parsing is left to the scheduled incremental backfill.
  */
-export function needsBackfill(dbManager: DatabaseManager, sessionsDir: string, now = new Date()): boolean {
+export function needsBackfill(
+  dbManager: DatabaseManager,
+  sessionsDir: string,
+  now = new Date(),
+  retentionCutoffMs = 0,
+): boolean {
   const db = dbManager.getDb();
   const files = getSessionFiles(sessionsDir);
   const indexed = db.prepare('SELECT COUNT(*) as count FROM sessions').get() as { count: number };
 
-  if (files.length > indexed.count) {
-    return true;
+  if (retentionCutoffMs <= 0) {
+    // Retention disabled: keep the historical cheap path — a plain file-count
+    // vs row-count comparison decides before any per-file stat work, so
+    // large session directories stay fast on startup.
+    if (files.length > indexed.count) {
+      return true;
+    }
+
+    for (const file of files) {
+      try {
+        const metadata = getSessionFileMetadata(file);
+        if (storedSessionFileMatches(dbManager, metadata)) continue;
+        return true;
+      } catch {
+        // An unreadable or malformed session file still needs indexing.
+        return true;
+      }
+    }
+
+    return !isRecentBackfillTimestamp(getLastBackfillTimestamp(dbManager), now.getTime());
   }
 
+  // Retention enabled: one metadata pass decides everything — a retained file
+  // with stale stored metadata demands a backfill, and when no file is inside
+  // the window there is no work at all. Returning here (instead of falling
+  // through to the periodic timestamp check) keeps an all-expired store from
+  // scheduling an empty backfill on every startup before a timestamp is ever
+  // written.
+  let hasRetainedFile = false;
   for (const file of files) {
     try {
       const metadata = getSessionFileMetadata(file);
-      if (storedSessionFileMatches(dbManager, metadata)) continue;
-      return true;
+      if (!isWithinRetention(metadata.mtimeMs, retentionCutoffMs)) continue;
+      hasRetainedFile = true;
+      if (!storedSessionFileMatches(dbManager, metadata)) return true;
     } catch {
       return true;
     }
   }
-
+  if (!hasRetainedFile) {
+    return false;
+  }
   return !isRecentBackfillTimestamp(getLastBackfillTimestamp(dbManager), now.getTime());
 }
 
@@ -509,3 +591,83 @@ export function getSessionStats(dbManager: DatabaseManager): {
     projects,
   };
 }
+
+/**
+ * Compute the retention cutoff (ms epoch) from a retention window in days.
+ * Returns 0 (no cutoff) when retention is undefined/zero/disabled.
+ */
+export function retentionCutoffMs(retentionDays: number | undefined): number {
+  if (!retentionDays || retentionDays <= 0) return 0;
+  return Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+}
+
+/**
+ * Delete sessions outside the retention window, along with their messages,
+ * to bound the growth of the session index database (see #183).
+ *
+ * A session is eligible for pruning when its session file's last-mod time is
+ * older than the window (falling back to started_at when no file metadata
+ * exists). This deliberately mirrors the backfill eligibility check in
+ * `needsBackfill`/`indexChangedSessions` (also keyed to file mtime), so a
+ * pruned session's on-disk JSONL file is never re-indexed by a later startup
+ * and never re-triggers a backfill. Retention and backfill therefore agree on
+ * the same eligible file set.
+ *
+ * `messages` and `session_files` reference `sessions`, but only `session_files`
+ * is declared `ON DELETE CASCADE`. Orphaned `messages` rows are deleted
+ * explicitly first so a `PRAGMA foreign_keys`-enabled delete never trips a
+ * FK constraint, and so an accurate `messagesRemoved` count is reported.
+ *
+ * Returns the number of sessions and messages removed.
+ */
+export function pruneOldSessions(
+  dbManager: DatabaseManager,
+  retentionDays: number,
+): { sessionsRemoved: number; messagesRemoved: number } {
+  const db = dbManager.getDb();
+  const cutoffMs = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  const cutoffIso = new Date(cutoffMs).toISOString();
+
+  // A session is pruned when it falls outside the retention window. We use the
+  // same signal the backfill eligibility uses -- the session file's last-mod
+  // time -- so that a pruned session's on-disk JSONL file is not re-indexed on
+  // a later startup. Sessions without file metadata (e.g. indexed before files
+  // were tracked) fall back to started_at. Both must be strictly older than
+  // the cutoff.
+  const eligibleSessionIds = db.prepare(`
+    SELECT s.id
+    FROM sessions s
+    LEFT JOIN session_files sf ON sf.session_id = s.id
+    WHERE
+      (sf.path IS NOT NULL AND sf.mtime_ms < ?)
+      OR (sf.path IS NULL AND s.started_at < ?)
+  `).all(cutoffMs, cutoffIso) as Array<{ id: string }>;
+
+  if (eligibleSessionIds.length === 0) {
+    return { sessionsRemoved: 0, messagesRemoved: 0 };
+  }
+
+  // Count messages upfront because SQLite does not surface cascade/orphan
+  // counts from DELETE ... RETURNING across tables.
+  const messageCount = db.prepare(
+    `SELECT COUNT(*) as cnt FROM messages WHERE session_id IN (${eligibleSessionIds.map(() => '?').join(',')})`,
+  ).get(...eligibleSessionIds.map((r) => r.id)) as { cnt: number };
+
+  const prune = () => {
+    const delMessages = db.prepare(
+      `DELETE FROM messages WHERE session_id IN (${eligibleSessionIds.map(() => '?').join(',')})`,
+    ).run(...eligibleSessionIds.map((r) => r.id));
+    const delSessions = db.prepare(
+      `DELETE FROM sessions WHERE id IN (${eligibleSessionIds.map(() => '?').join(',')})`,
+    ).run(...eligibleSessionIds.map((r) => r.id));
+    return { messagesRemoved: delMessages.changes, sessionsRemoved: delSessions.changes };
+  };
+
+  const result = db.transaction ? db.transaction(prune)() : prune();
+  // Fall back to the pre-counted values if the transaction changed nothing
+  // unexpectedly (defensive; DELETE should always affect the counted rows).
+  return {
+    sessionsRemoved: result.sessionsRemoved || eligibleSessionIds.length,
+    messagesRemoved: result.messagesRemoved || messageCount.cnt,
+  };
+}

package/dist/extensions/pi-hermes-memory/src/store/session-search.ts

@@ -2,6 +2,7 @@ import { DatabaseManager } from './db.js';
 import {
   buildFallbackFts5Query,
   buildNaturalLanguageFallbackQuery,
+  collectLikeTerms,
   hasExplicitFts5Operator,
   isFts5QueryError,
   normalizeFts5Query,
@@ -38,30 +39,10 @@ type SearchMatch =
   | { type: 'fts'; query: string }
   | { type: 'like'; terms: string[] };
 
-const QUERY_TOKEN_PATTERN = /"([^"]*)"|(\S+)/g;
-const NATURAL_LANGUAGE_CONNECTORS = new Set(['and', 'or', 'not', 'near']);
-
 function escapeLikePattern(text: string): string {
   return text.replace(/[\\%_]/g, '\\$&');
 }
 
-function collectLikeTerms(query: string): string[] {
-  const terms: string[] = [];
-
-  for (const match of query.matchAll(QUERY_TOKEN_PATTERN)) {
-    const phrase = match[1];
-    const term = match[2];
-    if (phrase === undefined && term && NATURAL_LANGUAGE_CONNECTORS.has(term.toLowerCase())) {
-      continue;
-    }
-
-    const rawValue = phrase ?? term ?? '';
-    if (rawValue.length > 0) terms.push(rawValue);
-  }
-
-  return terms;
-}
-
 function mapRows(rows: Array<{
   session_id: string;
   project: string;
@@ -178,7 +159,12 @@ export function searchSessions(
 
   const normalizedQuery = normalizeFts5Query(query);
   if (normalizedQuery.length === 0) {
-    return [];
+    // Every term was a stop word or connector: nothing is left for FTS5 to
+    // match. Fall through to the scoped LIKE fallback instead of a hard [] so
+    // the query degrades to a literal substring search (the same fallback a
+    // normal query that matches nothing ends with).
+    const likeTerms = collectLikeTerms(query);
+    return executeSearch({ type: 'like', terms: likeTerms });
   }
 
   const exactResults = executeSearch({ type: 'fts', query: normalizedQuery });