openclaw-hybrid-memory 2026.6.21 → 2026.6.40

package/README.md

@@ -299,10 +299,20 @@ openclaw hybrid-mem analyze-maintenance-logs --since 7d --format json --out /tmp
 openclaw hybrid-mem analyze-maintenance-logs --since 24h --auto-fix --glitchtip --strict
 ```
 
-Rules are data-driven in [`services/maintenance-rules.json`](services/maintenance-rules.json), so operators can inspect or extend classifications without changing analyzer code. Findings are persisted in `maintenance-findings.db` table `maintenance_finding`, enabling week-over-week trend output for `--since 7d` / `--trend` runs.
+Rules are data-driven in [`services/maintenance-rules.json`](services/maintenance-rules.json), so operators can inspect or extend classifications without changing analyzer code. Findings are persisted in `maintenance-findings.db` table `maintenance_finding`, enabling week-over-week trend output for `--since 7d` / `--trend` runs. The digest collapses repeated fingerprints into `New` vs `Still failing`, suppresses stale historical fingerprints outside the current window from the primary findings, and avoids re-reporting already GlitchTip-reported fingerprints on every analyzer run. Known benign compatibility notices (for example missing `registerContextEngine` on older SDKs and Codex project-local config warnings) are collected in `noiseWarnings[]` and excluded from primary failure counts by default (#1833).
 
 The installer registers `hybrid-mem:maintenance-log-analyzer` to run after the nightly chain and announce the rendered digest to the operator.
 
+### Unified maintenance inventory
+
+Use the maintenance inventory report to see host crontab jobs and gateway `~/.openclaw/cron/jobs.json` jobs in one place, including scheduler ownership, timezone, guard/log paths, last-run state, and collision groups:
+
+```bash
+openclaw hybrid-mem maintenance inventory
+openclaw hybrid-mem maintenance inventory --format markdown
+openclaw hybrid-mem maintenance inventory --json
+```
+
 ### Auto-fix whitelist (#1199)
 
 `--auto-fix` applies **only** safe, idempotent actions implemented in [`services/maintenance-auto-fix.ts`](services/maintenance-auto-fix.ts):

package/backends/credentials-db.ts

@@ -5,7 +5,7 @@
  */
 
 import { createCipheriv, createDecipheriv, createHash, randomBytes, scryptSync } from "node:crypto";
-import { mkdirSync } from "node:fs";
+import { existsSync, mkdirSync, renameSync, unlinkSync } from "node:fs";
 import { dirname } from "node:path";
 import { DatabaseSync } from "node:sqlite";
 import type { CredentialType } from "../config.js";
@@ -30,6 +30,9 @@ const CRED_KDF_PLAINTEXT = 0; // no encryption (user secures by other means)
 /** Log once per vault path: legacy v1 KDF is weak; opening triggers migration to scrypt when possible. */
 const _v1KdfWarnedPaths = new Set<string>();
 
+/** Log once per vault path: key is configured but vault is plaintext (kdf_version=0). */
+const _plaintextKeyIgnoredWarnedPaths = new Set<string>();
+
 /** v1 only: legacy SHA-256 KDF (weak). Existing vaults cannot be decrypted with another KDF. */
 function deriveKeyV1Legacy(password: string): Buffer {
   // codeql[js/insufficient-password-hash]
@@ -170,11 +173,12 @@ export class CredentialsDB extends BaseSqliteStore {
       this.salt = Buffer.alloc(0);
       this.key = Buffer.alloc(0);
       this.password = null;
-      // Optionally warn that key is being ignored
-      if (encryptionKey.length >= 16) {
+      // Optionally warn that key is being ignored (once per process per path)
+      if (encryptionKey.length >= 16 && !_plaintextKeyIgnoredWarnedPaths.has(dbPath)) {
+        _plaintextKeyIgnoredWarnedPaths.add(dbPath);
         pluginLogger.warn(
           "Credentials vault is in plaintext mode (kdf_version=0). The configured encryption key is being ignored. " +
-            "To encrypt the existing vault at rest, run: openclaw hybrid-mem credentials encrypt-vault --yes",
+            "To encrypt the existing vault at rest, run: openclaw hybrid-mem credentials encrypt-vault --backup --verify --yes",
         );
       }
       return;
@@ -227,10 +231,13 @@ export class CredentialsDB extends BaseSqliteStore {
     configuredKeyPresent: boolean;
     keyIgnored: boolean;
     migrationRequired: boolean;
+    entryCount: number;
   } {
     const encryptedAtRest = this.kdfVersion !== CRED_KDF_PLAINTEXT;
     const keyIgnored = this.kdfVersion === CRED_KDF_PLAINTEXT && this.configuredKeyPresent;
     const migrationRequired = keyIgnored;
+    const entryCount = (this.liveDb.prepare("SELECT COUNT(*) as count FROM credentials").get() as { count: number })
+      .count;
     return {
       dbPath: this.dbPath,
       kdfVersion: this.kdfVersion,
@@ -238,6 +245,7 @@ export class CredentialsDB extends BaseSqliteStore {
       configuredKeyPresent: this.configuredKeyPresent,
       keyIgnored,
       migrationRequired,
+      entryCount,
     };
   }
 
@@ -297,6 +305,224 @@ export class CredentialsDB extends BaseSqliteStore {
     return { migrated: migratedCount, kdfVersion: this.kdfVersion };
   }
 
+  /**
+   * Safe vault encryption with optional backup and post-encryption verification.
+   *
+   * Steps:
+   * 1. If `backupPath` is set, create a consistent copy of the current plaintext DB via VACUUM INTO.
+   * 2. Encrypt the vault in-place via `enableEncryptionAtRest`.
+   * 3. If `verify` is true, attempt to decrypt every entry to confirm data integrity.
+   *    On failure, throws with the backup path so the operator can restore manually.
+   *
+   * Safety: does NOT auto-restore on verify failure — the in-place operation may have
+   * already committed. The caller should inspect `backupPath` and restore if needed.
+   */
+  encryptVaultSafe(
+    encryptionKey: string,
+    opts: { backupPath?: string; verify?: boolean } = {},
+  ): { migrated: number; kdfVersion: number; backupPath?: string; verified?: boolean } {
+    const { backupPath, verify } = opts;
+
+    const keyLooksEncrypted = encryptionKey.length >= 16;
+    if (!keyLooksEncrypted) {
+      throw new Error(
+        "Encryption key is missing or too short. Set credentials.encryptionKey (16+ chars) or OPENCLAW_CRED_KEY before encrypting the vault.",
+      );
+    }
+    if (this.kdfVersion !== CRED_KDF_PLAINTEXT) {
+      throw new Error(`Credentials vault is already encrypted (kdf_version=${this.kdfVersion}).`);
+    }
+
+    const newSalt = randomBytes(32);
+    const newKdfVersion = CRED_KDF_VERSION;
+    const newKey = deriveKey(encryptionKey, newSalt, newKdfVersion);
+
+    let migratedCount = 0;
+
+    // Perform backup and encryption in a single IMMEDIATE transaction to prevent
+    // writes between backup snapshot and encryption. This fixes the race condition
+    // where VACUUM INTO would snapshot at T1, then new rows could be written at T2,
+    // then encryption would start at T3, resulting in encrypted rows missing from backup.
+
+    // Preserve existing backup during retry by renaming it temporarily
+    let oldBackupPath: string | undefined;
+    if (backupPath) {
+      mkdirSync(dirname(backupPath), { recursive: true });
+      try {
+        // Check if old backup exists and rename it temporarily
+        if (existsSync(backupPath)) {
+          oldBackupPath = `${backupPath}.old.${Date.now()}`;
+          renameSync(backupPath, oldBackupPath);
+        }
+      } catch {
+        // If rename fails, proceed anyway (file might not exist)
+      }
+    }
+
+    const migrateWithBackup = createTransaction(
+      this.liveDb,
+      () => {
+        if (backupPath) {
+          // Create backup database and copy all data within this transaction
+          const backupDb = new DatabaseSync(backupPath);
+          try {
+            // Copy schema
+            backupDb.exec(`
+              CREATE TABLE vault_meta (
+                key TEXT PRIMARY KEY,
+                value BLOB NOT NULL
+              )
+            `);
+            backupDb.exec(`
+              CREATE TABLE credentials (
+                service TEXT NOT NULL,
+                type TEXT NOT NULL DEFAULT 'other',
+                value BLOB NOT NULL,
+                url TEXT,
+                notes TEXT,
+                created INTEGER NOT NULL,
+                updated INTEGER NOT NULL,
+                expires INTEGER,
+                PRIMARY KEY (service, type)
+              )
+            `);
+            backupDb.exec(`
+              CREATE INDEX idx_credentials_service ON credentials(service)
+            `);
+
+            // Copy vault_meta
+            const metaRows = this.liveDb.prepare("SELECT key, value FROM vault_meta").all() as Array<{
+              key: string;
+              value: Uint8Array | Buffer;
+            }>;
+            const insertMeta = backupDb.prepare("INSERT INTO vault_meta (key, value) VALUES (?, ?)");
+            for (const row of metaRows) {
+              insertMeta.run(row.key, row.value);
+            }
+
+            // Copy credentials
+            const credRows = this.liveDb.prepare("SELECT * FROM credentials").all() as Array<Record<string, unknown>>;
+            migratedCount = credRows.length;
+            const insertCred = backupDb.prepare(
+              "INSERT INTO credentials (service, type, value, url, notes, created, updated, expires) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+            );
+            for (const row of credRows) {
+              insertCred.run(
+                row.service,
+                row.type,
+                row.value,
+                row.url,
+                row.notes,
+                row.created,
+                row.updated,
+                row.expires,
+              );
+            }
+
+            backupDb.close();
+            tryRestrictSqliteDbFileMode(backupPath);
+          } catch (err) {
+            backupDb.close();
+            throw err;
+          }
+        } else {
+          // No backup, just count rows
+          const rows = this.liveDb.prepare("SELECT service, type FROM credentials").all() as Array<{
+            service: string;
+            type: string;
+          }>;
+          migratedCount = rows.length;
+        }
+
+        // Now encrypt within the same transaction
+        const rows = this.liveDb.prepare("SELECT service, type, value FROM credentials").all() as Array<{
+          service: string;
+          type: string;
+          value: Uint8Array | Buffer;
+        }>;
+        const updateStmt = this.liveDb.prepare("UPDATE credentials SET value = ? WHERE service = ? AND type = ?");
+        for (const r of rows) {
+          const plaintext = toBuffer(r.value).toString("utf8");
+          const encrypted = encryptValue(plaintext, newKey);
+          updateStmt.run(encrypted as unknown as Uint8Array, r.service, r.type);
+        }
+        this.liveDb
+          .prepare("INSERT OR REPLACE INTO vault_meta (key, value) VALUES ('kdf_version', ?)")
+          .run(Buffer.from([newKdfVersion]));
+        this.liveDb.prepare("INSERT OR REPLACE INTO vault_meta (key, value) VALUES ('salt', ?)").run(newSalt);
+      },
+      "IMMEDIATE",
+    );
+
+    let migrationSucceeded = false;
+    try {
+      migrateWithBackup();
+      migrationSucceeded = true;
+
+      this.kdfVersion = newKdfVersion;
+      this.salt = newSalt;
+      this.key = newKey;
+      this.password = null;
+      this.storesEncryptedValues = true;
+
+      const result = { migrated: migratedCount, kdfVersion: this.kdfVersion };
+
+      let verified: boolean | undefined;
+      if (verify) {
+        try {
+          const entries = this.listAll();
+          // Verify that all encrypted entries were successfully decrypted
+          if (entries.length !== result.migrated) {
+            throw new Error(
+              `Verification failed: only ${entries.length} of ${result.migrated} entries could be decrypted`,
+            );
+          }
+          for (const e of entries) {
+            const verifiedEntry = this.get(e.service, e.type as CredentialType);
+            if (!verifiedEntry) {
+              throw new Error(`Verification failed: missing credential ${e.service}/${e.type}`);
+            }
+          }
+          verified = true;
+        } catch (err) {
+          const backupNote = backupPath ? ` Plaintext backup is at: ${backupPath}` : "";
+          throw new Error(
+            `Vault encrypted but post-encryption verification failed: ${err instanceof Error ? err.message : String(err)}.${backupNote}`,
+          );
+        }
+      }
+
+      // Success: delete the old backup if it exists
+      if (oldBackupPath) {
+        try {
+          unlinkSync(oldBackupPath);
+        } catch {
+          // Best effort cleanup; don't fail on cleanup errors
+        }
+      }
+
+      return {
+        ...result,
+        ...(backupPath !== undefined ? { backupPath } : {}),
+        ...(verified !== undefined ? { verified } : {}),
+      };
+    } catch (err) {
+      // After successful migration, keep the new backup (at backupPath) that matches the encrypted state.
+      // Only restore the old backup if migration itself failed.
+      if (oldBackupPath && backupPath && !migrationSucceeded) {
+        try {
+          if (existsSync(backupPath)) {
+            unlinkSync(backupPath);
+          }
+          renameSync(oldBackupPath, backupPath);
+        } catch {
+          // If restore fails, leave both files (old backup is preserved with .old suffix)
+        }
+      }
+      throw err;
+    }
+  }
+
   /**
    * Insert or update a credential. On conflict (service, type), `updated` and value fields refresh;
    * `created` is preserved from the original row — intentional for "same key, rotated secret" flows (#894).

package/backends/vector-db/runtime-locks.ts

@@ -1,3 +1,9 @@
+/**
+ * Module-level cooperative locks for LanceDB readers, optimize exclusivity, and re-index
+ * exclusion. Refcount maps use `.get(path) ?? 0` intentionally: updates run in synchronous
+ * JS blocks (no await between read and write), which is safe on Node's single-threaded model.
+ * Async callers must re-check immediately before mutating LanceDB (#1841).
+ */
 export const activeReadersByPath = new Map<string, number>();
 export const optimizeExclusiveLockByPath = new Map<string, boolean>();
 export const reindexExclusiveLockByPath = new Map<string, number>();
@@ -38,3 +44,24 @@ export function decrementReindexLockCount(dbPath: string): void {
   }
   reindexExclusiveLockByPath.set(dbPath, next);
 }
+
+/** Returns true when a re-index operation currently holds the cooperative re-index lock. */
+export function isReindexLockHeld(dbPath: string): boolean {
+  return getReindexLockCount(dbPath) > 0;
+}
+
+/**
+ * Wait until no re-index lock is held. Re-validates synchronously after each drain so a lock
+ * acquired during an await does not leave callers proceeding into LanceDB mutations (#1841).
+ */
+export async function waitForReindexLockClear(dbPath: string, maxWaitMs = 30_000): Promise<void> {
+  const started = Date.now();
+  while (isReindexLockHeld(dbPath)) {
+    if (Date.now() - started > maxWaitMs) {
+      throw new Error(
+        `VectorDB operation blocked by active re-index lock for >${maxWaitMs}ms. Retry after re-index completes.`,
+      );
+    }
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+}

package/backends/vector-db/vector-db-class.ts

@@ -50,9 +50,11 @@ import {
   incrementActiveReaderCount,
   incrementReindexLockCount,
   isOptimizeLocked,
+  isReindexLockHeld,
   optimizeExclusiveLockByPath,
   reindexExclusiveLockByPath,
   setOptimizeLock,
+  waitForReindexLockClear,
 } from "./runtime-locks.js";
 
 export class VectorDB {
@@ -228,15 +230,7 @@ export class VectorDB {
   }
 
   private async waitForReindexLockRelease(maxWaitMs = 30_000): Promise<void> {
-    const started = Date.now();
-    while (getReindexLockCount(this.dbPath) > 0) {
-      if (Date.now() - started > maxWaitMs) {
-        throw new Error(
-          `VectorDB operation blocked by active re-index lock for >${maxWaitMs}ms. Retry after re-index completes.`,
-        );
-      }
-      await new Promise((resolve) => setTimeout(resolve, 50));
-    }
+    await waitForReindexLockClear(this.dbPath, maxWaitMs);
   }
 
   async runWithReindexLock<T>(fn: () => Promise<T>): Promise<T> {
@@ -1342,6 +1336,11 @@ export class VectorDB {
     return /retryable commit conflict/i.test(msg);
   }
 
+  /** TOCTOU guard after waitForReindexLockRelease; distinct from max-wait timeout errors. */
+  private isRetryableReindexLockBlockError(err: unknown): boolean {
+    return err instanceof Error && err.message === "VectorDB write blocked by active re-index lock";
+  }
+
   private async sleep(ms: number): Promise<void> {
     await new Promise((resolve) => setTimeout(resolve, ms));
   }
@@ -1359,13 +1358,21 @@ export class VectorDB {
     let lastErr: unknown;
     for (let attempt = 1; attempt <= maxAttempts; attempt++) {
       try {
+        await this.waitForReindexLockRelease();
+        if (isReindexLockHeld(this.dbPath)) {
+          throw new Error("VectorDB write blocked by active re-index lock");
+        }
         return await fn();
       } catch (err) {
         lastErr = err;
-        if (!this.isRetryableCommitConflictError(err) || attempt >= maxAttempts) throw err;
+        const isCommitConflict = this.isRetryableCommitConflictError(err);
+        const isReindexBlock = this.isRetryableReindexLockBlockError(err);
+        if (!isCommitConflict && !isReindexBlock) throw err;
+        if (attempt >= maxAttempts) throw err;
         const delayMs = this.getWriteConflictRetryDelayMs(attempt);
+        const reason = isReindexBlock ? "re-index lock block" : "retryable commit conflict";
         this.logWarn(
-          `memory-hybrid: ${operation} hit retryable commit conflict (attempt ${attempt}/${maxAttempts}) — retrying in ${delayMs}ms`,
+          `memory-hybrid: ${operation} hit ${reason} (attempt ${attempt}/${maxAttempts}) — retrying in ${delayMs}ms`,
         );
         await this.sleep(delayMs);
       }

package/backends/wal.ts

@@ -132,39 +132,46 @@ export class WriteAheadLog {
     try {
       // "a+" read+append so fdatasync works on more filesystems than read-only or append-only edge cases (issue #854).
       fh = await open(this.walPath, "a+");
-      await fh.datasync();
-    } catch (err) {
-      const code = (err as NodeJS.ErrnoException).code;
-      if (code === "EPERM" || code === "EINVAL") {
-        // CRITICAL FIX (#7): Some filesystems (e.g. NTFS via WSL2) reject fdatasync/fsync.
-        // Instead of silently degrading durability forever, try fsync() as a fallback.
-        // If both fail, throw an error so operators know their WAL has no crash safety.
+      try {
+        await fh.datasync();
+        return;
+      } catch (datasyncErr) {
+        const code = (datasyncErr as NodeJS.ErrnoException).code;
+        if (code !== "EPERM" && code !== "EINVAL") throw datasyncErr;
+        // Intentional fail-fast when neither datasync nor fsync can persist WAL writes (#7, #1846).
+        // Some filesystems (e.g. NTFS via WSL2) reject fdatasync; reopen and try fsync() as fallback.
+        await fh.close().catch(() => {});
+        fh = undefined;
         try {
-          if (!fh) {
-            throw new Error(`fallback fsync could not open WAL file after datasync() failed with ${code}`);
+          fh = await open(this.walPath, "a+");
+        } catch (openErr) {
+          throw openErr;
+        }
+        try {
+          await fh.sync();
+          if (!this.fsyncWarnEmitted) {
+            pluginLogger.warn(
+              `[WAL] datasync() unsupported (${code}), using fsync() fallback — durability available but may be slower`,
+            );
+            this.fsyncWarnEmitted = true;
           }
-          await fh.sync(); // Try fsync() instead of datasync()
-        } catch (fsyncErr) {
+          return;
+        } catch (syncErr) {
           if (!this.fsyncWarnEmitted) {
             pluginLogger.error(
               `[WAL] CRITICAL: Both datasync() and fsync() failed (${code}) — WAL durability unavailable on this filesystem. Data loss may occur on crash. Consider moving the database to a POSIX-compliant filesystem.`,
             );
             this.fsyncWarnEmitted = true;
           }
-          // Throw so the write operation fails fast rather than silently losing crash safety.
-          throw new Error(
-            `WAL fsync unavailable: ${code}: ${fsyncErr instanceof Error ? fsyncErr.message : String(fsyncErr)}`,
-          );
-        }
-        // fsync() succeeded as fallback
-        if (!this.fsyncWarnEmitted) {
-          pluginLogger.warn(
-            `[WAL] datasync() unsupported (${code}), using fsync() fallback — durability available but may be slower`,
-          );
-          this.fsyncWarnEmitted = true;
+          const causes = [datasyncErr, syncErr].filter((e): e is Error => e instanceof Error);
+          const err = new Error(`WAL fsync unavailable (${code}): datasync and fsync fallback both failed`, {
+            cause: syncErr instanceof Error ? syncErr : datasyncErr,
+          });
+          if (causes.length > 1) {
+            (err as Error & { causes: Error[] }).causes = causes;
+          }
+          throw err;
         }
-      } else {
-        throw err;
       }
     } finally {
       await fh?.close();
@@ -189,7 +196,7 @@ export class WriteAheadLog {
         operation: "wal-write",
         subsystem: "wal",
       });
-      throw new Error(`WAL write failed: ${err}`);
+      throw new Error(`WAL write failed: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
     } finally {
       // biome-ignore lint/style/noNonNullAssertion: Synchronous
       releaseLock!();
@@ -284,7 +291,7 @@ export class WriteAheadLog {
         operation: "wal-remove",
         subsystem: "wal",
       });
-      throw new Error(`WAL remove failed: ${err}`);
+      throw new Error(`WAL remove failed: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
     } finally {
       // biome-ignore lint/style/noNonNullAssertion: Synchronous
       releaseLock!();
@@ -300,7 +307,7 @@ export class WriteAheadLog {
         operation: "wal-clear",
         subsystem: "wal",
       });
-      throw new Error(`WAL clear failed: ${err}`);
+      throw new Error(`WAL clear failed: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
     }
   }