clawmem 0.7.2 → 0.8.1

package/src/store.ts

@@ -496,16 +496,35 @@ function initializeDatabase(db: Database): void {
       injected_paths TEXT NOT NULL DEFAULT '[]',
       estimated_tokens INTEGER NOT NULL DEFAULT 0,
       was_referenced INTEGER NOT NULL DEFAULT 0,
-      turn_index INTEGER NOT NULL DEFAULT 0
+      turn_index INTEGER NOT NULL DEFAULT 0,
+      query_text TEXT
     )
   `);
   db.exec(`CREATE INDEX IF NOT EXISTS idx_context_usage_session ON context_usage(session_id)`);
 
   // Migration: add turn_index to existing context_usage
-  const cuCols = db.prepare("PRAGMA table_info(context_usage)").all() as { name: string }[];
+  let cuCols = db.prepare("PRAGMA table_info(context_usage)").all() as { name: string }[];
   if (!cuCols.some(c => c.name === "turn_index")) {
     try { db.exec(`ALTER TABLE context_usage ADD COLUMN turn_index INTEGER NOT NULL DEFAULT 0`); } catch { /* exists */ }
+    cuCols = db.prepare("PRAGMA table_info(context_usage)").all() as { name: string }[];
+  }
+
+  // v0.8.1 Ext 6b: add nullable query_text column to existing context_usage
+  // so multi-turn lookback can persist the raw prompt alongside turn_index.
+  // The column is nullable and defaults to NULL — pre-migration rows are
+  // treated as "no prior query" by buildMultiTurnSurfacingQuery, preserving
+  // the current-prompt-only fallback for any session that predates v0.8.1.
+  if (!cuCols.some(c => c.name === "query_text")) {
+    try { db.exec(`ALTER TABLE context_usage ADD COLUMN query_text TEXT`); } catch { /* exists */ }
   }
+  // Cache the column presence for insertUsageFn so it can build the INSERT
+  // statement without running PRAGMA table_info on every write path.
+  contextUsageHasQueryTextCache.set(
+    db,
+    db.prepare("PRAGMA table_info(context_usage)")
+      .all()
+      .some((c) => (c as { name: string }).name === "query_text"),
+  );
 
   // Hook prompt dedupe: suppress duplicate/heartbeat prompts to reduce GPU churn.
   db.exec(`
@@ -854,12 +873,53 @@ function initializeDatabase(db: Database): void {
   if (!mrColNames.has("contradict_confidence")) {
     try { db.exec(`ALTER TABLE memory_relations ADD COLUMN contradict_confidence REAL`); } catch { /* column exists */ }
   }
+
+  // v0.8.0 Ext 5: Heavy maintenance lane journal. Every scheduled attempt
+  // writes one row — including skips — so operators can reconstruct why a
+  // lane did or did not run on any tick.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS maintenance_runs (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      lane TEXT NOT NULL,
+      phase TEXT NOT NULL,
+      status TEXT NOT NULL,
+      reason TEXT,
+      selected_count INTEGER NOT NULL DEFAULT 0,
+      processed_count INTEGER NOT NULL DEFAULT 0,
+      created_count INTEGER NOT NULL DEFAULT 0,
+      updated_count INTEGER NOT NULL DEFAULT 0,
+      rejected_count INTEGER NOT NULL DEFAULT 0,
+      null_call_count INTEGER NOT NULL DEFAULT 0,
+      started_at TEXT NOT NULL DEFAULT (datetime('now')),
+      finished_at TEXT,
+      metrics_json TEXT
+    )
+  `);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_maintenance_runs_lane_started ON maintenance_runs(lane, started_at DESC)`);
+
+  // v0.8.0 Ext 5: DB-backed worker lease table for multi-process exclusivity
+  // on the heavy lane. Lease holders fence via random token; expired leases
+  // are reclaimed via atomic upsert inside a transaction.
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS worker_leases (
+      worker_name TEXT PRIMARY KEY,
+      lease_token TEXT NOT NULL,
+      acquired_at TEXT NOT NULL,
+      expires_at TEXT NOT NULL
+    )
+  `);
 }
 
 
 // Per-database dimension cache (WeakMap keyed by db object — no collisions for in-memory DBs)
 const vecTableDimsCache = new WeakMap<Database, number>();
 
+// v0.8.1 Ext 6b: per-database cache for the query_text column presence on
+// context_usage. Set once at migration time so insertUsageFn can pick the
+// correct INSERT shape without running PRAGMA on every write. Falls back
+// to `false` (safe — equivalent to pre-migration behavior) when absent.
+const contextUsageHasQueryTextCache = new WeakMap<Database, boolean>();
+
 function ensureVecTableInternal(db: Database, dimensions: number): void {
   if (vecTableDimsCache.get(db) === dimensions) return;
 
@@ -1687,6 +1747,13 @@ export type UsageRecord = {
   estimatedTokens: number;
   wasReferenced: number;
   turnIndex?: number;
+  /**
+   * v0.8.1 Ext 6b: raw user prompt for this turn. Written when the caller
+   * wants the row to be usable for multi-turn lookback retrieval. Persisted
+   * via `insertUsageFn` only when the `query_text` column is present on
+   * `context_usage` (pre-migration stores degrade to "no prior query").
+   */
+  queryText?: string;
 };
 
 export type UsageRow = {
@@ -3904,10 +3971,33 @@ function getRecentSessionsFn(db: Database, limit: number): SessionRecord[] {
 // =============================================================================
 
 function insertUsageFn(db: Database, usage: UsageRecord): number {
-  db.prepare(`
-    INSERT INTO context_usage (session_id, timestamp, hook_name, injected_paths, estimated_tokens, was_referenced, turn_index)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
-  `).run(usage.sessionId, usage.timestamp, usage.hookName, JSON.stringify(usage.injectedPaths), usage.estimatedTokens, usage.wasReferenced, usage.turnIndex ?? 0);
+  // v0.8.1 Ext 6b: write query_text when the column is present AND the
+  // caller provided one. The column presence is cached at migration time
+  // in contextUsageHasQueryTextCache — missing entries default to false
+  // so ad-hoc DBs constructed outside createStore() degrade gracefully
+  // to the pre-v0.8.1 INSERT shape.
+  const hasQueryText = contextUsageHasQueryTextCache.get(db) ?? false;
+  if (hasQueryText) {
+    db.prepare(`
+      INSERT INTO context_usage
+        (session_id, timestamp, hook_name, injected_paths, estimated_tokens, was_referenced, turn_index, query_text)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      usage.sessionId,
+      usage.timestamp,
+      usage.hookName,
+      JSON.stringify(usage.injectedPaths),
+      usage.estimatedTokens,
+      usage.wasReferenced,
+      usage.turnIndex ?? 0,
+      usage.queryText ?? null,
+    );
+  } else {
+    db.prepare(`
+      INSERT INTO context_usage (session_id, timestamp, hook_name, injected_paths, estimated_tokens, was_referenced, turn_index)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `).run(usage.sessionId, usage.timestamp, usage.hookName, JSON.stringify(usage.injectedPaths), usage.estimatedTokens, usage.wasReferenced, usage.turnIndex ?? 0);
+  }
   // Return the rowid of the just-inserted row for recall event linkage
   const row = db.prepare("SELECT last_insert_rowid() as id").get() as { id: number };
   return row.id;

package/src/worker-lease.ts

@@ -0,0 +1,141 @@
+/**
+ * ClawMem Worker Lease (v0.8.0 Ext 5)
+ *
+ * DB-backed exclusive lease for heavy-lane workers. Uses the `worker_leases`
+ * table (schema in store.ts) instead of module globals so multiple processes
+ * sharing a vault cannot run heavy maintenance concurrently.
+ *
+ * Lease lifecycle:
+ *   1. acquireWorkerLease inserts or reclaims an expired row via transaction
+ *      and returns a random fencing token on success.
+ *   2. The holder runs its work.
+ *   3. releaseWorkerLease deletes the row only if the caller's token matches,
+ *      so a lease reclaimed by another worker after TTL expiry cannot be
+ *      torn down by the original holder.
+ *
+ * withWorkerLease wraps acquire/release around a callback; failure to acquire
+ * is a silent no-op (returns `{acquired: false}`) — callers should log a
+ * `skipped` journal row with reason `lease_unavailable`.
+ */
+
+import { randomBytes } from "node:crypto";
+import type { Store } from "./store.ts";
+
+export interface LeaseAcquireResult {
+  acquired: boolean;
+  token?: string;
+  expiresAt?: string;
+}
+
+function nowIso(now: Date = new Date()): string {
+  return now.toISOString();
+}
+
+function futureIso(now: Date, ttlMs: number): string {
+  return new Date(now.getTime() + ttlMs).toISOString();
+}
+
+/**
+ * Attempt to acquire an exclusive lease on `workerName` for `ttlMs`.
+ *
+ * Returns `{acquired: true, token, expiresAt}` on success, or
+ * `{acquired: false}` if another worker holds a live (non-expired) lease.
+ *
+ * Race-safe under multi-process contention: uses a single
+ * `INSERT ... ON CONFLICT DO UPDATE ... WHERE expires_at <= ?` statement
+ * so the "no row → insert" and "expired row → update" paths cannot
+ * both fire for two concurrent callers. SQLite's changes() reports 1
+ * iff THIS call either inserted a fresh row or reclaimed an expired row;
+ * 0 means a live lease was held by someone else.
+ *
+ * Any SQLITE_BUSY / constraint failure is translated to
+ * `{ acquired: false }` so the advertised non-throw contract holds for
+ * callers that are layering `shouldRunHeavyMaintenance` above this.
+ */
+export function acquireWorkerLease(
+  store: Store,
+  workerName: string,
+  ttlMs: number,
+  now: Date = new Date(),
+): LeaseAcquireResult {
+  if (ttlMs <= 0) {
+    throw new Error(`acquireWorkerLease: ttlMs must be positive, got ${ttlMs}`);
+  }
+  const token = randomBytes(16).toString("hex");
+  const acquiredAt = nowIso(now);
+  const expiresAt = futureIso(now, ttlMs);
+
+  try {
+    // Single-statement atomic acquire. The WHERE on the UPDATE clause
+    // only reclaims when the existing lease has expired (its expires_at
+    // <= our acquired_at); otherwise the ON CONFLICT DO UPDATE becomes
+    // a no-op and SQLite reports changes=0.
+    const result = store.db.prepare(
+      `INSERT INTO worker_leases
+         (worker_name, lease_token, acquired_at, expires_at)
+       VALUES (?, ?, ?, ?)
+       ON CONFLICT(worker_name) DO UPDATE SET
+         lease_token = excluded.lease_token,
+         acquired_at = excluded.acquired_at,
+         expires_at = excluded.expires_at
+       WHERE worker_leases.expires_at <= excluded.acquired_at`,
+    ).run(workerName, token, acquiredAt, expiresAt);
+
+    if (result.changes === 0) {
+      return { acquired: false };
+    }
+    return { acquired: true, token, expiresAt };
+  } catch (err) {
+    // Defensive fallback: any unexpected DB error (SQLITE_BUSY under
+    // extreme contention, constraint error from schema drift, etc.) is
+    // translated to a lease-unavailable result instead of bubbling up,
+    // so heavy-maintenance callers always get a deterministic
+    // "skipped/lease_unavailable" journal row.
+    console.error(
+      `[worker-lease] acquire error for ${workerName}: ${(err as Error).message}`,
+    );
+    return { acquired: false };
+  }
+}
+
+/**
+ * Release a lease if the caller's token still matches. Returns `true` if
+ * the lease was owned and deleted, `false` if a different token held it
+ * (e.g., TTL expired and another worker reclaimed).
+ */
+export function releaseWorkerLease(
+  store: Store,
+  workerName: string,
+  token: string,
+): boolean {
+  const result = store.db.prepare(
+    `DELETE FROM worker_leases WHERE worker_name = ? AND lease_token = ?`,
+  ).run(workerName, token);
+  return result.changes > 0;
+}
+
+/**
+ * Run `fn` under an exclusive lease on `workerName`. If the lease cannot
+ * be acquired, returns `{acquired: false}` without invoking `fn`. The
+ * lease is always released in a `finally` block, even if `fn` throws.
+ *
+ * Rethrows any error from `fn` — callers are responsible for translating
+ * exceptions into journal rows.
+ */
+export async function withWorkerLease<T>(
+  store: Store,
+  workerName: string,
+  ttlMs: number,
+  fn: () => Promise<T>,
+): Promise<{ acquired: boolean; result?: T }> {
+  const lease = acquireWorkerLease(store, workerName, ttlMs);
+  if (!lease.acquired || !lease.token) {
+    return { acquired: false };
+  }
+  try {
+    const result = await fn();
+    return { acquired: true, result };
+  } finally {
+    releaseWorkerLease(store, workerName, lease.token);
+  }
+}