@askexenow/exe-os 0.9.34 → 0.9.35

package/dist/hooks/ingest.js

@@ -2455,6 +2455,14 @@ async function ensureSchema() {
     );
   } catch {
   }
+  try {
+    await client.execute(
+      `CREATE INDEX IF NOT EXISTS idx_memories_scoped_content_hash
+         ON memories(content_hash, agent_id, project_name, memory_type)
+         WHERE content_hash IS NOT NULL`
+    );
+  } catch {
+  }
   await client.executeMultiple(`
     CREATE TABLE IF NOT EXISTS entities (
       id TEXT PRIMARY KEY,
@@ -3200,6 +3208,196 @@ var init_state_bus = __esm({
   }
 });
 
+// src/lib/memory-write-governor.ts
+import { createHash } from "crypto";
+function normalizeMemoryText(text) {
+  return text.replace(/\r\n/g, "\n").replace(/[ \t]+$/gm, "").replace(/\n{4,}/g, "\n\n\n").trim();
+}
+function classifyMemoryType(input2) {
+  if (input2.memory_type && input2.memory_type.trim()) return input2.memory_type.trim();
+  const tool = input2.tool_name.toLowerCase();
+  const text = input2.raw_text.toLowerCase();
+  if (tool.includes("store_decision") || tool.includes("decision")) return "decision";
+  if (tool.includes("commit") || text.includes("adr-") || text.includes("architectural decision")) return "adr";
+  if (tool.includes("store_behavior") || tool.includes("behavior")) return "behavior";
+  if (tool.includes("global_procedure") || text.includes("organization-wide procedures")) return "procedure";
+  if (tool.includes("send_whatsapp") || tool.includes("conversation")) return "conversation";
+  if (tool === "store_memory" || tool === "manual") return "observation";
+  return "raw";
+}
+function shouldDropMemory(text) {
+  const normalized = normalizeMemoryText(text);
+  if (normalized.length < 10) return { drop: true, reason: "too_short" };
+  if (NOISE_DROP_PATTERNS.some((pattern) => pattern.test(normalized))) {
+    return { drop: true, reason: "known_boilerplate_noise" };
+  }
+  return { drop: false };
+}
+function shouldSkipEmbedding(input2) {
+  const type = classifyMemoryType(input2);
+  if (HIGH_VALUE_SUPERSESSION_TYPES.has(type)) return false;
+  if (type === "raw" && input2.raw_text.length > 2e4) return true;
+  if (SKIP_EMBED_PATTERNS.some((pattern) => pattern.test(input2.raw_text))) return true;
+  return false;
+}
+function hashMemoryContent(text) {
+  return createHash("sha256").update(normalizeMemoryText(text)).digest("hex");
+}
+function scopedDedupArgs(input2) {
+  return [input2.contentHash, input2.agentId, input2.projectName, input2.memoryType];
+}
+function governMemoryRecord(record) {
+  const normalized = normalizeMemoryText(record.raw_text);
+  const memoryType = classifyMemoryType({
+    raw_text: normalized,
+    agent_id: record.agent_id,
+    project_name: record.project_name,
+    tool_name: record.tool_name,
+    memory_type: record.memory_type
+  });
+  const drop = shouldDropMemory(normalized);
+  const skipEmbedding = shouldSkipEmbedding({
+    raw_text: normalized,
+    agent_id: record.agent_id,
+    project_name: record.project_name,
+    tool_name: record.tool_name,
+    memory_type: memoryType
+  });
+  return {
+    record: {
+      ...record,
+      raw_text: normalized,
+      memory_type: memoryType,
+      vector: skipEmbedding ? null : record.vector
+    },
+    contentHash: hashMemoryContent(normalized),
+    shouldDrop: drop.drop,
+    dropReason: drop.reason,
+    skipEmbedding,
+    hygiene: {
+      dedup: true,
+      supersession: HIGH_VALUE_SUPERSESSION_TYPES.has(memoryType)
+    }
+  };
+}
+async function findScopedDuplicate(input2) {
+  const { getClient: getClient2 } = await Promise.resolve().then(() => (init_database(), database_exports));
+  const client = getClient2();
+  const args = scopedDedupArgs(input2);
+  let sql = `SELECT id FROM memories
+             WHERE content_hash = ?
+               AND agent_id = ?
+               AND project_name = ?
+               AND COALESCE(memory_type, 'raw') = ?
+               AND COALESCE(status, 'active') != 'deleted'`;
+  if (input2.excludeId) {
+    sql += " AND id != ?";
+    args.push(input2.excludeId);
+  }
+  sql += " ORDER BY timestamp DESC LIMIT 1";
+  const result = await client.execute({ sql, args });
+  return result.rows[0]?.id ? String(result.rows[0].id) : null;
+}
+async function runPostWriteMemoryHygiene(memoryId) {
+  try {
+    const { getClient: getClient2 } = await Promise.resolve().then(() => (init_database(), database_exports));
+    const client = getClient2();
+    const current = await client.execute({
+      sql: `SELECT id, agent_id, project_name, memory_type, content_hash, supersedes_id,
+                   importance, timestamp
+            FROM memories
+            WHERE id = ?
+            LIMIT 1`,
+      args: [memoryId]
+    });
+    const row = current.rows[0];
+    if (!row) return;
+    const memoryType = String(row.memory_type ?? "raw");
+    const contentHash = row.content_hash ? String(row.content_hash) : null;
+    const agentId = String(row.agent_id);
+    const projectName = String(row.project_name);
+    if (contentHash) {
+      await client.execute({
+        sql: `UPDATE memories
+              SET status = 'deleted',
+                  outcome = COALESCE(outcome, 'superseded')
+              WHERE id != ?
+                AND content_hash = ?
+                AND agent_id = ?
+                AND project_name = ?
+                AND COALESCE(memory_type, 'raw') = ?
+                AND COALESCE(status, 'active') = 'active'`,
+        args: [memoryId, contentHash, agentId, projectName, memoryType]
+      });
+    }
+    const supersedesId = row.supersedes_id ? String(row.supersedes_id) : null;
+    if (supersedesId && HIGH_VALUE_SUPERSESSION_TYPES.has(memoryType)) {
+      const old = await client.execute({
+        sql: `SELECT importance FROM memories WHERE id = ? LIMIT 1`,
+        args: [supersedesId]
+      });
+      const oldImportance = Number(old.rows[0]?.importance ?? 0);
+      const newImportance = Number(row.importance ?? 0);
+      await client.batch([
+        {
+          sql: `UPDATE memories
+                SET status = 'archived',
+                    outcome = COALESCE(outcome, 'superseded')
+                WHERE id = ?`,
+          args: [supersedesId]
+        },
+        {
+          sql: `UPDATE memories
+                SET importance = MAX(COALESCE(importance, 5), ?),
+                    parent_memory_id = COALESCE(parent_memory_id, ?)
+                WHERE id = ?`,
+          args: [Math.max(oldImportance, newImportance), supersedesId, memoryId]
+        }
+      ], "write");
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[memory-governor] post-write hygiene failed for ${memoryId}: ${err instanceof Error ? err.message : String(err)}
+`
+    );
+  }
+}
+function schedulePostWriteMemoryHygiene(memoryIds) {
+  if (memoryIds.length === 0) return;
+  const run = () => {
+    void Promise.all(memoryIds.map((id) => runPostWriteMemoryHygiene(id)));
+  };
+  if (typeof setImmediate === "function") setImmediate(run);
+  else setTimeout(run, 0);
+}
+var HIGH_VALUE_SUPERSESSION_TYPES, NOISE_DROP_PATTERNS, SKIP_EMBED_PATTERNS;
+var init_memory_write_governor = __esm({
+  "src/lib/memory-write-governor.ts"() {
+    "use strict";
+    HIGH_VALUE_SUPERSESSION_TYPES = /* @__PURE__ */ new Set([
+      "decision",
+      "adr",
+      "behavior",
+      "procedure"
+    ]);
+    NOISE_DROP_PATTERNS = [
+      /^\s*\[📋\s+\d+\s+reviews?\s+pending\b/im,
+      /^\s*<system-reminder>[\s\S]*?<\/system-reminder>\s*$/im,
+      /^\s*The UserPromptSubmit hook checks the DB for new tasks/im,
+      /^\s*Intercom is a speedup, not delivery/im,
+      /^\s*Context bar reads as USAGE not remaining/im
+    ];
+    SKIP_EMBED_PATTERNS = [
+      /tmux capture-pane\b/i,
+      /docker ps\b/i,
+      /docker images\b/i,
+      /git status\b/i,
+      /grep .*node_modules/i,
+      /npm (install|ci)\b[\s\S]*(added \d+ packages|audited \d+ packages)/i
+    ];
+  }
+});
+
 // src/lib/shard-manager.ts
 var shard_manager_exports = {};
 __export(shard_manager_exports, {
@@ -3364,7 +3562,8 @@ async function ensureShardSchema(client) {
   }
   for (const idx of [
     "CREATE INDEX IF NOT EXISTS idx_memories_tier ON memories(tier)",
-    "CREATE INDEX IF NOT EXISTS idx_memories_supersedes ON memories(supersedes_id) WHERE supersedes_id IS NOT NULL"
+    "CREATE INDEX IF NOT EXISTS idx_memories_supersedes ON memories(supersedes_id) WHERE supersedes_id IS NOT NULL",
+    "CREATE INDEX IF NOT EXISTS idx_memories_scoped_content_hash ON memories(content_hash, agent_id, project_name, memory_type) WHERE content_hash IS NOT NULL"
   ]) {
     try {
       await client.execute(idx);
@@ -3789,7 +3988,6 @@ __export(store_exports, {
   vectorToBlob: () => vectorToBlob,
   writeMemory: () => writeMemory
 });
-import { createHash } from "crypto";
 function isBusyError2(err) {
   if (err instanceof Error) {
     const msg = err.message.toLowerCase();
@@ -3903,17 +4101,24 @@ async function writeMemory(record) {
       `Expected ${EMBEDDING_DIM}-dim vector, got ${record.vector.length}`
     );
   }
-  const contentHash = createHash("md5").update(record.raw_text).digest("hex");
-  if (_pendingRecords.some((r) => r.content_hash === contentHash && r.agent_id === record.agent_id)) {
+  const governed = governMemoryRecord(record);
+  if (governed.shouldDrop) return;
+  record = governed.record;
+  const contentHash = governed.contentHash;
+  const memoryType = record.memory_type ?? "raw";
+  if (_pendingRecords.some(
+    (r) => r.content_hash === contentHash && r.agent_id === record.agent_id && r.project_name === record.project_name && (r.memory_type ?? "raw") === memoryType
+  )) {
     return;
   }
   try {
-    const client = getClient();
-    const existing = await client.execute({
-      sql: "SELECT id FROM memories WHERE content_hash = ? AND agent_id = ? LIMIT 1",
-      args: [contentHash, record.agent_id]
+    const existing = await findScopedDuplicate({
+      contentHash,
+      agentId: record.agent_id,
+      projectName: record.project_name,
+      memoryType
     });
-    if (existing.rows.length > 0) return;
+    if (existing) return;
   } catch {
   }
   const dbRow = {
@@ -3944,7 +4149,7 @@ async function writeMemory(record) {
     tier: record.tier ?? classifyTier(record),
     supersedes_id: record.supersedes_id ?? null,
     draft: record.draft ? 1 : 0,
-    memory_type: record.memory_type ?? "raw",
+    memory_type: memoryType,
     trajectory: record.trajectory ? JSON.stringify(record.trajectory) : null,
     content_hash: contentHash,
     intent: record.intent ?? null,
@@ -4103,6 +4308,7 @@ async function flushBatch() {
     const globalClient = getClient();
     const globalStmts = batch.map(buildStmt);
     await globalClient.batch(globalStmts, "write");
+    schedulePostWriteMemoryHygiene(batch.map((row) => row.id));
     _pendingRecords.splice(0, batch.length);
     try {
       const { isShardingEnabled: isShardingEnabled2, getReadyShardClient: getReadyShardClient2 } = await Promise.resolve().then(() => (init_shard_manager(), shard_manager_exports));
@@ -4361,6 +4567,7 @@ var init_store = __esm({
     init_keychain();
     init_config();
     init_state_bus();
+    init_memory_write_governor();
     INIT_MAX_RETRIES = 3;
     INIT_RETRY_DELAY_MS = 1e3;
     _pendingRecords = [];

package/dist/hooks/instructions-loaded.js

@@ -2290,6 +2290,14 @@ async function ensureSchema() {
     );
   } catch {
   }
+  try {
+    await client.execute(
+      `CREATE INDEX IF NOT EXISTS idx_memories_scoped_content_hash
+         ON memories(content_hash, agent_id, project_name, memory_type)
+         WHERE content_hash IS NOT NULL`
+    );
+  } catch {
+  }
   await client.executeMultiple(`
     CREATE TABLE IF NOT EXISTS entities (
       id TEXT PRIMARY KEY,
@@ -3035,6 +3043,196 @@ var init_state_bus = __esm({
   }
 });
 
+// src/lib/memory-write-governor.ts
+import { createHash as createHash2 } from "crypto";
+function normalizeMemoryText(text) {
+  return text.replace(/\r\n/g, "\n").replace(/[ \t]+$/gm, "").replace(/\n{4,}/g, "\n\n\n").trim();
+}
+function classifyMemoryType(input2) {
+  if (input2.memory_type && input2.memory_type.trim()) return input2.memory_type.trim();
+  const tool = input2.tool_name.toLowerCase();
+  const text = input2.raw_text.toLowerCase();
+  if (tool.includes("store_decision") || tool.includes("decision")) return "decision";
+  if (tool.includes("commit") || text.includes("adr-") || text.includes("architectural decision")) return "adr";
+  if (tool.includes("store_behavior") || tool.includes("behavior")) return "behavior";
+  if (tool.includes("global_procedure") || text.includes("organization-wide procedures")) return "procedure";
+  if (tool.includes("send_whatsapp") || tool.includes("conversation")) return "conversation";
+  if (tool === "store_memory" || tool === "manual") return "observation";
+  return "raw";
+}
+function shouldDropMemory(text) {
+  const normalized = normalizeMemoryText(text);
+  if (normalized.length < 10) return { drop: true, reason: "too_short" };
+  if (NOISE_DROP_PATTERNS.some((pattern) => pattern.test(normalized))) {
+    return { drop: true, reason: "known_boilerplate_noise" };
+  }
+  return { drop: false };
+}
+function shouldSkipEmbedding(input2) {
+  const type = classifyMemoryType(input2);
+  if (HIGH_VALUE_SUPERSESSION_TYPES.has(type)) return false;
+  if (type === "raw" && input2.raw_text.length > 2e4) return true;
+  if (SKIP_EMBED_PATTERNS.some((pattern) => pattern.test(input2.raw_text))) return true;
+  return false;
+}
+function hashMemoryContent(text) {
+  return createHash2("sha256").update(normalizeMemoryText(text)).digest("hex");
+}
+function scopedDedupArgs(input2) {
+  return [input2.contentHash, input2.agentId, input2.projectName, input2.memoryType];
+}
+function governMemoryRecord(record) {
+  const normalized = normalizeMemoryText(record.raw_text);
+  const memoryType = classifyMemoryType({
+    raw_text: normalized,
+    agent_id: record.agent_id,
+    project_name: record.project_name,
+    tool_name: record.tool_name,
+    memory_type: record.memory_type
+  });
+  const drop = shouldDropMemory(normalized);
+  const skipEmbedding = shouldSkipEmbedding({
+    raw_text: normalized,
+    agent_id: record.agent_id,
+    project_name: record.project_name,
+    tool_name: record.tool_name,
+    memory_type: memoryType
+  });
+  return {
+    record: {
+      ...record,
+      raw_text: normalized,
+      memory_type: memoryType,
+      vector: skipEmbedding ? null : record.vector
+    },
+    contentHash: hashMemoryContent(normalized),
+    shouldDrop: drop.drop,
+    dropReason: drop.reason,
+    skipEmbedding,
+    hygiene: {
+      dedup: true,
+      supersession: HIGH_VALUE_SUPERSESSION_TYPES.has(memoryType)
+    }
+  };
+}
+async function findScopedDuplicate(input2) {
+  const { getClient: getClient2 } = await Promise.resolve().then(() => (init_database(), database_exports));
+  const client = getClient2();
+  const args = scopedDedupArgs(input2);
+  let sql = `SELECT id FROM memories
+             WHERE content_hash = ?
+               AND agent_id = ?
+               AND project_name = ?
+               AND COALESCE(memory_type, 'raw') = ?
+               AND COALESCE(status, 'active') != 'deleted'`;
+  if (input2.excludeId) {
+    sql += " AND id != ?";
+    args.push(input2.excludeId);
+  }
+  sql += " ORDER BY timestamp DESC LIMIT 1";
+  const result = await client.execute({ sql, args });
+  return result.rows[0]?.id ? String(result.rows[0].id) : null;
+}
+async function runPostWriteMemoryHygiene(memoryId) {
+  try {
+    const { getClient: getClient2 } = await Promise.resolve().then(() => (init_database(), database_exports));
+    const client = getClient2();
+    const current = await client.execute({
+      sql: `SELECT id, agent_id, project_name, memory_type, content_hash, supersedes_id,
+                   importance, timestamp
+            FROM memories
+            WHERE id = ?
+            LIMIT 1`,
+      args: [memoryId]
+    });
+    const row = current.rows[0];
+    if (!row) return;
+    const memoryType = String(row.memory_type ?? "raw");
+    const contentHash2 = row.content_hash ? String(row.content_hash) : null;
+    const agentId = String(row.agent_id);
+    const projectName = String(row.project_name);
+    if (contentHash2) {
+      await client.execute({
+        sql: `UPDATE memories
+              SET status = 'deleted',
+                  outcome = COALESCE(outcome, 'superseded')
+              WHERE id != ?
+                AND content_hash = ?
+                AND agent_id = ?
+                AND project_name = ?
+                AND COALESCE(memory_type, 'raw') = ?
+                AND COALESCE(status, 'active') = 'active'`,
+        args: [memoryId, contentHash2, agentId, projectName, memoryType]
+      });
+    }
+    const supersedesId = row.supersedes_id ? String(row.supersedes_id) : null;
+    if (supersedesId && HIGH_VALUE_SUPERSESSION_TYPES.has(memoryType)) {
+      const old = await client.execute({
+        sql: `SELECT importance FROM memories WHERE id = ? LIMIT 1`,
+        args: [supersedesId]
+      });
+      const oldImportance = Number(old.rows[0]?.importance ?? 0);
+      const newImportance = Number(row.importance ?? 0);
+      await client.batch([
+        {
+          sql: `UPDATE memories
+                SET status = 'archived',
+                    outcome = COALESCE(outcome, 'superseded')
+                WHERE id = ?`,
+          args: [supersedesId]
+        },
+        {
+          sql: `UPDATE memories
+                SET importance = MAX(COALESCE(importance, 5), ?),
+                    parent_memory_id = COALESCE(parent_memory_id, ?)
+                WHERE id = ?`,
+          args: [Math.max(oldImportance, newImportance), supersedesId, memoryId]
+        }
+      ], "write");
+    }
+  } catch (err) {
+    process.stderr.write(
+      `[memory-governor] post-write hygiene failed for ${memoryId}: ${err instanceof Error ? err.message : String(err)}
+`
+    );
+  }
+}
+function schedulePostWriteMemoryHygiene(memoryIds) {
+  if (memoryIds.length === 0) return;
+  const run = () => {
+    void Promise.all(memoryIds.map((id) => runPostWriteMemoryHygiene(id)));
+  };
+  if (typeof setImmediate === "function") setImmediate(run);
+  else setTimeout(run, 0);
+}
+var HIGH_VALUE_SUPERSESSION_TYPES, NOISE_DROP_PATTERNS, SKIP_EMBED_PATTERNS;
+var init_memory_write_governor = __esm({
+  "src/lib/memory-write-governor.ts"() {
+    "use strict";
+    HIGH_VALUE_SUPERSESSION_TYPES = /* @__PURE__ */ new Set([
+      "decision",
+      "adr",
+      "behavior",
+      "procedure"
+    ]);
+    NOISE_DROP_PATTERNS = [
+      /^\s*\[📋\s+\d+\s+reviews?\s+pending\b/im,
+      /^\s*<system-reminder>[\s\S]*?<\/system-reminder>\s*$/im,
+      /^\s*The UserPromptSubmit hook checks the DB for new tasks/im,
+      /^\s*Intercom is a speedup, not delivery/im,
+      /^\s*Context bar reads as USAGE not remaining/im
+    ];
+    SKIP_EMBED_PATTERNS = [
+      /tmux capture-pane\b/i,
+      /docker ps\b/i,
+      /docker images\b/i,
+      /git status\b/i,
+      /grep .*node_modules/i,
+      /npm (install|ci)\b[\s\S]*(added \d+ packages|audited \d+ packages)/i
+    ];
+  }
+});
+
 // src/lib/shard-manager.ts
 var shard_manager_exports = {};
 __export(shard_manager_exports, {
@@ -3199,7 +3397,8 @@ async function ensureShardSchema(client) {
   }
   for (const idx of [
     "CREATE INDEX IF NOT EXISTS idx_memories_tier ON memories(tier)",
-    "CREATE INDEX IF NOT EXISTS idx_memories_supersedes ON memories(supersedes_id) WHERE supersedes_id IS NOT NULL"
+    "CREATE INDEX IF NOT EXISTS idx_memories_supersedes ON memories(supersedes_id) WHERE supersedes_id IS NOT NULL",
+    "CREATE INDEX IF NOT EXISTS idx_memories_scoped_content_hash ON memories(content_hash, agent_id, project_name, memory_type) WHERE content_hash IS NOT NULL"
   ]) {
     try {
       await client.execute(idx);
@@ -3624,7 +3823,6 @@ __export(store_exports, {
   vectorToBlob: () => vectorToBlob,
   writeMemory: () => writeMemory
 });
-import { createHash as createHash2 } from "crypto";
 function isBusyError2(err) {
   if (err instanceof Error) {
     const msg = err.message.toLowerCase();
@@ -3738,17 +3936,24 @@ async function writeMemory(record) {
       `Expected ${EMBEDDING_DIM}-dim vector, got ${record.vector.length}`
     );
   }
-  const contentHash2 = createHash2("md5").update(record.raw_text).digest("hex");
-  if (_pendingRecords.some((r) => r.content_hash === contentHash2 && r.agent_id === record.agent_id)) {
+  const governed = governMemoryRecord(record);
+  if (governed.shouldDrop) return;
+  record = governed.record;
+  const contentHash2 = governed.contentHash;
+  const memoryType = record.memory_type ?? "raw";
+  if (_pendingRecords.some(
+    (r) => r.content_hash === contentHash2 && r.agent_id === record.agent_id && r.project_name === record.project_name && (r.memory_type ?? "raw") === memoryType
+  )) {
     return;
   }
   try {
-    const client = getClient();
-    const existing = await client.execute({
-      sql: "SELECT id FROM memories WHERE content_hash = ? AND agent_id = ? LIMIT 1",
-      args: [contentHash2, record.agent_id]
+    const existing = await findScopedDuplicate({
+      contentHash: contentHash2,
+      agentId: record.agent_id,
+      projectName: record.project_name,
+      memoryType
     });
-    if (existing.rows.length > 0) return;
+    if (existing) return;
   } catch {
   }
   const dbRow = {
@@ -3779,7 +3984,7 @@ async function writeMemory(record) {
     tier: record.tier ?? classifyTier(record),
     supersedes_id: record.supersedes_id ?? null,
     draft: record.draft ? 1 : 0,
-    memory_type: record.memory_type ?? "raw",
+    memory_type: memoryType,
     trajectory: record.trajectory ? JSON.stringify(record.trajectory) : null,
     content_hash: contentHash2,
     intent: record.intent ?? null,
@@ -3938,6 +4143,7 @@ async function flushBatch() {
     const globalClient = getClient();
     const globalStmts = batch.map(buildStmt);
     await globalClient.batch(globalStmts, "write");
+    schedulePostWriteMemoryHygiene(batch.map((row) => row.id));
     _pendingRecords.splice(0, batch.length);
     try {
       const { isShardingEnabled: isShardingEnabled2, getReadyShardClient: getReadyShardClient2 } = await Promise.resolve().then(() => (init_shard_manager(), shard_manager_exports));
@@ -4196,6 +4402,7 @@ var init_store = __esm({
     init_keychain();
     init_config();
     init_state_bus();
+    init_memory_write_governor();
     INIT_MAX_RETRIES = 3;
     INIT_RETRY_DELAY_MS = 1e3;
     _pendingRecords = [];