@askexenow/exe-os 0.8.0

package/dist/lib/consolidation.js

@@ -0,0 +1,476 @@
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+
+// src/lib/config.ts
+import { readFile as readFile2, writeFile as writeFile2, mkdir as mkdir2 } from "fs/promises";
+import { readFileSync, existsSync as existsSync2, renameSync } from "fs";
+import path2 from "path";
+import os from "os";
+function resolveDataDir() {
+  if (process.env.EXE_OS_DIR) return process.env.EXE_OS_DIR;
+  if (process.env.EXE_MEM_DIR) return process.env.EXE_MEM_DIR;
+  const newDir = path2.join(os.homedir(), ".exe-os");
+  const legacyDir = path2.join(os.homedir(), ".exe-mem");
+  if (!existsSync2(newDir) && existsSync2(legacyDir)) {
+    try {
+      renameSync(legacyDir, newDir);
+      process.stderr.write(`[exe-os] Migrated data directory: ~/.exe-mem \u2192 ~/.exe-os
+`);
+    } catch {
+      return legacyDir;
+    }
+  }
+  return newDir;
+}
+var EXE_AI_DIR, DB_PATH, MODELS_DIR, CONFIG_PATH, LEGACY_LANCE_PATH, CURRENT_CONFIG_VERSION, DEFAULT_CONFIG;
+var init_config = __esm({
+  "src/lib/config.ts"() {
+    "use strict";
+    EXE_AI_DIR = resolveDataDir();
+    DB_PATH = path2.join(EXE_AI_DIR, "memories.db");
+    MODELS_DIR = path2.join(EXE_AI_DIR, "models");
+    CONFIG_PATH = path2.join(EXE_AI_DIR, "config.json");
+    LEGACY_LANCE_PATH = path2.join(EXE_AI_DIR, "local.lance");
+    CURRENT_CONFIG_VERSION = 1;
+    DEFAULT_CONFIG = {
+      config_version: CURRENT_CONFIG_VERSION,
+      dbPath: DB_PATH,
+      modelFile: "jina-embeddings-v5-small-q4_k_m.gguf",
+      embeddingDim: 1024,
+      batchSize: 20,
+      flushIntervalMs: 1e4,
+      autoIngestion: true,
+      autoRetrieval: true,
+      searchMode: "hybrid",
+      hookSearchMode: "hybrid",
+      fileGrepEnabled: true,
+      splashEffect: true,
+      consolidationEnabled: true,
+      consolidationIntervalMs: 6 * 60 * 60 * 1e3,
+      consolidationModel: "claude-haiku-4-5-20251001",
+      consolidationMaxCallsPerRun: 20,
+      selfQueryRouter: true,
+      selfQueryModel: "claude-haiku-4-5-20251001",
+      rerankerEnabled: true,
+      scalingRoadmap: {
+        rerankerAutoTrigger: {
+          enabled: true,
+          broadQueryMinCardinality: 5e4,
+          fetchTopK: 150,
+          returnTopK: 5
+        }
+      },
+      graphRagEnabled: true,
+      wikiEnabled: false,
+      wikiUrl: "",
+      wikiApiKey: "",
+      wikiSyncIntervalMs: 30 * 60 * 1e3,
+      wikiWorkspaceMapping: {
+        exe: "Executive",
+        yoshi: "Engineering",
+        mari: "Marketing",
+        tom: "Engineering",
+        sasha: "Production"
+      },
+      wikiAutoUpdate: true,
+      wikiAutoUpdateThreshold: 0.5,
+      wikiAutoUpdateCreateNew: true,
+      skillLearning: true,
+      skillThreshold: 3,
+      skillModel: "claude-haiku-4-5-20251001",
+      exeHeartbeat: {
+        enabled: true,
+        intervalSeconds: 60,
+        staleInProgressThresholdHours: 2
+      },
+      sessionLifecycle: {
+        idleKillEnabled: true,
+        idleKillTicksRequired: 3,
+        idleKillIntercomAckWindowMs: 1e4,
+        maxAutoInstances: 10
+      }
+    };
+  }
+});
+
+// src/lib/consolidation.ts
+import { randomUUID } from "crypto";
+
+// src/lib/database.ts
+import { createClient } from "@libsql/client";
+
+// src/lib/keychain.ts
+import { readFile, writeFile, unlink, mkdir, chmod } from "fs/promises";
+import { existsSync } from "fs";
+import path from "path";
+import crypto from "crypto";
+
+// src/lib/store.ts
+init_config();
+function vectorToBlob(vector) {
+  const f32 = vector instanceof Float32Array ? vector : new Float32Array(vector);
+  return JSON.stringify(Array.from(f32));
+}
+
+// src/lib/consolidation.ts
+async function selectUnconsolidated(client, limit = 200) {
+  const result = await client.execute({
+    sql: `SELECT id, agent_id, project_name, tool_name, raw_text, timestamp
+          FROM memories
+          WHERE consolidated = 0
+            AND timestamp >= datetime('now', '-7 days')
+          ORDER BY timestamp DESC
+          LIMIT ?`,
+    args: [limit]
+  });
+  return result.rows.map((row) => ({
+    id: row.id,
+    agent_id: row.agent_id,
+    project_name: row.project_name,
+    tool_name: row.tool_name,
+    raw_text: row.raw_text,
+    timestamp: row.timestamp
+  }));
+}
+function groupMemories(memories) {
+  const byProject = /* @__PURE__ */ new Map();
+  for (const mem of memories) {
+    const key = `${mem.agent_id}::${mem.project_name}`;
+    const list = byProject.get(key) ?? [];
+    list.push(mem);
+    byProject.set(key, list);
+  }
+  const clusters = [];
+  for (const [key, projectMemories] of byProject) {
+    const [agentId, projectName] = key.split("::");
+    const byDay = /* @__PURE__ */ new Map();
+    for (const mem of projectMemories) {
+      const day = mem.timestamp.slice(0, 10);
+      const list = byDay.get(day) ?? [];
+      list.push(mem);
+      byDay.set(day, list);
+    }
+    for (const [day, dayMemories] of byDay) {
+      if (dayMemories.length <= 15) {
+        clusters.push({
+          agentId,
+          projectName,
+          dateRange: day,
+          memories: dayMemories
+        });
+      } else {
+        for (let i = 0; i < dayMemories.length; i += 12) {
+          const chunk = dayMemories.slice(i, i + 12);
+          clusters.push({
+            agentId,
+            projectName,
+            dateRange: day,
+            memories: chunk
+          });
+        }
+      }
+    }
+  }
+  return clusters;
+}
+function buildConsolidationPrompt(cluster) {
+  const snippets = cluster.memories.map((m, i) => {
+    const text = m.raw_text.length > 200 ? m.raw_text.slice(0, 200) + "..." : m.raw_text;
+    return `${i + 1}. [${m.tool_name}] ${text}`;
+  }).join("\n");
+  return `You are reviewing a set of work memories from an AI coding agent.
+These are raw tool call records from ${cluster.dateRange}.
+
+Agent: ${cluster.agentId} | Project: ${cluster.projectName} | Date: ${cluster.dateRange}
+
+MEMORIES:
+${snippets}
+
+Extract EXACTLY THREE types of insights:
+
+1. KEY DECISIONS \u2014 choices that were made and why (at most 3)
+2. RECURRING PATTERNS \u2014 actions or approaches that repeated (at most 3)
+3. OPEN QUESTIONS \u2014 things that seem unresolved or risky (at most 2)
+
+Format each as a single sentence. Be specific \u2014 include file names,
+function names, and concrete details. Skip if no insight for a category.`;
+}
+async function consolidateCluster(cluster, model) {
+  const Anthropic = (await import("@anthropic-ai/sdk")).default;
+  const client = new Anthropic();
+  const prompt = buildConsolidationPrompt(cluster);
+  const response = await client.messages.create({
+    model,
+    max_tokens: 300,
+    messages: [{ role: "user", content: prompt }]
+  });
+  const textBlock = response.content.find((b) => b.type === "text");
+  return textBlock?.text ?? "";
+}
+async function storeConsolidation(client, cluster, synthesisText, embedFn) {
+  const consolidatedId = randomUUID();
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const rawText = `CONSOLIDATION [${cluster.dateRange}, ${cluster.projectName}]:
+
+${synthesisText}`;
+  let vector = null;
+  if (embedFn) {
+    try {
+      vector = await embedFn(rawText);
+    } catch {
+    }
+  }
+  const insertSql = vector ? `INSERT INTO memories
+        (id, agent_id, agent_role, session_id, timestamp,
+         tool_name, project_name, has_error, raw_text, vector, version, consolidated)
+        VALUES (?, ?, 'consolidation', 'daemon-consolidation', ?, 'consolidation', ?, 0, ?, vector32(?), 0, 1)` : `INSERT INTO memories
+        (id, agent_id, agent_role, session_id, timestamp,
+         tool_name, project_name, has_error, raw_text, vector, version, consolidated)
+        VALUES (?, ?, 'consolidation', 'daemon-consolidation', ?, 'consolidation', ?, 0, ?, NULL, 0, 1)`;
+  const insertArgs = vector ? [consolidatedId, cluster.agentId, now, cluster.projectName, rawText, vectorToBlob(vector)] : [consolidatedId, cluster.agentId, now, cluster.projectName, rawText];
+  await client.execute({ sql: insertSql, args: insertArgs });
+  const sourceIds = cluster.memories.map((m) => m.id);
+  const linkStmts = sourceIds.map((sourceId) => ({
+    sql: `INSERT INTO consolidations (id, consolidated_memory_id, source_memory_id, created_at)
+          VALUES (?, ?, ?, ?)`,
+    args: [randomUUID(), consolidatedId, sourceId, now]
+  }));
+  const placeholders = sourceIds.map(() => "?").join(",");
+  const markStmt = {
+    sql: `UPDATE memories SET consolidated = 1 WHERE id IN (${placeholders})`,
+    args: sourceIds
+  };
+  await client.batch([...linkStmts, markStmt], "write");
+  return { consolidatedMemoryId: consolidatedId, sourceIds, rawText };
+}
+var WIKI_FETCH_TIMEOUT_MS = 1e4;
+async function pushToWiki(consolidation, config) {
+  if (!config.wikiEnabled || !config.wikiAutoUpdate) {
+    return { updated: false };
+  }
+  const apiUrl = config.wikiUrl || process.env.EXE_WIKI_API_URL;
+  const apiKey = config.wikiApiKey || process.env.EXE_WIKI_API_KEY;
+  if (!apiUrl || !apiKey) {
+    return { updated: false };
+  }
+  const workspace = config.wikiWorkspaceMapping[consolidation.projectName] ?? consolidation.projectName;
+  try {
+    const listRes = await fetch(
+      `${apiUrl}/v1/workspace/${encodeURIComponent(workspace)}/documents`,
+      {
+        headers: { Authorization: `Bearer ${apiKey}` },
+        signal: AbortSignal.timeout(WIKI_FETCH_TIMEOUT_MS)
+      }
+    );
+    if (!listRes.ok) {
+      return { updated: false, error: `Workspace "${workspace}" not accessible (${listRes.status})` };
+    }
+    const listJson = await listRes.json();
+    const docs = listJson.documents ?? listJson.localFiles?.items ?? [];
+    const contentLines = consolidation.rawText.split("\n").filter((l) => l.trim() && !l.startsWith("CONSOLIDATION") && !l.match(/^[A-Z\s]+:$/)).join(" ");
+    const keywords = contentLines.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((w) => w.length > 3);
+    let bestMatch = null;
+    for (const doc of docs) {
+      if (!doc.id || !doc.title) continue;
+      const titleWords = doc.title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((w) => w.length > 3);
+      if (titleWords.length === 0) continue;
+      const matchCount = titleWords.filter(
+        (tw) => keywords.some((k) => k.includes(tw) || tw.includes(k))
+      ).length;
+      const score = matchCount / titleWords.length;
+      if (score > (bestMatch?.score ?? 0)) {
+        bestMatch = { id: doc.id, title: doc.title, score };
+      }
+    }
+    if (bestMatch && bestMatch.score >= config.wikiAutoUpdateThreshold) {
+      const updateRes = await fetch(`${apiUrl}/v1/document/raw-text`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`
+        },
+        body: JSON.stringify({
+          textContent: consolidation.rawText,
+          metadata: { title: bestMatch.title, appendTo: bestMatch.id },
+          workspaceSlugs: [workspace]
+        }),
+        signal: AbortSignal.timeout(WIKI_FETCH_TIMEOUT_MS)
+      });
+      if (updateRes.ok) {
+        process.stderr.write(
+          `[consolidation] Wiki auto-updated: ${workspace}/${bestMatch.title}
+`
+        );
+        return { updated: true, action: "updated", page: bestMatch.title };
+      }
+      return { updated: false, error: `Update failed (${updateRes.status})` };
+    }
+    if (config.wikiAutoUpdateCreateNew) {
+      const title = `Consolidated Insights \u2014 ${consolidation.projectName} (${(/* @__PURE__ */ new Date()).toISOString().slice(0, 10)})`;
+      const createRes = await fetch(`${apiUrl}/v1/document/raw-text`, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`
+        },
+        body: JSON.stringify({
+          textContent: consolidation.rawText,
+          metadata: { title },
+          workspaceSlugs: [workspace]
+        }),
+        signal: AbortSignal.timeout(WIKI_FETCH_TIMEOUT_MS)
+      });
+      if (createRes.ok) {
+        process.stderr.write(
+          `[consolidation] Wiki page created: ${workspace}/${title}
+`
+        );
+        return { updated: true, action: "created", page: title };
+      }
+      return { updated: false, error: `Create failed (${createRes.status})` };
+    }
+    return { updated: false };
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stderr.write(`[consolidation] Wiki push failed: ${msg}
+`);
+    return { updated: false, error: msg };
+  }
+}
+async function runConsolidation(client, options) {
+  const memories = await selectUnconsolidated(client);
+  if (memories.length < 20) {
+    return { clustersProcessed: 0, memoriesConsolidated: 0 };
+  }
+  const clusters = groupMemories(memories);
+  let clustersProcessed = 0;
+  let memoriesConsolidated = 0;
+  for (const cluster of clusters) {
+    if (clustersProcessed >= options.maxCalls) break;
+    if (cluster.memories.length < 3) continue;
+    try {
+      const isExe = cluster.agentId === "exe";
+      if (isExe) {
+        const synthesis = await consolidateCluster(cluster, options.model);
+        if (!synthesis.trim()) continue;
+        const result = await storeConsolidation(client, cluster, synthesis, options.embedFn);
+        if (options.wikiConfig) {
+          await pushToWiki(
+            { ...result, projectName: cluster.projectName },
+            options.wikiConfig
+          ).catch((err) => {
+            process.stderr.write(
+              `[consolidation] Wiki push error (non-fatal): ${err instanceof Error ? err.message : String(err)}
+`
+            );
+          });
+        }
+        const sourceIds = result.sourceIds;
+        if (sourceIds.length > 0) {
+          const placeholders = sourceIds.map(() => "?").join(",");
+          await client.execute({
+            sql: `UPDATE memories SET status = 'archived' WHERE id IN (${placeholders})`,
+            args: sourceIds
+          });
+        }
+      } else {
+        const dedupCount = await dedupCluster(client, cluster, options.embedFn);
+        memoriesConsolidated += dedupCount;
+        if (dedupCount === 0) continue;
+      }
+      clustersProcessed++;
+      memoriesConsolidated += isExe ? cluster.memories.length : 0;
+    } catch (err) {
+      process.stderr.write(
+        `[consolidation] Cluster failed (${cluster.projectName}/${cluster.dateRange}): ${err instanceof Error ? err.message : String(err)}
+`
+      );
+    }
+  }
+  return { clustersProcessed, memoriesConsolidated };
+}
+async function dedupCluster(client, cluster, embedFn) {
+  if (!embedFn || cluster.memories.length < 2) return 0;
+  const vectors = [];
+  for (const mem of cluster.memories) {
+    try {
+      const v = await embedFn(mem.raw_text.slice(0, 500));
+      vectors.push({ id: mem.id, vector: v });
+    } catch {
+    }
+  }
+  if (vectors.length < 2) return 0;
+  const toArchive = /* @__PURE__ */ new Set();
+  for (let i = 0; i < vectors.length; i++) {
+    if (toArchive.has(vectors[i].id)) continue;
+    for (let j = i + 1; j < vectors.length; j++) {
+      if (toArchive.has(vectors[j].id)) continue;
+      const sim = cosineSimilarity(vectors[i].vector, vectors[j].vector);
+      if (sim > 0.95) {
+        toArchive.add(vectors[j].id);
+      }
+    }
+  }
+  if (toArchive.size === 0) return 0;
+  const ids = [...toArchive];
+  const placeholders = ids.map(() => "?").join(",");
+  await client.execute({
+    sql: `UPDATE memories SET status = 'archived', consolidated = 1 WHERE id IN (${placeholders})`,
+    args: ids
+  });
+  const survivors = vectors.filter((v) => !toArchive.has(v.id)).map((v) => v.id);
+  if (survivors.length > 0) {
+    const survivorPlaceholders = survivors.map(() => "?").join(",");
+    await client.execute({
+      sql: `UPDATE memories SET confidence = MIN(1.0, COALESCE(confidence, 0.7) + 0.1) WHERE id IN (${survivorPlaceholders})`,
+      args: survivors
+    });
+  }
+  return ids.length;
+}
+function cosineSimilarity(a, b) {
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+async function isUserIdle(client, idleMinutes = 30) {
+  const result = await client.execute({
+    sql: `SELECT MAX(timestamp) as last_activity
+          FROM memories
+          WHERE tool_name != 'consolidation'
+            AND timestamp >= datetime('now', '-1 day')`,
+    args: []
+  });
+  const lastActivity = result.rows[0]?.last_activity;
+  if (!lastActivity) return true;
+  const lastMs = new Date(lastActivity).getTime();
+  const now = Date.now();
+  return now - lastMs >= idleMinutes * 60 * 1e3;
+}
+async function countUnconsolidated(client) {
+  const result = await client.execute({
+    sql: `SELECT COUNT(*) as cnt FROM memories
+          WHERE consolidated = 0
+            AND timestamp >= datetime('now', '-7 days')`,
+    args: []
+  });
+  return Number(result.rows[0]?.cnt ?? 0);
+}
+export {
+  buildConsolidationPrompt,
+  consolidateCluster,
+  countUnconsolidated,
+  groupMemories,
+  isUserIdle,
+  pushToWiki,
+  runConsolidation,
+  selectUnconsolidated,
+  storeConsolidation
+};

package/dist/lib/crypto.js

@@ -0,0 +1,51 @@
+// src/lib/crypto.ts
+import crypto from "crypto";
+var ALGORITHM = "aes-256-gcm";
+var IV_LENGTH = 12;
+var TAG_LENGTH = 16;
+var SYNC_HKDF_INFO = "exe-mem-sync-v2";
+var _syncKey = null;
+function initSyncCrypto(masterKey) {
+  if (masterKey.length !== 32) {
+    throw new Error(`Master key must be 32 bytes, got ${masterKey.length}`);
+  }
+  _syncKey = Buffer.from(
+    crypto.hkdfSync("sha256", masterKey, "", SYNC_HKDF_INFO, 32)
+  );
+}
+function isSyncCryptoInitialized() {
+  return _syncKey !== null;
+}
+function requireSyncKey() {
+  if (!_syncKey) {
+    throw new Error("Sync crypto not initialized. Call initSyncCrypto(masterKey) first.");
+  }
+  return _syncKey;
+}
+function encryptSyncBlob(data) {
+  const key = requireSyncKey();
+  const iv = crypto.randomBytes(IV_LENGTH);
+  const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
+  const encrypted = Buffer.concat([cipher.update(data), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  return Buffer.concat([iv, encrypted, tag]).toString("base64");
+}
+function decryptSyncBlob(ciphertext) {
+  const key = requireSyncKey();
+  const combined = Buffer.from(ciphertext, "base64");
+  if (combined.length < IV_LENGTH + TAG_LENGTH) {
+    throw new Error("Sync blob too short to contain IV + tag");
+  }
+  const iv = combined.subarray(0, IV_LENGTH);
+  const tag = combined.subarray(combined.length - TAG_LENGTH);
+  const encrypted = combined.subarray(IV_LENGTH, combined.length - TAG_LENGTH);
+  const decipher = crypto.createDecipheriv(ALGORITHM, key, iv);
+  decipher.setAuthTag(tag);
+  return Buffer.concat([decipher.update(encrypted), decipher.final()]);
+}
+export {
+  decryptSyncBlob,
+  encryptSyncBlob,
+  initSyncCrypto,
+  isSyncCryptoInitialized
+};