@askexenow/exe-os 0.8.0

package/dist/lib/embedder.js

@@ -0,0 +1,632 @@
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/lib/config.ts
+var config_exports = {};
+__export(config_exports, {
+  CONFIG_MIGRATIONS: () => CONFIG_MIGRATIONS,
+  CONFIG_PATH: () => CONFIG_PATH,
+  CURRENT_CONFIG_VERSION: () => CURRENT_CONFIG_VERSION,
+  DB_PATH: () => DB_PATH,
+  EXE_AI_DIR: () => EXE_AI_DIR,
+  LEGACY_LANCE_PATH: () => LEGACY_LANCE_PATH,
+  MODELS_DIR: () => MODELS_DIR,
+  loadConfig: () => loadConfig,
+  loadConfigFrom: () => loadConfigFrom,
+  loadConfigSync: () => loadConfigSync,
+  migrateConfig: () => migrateConfig,
+  saveConfig: () => saveConfig
+});
+import { readFile, writeFile, mkdir } from "fs/promises";
+import { readFileSync, existsSync, renameSync } from "fs";
+import path from "path";
+import os from "os";
+function resolveDataDir() {
+  if (process.env.EXE_OS_DIR) return process.env.EXE_OS_DIR;
+  if (process.env.EXE_MEM_DIR) return process.env.EXE_MEM_DIR;
+  const newDir = path.join(os.homedir(), ".exe-os");
+  const legacyDir = path.join(os.homedir(), ".exe-mem");
+  if (!existsSync(newDir) && existsSync(legacyDir)) {
+    try {
+      renameSync(legacyDir, newDir);
+      process.stderr.write(`[exe-os] Migrated data directory: ~/.exe-mem \u2192 ~/.exe-os
+`);
+    } catch {
+      return legacyDir;
+    }
+  }
+  return newDir;
+}
+function migrateLegacyConfig(raw) {
+  if ("r2" in raw) {
+    process.stderr.write(
+      "[exe-os] Warning: config.json contains deprecated 'r2' field from v1.0. R2 sync has been replaced in v1.1. The 'r2' field will be ignored.\n"
+    );
+    delete raw.r2;
+  }
+  if ("syncIntervalMs" in raw) {
+    delete raw.syncIntervalMs;
+  }
+  return raw;
+}
+function migrateConfig(raw) {
+  const fromVersion = typeof raw.config_version === "number" ? raw.config_version : 0;
+  let currentVersion = fromVersion;
+  let migrated = false;
+  if (currentVersion > CURRENT_CONFIG_VERSION) {
+    return { config: raw, migrated: false, fromVersion };
+  }
+  for (const migration of CONFIG_MIGRATIONS) {
+    if (currentVersion === migration.from && migration.to <= CURRENT_CONFIG_VERSION) {
+      raw = migration.migrate(raw);
+      currentVersion = migration.to;
+      migrated = true;
+    }
+  }
+  return { config: raw, migrated, fromVersion };
+}
+function normalizeScalingRoadmap(raw) {
+  const defaultAuto = DEFAULT_CONFIG.scalingRoadmap.rerankerAutoTrigger;
+  const userRoadmap = raw.scalingRoadmap ?? {};
+  const userAuto = userRoadmap.rerankerAutoTrigger ?? {};
+  if (userAuto.enabled === void 0 && raw.rerankerEnabled !== void 0) {
+    userAuto.enabled = raw.rerankerEnabled;
+  }
+  raw.scalingRoadmap = {
+    ...userRoadmap,
+    rerankerAutoTrigger: { ...defaultAuto, ...userAuto }
+  };
+}
+function normalizeSessionLifecycle(raw) {
+  const defaultSL = DEFAULT_CONFIG.sessionLifecycle;
+  const userSL = raw.sessionLifecycle ?? {};
+  raw.sessionLifecycle = { ...defaultSL, ...userSL };
+}
+async function loadConfig() {
+  const dir = process.env.EXE_OS_DIR ?? process.env.EXE_MEM_DIR ?? EXE_AI_DIR;
+  await mkdir(dir, { recursive: true });
+  const configPath = path.join(dir, "config.json");
+  if (!existsSync(configPath)) {
+    return { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db") };
+  }
+  const raw = await readFile(configPath, "utf-8");
+  try {
+    let parsed = JSON.parse(raw);
+    parsed = migrateLegacyConfig(parsed);
+    const { config: migratedCfg, migrated, fromVersion } = migrateConfig(parsed);
+    if (migrated) {
+      process.stderr.write(`[exe-os] Config migrated from v${fromVersion} to v${migratedCfg.config_version}
+`);
+      try {
+        await writeFile(configPath, JSON.stringify(migratedCfg, null, 2) + "\n");
+      } catch {
+      }
+    }
+    normalizeScalingRoadmap(migratedCfg);
+    normalizeSessionLifecycle(migratedCfg);
+    const config = { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db"), ...migratedCfg };
+    if (config.dbPath.startsWith("~")) {
+      config.dbPath = config.dbPath.replace(/^~/, os.homedir());
+    }
+    return config;
+  } catch {
+    return { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db") };
+  }
+}
+function loadConfigSync() {
+  const dir = process.env.EXE_OS_DIR ?? process.env.EXE_MEM_DIR ?? EXE_AI_DIR;
+  const configPath = path.join(dir, "config.json");
+  if (!existsSync(configPath)) {
+    return { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db") };
+  }
+  try {
+    const raw = readFileSync(configPath, "utf-8");
+    let parsed = JSON.parse(raw);
+    parsed = migrateLegacyConfig(parsed);
+    const { config: migratedCfg } = migrateConfig(parsed);
+    normalizeScalingRoadmap(migratedCfg);
+    normalizeSessionLifecycle(migratedCfg);
+    return { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db"), ...migratedCfg };
+  } catch {
+    return { ...DEFAULT_CONFIG, dbPath: path.join(dir, "memories.db") };
+  }
+}
+async function saveConfig(config) {
+  const dir = process.env.EXE_OS_DIR ?? process.env.EXE_MEM_DIR ?? EXE_AI_DIR;
+  await mkdir(dir, { recursive: true });
+  const configPath = path.join(dir, "config.json");
+  await writeFile(configPath, JSON.stringify(config, null, 2) + "\n");
+}
+async function loadConfigFrom(configPath) {
+  const raw = await readFile(configPath, "utf-8");
+  try {
+    let parsed = JSON.parse(raw);
+    parsed = migrateLegacyConfig(parsed);
+    const { config: migratedCfg } = migrateConfig(parsed);
+    normalizeScalingRoadmap(migratedCfg);
+    normalizeSessionLifecycle(migratedCfg);
+    return { ...DEFAULT_CONFIG, ...migratedCfg };
+  } catch {
+    return { ...DEFAULT_CONFIG };
+  }
+}
+var EXE_AI_DIR, DB_PATH, MODELS_DIR, CONFIG_PATH, LEGACY_LANCE_PATH, CURRENT_CONFIG_VERSION, DEFAULT_CONFIG, CONFIG_MIGRATIONS;
+var init_config = __esm({
+  "src/lib/config.ts"() {
+    "use strict";
+    EXE_AI_DIR = resolveDataDir();
+    DB_PATH = path.join(EXE_AI_DIR, "memories.db");
+    MODELS_DIR = path.join(EXE_AI_DIR, "models");
+    CONFIG_PATH = path.join(EXE_AI_DIR, "config.json");
+    LEGACY_LANCE_PATH = path.join(EXE_AI_DIR, "local.lance");
+    CURRENT_CONFIG_VERSION = 1;
+    DEFAULT_CONFIG = {
+      config_version: CURRENT_CONFIG_VERSION,
+      dbPath: DB_PATH,
+      modelFile: "jina-embeddings-v5-small-q4_k_m.gguf",
+      embeddingDim: 1024,
+      batchSize: 20,
+      flushIntervalMs: 1e4,
+      autoIngestion: true,
+      autoRetrieval: true,
+      searchMode: "hybrid",
+      hookSearchMode: "hybrid",
+      fileGrepEnabled: true,
+      splashEffect: true,
+      consolidationEnabled: true,
+      consolidationIntervalMs: 6 * 60 * 60 * 1e3,
+      consolidationModel: "claude-haiku-4-5-20251001",
+      consolidationMaxCallsPerRun: 20,
+      selfQueryRouter: true,
+      selfQueryModel: "claude-haiku-4-5-20251001",
+      rerankerEnabled: true,
+      scalingRoadmap: {
+        rerankerAutoTrigger: {
+          enabled: true,
+          broadQueryMinCardinality: 5e4,
+          fetchTopK: 150,
+          returnTopK: 5
+        }
+      },
+      graphRagEnabled: true,
+      wikiEnabled: false,
+      wikiUrl: "",
+      wikiApiKey: "",
+      wikiSyncIntervalMs: 30 * 60 * 1e3,
+      wikiWorkspaceMapping: {
+        exe: "Executive",
+        yoshi: "Engineering",
+        mari: "Marketing",
+        tom: "Engineering",
+        sasha: "Production"
+      },
+      wikiAutoUpdate: true,
+      wikiAutoUpdateThreshold: 0.5,
+      wikiAutoUpdateCreateNew: true,
+      skillLearning: true,
+      skillThreshold: 3,
+      skillModel: "claude-haiku-4-5-20251001",
+      exeHeartbeat: {
+        enabled: true,
+        intervalSeconds: 60,
+        staleInProgressThresholdHours: 2
+      },
+      sessionLifecycle: {
+        idleKillEnabled: true,
+        idleKillTicksRequired: 3,
+        idleKillIntercomAckWindowMs: 1e4,
+        maxAutoInstances: 10
+      }
+    };
+    CONFIG_MIGRATIONS = [
+      {
+        from: 0,
+        to: 1,
+        migrate: (cfg) => {
+          cfg.config_version = 1;
+          return cfg;
+        }
+      }
+    ];
+  }
+});
+
+// src/types/memory.ts
+var EMBEDDING_DIM = 1024;
+
+// src/lib/exe-daemon-client.ts
+init_config();
+import net from "net";
+import { spawn } from "child_process";
+import { randomUUID } from "crypto";
+import { existsSync as existsSync2, unlinkSync, readFileSync as readFileSync2, openSync, closeSync, statSync } from "fs";
+import path2 from "path";
+import { fileURLToPath } from "url";
+var SOCKET_PATH = process.env.EXE_DAEMON_SOCK ?? process.env.EXE_EMBED_SOCK ?? path2.join(EXE_AI_DIR, "exed.sock");
+var PID_PATH = process.env.EXE_DAEMON_PID ?? process.env.EXE_EMBED_PID ?? path2.join(EXE_AI_DIR, "exed.pid");
+var SPAWN_LOCK_PATH = path2.join(EXE_AI_DIR, "exed-spawn.lock");
+var SPAWN_LOCK_STALE_MS = 3e4;
+var CONNECT_TIMEOUT_MS = 15e3;
+var REQUEST_TIMEOUT_MS = 3e4;
+var _socket = null;
+var _connected = false;
+var _buffer = "";
+var _requestCount = 0;
+var HEALTH_CHECK_INTERVAL = 100;
+var _pending = /* @__PURE__ */ new Map();
+function handleData(chunk) {
+  _buffer += chunk.toString();
+  let newlineIdx;
+  while ((newlineIdx = _buffer.indexOf("\n")) !== -1) {
+    const line = _buffer.slice(0, newlineIdx).trim();
+    _buffer = _buffer.slice(newlineIdx + 1);
+    if (!line) continue;
+    try {
+      const response = JSON.parse(line);
+      const entry = _pending.get(response.id);
+      if (entry) {
+        clearTimeout(entry.timer);
+        _pending.delete(response.id);
+        entry.resolve(response);
+      }
+    } catch {
+    }
+  }
+}
+function cleanupStaleFiles() {
+  if (existsSync2(PID_PATH)) {
+    try {
+      const pid = parseInt(readFileSync2(PID_PATH, "utf8").trim(), 10);
+      if (pid > 0) {
+        try {
+          process.kill(pid, 0);
+          return;
+        } catch {
+        }
+      }
+    } catch {
+    }
+    try {
+      unlinkSync(PID_PATH);
+    } catch {
+    }
+    try {
+      unlinkSync(SOCKET_PATH);
+    } catch {
+    }
+  }
+}
+function findPackageRoot() {
+  let dir = path2.dirname(fileURLToPath(import.meta.url));
+  const { root } = path2.parse(dir);
+  while (dir !== root) {
+    if (existsSync2(path2.join(dir, "package.json"))) return dir;
+    dir = path2.dirname(dir);
+  }
+  return null;
+}
+function spawnDaemon() {
+  const pkgRoot = findPackageRoot();
+  if (!pkgRoot) {
+    process.stderr.write("[exed-client] WARN: cannot find package root\n");
+    return;
+  }
+  const daemonPath = path2.join(pkgRoot, "dist", "lib", "exe-daemon.js");
+  if (!existsSync2(daemonPath)) {
+    process.stderr.write(`[exed-client] WARN: daemon script not found at ${daemonPath}
+`);
+    return;
+  }
+  const resolvedPath = daemonPath;
+  process.stderr.write(`[exed-client] Spawning daemon: ${resolvedPath}
+`);
+  const logPath = path2.join(path2.dirname(SOCKET_PATH), "exed.log");
+  let stderrFd = "ignore";
+  try {
+    stderrFd = openSync(logPath, "a");
+  } catch {
+  }
+  const child = spawn(process.execPath, [resolvedPath], {
+    detached: true,
+    stdio: ["ignore", "ignore", stderrFd],
+    env: {
+      ...process.env,
+      EXE_DAEMON_SOCK: SOCKET_PATH,
+      EXE_DAEMON_PID: PID_PATH
+    }
+  });
+  child.unref();
+  if (typeof stderrFd === "number") {
+    try {
+      closeSync(stderrFd);
+    } catch {
+    }
+  }
+}
+function acquireSpawnLock() {
+  try {
+    const fd = openSync(SPAWN_LOCK_PATH, "wx");
+    closeSync(fd);
+    return true;
+  } catch {
+    try {
+      const stat = statSync(SPAWN_LOCK_PATH);
+      if (Date.now() - stat.mtimeMs > SPAWN_LOCK_STALE_MS) {
+        try {
+          unlinkSync(SPAWN_LOCK_PATH);
+        } catch {
+        }
+        try {
+          const fd = openSync(SPAWN_LOCK_PATH, "wx");
+          closeSync(fd);
+          return true;
+        } catch {
+        }
+      }
+    } catch {
+    }
+    return false;
+  }
+}
+function releaseSpawnLock() {
+  try {
+    unlinkSync(SPAWN_LOCK_PATH);
+  } catch {
+  }
+}
+function connectToSocket() {
+  return new Promise((resolve) => {
+    if (_socket && _connected) {
+      resolve(true);
+      return;
+    }
+    const socket = net.createConnection({ path: SOCKET_PATH });
+    const connectTimeout = setTimeout(() => {
+      socket.destroy();
+      resolve(false);
+    }, 2e3);
+    socket.on("connect", () => {
+      clearTimeout(connectTimeout);
+      _socket = socket;
+      _connected = true;
+      _buffer = "";
+      socket.on("data", handleData);
+      socket.on("close", () => {
+        _connected = false;
+        _socket = null;
+        for (const [id, entry] of _pending) {
+          clearTimeout(entry.timer);
+          _pending.delete(id);
+          entry.resolve({ error: "Connection closed" });
+        }
+      });
+      socket.on("error", () => {
+        _connected = false;
+        _socket = null;
+      });
+      resolve(true);
+    });
+    socket.on("error", () => {
+      clearTimeout(connectTimeout);
+      resolve(false);
+    });
+  });
+}
+async function connectEmbedDaemon() {
+  if (_socket && _connected) return true;
+  if (await connectToSocket()) return true;
+  if (acquireSpawnLock()) {
+    try {
+      cleanupStaleFiles();
+      spawnDaemon();
+    } finally {
+      releaseSpawnLock();
+    }
+  }
+  const start = Date.now();
+  let delay = 100;
+  while (Date.now() - start < CONNECT_TIMEOUT_MS) {
+    await new Promise((r) => setTimeout(r, delay));
+    if (await connectToSocket()) return true;
+    delay = Math.min(delay * 2, 3e3);
+  }
+  return false;
+}
+function sendRequest(texts, priority) {
+  return new Promise((resolve) => {
+    if (!_socket || !_connected) {
+      resolve({ error: "Not connected" });
+      return;
+    }
+    const id = randomUUID();
+    const timer = setTimeout(() => {
+      _pending.delete(id);
+      resolve({ error: "Request timeout" });
+    }, REQUEST_TIMEOUT_MS);
+    _pending.set(id, { resolve, timer });
+    try {
+      _socket.write(JSON.stringify({ id, texts, priority }) + "\n");
+    } catch {
+      clearTimeout(timer);
+      _pending.delete(id);
+      resolve({ error: "Write failed" });
+    }
+  });
+}
+async function pingDaemon() {
+  if (!_socket || !_connected) return null;
+  return new Promise((resolve) => {
+    const id = randomUUID();
+    const timer = setTimeout(() => {
+      _pending.delete(id);
+      resolve(null);
+    }, 5e3);
+    _pending.set(id, {
+      resolve: (data) => {
+        if (data.health) {
+          resolve(data.health);
+        } else {
+          resolve(null);
+        }
+      },
+      timer
+    });
+    try {
+      _socket.write(JSON.stringify({ id, type: "health" }) + "\n");
+    } catch {
+      clearTimeout(timer);
+      _pending.delete(id);
+      resolve(null);
+    }
+  });
+}
+function killAndRespawnDaemon() {
+  process.stderr.write("[exed-client] Killing daemon for restart...\n");
+  if (existsSync2(PID_PATH)) {
+    try {
+      const pid = parseInt(readFileSync2(PID_PATH, "utf8").trim(), 10);
+      if (pid > 0) {
+        try {
+          process.kill(pid, "SIGKILL");
+        } catch {
+        }
+      }
+    } catch {
+    }
+  }
+  if (_socket) {
+    _socket.destroy();
+    _socket = null;
+  }
+  _connected = false;
+  _buffer = "";
+  try {
+    unlinkSync(PID_PATH);
+  } catch {
+  }
+  try {
+    unlinkSync(SOCKET_PATH);
+  } catch {
+  }
+  spawnDaemon();
+}
+async function embedViaClient(text, priority = "high") {
+  if (!_connected && !await connectEmbedDaemon()) return null;
+  _requestCount++;
+  if (_requestCount % HEALTH_CHECK_INTERVAL === 0) {
+    const health = await pingDaemon();
+    if (!health) {
+      process.stderr.write(`[exed-client] Periodic health check failed at request ${_requestCount} \u2014 restarting daemon
+`);
+      killAndRespawnDaemon();
+      const start = Date.now();
+      let delay = 200;
+      while (Date.now() - start < CONNECT_TIMEOUT_MS) {
+        await new Promise((r) => setTimeout(r, delay));
+        if (await connectToSocket()) break;
+        delay = Math.min(delay * 2, 3e3);
+      }
+      if (!_connected) return null;
+    }
+  }
+  const result = await sendRequest([text], priority);
+  if (!result.error && result.vectors?.[0]) return result.vectors[0];
+  if (result.error) {
+    process.stderr.write(`[exed-client] Embed failed (${result.error}) \u2014 attempting restart
+`);
+    killAndRespawnDaemon();
+    const start = Date.now();
+    let delay = 200;
+    while (Date.now() - start < CONNECT_TIMEOUT_MS) {
+      await new Promise((r) => setTimeout(r, delay));
+      if (await connectToSocket()) break;
+      delay = Math.min(delay * 2, 3e3);
+    }
+    if (!_connected) return null;
+    const retry = await sendRequest([text], priority);
+    if (!retry.error && retry.vectors?.[0]) return retry.vectors[0];
+    process.stderr.write(`[exed-client] Embed retry also failed: ${retry.error ?? "no vector"}
+`);
+  }
+  return null;
+}
+function disconnectClient() {
+  if (_socket) {
+    _socket.destroy();
+    _socket = null;
+  }
+  _connected = false;
+  _buffer = "";
+  for (const [id, entry] of _pending) {
+    clearTimeout(entry.timer);
+    _pending.delete(id);
+    entry.resolve({ error: "Client disconnected" });
+  }
+}
+
+// src/lib/embedder.ts
+async function getEmbedder() {
+  const ok = await connectEmbedDaemon();
+  if (!ok) {
+    throw new Error(
+      "Could not connect to embedding daemon. Ensure the model is installed (run /exe-setup)."
+    );
+  }
+}
+async function embed(text) {
+  const priority = process.env.EXE_EMBED_PRIORITY === "low" ? "low" : "high";
+  const vector = await embedViaClient(text, priority);
+  if (!vector) {
+    throw new Error(
+      "Embedding failed: daemon unavailable. Run /exe-setup to verify model installation."
+    );
+  }
+  if (vector.length !== EMBEDDING_DIM) {
+    throw new Error(
+      `Embedding dimension mismatch: expected ${EMBEDDING_DIM}, got ${vector.length}. Ensure the correct Jina v5-small Q4_K_M GGUF model is installed.`
+    );
+  }
+  return vector;
+}
+async function disposeEmbedder() {
+  disconnectClient();
+}
+async function embedDirect(text) {
+  const llamaCpp = await import("node-llama-cpp");
+  const { MODELS_DIR: MODELS_DIR2 } = await Promise.resolve().then(() => (init_config(), config_exports));
+  const { existsSync: existsSync3 } = await import("fs");
+  const path3 = await import("path");
+  const modelPath = path3.join(MODELS_DIR2, "jina-embeddings-v5-small-q4_k_m.gguf");
+  if (!existsSync3(modelPath)) {
+    throw new Error(`Embedding model not found at ${modelPath}. Run '/exe-setup' to download it.`);
+  }
+  const llama = await llamaCpp.getLlama();
+  const model = await llama.loadModel({ modelPath });
+  const context = await model.createEmbeddingContext();
+  try {
+    const embedding = await context.getEmbeddingFor(text);
+    const vector = Array.from(embedding.vector);
+    if (vector.length !== EMBEDDING_DIM) {
+      throw new Error(
+        `Embedding dimension mismatch: expected ${EMBEDDING_DIM}, got ${vector.length}.`
+      );
+    }
+    return vector;
+  } finally {
+    await context.dispose();
+    await model.dispose();
+  }
+}
+export {
+  disposeEmbedder,
+  embed,
+  embedDirect,
+  getEmbedder
+};