@askexenow/exe-os 0.9.67 → 0.9.68

package/dist/bin/age-ontology-load.js

@@ -2,6 +2,255 @@
 
 // src/bin/age-ontology-load.ts
 import { Client } from "pg";
+
+// src/lib/background-jobs.ts
+import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync, unlinkSync } from "fs";
+import { execFileSync } from "child_process";
+import os2 from "os";
+import path2 from "path";
+
+// src/lib/config.ts
+import { readFile, writeFile } from "fs/promises";
+import { readFileSync, existsSync as existsSync2, renameSync } from "fs";
+import path from "path";
+import os from "os";
+
+// src/lib/secure-files.ts
+import { chmodSync, existsSync, mkdirSync } from "fs";
+import { chmod, mkdir } from "fs/promises";
+
+// src/lib/config.ts
+function resolveDataDir() {
+  if (process.env.EXE_OS_DIR) return process.env.EXE_OS_DIR;
+  if (process.env.EXE_MEM_DIR) return process.env.EXE_MEM_DIR;
+  const newDir = path.join(os.homedir(), ".exe-os");
+  const legacyDir = path.join(os.homedir(), ".exe-mem");
+  if (!existsSync2(newDir) && existsSync2(legacyDir)) {
+    try {
+      renameSync(legacyDir, newDir);
+      process.stderr.write(`[exe-os] Migrated data directory: ~/.exe-mem \u2192 ~/.exe-os
+`);
+    } catch {
+      return legacyDir;
+    }
+  }
+  return newDir;
+}
+var EXE_AI_DIR = resolveDataDir();
+var DB_PATH = path.join(EXE_AI_DIR, "memories.db");
+var MODELS_DIR = path.join(EXE_AI_DIR, "models");
+var CONFIG_PATH = path.join(EXE_AI_DIR, "config.json");
+var LEGACY_LANCE_PATH = path.join(EXE_AI_DIR, "local.lance");
+var CURRENT_CONFIG_VERSION = 1;
+var DEFAULT_CONFIG = {
+  config_version: CURRENT_CONFIG_VERSION,
+  dbPath: DB_PATH,
+  modelFile: "jina-embeddings-v5-small-q4_k_m.gguf",
+  embeddingDim: 1024,
+  batchSize: 20,
+  flushIntervalMs: 1e4,
+  autoIngestion: true,
+  autoRetrieval: true,
+  searchMode: "hybrid",
+  hookSearchMode: "hybrid",
+  fileGrepEnabled: true,
+  splashEffect: true,
+  consolidationEnabled: true,
+  consolidationIntervalMs: 6 * 60 * 60 * 1e3,
+  consolidationModel: "claude-haiku-4-5-20251001",
+  consolidationMaxCallsPerRun: 20,
+  selfQueryRouter: true,
+  selfQueryModel: "claude-haiku-4-5-20251001",
+  rerankerEnabled: true,
+  scalingRoadmap: {
+    rerankerAutoTrigger: {
+      enabled: true,
+      broadQueryMinCardinality: 5e4,
+      fetchTopK: 200,
+      returnTopK: 20
+    }
+  },
+  graphRagEnabled: true,
+  wikiEnabled: false,
+  wikiUrl: "",
+  wikiApiKey: "",
+  wikiSyncIntervalMs: 30 * 60 * 1e3,
+  wikiWorkspaceMapping: {},
+  wikiAutoUpdate: true,
+  wikiAutoUpdateThreshold: 0.5,
+  wikiAutoUpdateCreateNew: true,
+  skillLearning: true,
+  skillThreshold: 3,
+  skillModel: "claude-haiku-4-5-20251001",
+  exeHeartbeat: {
+    enabled: true,
+    intervalSeconds: 60,
+    staleInProgressThresholdHours: 2
+  },
+  sessionLifecycle: {
+    idleKillEnabled: true,
+    idleKillTicksRequired: 3,
+    idleKillIntercomAckWindowMs: 1e4,
+    maxAutoInstances: 10
+  },
+  autoUpdate: {
+    checkOnBoot: true,
+    autoInstall: false,
+    checkIntervalMs: 24 * 60 * 60 * 1e3
+  },
+  orchestration: {
+    phase: "phase_1_coo",
+    phaseSetBy: "default"
+  }
+};
+
+// src/lib/background-jobs.ts
+var JOB_DIR = path2.join(EXE_AI_DIR, "jobs");
+var JOBS_FILE = path2.join(JOB_DIR, "jobs.json");
+var LOCK_DIR = path2.join(JOB_DIR, "locks");
+var DEFAULT_LOCK_TTL_MS = 6 * 60 * 60 * 1e3;
+var MAX_HISTORY = 200;
+function ensureDirs() {
+  mkdirSync2(LOCK_DIR, { recursive: true });
+}
+function now() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function isAlive(pid) {
+  if (!pid || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function readJobsRaw() {
+  ensureDirs();
+  if (!existsSync3(JOBS_FILE)) return [];
+  try {
+    const parsed = JSON.parse(readFileSync2(JOBS_FILE, "utf8"));
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+function writeJobsRaw(jobs) {
+  ensureDirs();
+  const running = jobs.filter((j) => j.status === "running");
+  const rest = jobs.filter((j) => j.status !== "running").slice(-MAX_HISTORY);
+  writeFileSync(JOBS_FILE, JSON.stringify([...rest, ...running], null, 2) + "\n");
+}
+function lockPath(type) {
+  return path2.join(LOCK_DIR, `${type.replace(/[^a-zA-Z0-9_.-]/g, "_")}.lock`);
+}
+function acquireJobLock(type, ttlMs = DEFAULT_LOCK_TTL_MS) {
+  ensureDirs();
+  const file = lockPath(type);
+  if (existsSync3(file)) {
+    try {
+      const lock = JSON.parse(readFileSync2(file, "utf8"));
+      const age = Date.now() - Date.parse(lock.updatedAt ?? "");
+      if (lock.pid && isAlive(lock.pid) && Number.isFinite(age) && age < ttlMs) return false;
+    } catch {
+    }
+    try {
+      unlinkSync(file);
+    } catch {
+    }
+  }
+  try {
+    writeFileSync(file, JSON.stringify({ pid: process.pid, updatedAt: now() }, null, 2) + "\n", { flag: "wx" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function releaseJobLock(type) {
+  const file = lockPath(type);
+  try {
+    if (!existsSync3(file)) return;
+    const lock = JSON.parse(readFileSync2(file, "utf8"));
+    if (lock.pid === process.pid || !lock.pid || !isAlive(lock.pid)) unlinkSync(file);
+  } catch {
+    try {
+      unlinkSync(file);
+    } catch {
+    }
+  }
+}
+function startManagedJob(options) {
+  const lowPriority = options.lowPriority ?? true;
+  if (!acquireJobLock(options.type, options.lockTtlMs)) return null;
+  if (lowPriority) {
+    try {
+      os2.setPriority(process.pid, 10);
+    } catch {
+    }
+  }
+  const id = `${options.type}-${Date.now()}-${process.pid}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
+  const record = {
+    id,
+    type: options.type,
+    name: options.name,
+    pid: process.pid,
+    command: options.command ?? process.argv.join(" "),
+    cwd: process.cwd(),
+    status: "running",
+    startedAt: now(),
+    updatedAt: now(),
+    lastHeartbeatAt: now(),
+    cancelCommand: `exe-os jobs cancel ${id}`,
+    lowPriority
+  };
+  const upsert = (patch) => {
+    const jobs = readJobsRaw().filter((j) => j.id !== id);
+    Object.assign(record, patch, { updatedAt: now() });
+    writeJobsRaw([...jobs, record]);
+    const file = lockPath(options.type);
+    try {
+      writeFileSync(file, JSON.stringify({ pid: process.pid, jobId: id, updatedAt: record.updatedAt }, null, 2) + "\n");
+    } catch {
+    }
+  };
+  upsert({});
+  const timer = setInterval(() => upsert({ lastHeartbeatAt: now() }), 3e4);
+  timer.unref?.();
+  const cleanup = (status, error) => {
+    clearInterval(timer);
+    upsert({ status, error, lastHeartbeatAt: now() });
+    releaseJobLock(options.type);
+  };
+  process.once("SIGTERM", () => {
+    cleanup("cancelled");
+    process.exit(0);
+  });
+  process.once("SIGINT", () => {
+    cleanup("cancelled");
+    process.exit(130);
+  });
+  process.once("exit", () => releaseJobLock(options.type));
+  return {
+    id,
+    update(progress) {
+      upsert({ progressCurrent: progress.current, progressTotal: progress.total, progressLabel: progress.label, lastHeartbeatAt: now() });
+    },
+    complete() {
+      cleanup("completed");
+    },
+    fail(err) {
+      cleanup("failed", err instanceof Error ? err.message : String(err));
+    },
+    cancel() {
+      cleanup("cancelled");
+    }
+  };
+}
+async function politeBatchPause(ms = 250) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/bin/age-ontology-load.ts
 function q(value) {
   return `'${String(value ?? "").replace(/\u0000/g, "").slice(0, 500).replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
 }
@@ -9,6 +258,11 @@ function sqlString(value) {
   return `'${value.replace(/'/g, "''")}'`;
 }
 async function main() {
+  const job = startManagedJob({ type: "age-ontology-load", name: "Apache AGE ontology loader", lowPriority: true });
+  if (!job) {
+    process.stderr.write("[age-ontology-load] Another AGE ontology load is already running.\n");
+    return;
+  }
   const sourceUrl = process.env.DATABASE_URL || process.env.EXED_DATABASE_URL;
   const ageUrl = process.env.AGE_DATABASE_URL;
   if (!sourceUrl) throw new Error("DATABASE_URL or EXED_DATABASE_URL is required for canonical source");
@@ -39,6 +293,10 @@ async function main() {
       const cypher = `CREATE (:Entity {id: ${q(entity.id)}, name: ${q(entity.name)}, type: ${q(entity.type)}})`;
       await age.query(`SELECT * FROM cypher(${sqlString(graph)}, $$ ${cypher} $$) AS (v agtype)`);
       nodeCount++;
+      if (nodeCount % 250 === 0) {
+        job.update({ current: nodeCount, total: limit, label: `Loaded ${nodeCount} AGE nodes` });
+        await politeBatchPause(250);
+      }
     }
     const relationships = await source.query(`SELECT source_entity_id, target_entity_id, type, confidence FROM memory.relationships ORDER BY id LIMIT $1`, [limit * 2]);
     let edgeCount = 0;
@@ -46,9 +304,14 @@ async function main() {
       const cypher = `MATCH (a:Entity {id: ${q(rel.source_entity_id)}}), (b:Entity {id: ${q(rel.target_entity_id)}}) CREATE (a)-[:RELATED {type: ${q(rel.type)}}]->(b)`;
       await age.query(`SELECT * FROM cypher(${sqlString(graph)}, $$ ${cypher} $$) AS (e agtype)`);
       edgeCount++;
+      if (edgeCount % 500 === 0) {
+        job.update({ current: nodeCount + edgeCount, total: limit * 3, label: `Loaded ${nodeCount} nodes, ${edgeCount} edges` });
+        await politeBatchPause(250);
+      }
     }
     process.stderr.write(`[age-ontology-load] Loaded ${nodeCount} nodes and ${edgeCount} edges into ${graph}.
 `);
+    job.complete();
   } finally {
     await age.end();
     await source.end();

package/dist/bin/agentic-ontology-backfill.js

@@ -3202,10 +3202,10 @@ function evictLRU() {
   }
 }
 function evictIdleShards() {
-  const now = Date.now();
+  const now2 = Date.now();
   const toEvict = [];
   for (const [name, lastAccess] of _shardLastAccess) {
-    if (now - lastAccess > SHARD_IDLE_MS) {
+    if (now2 - lastAccess > SHARD_IDLE_MS) {
       toEvict.push(name);
     }
   }
@@ -3485,22 +3485,22 @@ ${sections.join("\n\n")}
 }
 async function storeGlobalProcedure(input) {
   const id = randomUUID2();
-  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const now2 = (/* @__PURE__ */ new Date()).toISOString();
   const client = getClient();
   await client.execute({
     sql: `INSERT INTO company_procedures (id, title, content, priority, domain, active, created_at, updated_at)
           VALUES (?, ?, ?, ?, ?, 1, ?, ?)`,
-    args: [id, input.title, input.content, input.priority ?? "p0", input.domain ?? null, now, now]
+    args: [id, input.title, input.content, input.priority ?? "p0", input.domain ?? null, now2, now2]
   });
   await loadGlobalProcedures();
   return id;
 }
 async function deactivateGlobalProcedure(id) {
-  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const now2 = (/* @__PURE__ */ new Date()).toISOString();
   const client = getClient();
   const result = await client.execute({
     sql: "UPDATE company_procedures SET active = 0, updated_at = ? WHERE id = ?",
-    args: [now, id]
+    args: [now2, id]
   });
   await loadGlobalProcedures();
   return result.rowsAffected > 0;
@@ -3593,7 +3593,7 @@ function extractMemoryCards(row) {
 async function insertMemoryCardsForBatch(rows) {
   const cards = rows.flatMap(extractMemoryCards);
   if (cards.length === 0) return 0;
-  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const now2 = (/* @__PURE__ */ new Date()).toISOString();
   const client = getClient();
   const stmts = cards.map((card) => ({
     sql: `INSERT OR IGNORE INTO memory_cards
@@ -3614,7 +3614,7 @@ async function insertMemoryCardsForBatch(rows) {
       card.content,
       card.source_ref,
       card.confidence,
-      now
+      now2
     ]
   }));
   await client.batch(stmts, "write");
@@ -3851,7 +3851,7 @@ async function insertOntologyForMemory(row, client) {
   const intention = inferIntention(row);
   const outcome = inferOutcome(row);
   const eventId = stableId2("event", row.id);
-  const now = (/* @__PURE__ */ new Date()).toISOString();
+  const now2 = (/* @__PURE__ */ new Date()).toISOString();
   await db.execute({
     sql: `INSERT INTO agent_sessions (id, agent_id, project_name, started_at, last_event_at, event_count, properties)
           VALUES (?, ?, ?, ?, ?, 1, ?)
@@ -3880,7 +3880,7 @@ async function insertOntologyForMemory(row, client) {
       row.id,
       row.has_error ? "negative" : outcome === "success_signal" ? "positive" : "neutral",
       JSON.stringify(ontologyPayload(row)),
-      now
+      now2
     ]
   });
   const semantic = inferSemanticLabel(row);
@@ -3898,8 +3898,8 @@ async function insertOntologyForMemory(row, client) {
       semantic.schemaVersion,
       semantic.confidence,
       JSON.stringify(semantic),
-      now,
-      now
+      now2,
+      now2
     ]
   });
   for (const statement of extractGoalCandidates(row)) {
@@ -3910,19 +3910,19 @@ async function insertOntologyForMemory(row, client) {
              parent_goal_id, due_at, achieved_at, supersedes_id, created_at, updated_at, source_memory_id)
             VALUES (?, ?, ?, ?, 'open', 5, NULL, NULL, NULL, NULL, NULL, ?, ?, ?)
             ON CONFLICT(id) DO UPDATE SET updated_at = excluded.updated_at`,
-      args: [goalId, statement, row.agent_id, row.project_name, now, now, row.id]
+      args: [goalId, statement, row.agent_id, row.project_name, now2, now2, row.id]
     });
     await db.execute({
       sql: `INSERT OR IGNORE INTO agent_goal_links
             (id, goal_id, link_type, target_id, target_type, created_at)
             VALUES (?, ?, 'evidence', ?, 'memory', ?)`,
-      args: [stableId2("goal_link", goalId, row.id, "memory"), goalId, row.id, now]
+      args: [stableId2("goal_link", goalId, row.id, "memory"), goalId, row.id, now2]
     });
     await db.execute({
       sql: `INSERT OR IGNORE INTO agent_goal_links
             (id, goal_id, link_type, target_id, target_type, created_at)
             VALUES (?, ?, 'event', ?, 'event', ?)`,
-      args: [stableId2("goal_link", goalId, eventId, "event"), goalId, eventId, now]
+      args: [stableId2("goal_link", goalId, eventId, "event"), goalId, eventId, now2]
     });
   }
 }
@@ -4106,8 +4106,8 @@ function deriveMachineKey() {
 }
 function readMachineId() {
   try {
-    const { readFileSync: readFileSync5 } = __require("fs");
-    return readFileSync5("/etc/machine-id", "utf-8").trim();
+    const { readFileSync: readFileSync6 } = __require("fs");
+    return readFileSync6("/etc/machine-id", "utf-8").trim();
   } catch {
     return "";
   }
@@ -4657,8 +4657,166 @@ function vectorToBlob(vector) {
 // src/bin/agentic-ontology-backfill.ts
 init_database();
 init_agentic_ontology();
+
+// src/lib/background-jobs.ts
+init_config();
+import { existsSync as existsSync8, mkdirSync as mkdirSync3, readFileSync as readFileSync5, writeFileSync as writeFileSync3, unlinkSync as unlinkSync3 } from "fs";
+import { execFileSync } from "child_process";
+import os6 from "os";
+import path8 from "path";
+var JOB_DIR = path8.join(EXE_AI_DIR, "jobs");
+var JOBS_FILE = path8.join(JOB_DIR, "jobs.json");
+var LOCK_DIR = path8.join(JOB_DIR, "locks");
+var DEFAULT_LOCK_TTL_MS = 6 * 60 * 60 * 1e3;
+var MAX_HISTORY = 200;
+function ensureDirs() {
+  mkdirSync3(LOCK_DIR, { recursive: true });
+}
+function now() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function isAlive(pid) {
+  if (!pid || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function readJobsRaw() {
+  ensureDirs();
+  if (!existsSync8(JOBS_FILE)) return [];
+  try {
+    const parsed = JSON.parse(readFileSync5(JOBS_FILE, "utf8"));
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+function writeJobsRaw(jobs) {
+  ensureDirs();
+  const running = jobs.filter((j) => j.status === "running");
+  const rest = jobs.filter((j) => j.status !== "running").slice(-MAX_HISTORY);
+  writeFileSync3(JOBS_FILE, JSON.stringify([...rest, ...running], null, 2) + "\n");
+}
+function lockPath(type) {
+  return path8.join(LOCK_DIR, `${type.replace(/[^a-zA-Z0-9_.-]/g, "_")}.lock`);
+}
+function acquireJobLock(type, ttlMs = DEFAULT_LOCK_TTL_MS) {
+  ensureDirs();
+  const file = lockPath(type);
+  if (existsSync8(file)) {
+    try {
+      const lock = JSON.parse(readFileSync5(file, "utf8"));
+      const age = Date.now() - Date.parse(lock.updatedAt ?? "");
+      if (lock.pid && isAlive(lock.pid) && Number.isFinite(age) && age < ttlMs) return false;
+    } catch {
+    }
+    try {
+      unlinkSync3(file);
+    } catch {
+    }
+  }
+  try {
+    writeFileSync3(file, JSON.stringify({ pid: process.pid, updatedAt: now() }, null, 2) + "\n", { flag: "wx" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function releaseJobLock(type) {
+  const file = lockPath(type);
+  try {
+    if (!existsSync8(file)) return;
+    const lock = JSON.parse(readFileSync5(file, "utf8"));
+    if (lock.pid === process.pid || !lock.pid || !isAlive(lock.pid)) unlinkSync3(file);
+  } catch {
+    try {
+      unlinkSync3(file);
+    } catch {
+    }
+  }
+}
+function startManagedJob(options) {
+  const lowPriority = options.lowPriority ?? true;
+  if (!acquireJobLock(options.type, options.lockTtlMs)) return null;
+  if (lowPriority) {
+    try {
+      os6.setPriority(process.pid, 10);
+    } catch {
+    }
+  }
+  const id = `${options.type}-${Date.now()}-${process.pid}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
+  const record = {
+    id,
+    type: options.type,
+    name: options.name,
+    pid: process.pid,
+    command: options.command ?? process.argv.join(" "),
+    cwd: process.cwd(),
+    status: "running",
+    startedAt: now(),
+    updatedAt: now(),
+    lastHeartbeatAt: now(),
+    cancelCommand: `exe-os jobs cancel ${id}`,
+    lowPriority
+  };
+  const upsert = (patch) => {
+    const jobs = readJobsRaw().filter((j) => j.id !== id);
+    Object.assign(record, patch, { updatedAt: now() });
+    writeJobsRaw([...jobs, record]);
+    const file = lockPath(options.type);
+    try {
+      writeFileSync3(file, JSON.stringify({ pid: process.pid, jobId: id, updatedAt: record.updatedAt }, null, 2) + "\n");
+    } catch {
+    }
+  };
+  upsert({});
+  const timer = setInterval(() => upsert({ lastHeartbeatAt: now() }), 3e4);
+  timer.unref?.();
+  const cleanup = (status, error) => {
+    clearInterval(timer);
+    upsert({ status, error, lastHeartbeatAt: now() });
+    releaseJobLock(options.type);
+  };
+  process.once("SIGTERM", () => {
+    cleanup("cancelled");
+    process.exit(0);
+  });
+  process.once("SIGINT", () => {
+    cleanup("cancelled");
+    process.exit(130);
+  });
+  process.once("exit", () => releaseJobLock(options.type));
+  return {
+    id,
+    update(progress) {
+      upsert({ progressCurrent: progress.current, progressTotal: progress.total, progressLabel: progress.label, lastHeartbeatAt: now() });
+    },
+    complete() {
+      cleanup("completed");
+    },
+    fail(err) {
+      cleanup("failed", err instanceof Error ? err.message : String(err));
+    },
+    cancel() {
+      cleanup("cancelled");
+    }
+  };
+}
+async function politeBatchPause(ms = 250) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// src/bin/agentic-ontology-backfill.ts
 var BATCH_SIZE = 500;
 async function main() {
+  const job = startManagedJob({ type: "agentic-ontology-backfill", name: "Agentic ontology backfill", lowPriority: true });
+  if (!job) {
+    process.stderr.write("[agentic-ontology-backfill] Another ontology backfill is already running.\n");
+    return;
+  }
   await initStore({ lightweight: true });
   const client = getClient();
   let offset = 0;
@@ -4695,10 +4853,13 @@ async function main() {
     total += inserted;
     process.stderr.write(`[agentic-ontology-backfill] +${inserted} memories projected (${total} total)
 `);
+    job.update({ current: total, label: `Projected ${total} memories` });
+    await politeBatchPause(250);
     offset = 0;
   }
   process.stderr.write(`[agentic-ontology-backfill] Complete: ${total} memories projected.
 `);
+  job.complete();
   await disposeStore();
 }
 main().catch((err) => {