@askexenow/exe-os 0.9.67 → 0.9.69

package/dist/bin/postgres-agentic-semantic-backfill.js

@@ -1,4 +1,113 @@
 #!/usr/bin/env node
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+
+// src/lib/secure-files.ts
+import { chmodSync, existsSync, mkdirSync } from "fs";
+import { chmod, mkdir } from "fs/promises";
+var init_secure_files = __esm({
+  "src/lib/secure-files.ts"() {
+    "use strict";
+  }
+});
+
+// src/lib/config.ts
+import { readFile, writeFile } from "fs/promises";
+import { readFileSync, existsSync as existsSync2, renameSync } from "fs";
+import path from "path";
+import os from "os";
+function resolveDataDir() {
+  if (process.env.EXE_OS_DIR) return process.env.EXE_OS_DIR;
+  if (process.env.EXE_MEM_DIR) return process.env.EXE_MEM_DIR;
+  const newDir = path.join(os.homedir(), ".exe-os");
+  const legacyDir = path.join(os.homedir(), ".exe-mem");
+  if (!existsSync2(newDir) && existsSync2(legacyDir)) {
+    try {
+      renameSync(legacyDir, newDir);
+      process.stderr.write(`[exe-os] Migrated data directory: ~/.exe-mem \u2192 ~/.exe-os
+`);
+    } catch {
+      return legacyDir;
+    }
+  }
+  return newDir;
+}
+var EXE_AI_DIR, DB_PATH, MODELS_DIR, CONFIG_PATH, LEGACY_LANCE_PATH, CURRENT_CONFIG_VERSION, DEFAULT_CONFIG;
+var init_config = __esm({
+  "src/lib/config.ts"() {
+    "use strict";
+    init_secure_files();
+    EXE_AI_DIR = resolveDataDir();
+    DB_PATH = path.join(EXE_AI_DIR, "memories.db");
+    MODELS_DIR = path.join(EXE_AI_DIR, "models");
+    CONFIG_PATH = path.join(EXE_AI_DIR, "config.json");
+    LEGACY_LANCE_PATH = path.join(EXE_AI_DIR, "local.lance");
+    CURRENT_CONFIG_VERSION = 1;
+    DEFAULT_CONFIG = {
+      config_version: CURRENT_CONFIG_VERSION,
+      dbPath: DB_PATH,
+      modelFile: "jina-embeddings-v5-small-q4_k_m.gguf",
+      embeddingDim: 1024,
+      batchSize: 20,
+      flushIntervalMs: 1e4,
+      autoIngestion: true,
+      autoRetrieval: true,
+      searchMode: "hybrid",
+      hookSearchMode: "hybrid",
+      fileGrepEnabled: true,
+      splashEffect: true,
+      consolidationEnabled: true,
+      consolidationIntervalMs: 6 * 60 * 60 * 1e3,
+      consolidationModel: "claude-haiku-4-5-20251001",
+      consolidationMaxCallsPerRun: 20,
+      selfQueryRouter: true,
+      selfQueryModel: "claude-haiku-4-5-20251001",
+      rerankerEnabled: true,
+      scalingRoadmap: {
+        rerankerAutoTrigger: {
+          enabled: true,
+          broadQueryMinCardinality: 5e4,
+          fetchTopK: 200,
+          returnTopK: 20
+        }
+      },
+      graphRagEnabled: true,
+      wikiEnabled: false,
+      wikiUrl: "",
+      wikiApiKey: "",
+      wikiSyncIntervalMs: 30 * 60 * 1e3,
+      wikiWorkspaceMapping: {},
+      wikiAutoUpdate: true,
+      wikiAutoUpdateThreshold: 0.5,
+      wikiAutoUpdateCreateNew: true,
+      skillLearning: true,
+      skillThreshold: 3,
+      skillModel: "claude-haiku-4-5-20251001",
+      exeHeartbeat: {
+        enabled: true,
+        intervalSeconds: 60,
+        staleInProgressThresholdHours: 2
+      },
+      sessionLifecycle: {
+        idleKillEnabled: true,
+        idleKillTicksRequired: 3,
+        idleKillIntercomAckWindowMs: 1e4,
+        maxAutoInstances: 10
+      },
+      autoUpdate: {
+        checkOnBoot: true,
+        autoInstall: false,
+        checkIntervalMs: 24 * 60 * 60 * 1e3
+      },
+      orchestration: {
+        phase: "phase_1_coo",
+        phaseSetBy: "default"
+      }
+    };
+  }
+});
 
 // src/bin/postgres-agentic-semantic-backfill.ts
 import { Client } from "pg";
@@ -136,6 +245,157 @@ function inferSemanticLabel(row) {
   };
 }
 
+// src/lib/background-jobs.ts
+init_config();
+import { existsSync as existsSync3, mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync, unlinkSync } from "fs";
+import { execFileSync } from "child_process";
+import os2 from "os";
+import path2 from "path";
+var JOB_DIR = path2.join(EXE_AI_DIR, "jobs");
+var JOBS_FILE = path2.join(JOB_DIR, "jobs.json");
+var LOCK_DIR = path2.join(JOB_DIR, "locks");
+var DEFAULT_LOCK_TTL_MS = 6 * 60 * 60 * 1e3;
+var MAX_HISTORY = 200;
+function ensureDirs() {
+  mkdirSync2(LOCK_DIR, { recursive: true });
+}
+function now() {
+  return (/* @__PURE__ */ new Date()).toISOString();
+}
+function isAlive(pid) {
+  if (!pid || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function readJobsRaw() {
+  ensureDirs();
+  if (!existsSync3(JOBS_FILE)) return [];
+  try {
+    const parsed = JSON.parse(readFileSync2(JOBS_FILE, "utf8"));
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+function writeJobsRaw(jobs) {
+  ensureDirs();
+  const running = jobs.filter((j) => j.status === "running");
+  const rest = jobs.filter((j) => j.status !== "running").slice(-MAX_HISTORY);
+  writeFileSync(JOBS_FILE, JSON.stringify([...rest, ...running], null, 2) + "\n");
+}
+function lockPath(type) {
+  return path2.join(LOCK_DIR, `${type.replace(/[^a-zA-Z0-9_.-]/g, "_")}.lock`);
+}
+function acquireJobLock(type, ttlMs = DEFAULT_LOCK_TTL_MS) {
+  ensureDirs();
+  const file = lockPath(type);
+  if (existsSync3(file)) {
+    try {
+      const lock = JSON.parse(readFileSync2(file, "utf8"));
+      const age = Date.now() - Date.parse(lock.updatedAt ?? "");
+      if (lock.pid && isAlive(lock.pid) && Number.isFinite(age) && age < ttlMs) return false;
+    } catch {
+    }
+    try {
+      unlinkSync(file);
+    } catch {
+    }
+  }
+  try {
+    writeFileSync(file, JSON.stringify({ pid: process.pid, updatedAt: now() }, null, 2) + "\n", { flag: "wx" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+function releaseJobLock(type) {
+  const file = lockPath(type);
+  try {
+    if (!existsSync3(file)) return;
+    const lock = JSON.parse(readFileSync2(file, "utf8"));
+    if (lock.pid === process.pid || !lock.pid || !isAlive(lock.pid)) unlinkSync(file);
+  } catch {
+    try {
+      unlinkSync(file);
+    } catch {
+    }
+  }
+}
+function startManagedJob(options) {
+  const lowPriority = options.lowPriority ?? true;
+  if (!acquireJobLock(options.type, options.lockTtlMs)) return null;
+  if (lowPriority) {
+    try {
+      os2.setPriority(process.pid, 10);
+    } catch {
+    }
+  }
+  const id = `${options.type}-${Date.now()}-${process.pid}`.replace(/[^a-zA-Z0-9_.-]/g, "_");
+  const record = {
+    id,
+    type: options.type,
+    name: options.name,
+    pid: process.pid,
+    command: options.command ?? process.argv.join(" "),
+    cwd: process.cwd(),
+    status: "running",
+    startedAt: now(),
+    updatedAt: now(),
+    lastHeartbeatAt: now(),
+    cancelCommand: `exe-os jobs cancel ${id}`,
+    lowPriority
+  };
+  const upsert = (patch) => {
+    const jobs = readJobsRaw().filter((j) => j.id !== id);
+    Object.assign(record, patch, { updatedAt: now() });
+    writeJobsRaw([...jobs, record]);
+    const file = lockPath(options.type);
+    try {
+      writeFileSync(file, JSON.stringify({ pid: process.pid, jobId: id, updatedAt: record.updatedAt }, null, 2) + "\n");
+    } catch {
+    }
+  };
+  upsert({});
+  const timer = setInterval(() => upsert({ lastHeartbeatAt: now() }), 3e4);
+  timer.unref?.();
+  const cleanup = (status, error) => {
+    clearInterval(timer);
+    upsert({ status, error, lastHeartbeatAt: now() });
+    releaseJobLock(options.type);
+  };
+  process.once("SIGTERM", () => {
+    cleanup("cancelled");
+    process.exit(0);
+  });
+  process.once("SIGINT", () => {
+    cleanup("cancelled");
+    process.exit(130);
+  });
+  process.once("exit", () => releaseJobLock(options.type));
+  return {
+    id,
+    update(progress) {
+      upsert({ progressCurrent: progress.current, progressTotal: progress.total, progressLabel: progress.label, lastHeartbeatAt: now() });
+    },
+    complete() {
+      cleanup("completed");
+    },
+    fail(err) {
+      cleanup("failed", err instanceof Error ? err.message : String(err));
+    },
+    cancel() {
+      cleanup("cancelled");
+    }
+  };
+}
+async function politeBatchPause(ms = 250) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 // src/bin/postgres-agentic-semantic-backfill.ts
 function arg(name) {
   const i = process.argv.indexOf(name);
@@ -160,6 +420,11 @@ function scrubJson(value) {
   return JSON.stringify(value).replace(/\u0000/g, "").replace(/\\u0000/gi, "").replace(/\\\\u0000/gi, "").replace(/\\u00[0-1][0-9a-f]/gi, " ");
 }
 async function main() {
+  const job = startManagedJob({ type: "postgres-agentic-semantic-backfill", name: "Postgres semantic label backfill", lowPriority: true });
+  if (!job) {
+    process.stderr.write("[postgres-agentic-semantic-backfill] Another Postgres semantic backfill is already running.\n");
+    return;
+  }
   const url = process.env.DATABASE_URL || process.env.EXED_DATABASE_URL;
   if (!url) throw new Error("DATABASE_URL or EXED_DATABASE_URL is required");
   const limit = Number(arg("--limit") ?? "20000");
@@ -221,11 +486,16 @@ async function main() {
         ]);
       }
       count++;
-      if (count % 5e3 === 0) process.stderr.write(`[postgres-agentic-semantic-backfill] labeled ${count}
+      if (count % 5e3 === 0) {
+        process.stderr.write(`[postgres-agentic-semantic-backfill] labeled ${count}
 `);
+        job.update({ current: count, total: limit, label: `Labeled ${count}/${limit}` });
+        await politeBatchPause(1e3);
+      }
     }
     process.stderr.write(`[postgres-agentic-semantic-backfill] Complete: ${count} semantic labels.
 `);
+    job.complete();
   } finally {
     await client.end();
   }