engrm 0.4.43 → 0.4.44

package/dist/cli.js

@@ -21,8 +21,8 @@ var __require = /* @__PURE__ */ createRequire(import.meta.url);
 import { existsSync as existsSync8, mkdirSync as mkdirSync4, readFileSync as readFileSync8, statSync } from "fs";
 import { hostname as hostname3, homedir as homedir5, networkInterfaces as networkInterfaces3 } from "os";
 import { dirname as dirname5, join as join8 } from "path";
-import { createHash as createHash4 } from "crypto";
-import { fileURLToPath as fileURLToPath4 } from "url";
+import { createHash as createHash5 } from "crypto";
+import { fileURLToPath as fileURLToPath4, pathToFileURL } from "url";
 
 // src/config.ts
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
@@ -880,6 +880,7 @@ function ensureObservationTypes(db) {
         DROP TABLE observations;
         ALTER TABLE observations_repair RENAME TO observations;
         CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id);
+        CREATE INDEX IF NOT EXISTS idx_observations_project_lifecycle ON observations(project_id, lifecycle);
         CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
         CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch);
         CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
@@ -1160,6 +1161,7 @@ class MemDatabase {
     this.db = openDatabase(dbPath);
     this.db.exec("PRAGMA journal_mode = WAL");
     this.db.exec("PRAGMA foreign_keys = ON");
+    this.db.exec("PRAGMA busy_timeout = 5000");
     this.vecAvailable = this.loadVecExtension();
     runMigrations(this.db);
     ensureObservationTypes(this.db);
@@ -1181,8 +1183,16 @@ class MemDatabase {
     this.db.close();
   }
   upsertProject(project) {
+    const canonicalId = project.canonical_id?.trim();
+    const name = project.name?.trim();
+    if (!canonicalId) {
+      throw new Error("Project canonical_id is required");
+    }
+    if (!name) {
+      throw new Error("Project name is required");
+    }
     const now = Math.floor(Date.now() / 1000);
-    const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(project.canonical_id);
+    const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(canonicalId);
     if (existing) {
       this.db.query(`UPDATE projects SET
             local_path = COALESCE(?, local_path),
@@ -1197,7 +1207,7 @@ class MemDatabase {
       };
     }
     const result = this.db.query(`INSERT INTO projects (canonical_id, name, local_path, remote_url, first_seen_epoch, last_active_epoch)
-        VALUES (?, ?, ?, ?, ?, ?)`).run(project.canonical_id, project.name, project.local_path ?? null, project.remote_url ?? null, now, now);
+        VALUES (?, ?, ?, ?, ?, ?)`).run(canonicalId, name, project.local_path ?? null, project.remote_url ?? null, now, now);
     return this.db.query("SELECT * FROM projects WHERE id = ?").get(Number(result.lastInsertRowid));
   }
   getProjectByCanonicalId(canonicalId) {
@@ -1890,6 +1900,33 @@ function getOutboxStats(db) {
   }
   return stats;
 }
+function getOutboxFailureSummaries(db, limit = 5) {
+  return db.db.query(`SELECT COALESCE(last_error, '') as error, COUNT(*) as count
+       FROM sync_outbox
+       WHERE status = 'failed'
+       GROUP BY COALESCE(last_error, '')
+       ORDER BY count DESC, error ASC
+       LIMIT ?`).all(limit).filter((row) => row.error.length > 0);
+}
+function classifyOutboxFailure(error) {
+  const normalized = error.toLowerCase();
+  if (normalized.includes("401") || normalized.includes("invalid or missing credentials")) {
+    return "auth";
+  }
+  if (normalized.includes("429") || normalized.includes("rate limit")) {
+    return "rate_limit";
+  }
+  if (normalized.includes("timeout") || normalized.includes("abort")) {
+    return "timeout";
+  }
+  if (normalized.includes("network") || normalized.includes("fetch") || normalized.includes("econn")) {
+    return "network";
+  }
+  if (normalized.includes("400") || normalized.includes("422") || normalized.includes("validation")) {
+    return "validation";
+  }
+  return "other";
+}
 
 // src/intelligence/value-signals.ts
 var LESSON_TYPES = new Set(["bugfix", "decision", "pattern"]);
@@ -2475,7 +2512,7 @@ async function provision(baseUrl, request) {
 import { randomBytes } from "node:crypto";
 import { createServer } from "node:http";
 import { execFile } from "node:child_process";
-var CALLBACK_TIMEOUT_MS = 120000;
+var CALLBACK_TIMEOUT_MS = 600000;
 async function runBrowserAuth(candengoUrl) {
   const state = randomBytes(16).toString("hex");
   const { port, waitForCallback, stop } = await startCallbackServer(state);
@@ -2810,13 +2847,16 @@ function registerCodexHooks() {
   return { path: CODEX_HOOKS, added: true };
 }
 function registerOpenCode() {
+  const runtime = findRuntime();
   const root = findPackageRoot();
+  const dist = isBuiltDist();
   const pluginSource = join2(root, "opencode", "plugin", "engrm-opencode.js");
+  const serverPath = dist ? join2(root, "dist", "server.js") : join2(root, "src", "server.ts");
   const config = readJsonFile(OPENCODE_CONFIG);
   const mcp = config["mcp"] ?? {};
   mcp["engrm"] = {
     type: "local",
-    command: ["engrm", "serve"],
+    command: dist ? [runtime, serverPath] : [runtime, "run", serverPath],
     enabled: true,
     timeout: 5000
   };
@@ -3326,10 +3366,11 @@ function readProjectConfigFile(directory) {
   }
 }
 function detectProject(directory) {
-  const remoteUrl = getGitRemoteUrl(directory);
+  const resolvedDirectory = resolve(directory);
+  const remoteUrl = getGitRemoteUrl(resolvedDirectory);
   if (remoteUrl) {
     const canonicalId = normaliseGitRemoteUrl(remoteUrl);
-    const repoRoot = getGitTopLevel(directory) ?? directory;
+    const repoRoot = getGitTopLevel(resolvedDirectory) ?? resolvedDirectory;
     return {
       canonical_id: canonicalId,
       name: projectNameFromCanonicalId(canonicalId),
@@ -3337,21 +3378,22 @@ function detectProject(directory) {
       local_path: repoRoot
     };
   }
-  const configFile = readProjectConfigFile(directory);
+  const configFile = readProjectConfigFile(resolvedDirectory);
   if (configFile) {
     return {
       canonical_id: configFile.project_id,
       name: configFile.name ?? projectNameFromCanonicalId(configFile.project_id),
       remote_url: null,
-      local_path: directory
+      local_path: resolvedDirectory
     };
   }
-  const dirName = basename(directory);
+  const dirName = basename(resolvedDirectory);
+  const safeDirName = !dirName || dirName === "/" || dirName === "." ? "root" : dirName;
   return {
-    canonical_id: `local/${dirName}`,
-    name: dirName,
+    canonical_id: `local/${safeDirName}`,
+    name: safeDirName,
     remote_url: null,
-    local_path: directory
+    local_path: resolvedDirectory
   };
 }
 function detectProjectForPath(filePath, fallbackCwd) {
@@ -4323,6 +4365,117 @@ function hasOpenClawMcpRegistration(content) {
   }
 }
 
+// src/sync/auth.ts
+import { createHash as createHash4 } from "node:crypto";
+
+// src/storage/outbox.ts
+function getPendingEntries(db, limit = 50) {
+  const now = Math.floor(Date.now() / 1000);
+  return db.db.query(`SELECT * FROM sync_outbox
+      WHERE (status = 'pending')
+        OR (status = 'failed' AND retry_count < max_retries AND (next_retry_epoch IS NULL OR next_retry_epoch <= ?))
+      ORDER BY created_at_epoch ASC
+      LIMIT ?`).all(now, limit);
+}
+function markSyncing(db, entryId) {
+  const now = Math.floor(Date.now() / 1000);
+  db.db.query("UPDATE sync_outbox SET status = 'syncing', next_retry_epoch = ? WHERE id = ?").run(now, entryId);
+}
+function markSynced(db, entryId) {
+  const now = Math.floor(Date.now() / 1000);
+  db.db.query("UPDATE sync_outbox SET status = 'synced', synced_at_epoch = ?, next_retry_epoch = NULL, last_error = NULL WHERE id = ?").run(now, entryId);
+}
+function markFailed(db, entryId, error) {
+  const now = Math.floor(Date.now() / 1000);
+  db.db.query(`UPDATE sync_outbox SET
+        status = 'failed',
+        retry_count = retry_count + 1,
+        last_error = ?,
+        next_retry_epoch = ? + MIN(30 * (1 << retry_count), 3600)
+      WHERE id = ?`).run(error, now, entryId);
+}
+function getOutboxStats2(db) {
+  const rows = db.db.query("SELECT status, COUNT(*) as count FROM sync_outbox GROUP BY status").all();
+  const stats = {
+    pending: 0,
+    syncing: 0,
+    synced: 0,
+    failed: 0
+  };
+  for (const row of rows) {
+    stats[row.status] = row.count;
+  }
+  return stats;
+}
+function getOutboxFailureSummaries2(db, limit = 5) {
+  return db.db.query(`SELECT COALESCE(last_error, '') as error, COUNT(*) as count
+       FROM sync_outbox
+       WHERE status = 'failed'
+       GROUP BY COALESCE(last_error, '')
+       ORDER BY count DESC, error ASC
+       LIMIT ?`).all(limit).filter((row) => row.error.length > 0);
+}
+function classifyOutboxFailure2(error) {
+  const normalized = error.toLowerCase();
+  if (normalized.includes("401") || normalized.includes("invalid or missing credentials")) {
+    return "auth";
+  }
+  if (normalized.includes("429") || normalized.includes("rate limit")) {
+    return "rate_limit";
+  }
+  if (normalized.includes("timeout") || normalized.includes("abort")) {
+    return "timeout";
+  }
+  if (normalized.includes("network") || normalized.includes("fetch") || normalized.includes("econn")) {
+    return "network";
+  }
+  if (normalized.includes("400") || normalized.includes("422") || normalized.includes("validation")) {
+    return "validation";
+  }
+  return "other";
+}
+function resetFailedEntries(db) {
+  const result = db.db.query(`UPDATE sync_outbox
+       SET status = 'pending',
+           retry_count = 0,
+           last_error = NULL,
+           next_retry_epoch = NULL
+       WHERE status = 'failed'`).run();
+  return result.changes;
+}
+function resetFailedEntriesMatching(db, predicate) {
+  const rows = db.db.query(`SELECT id, last_error
+       FROM sync_outbox
+       WHERE status = 'failed'`).all();
+  const matchingIds = rows.filter((row) => row.last_error && predicate(row.last_error)).map((row) => row.id);
+  if (matchingIds.length === 0)
+    return 0;
+  const placeholders = matchingIds.map(() => "?").join(", ");
+  const result = db.db.query(`UPDATE sync_outbox
+       SET status = 'pending',
+           retry_count = 0,
+           last_error = NULL,
+           next_retry_epoch = NULL
+       WHERE id IN (${placeholders})`).run(...matchingIds);
+  return result.changes;
+}
+function resetSyncingEntries(db) {
+  const result = db.db.query(`UPDATE sync_outbox
+       SET status = 'pending',
+           next_retry_epoch = NULL
+      WHERE status = 'syncing'`).run();
+  return result.changes;
+}
+function resetStaleSyncingEntries(db, maxAgeSeconds = 300) {
+  const cutoff = Math.floor(Date.now() / 1000) - maxAgeSeconds;
+  const result = db.db.query(`UPDATE sync_outbox
+       SET status = 'pending',
+           next_retry_epoch = NULL
+       WHERE status = 'syncing'
+         AND (next_retry_epoch IS NULL OR next_retry_epoch <= ?)`).run(cutoff);
+  return result.changes;
+}
+
 // src/sync/auth.ts
 var LEGACY_PUBLIC_HOSTS = new Set(["www.candengo.com", "candengo.com"]);
 function normalizeBaseUrl(url) {
@@ -4339,6 +4492,46 @@ function normalizeBaseUrl(url) {
     return trimmed.replace(/\/$/, "");
   }
 }
+function getApiKey(config) {
+  const envKey = process.env.ENGRM_TOKEN;
+  if (envKey && envKey.startsWith("cvk_"))
+    return envKey;
+  if (config.candengo_api_key && config.candengo_api_key.length > 0) {
+    return config.candengo_api_key;
+  }
+  return null;
+}
+function getBaseUrl(config) {
+  if (config.candengo_url && config.candengo_url.length > 0) {
+    return normalizeBaseUrl(config.candengo_url);
+  }
+  return null;
+}
+function getAuthFingerprint(config) {
+  const apiKey = getApiKey(config);
+  const baseUrl = getBaseUrl(config);
+  if (!apiKey || !baseUrl)
+    return null;
+  return createHash4("sha256").update(`${baseUrl}
+${apiKey}
+${config.namespace}
+${config.site_id}`).digest("hex");
+}
+function recoverOutboxAfterSuccessfulAuth(db, config) {
+  const fingerprint = getAuthFingerprint(config);
+  const staleSyncingReset = resetStaleSyncingEntries(db);
+  const authFailedReset = resetFailedEntriesMatching(db, (error) => classifyOutboxFailure2(error) === "auth");
+  if (fingerprint) {
+    db.setSyncState("sync_auth_fingerprint", fingerprint);
+  }
+  return {
+    fingerprintChanged: false,
+    failedReset: 0,
+    authFailedReset,
+    syncingReset: 0,
+    staleSyncingReset
+  };
+}
 
 // src/cli.ts
 var LEGACY_CODEX_SERVER_NAME3 = `candengo-${"mem"}`;
@@ -4350,6 +4543,9 @@ switch (command) {
   case "init":
     await handleInit(args.slice(1));
     break;
+  case "serve":
+    await handleServe();
+    break;
   case "status":
     handleStatus();
     break;
@@ -4415,6 +4611,12 @@ async function handleInit(flags) {
   await initWithBrowser(url);
   await maybeInstallPack(flags);
 }
+async function handleServe() {
+  const packageRoot = join8(THIS_DIR, "..");
+  const serverPath = IS_BUILT_DIST ? join8(packageRoot, "dist", "server.js") : join8(packageRoot, "src", "server.ts");
+  await import(pathToFileURL(serverPath).href);
+  await new Promise(() => {});
+}
 async function maybeInstallPack(flags) {
   const packFlag = flags.find((f) => f.startsWith("--pack"));
   if (!packFlag)
@@ -4510,11 +4712,27 @@ function writeConfigFromProvision(baseUrl, result) {
   ensureConfigDir();
   let existingDeviceId;
   let existingSentinel;
+  let existingObserver;
+  let existingTranscriptAnalysis;
+  let existingHttp;
+  let existingFleet;
+  let existingToolProfile;
+  let existingSync;
+  let existingSearch;
+  let existingScrubbing;
   if (configExists()) {
     try {
       const existing = loadConfig();
       existingDeviceId = existing.device_id;
       existingSentinel = existing.sentinel;
+      existingObserver = existing.observer;
+      existingTranscriptAnalysis = existing.transcript_analysis;
+      existingHttp = existing.http;
+      existingFleet = existing.fleet;
+      existingToolProfile = existing.tool_profile;
+      existingSync = existing.sync;
+      existingSearch = existing.search;
+      existingScrubbing = existing.scrubbing;
     } catch {}
   }
   const config = {
@@ -4526,17 +4744,17 @@ function writeConfigFromProvision(baseUrl, result) {
     user_email: result.user_email,
     device_id: existingDeviceId || generateDeviceId3(),
     teams: result.teams ?? [],
-    sync: {
+    sync: existingSync ?? {
       enabled: true,
       interval_seconds: 30,
       batch_size: 50
     },
-    search: {
+    search: existingSearch ?? {
       default_limit: 10,
       local_boost: 1.2,
       scope: "all"
     },
-    scrubbing: {
+    scrubbing: existingScrubbing ?? {
       enabled: true,
       custom_patterns: [],
       default_sensitivity: "shared"
@@ -4552,17 +4770,29 @@ function writeConfigFromProvision(baseUrl, result) {
       daily_limit: 100,
       tier: "free"
     },
-    observer: {
+    observer: existingObserver ?? {
       enabled: true,
       mode: "per_event",
       model: "haiku"
     },
-    transcript_analysis: {
+    transcript_analysis: existingTranscriptAnalysis ?? {
       enabled: false
-    }
+    },
+    http: existingHttp ?? {
+      enabled: false,
+      port: 3767,
+      bearer_tokens: []
+    },
+    fleet: existingFleet ?? {
+      project_name: "shared-experience",
+      namespace: "",
+      api_key: ""
+    },
+    tool_profile: existingToolProfile ?? "full"
   };
   saveConfig(config);
   const db = new MemDatabase(getDbPath());
+  recoverOutboxAfterSuccessfulAuth(db, config);
   db.close();
   console.log(`Configuration saved to ${getSettingsPath()}`);
   console.log(`Database initialised at ${getDbPath()}`);
@@ -4878,7 +5108,7 @@ function handleStatus() {
         }
         console.log(`    Value:         ${signalParts.join(", ")}`);
         if (signals.security_findings_count > 0 || signals.delivery_review_ready) {
-          console.log(`    Review/Safety: ${signals.delivery_review_ready ? "delivery-ready" : "not ready"}, ` + `${signals.security_findings_count} finding${signals.security_findings_count === 1 ? "" : "s"}`);
+          console.log(`    Review/Safety: ${signals.delivery_review_ready ? "delivery-ready" : "not ready"}, ${signals.security_findings_count} finding${signals.security_findings_count === 1 ? "" : "s"}`);
         }
       } catch {}
       try {
@@ -4899,6 +5129,11 @@ function handleStatus() {
       console.log(`
   Sync`);
       console.log(`    Outbox:        ${outbox["pending"] ?? 0} pending, ${outbox["failed"] ?? 0} failed, ${outbox["synced"] ?? 0} synced`);
+      const topFailures = getOutboxFailureSummaries(db, 2);
+      if (topFailures.length > 0) {
+        const failureSummary = topFailures.map((row) => `${classifyOutboxFailure(row.error)} ${row.count}`).join(", ");
+        console.log(`    Failures:      ${failureSummary}`);
+      }
       try {
         const lastPush = db.db.query("SELECT value FROM sync_state WHERE key = ?").get("last_push_epoch");
         const lastPull = db.db.query("SELECT value FROM sync_state WHERE key = ?").get("last_pull_epoch");
@@ -4979,7 +5214,7 @@ function generateDeviceId3() {
       break;
   }
   const material = `${host}:${mac || "no-mac"}`;
-  const suffix = createHash4("sha256").update(material).digest("hex").slice(0, 8);
+  const suffix = createHash5("sha256").update(material).digest("hex").slice(0, 8);
   return `${host}-${suffix}`;
 }
 async function handleInstallPack(flags) {
@@ -5393,7 +5628,7 @@ async function handleDoctor() {
     const dbPath = getDbPath();
     if (existsSync8(dbPath)) {
       const stats = statSync(dbPath);
-      const sizeMB = stats.size / (1024 * 1024);
+      const sizeMB = stats.size / 1048576;
       const sizeStr = sizeMB >= 1 ? `${sizeMB.toFixed(1)} MB` : `${(stats.size / 1024).toFixed(0)} KB`;
       info(`Database size: ${sizeStr}`);
     }
@@ -5529,7 +5764,7 @@ And add to ~/.config/opencode/opencode.json:`);
   "mcp": {
     "engrm": {
       "type": "local",
-      "command": ["engrm", "serve"],
+      "command": ${JSON.stringify(IS_BUILT_DIST ? [process.execPath, join8(packageRoot, "dist", "server.js")] : ["bun", "run", join8(packageRoot, "src", "server.ts")])},
       "enabled": true,
       "timeout": 5000
     }
@@ -5558,6 +5793,7 @@ function printUsage() {
   console.log(`Engrm \u2014 Memory layer for AI coding agents
 `);
   console.log("Usage:");
+  console.log("  engrm serve                 Run the MCP server over stdio");
   console.log("  engrm init                  Setup via browser (recommended)");
   console.log("  engrm init --token=cmt_xxx  Setup from provisioning token");
   console.log("  engrm init --pack=<name>    Setup + install a starter pack");

package/dist/hooks/elicitation-result.js

@@ -419,10 +419,11 @@ function readProjectConfigFile(directory) {
   }
 }
 function detectProject(directory) {
-  const remoteUrl = getGitRemoteUrl(directory);
+  const resolvedDirectory = resolve(directory);
+  const remoteUrl = getGitRemoteUrl(resolvedDirectory);
   if (remoteUrl) {
     const canonicalId = normaliseGitRemoteUrl(remoteUrl);
-    const repoRoot = getGitTopLevel(directory) ?? directory;
+    const repoRoot = getGitTopLevel(resolvedDirectory) ?? resolvedDirectory;
     return {
       canonical_id: canonicalId,
       name: projectNameFromCanonicalId(canonicalId),
@@ -430,21 +431,22 @@ function detectProject(directory) {
       local_path: repoRoot
     };
   }
-  const configFile = readProjectConfigFile(directory);
+  const configFile = readProjectConfigFile(resolvedDirectory);
   if (configFile) {
     return {
       canonical_id: configFile.project_id,
       name: configFile.name ?? projectNameFromCanonicalId(configFile.project_id),
       remote_url: null,
-      local_path: directory
+      local_path: resolvedDirectory
     };
   }
-  const dirName = basename(directory);
+  const dirName = basename(resolvedDirectory);
+  const safeDirName = !dirName || dirName === "/" || dirName === "." ? "root" : dirName;
   return {
-    canonical_id: `local/${dirName}`,
-    name: dirName,
+    canonical_id: `local/${safeDirName}`,
+    name: safeDirName,
     remote_url: null,
-    local_path: directory
+    local_path: resolvedDirectory
   };
 }
 function detectProjectForPath(filePath, fallbackCwd) {
@@ -1752,6 +1754,7 @@ function ensureObservationTypes(db) {
         DROP TABLE observations;
         ALTER TABLE observations_repair RENAME TO observations;
         CREATE INDEX IF NOT EXISTS idx_observations_project ON observations(project_id);
+        CREATE INDEX IF NOT EXISTS idx_observations_project_lifecycle ON observations(project_id, lifecycle);
         CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
         CREATE INDEX IF NOT EXISTS idx_observations_created ON observations(created_at_epoch);
         CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
@@ -2032,6 +2035,7 @@ class MemDatabase {
     this.db = openDatabase(dbPath);
     this.db.exec("PRAGMA journal_mode = WAL");
     this.db.exec("PRAGMA foreign_keys = ON");
+    this.db.exec("PRAGMA busy_timeout = 5000");
     this.vecAvailable = this.loadVecExtension();
     runMigrations(this.db);
     ensureObservationTypes(this.db);
@@ -2053,8 +2057,16 @@ class MemDatabase {
     this.db.close();
   }
   upsertProject(project) {
+    const canonicalId = project.canonical_id?.trim();
+    const name = project.name?.trim();
+    if (!canonicalId) {
+      throw new Error("Project canonical_id is required");
+    }
+    if (!name) {
+      throw new Error("Project name is required");
+    }
     const now = Math.floor(Date.now() / 1000);
-    const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(project.canonical_id);
+    const existing = this.db.query("SELECT * FROM projects WHERE canonical_id = ?").get(canonicalId);
     if (existing) {
       this.db.query(`UPDATE projects SET
             local_path = COALESCE(?, local_path),
@@ -2069,7 +2081,7 @@ class MemDatabase {
       };
     }
     const result = this.db.query(`INSERT INTO projects (canonical_id, name, local_path, remote_url, first_seen_epoch, last_active_epoch)
-        VALUES (?, ?, ?, ?, ?, ?)`).run(project.canonical_id, project.name, project.local_path ?? null, project.remote_url ?? null, now, now);
+        VALUES (?, ?, ?, ?, ?, ?)`).run(canonicalId, name, project.local_path ?? null, project.remote_url ?? null, now, now);
     return this.db.query("SELECT * FROM projects WHERE id = ?").get(Number(result.lastInsertRowid));
   }
   getProjectByCanonicalId(canonicalId) {