engrm 0.4.45 → 0.4.47

package/dist/hooks/stop.js

@@ -320,11 +320,20 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { homedir, hostname, networkInterfaces } from "node:os";
 import { join } from "node:path";
 import { createHash } from "node:crypto";
-var CONFIG_DIR = join(homedir(), ".engrm");
-var SETTINGS_PATH = join(CONFIG_DIR, "settings.json");
-var DB_PATH = join(CONFIG_DIR, "engrm.db");
+function resolveConfigDir() {
+  return process.env["ENGRM_CONFIG_DIR"]?.trim() || join(homedir(), ".engrm");
+}
+function resolveSettingsPath() {
+  return join(resolveConfigDir(), "settings.json");
+}
+function resolveDbPath() {
+  return join(resolveConfigDir(), "engrm.db");
+}
+function resolveAuthBackupPath() {
+  return join(resolveConfigDir(), "auth-backup.json");
+}
 function getDbPath() {
-  return DB_PATH;
+  return resolveDbPath();
 }
 function generateDeviceId() {
   const host = hostname().toLowerCase().replace(/[^a-z0-9-]/g, "");
@@ -347,7 +356,7 @@ function generateDeviceId() {
   return `${host}-${suffix}`;
 }
 function createDefaultConfig() {
-  return {
+  const merged = {
     candengo_url: "",
     candengo_api_key: "",
     site_id: "",
@@ -402,24 +411,26 @@ function createDefaultConfig() {
     },
     tool_profile: "full"
   };
+  return merged;
 }
 function loadConfig() {
-  if (!existsSync(SETTINGS_PATH)) {
-    throw new Error(`Config not found at ${SETTINGS_PATH}. Run 'engrm init --manual' to configure.`);
+  const settingsPath = resolveSettingsPath();
+  if (!existsSync(settingsPath)) {
+    throw new Error(`Config not found at ${settingsPath}. Run 'engrm init --manual' to configure.`);
   }
-  const raw = readFileSync(SETTINGS_PATH, "utf-8");
+  const raw = readFileSync(settingsPath, "utf-8");
   let parsed;
   try {
     parsed = JSON.parse(raw);
   } catch {
-    throw new Error(`Invalid JSON in ${SETTINGS_PATH}`);
+    throw new Error(`Invalid JSON in ${settingsPath}`);
   }
   if (typeof parsed !== "object" || parsed === null) {
-    throw new Error(`Config at ${SETTINGS_PATH} is not a JSON object`);
+    throw new Error(`Config at ${settingsPath} is not a JSON object`);
   }
   const config = parsed;
   const defaults = createDefaultConfig();
-  return {
+  const merged = {
     candengo_url: asString(config["candengo_url"], defaults.candengo_url),
     candengo_api_key: asString(config["candengo_api_key"], defaults.candengo_api_key),
     site_id: asString(config["site_id"], defaults.site_id),
@@ -474,16 +485,27 @@ function loadConfig() {
     },
     tool_profile: asToolProfile(config["tool_profile"], defaults.tool_profile)
   };
+  if (looksLikePlaceholderAuth(merged)) {
+    return restoreAuthBackup(merged) ?? merged;
+  }
+  return merged;
 }
 function saveConfig(config) {
-  if (!existsSync(CONFIG_DIR)) {
-    mkdirSync(CONFIG_DIR, { recursive: true });
+  const configDir = resolveConfigDir();
+  const settingsPath = resolveSettingsPath();
+  const authBackupPath = resolveAuthBackupPath();
+  if (!existsSync(configDir)) {
+    mkdirSync(configDir, { recursive: true });
   }
-  writeFileSync(SETTINGS_PATH, JSON.stringify(config, null, 2) + `
+  writeFileSync(settingsPath, JSON.stringify(config, null, 2) + `
+`, "utf-8");
+  if (!looksLikePlaceholderAuth(config)) {
+    writeFileSync(authBackupPath, JSON.stringify(extractAuthBackup(config), null, 2) + `
 `, "utf-8");
+  }
 }
 function configExists() {
-  return existsSync(SETTINGS_PATH);
+  return existsSync(resolveSettingsPath());
 }
 function asString(value, fallback) {
   return typeof value === "string" ? value : fallback;
@@ -537,6 +559,50 @@ function asTeams(value, fallback) {
     return fallback;
   return value.filter((t) => typeof t === "object" && t !== null && typeof t.id === "string" && typeof t.name === "string" && typeof t.namespace === "string");
 }
+function looksLikePlaceholderAuth(config) {
+  const apiKey = config.candengo_api_key.trim();
+  const siteId = config.site_id.trim();
+  const namespace = config.namespace.trim();
+  const email = config.user_email.trim().toLowerCase();
+  if (apiKey === "cvk_org" && siteId === "site-1" && namespace === "org-ns")
+    return true;
+  if (siteId === "site-1" && namespace === "org-ns" && email.endsWith("@example.com"))
+    return true;
+  return false;
+}
+function extractAuthBackup(config) {
+  return {
+    candengo_url: config.candengo_url,
+    candengo_api_key: config.candengo_api_key,
+    site_id: config.site_id,
+    namespace: config.namespace,
+    user_id: config.user_id,
+    user_email: config.user_email,
+    teams: config.teams
+  };
+}
+function restoreAuthBackup(config) {
+  const authBackupPath = resolveAuthBackupPath();
+  if (!existsSync(authBackupPath))
+    return null;
+  try {
+    const raw = readFileSync(authBackupPath, "utf-8");
+    const parsed = JSON.parse(raw);
+    const restored = {
+      ...config,
+      candengo_url: asString(parsed["candengo_url"], config.candengo_url),
+      candengo_api_key: asString(parsed["candengo_api_key"], config.candengo_api_key),
+      site_id: asString(parsed["site_id"], config.site_id),
+      namespace: asString(parsed["namespace"], config.namespace),
+      user_id: asString(parsed["user_id"], config.user_id),
+      user_email: asString(parsed["user_email"], config.user_email),
+      teams: asTeams(parsed["teams"], config.teams)
+    };
+    return looksLikePlaceholderAuth(restored) ? null : restored;
+  } catch {
+    return null;
+  }
+}
 
 // src/storage/migrations.ts
 var MIGRATIONS = [
@@ -1224,6 +1290,20 @@ function ensureChatMessageColumns(db) {
     db.exec("PRAGMA user_version = 17");
   }
 }
+function ensureObservationVectorTable(db) {
+  if (!isVecExtensionLoaded(db))
+    return;
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS vec_observations USING vec0(
+      observation_id INTEGER PRIMARY KEY,
+      embedding FLOAT[384]
+    );
+  `);
+  const current = getSchemaVersion(db);
+  if (current < 4) {
+    db.exec("PRAGMA user_version = 4");
+  }
+}
 function ensureChatVectorTable(db) {
   if (!isVecExtensionLoaded(db))
     return;
@@ -1372,6 +1452,7 @@ class MemDatabase {
     ensureObservationTypes(this.db);
     ensureSessionSummaryColumns(this.db);
     ensureChatMessageColumns(this.db);
+    ensureObservationVectorTable(this.db);
     ensureChatVectorTable(this.db);
     ensureSyncOutboxSupportsChatMessages(this.db);
   }
@@ -2316,6 +2397,10 @@ function resetStaleSyncingEntries(db, maxAgeSeconds = 300) {
 
 // src/sync/auth.ts
 var LEGACY_PUBLIC_HOSTS = new Set(["www.candengo.com", "candengo.com"]);
+var PLACEHOLDER_API_KEYS = new Set(["cvk_org"]);
+var PLACEHOLDER_SITE_IDS = new Set(["site-1"]);
+var PLACEHOLDER_NAMESPACES = new Set(["org-ns", "fleet-ns"]);
+var PLACEHOLDER_EMAIL_SUFFIXES = ["@example.com"];
 function normalizeBaseUrl(url) {
   const trimmed = url.trim();
   if (!trimmed)
@@ -2334,7 +2419,7 @@ function getApiKey(config) {
   const envKey = process.env.ENGRM_TOKEN;
   if (envKey && envKey.startsWith("cvk_"))
     return envKey;
-  if (config.candengo_api_key && config.candengo_api_key.length > 0) {
+  if (config.candengo_api_key && config.candengo_api_key.length > 0 && !looksLikePlaceholderConfig(config)) {
     return config.candengo_api_key;
   }
   return null;
@@ -2355,6 +2440,23 @@ ${apiKey}
 ${config.namespace}
 ${config.site_id}`).digest("hex");
 }
+function looksLikePlaceholderConfig(config) {
+  const apiKey = config.candengo_api_key?.trim() ?? "";
+  const siteId = config.site_id?.trim() ?? "";
+  const namespace = config.namespace?.trim() ?? "";
+  const email = config.user_email?.trim().toLowerCase() ?? "";
+  if (PLACEHOLDER_API_KEYS.has(apiKey) && PLACEHOLDER_SITE_IDS.has(siteId) && PLACEHOLDER_NAMESPACES.has(namespace)) {
+    return true;
+  }
+  if (PLACEHOLDER_SITE_IDS.has(siteId) && PLACEHOLDER_NAMESPACES.has(namespace) && PLACEHOLDER_EMAIL_SUFFIXES.some((suffix) => email.endsWith(suffix))) {
+    return true;
+  }
+  return false;
+}
+function clearSyncPushBlock(db) {
+  db.setSyncState("sync_push_blocked_until", "0");
+  db.setSyncState("sync_push_block_reason", "");
+}
 function recoverOutboxAfterAuthChange(db, config) {
   const fingerprint = getAuthFingerprint(config);
   if (!fingerprint) {
@@ -2373,6 +2475,7 @@ function recoverOutboxAfterAuthChange(db, config) {
   const syncingReset = resetSyncingEntries(db);
   const staleSyncingReset = 0;
   db.setSyncState(key, fingerprint);
+  clearSyncPushBlock(db);
   return { fingerprintChanged: true, failedReset, authFailedReset: 0, syncingReset, staleSyncingReset };
 }
 function buildSourceId(config, localId, type = "obs") {
@@ -2938,10 +3041,16 @@ function buildSummaryVectorDocument(summary, config, project, targetOrObservatio
   };
 }
 async function pushOutbox(db, config, batchSize = 50, options = {}) {
+  resetStaleSyncingEntries(db);
+  if (isPushBlocked(db)) {
+    return { pushed: 0, failed: 0, skipped: 0, blocked: true };
+  }
   const entries = getPendingEntries(db, batchSize);
   let pushed = 0;
   let failed = 0;
   let skipped = 0;
+  let authFailures = 0;
+  let rateLimitFailures = 0;
   const batch = [];
   for (const entry of entries) {
     if (entry.record_type === "summary") {
@@ -3076,14 +3185,44 @@ async function pushOutbox(db, config, batchSize = 50, options = {}) {
           markSynced(db, entryId);
           pushed++;
         } catch (err) {
-          markFailed(db, entryId, err instanceof Error ? err.message : String(err));
+          const message = err instanceof Error ? err.message : String(err);
+          const kind = classifyOutboxFailure(message);
+          if (kind === "auth")
+            authFailures++;
+          if (kind === "rate_limit")
+            rateLimitFailures++;
+          markFailed(db, entryId, message);
           failed++;
         }
       }
     }
   }
+  updatePushCooldown(db, { pushed, authFailures, rateLimitFailures });
   return { pushed, failed, skipped };
 }
+var PUSH_BLOCK_UNTIL_KEY = "sync_push_blocked_until";
+var PUSH_BLOCK_REASON_KEY = "sync_push_block_reason";
+function isPushBlocked(db) {
+  const blockedUntil = parseInt(db.getSyncState(PUSH_BLOCK_UNTIL_KEY) ?? "0", 10);
+  return Number.isFinite(blockedUntil) && blockedUntil > Math.floor(Date.now() / 1000);
+}
+function updatePushCooldown(db, result) {
+  const now = Math.floor(Date.now() / 1000);
+  if (result.authFailures > 0) {
+    db.setSyncState(PUSH_BLOCK_UNTIL_KEY, String(now + 365 * 24 * 60 * 60));
+    db.setSyncState(PUSH_BLOCK_REASON_KEY, "auth");
+    return;
+  }
+  if (result.rateLimitFailures > 0) {
+    db.setSyncState(PUSH_BLOCK_UNTIL_KEY, String(now + 2 * 60));
+    db.setSyncState(PUSH_BLOCK_REASON_KEY, "rate_limit");
+    return;
+  }
+  if (result.pushed > 0) {
+    db.setSyncState(PUSH_BLOCK_UNTIL_KEY, "0");
+    db.setSyncState(PUSH_BLOCK_REASON_KEY, "");
+  }
+}
 function maybeScrubFleetDocument(doc, target) {
   if (!target.isFleet)
     return doc;
@@ -3368,7 +3507,7 @@ function buildBeacon(db, config, sessionId, metrics) {
     sentinel_used: valueSignals.security_findings_count > 0,
     risk_score: riskScore,
     stacks_detected: stacks,
-    client_version: "0.4.45",
+    client_version: "0.4.47",
     context_observations_injected: metrics?.contextObsInjected ?? 0,
     context_total_available: metrics?.contextTotalAvailable ?? 0,
     recall_attempts: metrics?.recallAttempts ?? 0,
@@ -3559,7 +3698,7 @@ function detectProjectFromTouchedPaths(paths, fallbackCwd) {
 
 // src/capture/transcript.ts
 import { createHash as createHash4 } from "node:crypto";
-import { readFileSync as readFileSync4, existsSync as existsSync4 } from "node:fs";
+import { readFileSync as readFileSync4, existsSync as existsSync4, statSync, readdirSync } from "node:fs";
 import { join as join5 } from "node:path";
 import { homedir as homedir3 } from "node:os";
 
@@ -4156,7 +4295,11 @@ function resolveTranscriptPath(sessionId, cwd, transcriptPath) {
   if (transcriptPath)
     return transcriptPath;
   const encodedCwd = cwd.replace(/\//g, "-");
-  return join5(homedir3(), ".claude", "projects", encodedCwd, `${sessionId}.jsonl`);
+  const directPath = join5(homedir3(), ".claude", "projects", encodedCwd, `${sessionId}.jsonl`);
+  if (existsSync4(directPath))
+    return directPath;
+  const discovered = findTranscriptPathBySessionId(sessionId);
+  return discovered ?? directPath;
 }
 function readTranscript(sessionId, cwd, transcriptPath) {
   const path = resolveTranscriptPath(sessionId, cwd, transcriptPath);
@@ -4179,10 +4322,10 @@ function readTranscript(sessionId, cwd, transcriptPath) {
     } catch {
       continue;
     }
-    const role = entry.role;
+    const role = getTranscriptRole(entry);
     if (role !== "user" && role !== "assistant")
       continue;
-    const content = entry.content;
+    const content = getTranscriptContent(entry);
     if (typeof content === "string") {
       messages.push({ role, text: content });
       continue;
@@ -4202,6 +4345,66 @@ function readTranscript(sessionId, cwd, transcriptPath) {
   }
   return messages;
 }
+function readTranscriptToolEvents(sessionId, cwd, transcriptPath) {
+  const path = resolveTranscriptPath(sessionId, cwd, transcriptPath);
+  if (!existsSync4(path))
+    return [];
+  let raw;
+  try {
+    raw = readFileSync4(path, "utf-8");
+  } catch {
+    return [];
+  }
+  const toolEvents = [];
+  const toolEventIndexes = new Map;
+  for (const line of raw.split(`
+`)) {
+    if (!line.trim())
+      continue;
+    let entry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    const createdAtEpoch = parseTranscriptTimestamp(entry);
+    const content = getTranscriptContent(entry);
+    if (!Array.isArray(content))
+      continue;
+    for (const block of content) {
+      if (!block || typeof block !== "object")
+        continue;
+      if (block.type === "tool_result" && typeof block.tool_use_id === "string") {
+        const preview = extractToolResultPreview(block.content);
+        const index = toolEventIndexes.get(block.tool_use_id);
+        if (preview && index !== undefined) {
+          toolEvents[index] = {
+            ...toolEvents[index],
+            tool_response_preview: preview
+          };
+        }
+        continue;
+      }
+      if (block.type !== "tool_use")
+        continue;
+      const input = block.input && typeof block.input === "object" ? block.input : {};
+      const toolUseId = typeof block.id === "string" ? block.id : null;
+      const nextEvent = {
+        tool_name: typeof block.name === "string" ? block.name : "Unknown",
+        tool_input_json: JSON.stringify(input),
+        tool_response_preview: null,
+        file_path: extractToolFilePath(input),
+        command: typeof input.command === "string" ? input.command : null,
+        created_at_epoch: createdAtEpoch
+      };
+      toolEvents.push(nextEvent);
+      if (toolUseId) {
+        toolEventIndexes.set(toolUseId, toolEvents.length - 1);
+      }
+    }
+  }
+  return toolEvents;
+}
 function resolveHistoryPath(historyPath) {
   if (historyPath)
     return historyPath;
@@ -4267,9 +4470,22 @@ function readHistoryFallback(sessionId, cwd, opts) {
     createdAtEpoch: entry.timestamp
   })));
 }
-async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
+async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath, options = {}) {
+  const embed = options.embed ?? true;
   const session = db.getSessionById(sessionId);
-  const transcriptMessages = readTranscript(sessionId, cwd, transcriptPath).map((message) => ({
+  const resolvedTranscriptPath = resolveTranscriptPath(sessionId, cwd, transcriptPath);
+  const syncCursorKey = `transcript_sync_cursor:${sessionId}`;
+  if (existsSync4(resolvedTranscriptPath)) {
+    try {
+      const stat = statSync(resolvedTranscriptPath);
+      const cursor = `${stat.size}:${Math.floor(stat.mtimeMs)}`;
+      if (db.getSyncState(syncCursorKey) === cursor) {
+        return { imported: 0, total: 0 };
+      }
+      db.setSyncState(syncCursorKey, cursor);
+    } catch {}
+  }
+  const transcriptMessages = readTranscript(sessionId, cwd, resolvedTranscriptPath).map((message) => ({
     ...message,
     text: message.text.trim()
   })).filter((message) => message.text.length > 0);
@@ -4331,7 +4547,7 @@ async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
         created_at_epoch: createdAtEpoch
       });
     }
-    if (db.vecAvailable) {
+    if (embed && db.vecAvailable) {
       const embedding = await embedText(composeChatEmbeddingText(message.text));
       if (embedding) {
         db.vecChatInsert(row.id, embedding);
@@ -4341,6 +4557,35 @@ async function syncTranscriptChat(db, config, sessionId, cwd, transcriptPath) {
   }
   return { imported, total: messages.length };
 }
+function syncTranscriptToolEvents(db, config, sessionId, cwd, transcriptPath) {
+  const session = db.getSessionById(sessionId);
+  if (!session)
+    return { imported: 0, total: 0 };
+  if (db.getSessionToolEvents(sessionId, 1).length > 0) {
+    return { imported: 0, total: 0 };
+  }
+  const toolEvents = readTranscriptToolEvents(sessionId, cwd, transcriptPath);
+  if (toolEvents.length === 0)
+    return { imported: 0, total: 0 };
+  let imported = 0;
+  for (const event of toolEvents) {
+    db.insertToolEvent({
+      session_id: sessionId,
+      project_id: session.project_id,
+      tool_name: event.tool_name,
+      tool_input_json: event.tool_input_json,
+      tool_response_preview: event.tool_response_preview,
+      file_path: event.file_path,
+      command: event.command,
+      user_id: config.user_id,
+      device_id: config.device_id,
+      agent: "claude-code",
+      created_at_epoch: event.created_at_epoch ?? undefined
+    });
+    imported++;
+  }
+  return { imported, total: toolEvents.length };
+}
 function dedupeHistoryMessages(messages) {
   const deduped = [];
   for (const message of messages) {
@@ -4358,6 +4603,59 @@ function buildHistorySourceId(sessionId, createdAtEpoch, text) {
   const digest = createHash4("sha1").update(text).digest("hex").slice(0, 12);
   return `history:${sessionId}:${createdAtEpoch}:${digest}`;
 }
+function getTranscriptRole(entry) {
+  return entry.role ?? entry.message?.role ?? entry.type ?? entry.message?.type;
+}
+function getTranscriptContent(entry) {
+  return entry.content ?? entry.message?.content;
+}
+function parseTranscriptTimestamp(entry) {
+  const raw = entry.timestamp ?? entry.message?.timestamp;
+  if (typeof raw !== "string")
+    return null;
+  const epoch = Date.parse(raw);
+  return Number.isFinite(epoch) ? Math.floor(epoch / 1000) : null;
+}
+function extractToolResultPreview(content) {
+  if (typeof content === "string")
+    return content.slice(0, 4000);
+  if (Array.isArray(content)) {
+    const text = content.map((item) => {
+      if (typeof item === "string")
+        return item;
+      if (item && typeof item === "object" && typeof item.text === "string")
+        return item.text;
+      return "";
+    }).filter(Boolean).join(`
+`);
+    return text ? text.slice(0, 4000) : null;
+  }
+  return null;
+}
+function extractToolFilePath(input) {
+  for (const key of ["file_path", "path", "target_file"]) {
+    if (typeof input[key] === "string")
+      return input[key];
+  }
+  return null;
+}
+function findTranscriptPathBySessionId(sessionId) {
+  const projectsDir = join5(homedir3(), ".claude", "projects");
+  if (!existsSync4(projectsDir))
+    return null;
+  try {
+    for (const entry of readdirSync(projectsDir, { withFileTypes: true })) {
+      if (!entry.isDirectory())
+        continue;
+      const candidate = join5(projectsDir, entry.name, `${sessionId}.jsonl`);
+      if (existsSync4(candidate))
+        return candidate;
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
 function truncateTranscript(messages, maxBytes = 50000) {
   const lines = [];
   for (const msg of messages) {
@@ -4931,7 +5229,8 @@ async function main() {
   try {
     if (event.session_id) {
       db.completeSession(event.session_id);
-      await syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path);
+      syncTranscriptToolEvents(db, config, event.session_id, event.cwd, event.transcript_path);
+      await syncTranscriptChat(db, config, event.session_id, event.cwd, event.transcript_path, { embed: false });
       if (event.last_assistant_message) {
         try {
           const detected = detectProject(event.cwd);
@@ -5030,7 +5329,7 @@ async function main() {
         }
       } catch {}
     }
-    await pushOnce(db, config, { timeoutMs: 4000 });
+    await pushOnce(db, config, { timeoutMs: 1500 });
     try {
       if (event.session_id) {
         const metrics = readSessionMetrics(event.session_id);