@askexenow/exe-os 0.9.227 → 0.9.229

package/dist/bin/backfill-conversations.js

@@ -0,0 +1,399 @@
+#!/usr/bin/env node
+import {
+  connectEmbedDaemon,
+  embedViaClient
+} from "../chunk-L24HYX62.js";
+import "../chunk-YADYM2DE.js";
+import {
+  flushBatch,
+  initStore,
+  writeMemory
+} from "../chunk-HTCBLSIB.js";
+import "../chunk-CHCA3ZM2.js";
+import "../chunk-IW53DAYO.js";
+import "../chunk-F7LU65PQ.js";
+import "../chunk-MP2AFCGL.js";
+import "../chunk-PWMMIGVQ.js";
+import {
+  getClient,
+  isCoordinatorName
+} from "../chunk-UJOSFVWG.js";
+import "../chunk-QHIGMP2M.js";
+import "../chunk-FXU7JOXK.js";
+import "../chunk-7NNXBDOO.js";
+import "../chunk-TNTFEWU5.js";
+import "../chunk-LYH5HE24.js";
+import {
+  isMainModule
+} from "../chunk-6Y4B3QF6.js";
+import "../chunk-R5U7XKVJ.js";
+
+// src/bin/backfill-conversations.ts
+import crypto from "crypto";
+import { createReadStream } from "fs";
+import { readdir, stat } from "fs/promises";
+import path from "path";
+import { createInterface } from "readline";
+import { homedir } from "os";
+import { parseArgs } from "util";
+var TOOL_NAME = "backfill-conversation";
+var MIN_MESSAGES = 3;
+var MAX_SUMMARY_LENGTH = 4e3;
+var MAX_WALK_DEPTH = 10;
+async function findJsonlFiles(sinceDate, projectFilter) {
+  const projectsDir = path.join(homedir(), ".claude", "projects");
+  const files = [];
+  async function walk(dir, depth = 0) {
+    if (depth > MAX_WALK_DEPTH) return;
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (entry.name === "subagents" || entry.name === "tool-results") continue;
+        await walk(full, depth + 1);
+      } else if (entry.name.endsWith(".jsonl")) {
+        try {
+          const s = await stat(full);
+          if (sinceDate && s.mtimeMs < sinceDate.getTime()) continue;
+          files.push(full);
+        } catch {
+        }
+      }
+    }
+  }
+  if (projectFilter) {
+    let projectDirs;
+    try {
+      projectDirs = await readdir(projectsDir, { withFileTypes: true });
+    } catch {
+      return files;
+    }
+    for (const entry of projectDirs) {
+      if (!entry.isDirectory()) continue;
+      const decoded = decodeProjectDir(entry.name);
+      if (decoded.toLowerCase().includes(projectFilter.toLowerCase())) {
+        await walk(path.join(projectsDir, entry.name));
+      }
+    }
+  } else {
+    await walk(projectsDir);
+  }
+  return files;
+}
+function decodeProjectDir(dirName) {
+  const homeEncoded = homedir().replaceAll("/", "-");
+  if (dirName.startsWith(homeEncoded + "-")) {
+    return dirName.slice(homeEncoded.length + 1);
+  }
+  if (dirName === homeEncoded) return "home";
+  return dirName;
+}
+function projectNameFromPath(filePath) {
+  const projectsDir = path.join(homedir(), ".claude", "projects");
+  const relative = path.relative(projectsDir, filePath);
+  const projectDir = relative.split(path.sep)[0] ?? "unknown";
+  return decodeProjectDir(projectDir);
+}
+async function parseConversation(filePath) {
+  const conv = {
+    sessionId: path.basename(filePath, ".jsonl"),
+    projectName: projectNameFromPath(filePath),
+    cwd: void 0,
+    startTime: void 0,
+    endTime: void 0,
+    userMessages: [],
+    toolCounts: {},
+    filesTouched: /* @__PURE__ */ new Set(),
+    errorCount: 0,
+    totalMessages: 0,
+    agentId: "default"
+  };
+  const rl = createInterface({
+    input: createReadStream(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let entry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (entry.cwd && typeof entry.cwd === "string") {
+      conv.cwd = entry.cwd;
+    }
+    const ts = entry.timestamp;
+    if (ts) {
+      if (!conv.startTime || ts < conv.startTime) conv.startTime = ts;
+      if (!conv.endTime || ts > conv.endTime) conv.endTime = ts;
+    }
+    const entryType = entry.type;
+    if (entryType === "user") {
+      conv.totalMessages++;
+      const message = entry.message;
+      if (message?.content) {
+        const text = extractUserText(message.content);
+        if (text && text.length > 10) {
+          conv.userMessages.push(text);
+        }
+      }
+    } else if (entryType === "assistant") {
+      conv.totalMessages++;
+      const message = entry.message;
+      if (message?.content && Array.isArray(message.content)) {
+        for (const block of message.content) {
+          if (typeof block !== "object" || block === null) continue;
+          const b = block;
+          if (b.type === "tool_use") {
+            const toolName = b.name;
+            conv.toolCounts[toolName] = (conv.toolCounts[toolName] || 0) + 1;
+            const input = b.input;
+            if (input?.file_path && typeof input.file_path === "string") {
+              if (toolName === "Write" || toolName === "Edit") {
+                conv.filesTouched.add(input.file_path);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  if (conv.cwd) {
+    conv.projectName = path.basename(conv.cwd);
+    const worktreeMatch = conv.cwd.match(/\.worktrees\/([^/]+)/);
+    if (worktreeMatch?.[1]) {
+      conv.agentId = worktreeMatch[1];
+    }
+  }
+  return conv;
+}
+function extractUserText(content) {
+  if (typeof content === "string") return content;
+  if (Array.isArray(content)) {
+    const parts = [];
+    for (const block of content) {
+      if (typeof block === "string") {
+        parts.push(block);
+      } else if (typeof block === "object" && block !== null) {
+        const b = block;
+        if (b.type === "text" && typeof b.text === "string") {
+          parts.push(b.text);
+        }
+      }
+    }
+    return parts.join("\n");
+  }
+  return "";
+}
+function buildSummary(conv) {
+  const parts = [];
+  parts.push(`Session: ${conv.sessionId}`);
+  parts.push(`Project: ${conv.projectName}`);
+  if (conv.startTime) {
+    parts.push(`Time: ${conv.startTime}${conv.endTime ? ` \u2192 ${conv.endTime}` : ""}`);
+  }
+  parts.push(`Messages: ${conv.totalMessages}`);
+  if (conv.agentId !== "default") {
+    parts.push(`Agent: ${conv.agentId}`);
+  }
+  parts.push("");
+  if (conv.userMessages.length > 0) {
+    parts.push("## What was asked");
+    const prompts = conv.userMessages.slice(0, 5);
+    for (const prompt of prompts) {
+      const truncated = prompt.length > 300 ? prompt.slice(0, 300) + "..." : prompt;
+      const cleaned = truncated.replace(/<system-reminder>[\s\S]*?<\/system-reminder>/g, "").trim();
+      if (cleaned) parts.push(`- ${cleaned}`);
+    }
+    if (conv.userMessages.length > 5) {
+      parts.push(`- ... and ${conv.userMessages.length - 5} more prompts`);
+    }
+    parts.push("");
+  }
+  const toolEntries = Object.entries(conv.toolCounts).sort((a, b) => b[1] - a[1]);
+  if (toolEntries.length > 0) {
+    parts.push("## Tools used");
+    for (const [tool, count] of toolEntries) {
+      parts.push(`- ${tool}: ${count}`);
+    }
+    parts.push("");
+  }
+  if (conv.filesTouched.size > 0) {
+    parts.push("## Files modified");
+    const fileList = [...conv.filesTouched].sort();
+    const shown = fileList.slice(0, 20);
+    for (const f of shown) {
+      parts.push(`- ${f}`);
+    }
+    if (fileList.length > 20) {
+      parts.push(`- ... and ${fileList.length - 20} more files`);
+    }
+    parts.push("");
+  }
+  if (conv.errorCount > 0) {
+    parts.push(`Errors: ${conv.errorCount}`);
+  }
+  let summary = parts.join("\n");
+  if (summary.length > MAX_SUMMARY_LENGTH) {
+    summary = summary.slice(0, MAX_SUMMARY_LENGTH);
+  }
+  return summary;
+}
+async function loadExistingSourcePaths() {
+  const client = getClient();
+  const paths = /* @__PURE__ */ new Set();
+  let offset = 0;
+  const batchSize = 500;
+  while (true) {
+    const result = await client.execute({
+      sql: `SELECT source_path FROM memories
+            WHERE tool_name = ? AND source_path IS NOT NULL
+            ORDER BY id LIMIT ? OFFSET ?`,
+      args: [TOOL_NAME, batchSize, offset]
+    });
+    if (result.rows.length === 0) break;
+    for (const row of result.rows) {
+      paths.add(row.source_path);
+    }
+    offset += batchSize;
+  }
+  return paths;
+}
+async function backfillConversations(options) {
+  const stats = {
+    filesScanned: 0,
+    conversationsStored: 0,
+    skippedDedup: 0,
+    skippedTooShort: 0,
+    embedFailed: 0
+  };
+  const sinceDate = options.since ? new Date(options.since) : void 0;
+  if (sinceDate && isNaN(sinceDate.getTime())) {
+    throw new Error(`Invalid --since date: ${options.since}`);
+  }
+  process.stderr.write("[backfill-conversations] Initializing store...\n");
+  await initStore();
+  let daemonConnected = false;
+  if (!options.dryRun) {
+    daemonConnected = await connectEmbedDaemon();
+    if (!daemonConnected) {
+      process.stderr.write(
+        "[backfill-conversations] WARNING: Daemon unavailable \u2014 vectors will be NULL (backfill-vectors can fix later)\n"
+      );
+    }
+  }
+  process.stderr.write("[backfill-conversations] Loading already-ingested conversations...\n");
+  const existingPaths = options.dryRun ? /* @__PURE__ */ new Set() : await loadExistingSourcePaths();
+  process.stderr.write(`[backfill-conversations] ${existingPaths.size} conversations already ingested
+`);
+  const files = await findJsonlFiles(sinceDate, options.project);
+  process.stderr.write(`[backfill-conversations] Found ${files.length} JSONL files to process
+`);
+  process.env.EXE_EMBED_PRIORITY = "low";
+  const BATCH_SIZE = 50;
+  const MAX_DEDUP_SIZE = 5e4;
+  for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
+    const batch = files.slice(batchStart, batchStart + BATCH_SIZE);
+    for (const file of batch) {
+      stats.filesScanned++;
+      if (existingPaths.size < MAX_DEDUP_SIZE && existingPaths.has(file)) {
+        stats.skippedDedup++;
+        continue;
+      }
+      const conv = await parseConversation(file);
+      if (conv.totalMessages < MIN_MESSAGES) {
+        stats.skippedTooShort++;
+        continue;
+      }
+      const summary = buildSummary(conv);
+      if (options.dryRun) {
+        process.stdout.write(`
+\u2500\u2500\u2500 ${file} \u2500\u2500\u2500
+`);
+        process.stdout.write(`Project: ${conv.projectName} | Messages: ${conv.totalMessages}`);
+        process.stdout.write(` | Tools: ${Object.keys(conv.toolCounts).length}`);
+        process.stdout.write(` | Files: ${conv.filesTouched.size}
+`);
+        const firstPrompt = conv.userMessages[0];
+        if (firstPrompt) {
+          process.stdout.write(`First prompt: ${firstPrompt.slice(0, 120)}
+`);
+        }
+        stats.conversationsStored++;
+        continue;
+      }
+      let vector = null;
+      if (daemonConnected) {
+        try {
+          vector = await embedViaClient(summary, "low");
+          if (!vector) stats.embedFailed++;
+        } catch {
+          stats.embedFailed++;
+        }
+      }
+      await writeMemory({
+        id: crypto.randomUUID(),
+        agent_id: conv.agentId,
+        agent_role: isCoordinatorName(conv.agentId) ? "COO" : "specialist",
+        session_id: conv.sessionId,
+        timestamp: conv.startTime ?? (/* @__PURE__ */ new Date()).toISOString(),
+        tool_name: TOOL_NAME,
+        project_name: conv.projectName,
+        has_error: conv.errorCount > 0,
+        raw_text: summary,
+        vector,
+        source_path: file,
+        source_type: "conversation"
+      });
+      if (existingPaths.size < MAX_DEDUP_SIZE) {
+        existingPaths.add(file);
+      }
+      stats.conversationsStored++;
+      if (stats.filesScanned % 50 === 0) {
+        process.stderr.write(
+          `[backfill-conversations] Progress: ${stats.filesScanned}/${files.length} files, ${stats.conversationsStored} stored
+`
+        );
+        await flushBatch();
+      }
+    }
+  }
+  if (!options.dryRun) {
+    await flushBatch();
+  }
+  process.stderr.write(
+    `[backfill-conversations] Done. Scanned: ${stats.filesScanned}, Stored: ${stats.conversationsStored}, Dedup: ${stats.skippedDedup}, TooShort: ${stats.skippedTooShort}, EmbedFail: ${stats.embedFailed}
+`
+  );
+  return stats;
+}
+if (isMainModule(import.meta.url)) {
+  const { values } = parseArgs({
+    options: {
+      since: { type: "string" },
+      project: { type: "string" },
+      "dry-run": { type: "boolean", default: false }
+    },
+    strict: true
+  });
+  backfillConversations({
+    since: values.since,
+    project: values.project,
+    dryRun: values["dry-run"]
+  }).then((result) => {
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(0);
+  }).catch((err) => {
+    console.error("Backfill failed:", err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  });
+}
+export {
+  backfillConversations
+};

package/dist/bin/backfill-responses.js

@@ -0,0 +1,227 @@
+#!/usr/bin/env node
+import {
+  connectEmbedDaemon,
+  embedViaClient
+} from "../chunk-L24HYX62.js";
+import "../chunk-YADYM2DE.js";
+import {
+  flushBatch,
+  initStore,
+  writeMemory
+} from "../chunk-HTCBLSIB.js";
+import "../chunk-CHCA3ZM2.js";
+import "../chunk-IW53DAYO.js";
+import "../chunk-F7LU65PQ.js";
+import "../chunk-MP2AFCGL.js";
+import "../chunk-PWMMIGVQ.js";
+import {
+  getClient,
+  isCoordinatorName
+} from "../chunk-UJOSFVWG.js";
+import "../chunk-QHIGMP2M.js";
+import "../chunk-FXU7JOXK.js";
+import "../chunk-7NNXBDOO.js";
+import "../chunk-TNTFEWU5.js";
+import "../chunk-LYH5HE24.js";
+import {
+  isMainModule
+} from "../chunk-6Y4B3QF6.js";
+import "../chunk-R5U7XKVJ.js";
+
+// src/bin/backfill-responses.ts
+import crypto from "crypto";
+import { createReadStream } from "fs";
+import { readdir, stat } from "fs/promises";
+import path from "path";
+import { createInterface } from "readline";
+import { homedir } from "os";
+var MIN_LENGTH = 100;
+var MAX_LENGTH = 5e3;
+var SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1e3;
+process.env.EXE_EMBED_PRIORITY = "low";
+var MAX_WALK_DEPTH = 10;
+async function findRecentJsonlFiles() {
+  const projectsDir = path.join(homedir(), ".claude", "projects");
+  const cutoff = Date.now() - SEVEN_DAYS_MS;
+  const files = [];
+  async function walk(dir, depth = 0) {
+    if (depth > MAX_WALK_DEPTH) return;
+    let entries;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const entry of entries) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(full, depth + 1);
+      } else if (entry.name.endsWith(".jsonl")) {
+        try {
+          const s = await stat(full);
+          if (s.mtimeMs >= cutoff) files.push(full);
+        } catch {
+        }
+      }
+    }
+  }
+  await walk(projectsDir);
+  return files;
+}
+function projectNameFromPath(filePath) {
+  const projectsDir = path.join(homedir(), ".claude", "projects");
+  const relative = path.relative(projectsDir, filePath);
+  const projectDir = relative.split(path.sep)[0] ?? "";
+  const homeEncoded = homedir().replaceAll("/", "-");
+  if (projectDir.startsWith(homeEncoded + "-")) {
+    return projectDir.slice(homeEncoded.length + 1);
+  }
+  if (projectDir === homeEncoded) return "home";
+  return projectDir;
+}
+function agentIdFromPath(filePath) {
+  return filePath.includes("/subagents/") ? "subagent" : "exe";
+}
+function extractText(content) {
+  const texts = [];
+  for (const block of content) {
+    if (typeof block === "object" && block !== null && block.type === "text" && "text" in block) {
+      texts.push(block.text);
+    }
+  }
+  return texts.join("\n");
+}
+function hashText(text) {
+  return crypto.createHash("sha256").update(text).digest("hex");
+}
+async function loadExistingHashes() {
+  const client = getClient();
+  const cutoff = new Date(Date.now() - SEVEN_DAYS_MS).toISOString();
+  const hashes = /* @__PURE__ */ new Set();
+  let offset = 0;
+  const batchSize = 500;
+  while (true) {
+    const result = await client.execute({
+      sql: `SELECT raw_text FROM memories
+            WHERE tool_name = 'AssistantResponse' AND timestamp > ?
+            ORDER BY id LIMIT ? OFFSET ?`,
+      args: [cutoff, batchSize, offset]
+    });
+    if (result.rows.length === 0) break;
+    for (const row of result.rows) {
+      hashes.add(hashText(row.raw_text));
+    }
+    offset += batchSize;
+  }
+  return hashes;
+}
+async function processFile(filePath, seenHashes, daemonConnected, stats) {
+  const fallbackProject = projectNameFromPath(filePath);
+  const agentId = agentIdFromPath(filePath);
+  let fileCwd;
+  const rl = createInterface({
+    input: createReadStream(filePath, { encoding: "utf8" }),
+    crlfDelay: Infinity
+  });
+  for await (const line of rl) {
+    if (!line.trim()) continue;
+    let entry;
+    try {
+      entry = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (entry.cwd && typeof entry.cwd === "string") fileCwd = entry.cwd;
+    if (entry.type !== "assistant") continue;
+    const message = entry.message;
+    if (!message?.content || !Array.isArray(message.content)) continue;
+    const text = extractText(message.content);
+    if (text.length < MIN_LENGTH) {
+      stats.skippedShort++;
+      continue;
+    }
+    const truncated = text.length > MAX_LENGTH ? text.slice(0, MAX_LENGTH) : text;
+    const hash = hashText(truncated);
+    if (seenHashes.has(hash)) {
+      stats.skippedDedup++;
+      continue;
+    }
+    seenHashes.add(hash);
+    const resolvedProject = fileCwd ? path.basename(fileCwd) : fallbackProject;
+    let vector = null;
+    if (daemonConnected) {
+      try {
+        vector = await embedViaClient(truncated, "low");
+        if (!vector) stats.embedFailed++;
+      } catch {
+        stats.embedFailed++;
+      }
+    }
+    await writeMemory({
+      id: crypto.randomUUID(),
+      agent_id: agentId,
+      agent_role: isCoordinatorName(agentId) ? "COO" : "specialist",
+      session_id: entry.sessionId ?? "backfill",
+      timestamp: entry.timestamp ?? (/* @__PURE__ */ new Date()).toISOString(),
+      tool_name: "AssistantResponse",
+      project_name: resolvedProject,
+      has_error: false,
+      raw_text: truncated,
+      vector
+    });
+    stats.responsesStored++;
+  }
+}
+async function backfillResponses() {
+  const stats = {
+    filesProcessed: 0,
+    responsesStored: 0,
+    skippedDedup: 0,
+    skippedShort: 0,
+    embedFailed: 0
+  };
+  process.stderr.write("[backfill-responses] Initializing store...\n");
+  await initStore();
+  const daemonConnected = await connectEmbedDaemon();
+  if (!daemonConnected) {
+    process.stderr.write(
+      "[backfill-responses] WARNING: Daemon unavailable \u2014 vectors will be NULL (backfill-vectors can fix later)\n"
+    );
+  }
+  process.stderr.write("[backfill-responses] Loading existing response hashes for dedup...\n");
+  const seenHashes = await loadExistingHashes();
+  process.stderr.write(`[backfill-responses] ${seenHashes.size} existing responses loaded
+`);
+  const files = await findRecentJsonlFiles();
+  process.stderr.write(`[backfill-responses] Found ${files.length} JSONL files from last 7 days
+`);
+  for (const file of files) {
+    await processFile(file, seenHashes, daemonConnected, stats);
+    stats.filesProcessed++;
+    if (stats.filesProcessed % 20 === 0) {
+      process.stderr.write(
+        `[backfill-responses] Progress: ${stats.filesProcessed}/${files.length} files, ${stats.responsesStored} stored
+`
+      );
+      await flushBatch();
+    }
+  }
+  await flushBatch();
+  process.stderr.write(
+    `[backfill-responses] Done. Files: ${stats.filesProcessed}, Stored: ${stats.responsesStored}, Dedup: ${stats.skippedDedup}, Short: ${stats.skippedShort}, EmbedFail: ${stats.embedFailed}
+`
+  );
+  return stats;
+}
+if (isMainModule(import.meta.url)) {
+  backfillResponses().then((result) => {
+    console.log(JSON.stringify(result, null, 2));
+    process.exit(0);
+  }).catch((err) => {
+    console.error("Backfill failed:", err instanceof Error ? err.message : String(err));
+    process.exit(1);
+  });
+}
+export {
+  backfillResponses
+};