agent-trajectories 0.5.5 → 0.5.7

package/dist/cli/index.js

@@ -404,8 +404,16 @@ function abandonTrajectory(trajectory, reason) {
 }
 
 // src/storage/file.ts
+import { randomUUID } from "crypto";
 import { existsSync } from "fs";
-import { mkdir, readFile, readdir, unlink, writeFile } from "fs/promises";
+import {
+  mkdir,
+  readFile,
+  readdir,
+  rename,
+  unlink,
+  writeFile
+} from "fs/promises";
 import { join } from "path";
 
 // src/export/markdown.ts
@@ -586,6 +594,16 @@ function getSearchPaths() {
   }
   return [join(process.cwd(), ".trajectories")];
 }
+var indexLocks = /* @__PURE__ */ new Map();
+function withIndexLock(path2, task) {
+  const prev = indexLocks.get(path2) ?? Promise.resolve();
+  const next = prev.then(task, task);
+  indexLocks.set(
+    path2,
+    next.catch(() => void 0)
+  );
+  return next;
+}
 var FileStorage = class {
   baseDir;
   trajectoriesDir;
@@ -612,10 +630,10 @@ var FileStorage = class {
     await mkdir(this.activeDir, { recursive: true });
     await mkdir(this.completedDir, { recursive: true });
     if (!existsSync(this.indexPath)) {
-      await this.saveIndex({
-        version: 1,
-        lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
-        trajectories: {}
+      await withIndexLock(this.indexPath, async () => {
+        if (!existsSync(this.indexPath)) {
+          await this.saveIndex(this.emptyIndex());
+        }
       });
     }
     await this.reconcileIndex();
@@ -643,49 +661,51 @@ var FileStorage = class {
       skippedSchemaViolation: 0,
       skippedIoError: 0
     };
-    const index = await this.loadIndex();
-    const before = Object.keys(index.trajectories).length;
-    const discovered = [];
-    try {
-      const activeFiles = await readdir(this.activeDir);
-      for (const file of activeFiles) {
-        if (!file.endsWith(".json")) continue;
-        discovered.push(join(this.activeDir, file));
+    await withIndexLock(this.indexPath, async () => {
+      const index = await this.loadIndex();
+      const before = Object.keys(index.trajectories).length;
+      const discovered = [];
+      try {
+        const activeFiles = await readdir(this.activeDir);
+        for (const file of activeFiles) {
+          if (!file.endsWith(".json")) continue;
+          discovered.push(join(this.activeDir, file));
+        }
+      } catch (error) {
+        if (error.code !== "ENOENT") throw error;
       }
-    } catch (error) {
-      if (error.code !== "ENOENT") throw error;
-    }
-    await this.walkJsonFilesInto(this.completedDir, discovered);
-    for (const filePath of discovered) {
-      summary.scanned += 1;
-      const result = await this.readTrajectoryFile(filePath);
-      if (!result.ok) {
-        if (result.reason === "malformed_json") {
-          summary.skippedMalformedJson += 1;
-        } else if (result.reason === "schema_violation") {
-          summary.skippedSchemaViolation += 1;
-        } else {
-          summary.skippedIoError += 1;
+      await this.walkJsonFilesInto(this.completedDir, discovered);
+      for (const filePath of discovered) {
+        summary.scanned += 1;
+        const result = await this.readTrajectoryFile(filePath);
+        if (!result.ok) {
+          if (result.reason === "malformed_json") {
+            summary.skippedMalformedJson += 1;
+          } else if (result.reason === "schema_violation") {
+            summary.skippedSchemaViolation += 1;
+          } else {
+            summary.skippedIoError += 1;
+          }
+          continue;
         }
-        continue;
+        const trajectory = result.trajectory;
+        if (index.trajectories[trajectory.id]) {
+          summary.alreadyIndexed += 1;
+          continue;
+        }
+        index.trajectories[trajectory.id] = {
+          title: trajectory.task.title,
+          status: trajectory.status,
+          startedAt: trajectory.startedAt,
+          completedAt: trajectory.completedAt,
+          path: filePath
+        };
+        summary.added += 1;
       }
-      const trajectory = result.trajectory;
-      if (index.trajectories[trajectory.id]) {
-        summary.alreadyIndexed += 1;
-        continue;
+      if (Object.keys(index.trajectories).length !== before) {
+        await this.saveIndex(index);
       }
-      index.trajectories[trajectory.id] = {
-        title: trajectory.task.title,
-        status: trajectory.status,
-        startedAt: trajectory.startedAt,
-        completedAt: trajectory.completedAt,
-        path: filePath
-      };
-      summary.added += 1;
-    }
-    if (Object.keys(index.trajectories).length !== before) {
-      await this.saveIndex(index);
-    }
+    });
     const hadSkips = summary.skippedMalformedJson + summary.skippedSchemaViolation + summary.skippedIoError > 0;
     if (summary.added > 0 || hadSkips) {
       const parts = [`reconciled ${summary.added}/${summary.scanned}`];
@@ -888,17 +908,19 @@ var FileStorage = class {
     if (existsSync(activePath)) {
       await unlink(activePath);
     }
-    const index = await this.loadIndex();
-    const entry = index.trajectories[id];
-    if (entry?.path && existsSync(entry.path)) {
-      await unlink(entry.path);
-      const mdPath = entry.path.replace(".json", ".md");
-      if (existsSync(mdPath)) {
-        await unlink(mdPath);
+    await withIndexLock(this.indexPath, async () => {
+      const index = await this.loadIndex();
+      const entry = index.trajectories[id];
+      if (entry?.path && existsSync(entry.path)) {
+        await unlink(entry.path);
+        const mdPath = entry.path.replace(".json", ".md");
+        if (existsSync(mdPath)) {
+          await unlink(mdPath);
+        }
       }
-    }
-    delete index.trajectories[id];
-    await this.saveIndex(index);
+      delete index.trajectories[id];
+      await this.saveIndex(index);
+    });
   }
   /**
    * Search trajectories by text
@@ -976,10 +998,23 @@ var FileStorage = class {
     const result = await this.readTrajectoryFile(path2);
     return result.ok ? result.trajectory : null;
   }
+  /**
+   * Read and parse the on-disk index.
+   *
+   * Tolerances (belt-and-braces against the read/write race):
+   *   - ENOENT: first-run, return an empty index silently.
+   *   - Empty file: a concurrent writer truncated index.json in "w" mode
+   *     right before we read. Return an empty index silently — this is
+   *     not a real corruption, just an interleaving the mutex + atomic
+   *     rename should already prevent. Logging here would be noise.
+   *   - Non-empty but malformed JSON: genuinely corrupted on disk (hand
+   *     edit, disk error, etc). Log it and return an empty index so the
+   *     caller can recover, but keep the log so the problem is visible.
+   */
   async loadIndex() {
+    let content;
     try {
-      const content = await readFile(this.indexPath, "utf-8");
-      return JSON.parse(content);
+      content = await readFile(this.indexPath, "utf-8");
     } catch (error) {
       if (error.code !== "ENOENT") {
         console.error(
@@ -987,27 +1022,56 @@ var FileStorage = class {
           error
         );
       }
-      return {
-        version: 1,
-        lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
-        trajectories: {}
-      };
+      return this.emptyIndex();
+    }
+    if (content.length === 0) {
+      return this.emptyIndex();
+    }
+    try {
+      return JSON.parse(content);
+    } catch (error) {
+      console.error(
+        "Error loading trajectory index, using empty index:",
+        error
+      );
+      return this.emptyIndex();
     }
   }
+  emptyIndex() {
+    return {
+      version: 1,
+      lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
+      trajectories: {}
+    };
+  }
+  /**
+   * Atomic write: stage into a process-unique temp path in the same directory
+   * and then rename over the live file. `rename` is atomic on POSIX, so
+   * concurrent readers in any process either see the old complete file or
+   * the new complete file — never a half-written / zero-byte state.
+   *
+   * Callers MUST hold `withIndexLock(this.indexPath, ...)` so the in-process
+   * read-modify-write cycle stays serialized; the unique temp name also keeps
+   * parallel writers in other processes from colliding on a shared tmp path.
+   */
   async saveIndex(index) {
     index.lastUpdated = (/* @__PURE__ */ new Date()).toISOString();
-    await writeFile(this.indexPath, JSON.stringify(index, null, 2), "utf-8");
+    const tmpPath = `${this.indexPath}.${process.pid}.${randomUUID()}.tmp`;
+    await writeFile(tmpPath, JSON.stringify(index, null, 2), "utf-8");
+    await rename(tmpPath, this.indexPath);
   }
   async updateIndex(trajectory, filePath) {
-    const index = await this.loadIndex();
-    index.trajectories[trajectory.id] = {
-      title: trajectory.task.title,
-      status: trajectory.status,
-      startedAt: trajectory.startedAt,
-      completedAt: trajectory.completedAt,
-      path: filePath
-    };
-    await this.saveIndex(index);
+    await withIndexLock(this.indexPath, async () => {
+      const index = await this.loadIndex();
+      index.trajectories[trajectory.id] = {
+        title: trajectory.task.title,
+        status: trajectory.status,
+        startedAt: trajectory.startedAt,
+        completedAt: trajectory.completedAt,
+        path: filePath
+      };
+      await this.saveIndex(index);
+    });
   }
 };
 
@@ -1032,7 +1096,13 @@ function registerAbandonCommand(program2) {
 
 // src/cli/commands/compact.ts
 import { execFileSync } from "child_process";
-import { existsSync as existsSync3, mkdirSync, readFileSync as readFileSync2, writeFileSync } from "fs";
+import {
+  existsSync as existsSync3,
+  mkdirSync,
+  readFileSync as readFileSync2,
+  unlinkSync,
+  writeFileSync
+} from "fs";
 import { dirname, join as join4 } from "path";
 
 // src/compact/config.ts
@@ -2178,7 +2248,10 @@ function registerCompactCommand(program2) {
   ).option("--all", "Include all trajectories, even previously compacted ones").option("--llm", "Use LLM-based compaction when a provider is available").option("--no-llm", "Disable LLM-based compaction").option("--mechanical", "Force the original mechanical compaction flow").option(
     "--focus <areas>",
     "Comma-separated focus areas to emphasize in LLM compaction"
-  ).option("--markdown", "Also write a Markdown companion file").option("--no-markdown", "Skip writing a Markdown companion file").option("--dry-run", "Preview what would be compacted without saving").option("--output <path>", "Output path for compacted trajectory").action(async (options) => {
+  ).option("--markdown", "Also write a Markdown companion file").option("--no-markdown", "Skip writing a Markdown companion file").option(
+    "--discard-sources",
+    "After saving the compaction, delete source trajectory JSON/MD/trace files and remove their index entries"
+  ).option("--dry-run", "Preview what would be compacted without saving").option("--output <path>", "Output path for compacted trajectory").action(async (options) => {
     const trajectories = await loadTrajectories(options);
     if (trajectories.length === 0) {
       if (options.all || options.since || options.ids || options.workflow || options.pr || options.branch || options.commits) {
@@ -2217,7 +2290,15 @@ function registerCompactCommand(program2) {
         outputPath2,
         markdownEnabled
       );
-      await markTrajectoriesAsCompacted(trajectories, mechanicalCompacted.id);
+      if (options.discardSources) {
+        const discardSummary = discardSourceTrajectories(trajectories);
+        printDiscardSummary(discardSummary);
+      } else {
+        await markTrajectoriesAsCompacted(
+          trajectories,
+          mechanicalCompacted.id
+        );
+      }
       console.log(`
 Compacted trajectory saved to: ${outputPath2}`);
       if (markdownEnabled) {
@@ -2270,7 +2351,12 @@ Compacted trajectory saved to: ${outputPath2}`);
     };
     const outputPath = options.output || getDefaultOutputPath(compacted, options.workflow);
     saveCompactionArtifacts(compacted, outputPath, markdownEnabled);
-    await markTrajectoriesAsCompacted(trajectories, compacted.id);
+    if (options.discardSources) {
+      const discardSummary = discardSourceTrajectories(trajectories);
+      printDiscardSummary(discardSummary);
+    } else {
+      await markTrajectoriesAsCompacted(trajectories, compacted.id);
+    }
     console.log(`
 Compacted trajectory saved to: ${outputPath}`);
     if (markdownEnabled) {
@@ -2418,6 +2504,74 @@ async function markTrajectoriesAsCompacted(trajectories, compactedIntoId) {
     }
   }
 }
+function discardSourceTrajectories(trajectories) {
+  const sourceIds = new Set(trajectories.map((trajectory) => trajectory.id));
+  const summary = {
+    removedIndexEntries: 0,
+    deletedJsonFiles: 0,
+    deletedMarkdownFiles: 0,
+    deletedTraceFiles: 0
+  };
+  for (const searchPath of getSearchPaths()) {
+    const indexPath = join4(searchPath, "index.json");
+    if (!existsSync3(indexPath)) continue;
+    let index;
+    try {
+      const indexContent = readFileSync2(indexPath, "utf-8");
+      const parsedIndex = JSON.parse(indexContent);
+      if (!isTrajectoryIndex(parsedIndex)) {
+        continue;
+      }
+      index = parsedIndex;
+    } catch {
+      continue;
+    }
+    let updated = false;
+    for (const id of sourceIds) {
+      const entry = index.trajectories[id];
+      if (!entry) continue;
+      if (deleteFileIfExists(entry.path)) {
+        summary.deletedJsonFiles += 1;
+      }
+      if (deleteFileIfExists(getMarkdownOutputPath(entry.path))) {
+        summary.deletedMarkdownFiles += 1;
+      }
+      if (deleteFileIfExists(getTraceOutputPath(entry.path))) {
+        summary.deletedTraceFiles += 1;
+      }
+      delete index.trajectories[id];
+      summary.removedIndexEntries += 1;
+      updated = true;
+    }
+    if (updated) {
+      index.lastUpdated = (/* @__PURE__ */ new Date()).toISOString();
+      writeFileSync(indexPath, JSON.stringify(index, null, 2));
+    }
+  }
+  return summary;
+}
+function deleteFileIfExists(path2) {
+  if (!existsSync3(path2)) {
+    return false;
+  }
+  unlinkSync(path2);
+  return true;
+}
+function isTrajectoryIndex(value) {
+  if (value === null || typeof value !== "object") {
+    return false;
+  }
+  const candidate = value;
+  return candidate.trajectories !== null && typeof candidate.trajectories === "object" && !Array.isArray(candidate.trajectories);
+}
+function getTraceOutputPath(outputPath) {
+  return outputPath.endsWith(".json") ? outputPath.slice(0, -".json".length).concat(".trace.json") : `${outputPath}.trace.json`;
+}
+function printDiscardSummary(summary) {
+  console.log(
+    `Discarded source trajectories: ${summary.removedIndexEntries} index entries, ${summary.deletedJsonFiles} JSON files, ${summary.deletedMarkdownFiles} Markdown files, ${summary.deletedTraceFiles} trace files`
+  );
+}
 function parseRelativeDate(input) {
   const match = input.match(/^(\d+)([dwmh])$/);
   if (match) {