@arcreflex/agent-transcripts 0.1.9 → 0.1.11

package/src/archive.ts

@@ -0,0 +1,267 @@
+/**
+ * Archive module: persistent storage for parsed transcripts.
+ *
+ * Archive entries live at {archiveDir}/{sessionId}.json and contain
+ * the full parsed transcripts plus metadata for freshness checks.
+ */
+
+import { join } from "path";
+import { homedir } from "os";
+import { mkdir, readdir, rename, unlink } from "fs/promises";
+import type { Adapter, DiscoveredSession, Transcript } from "./types.ts";
+import { extractSessionId } from "./utils/naming.ts";
+
+export const DEFAULT_ARCHIVE_DIR = join(
+  homedir(),
+  ".local/share/agent-transcripts/archive",
+);
+
+const ARCHIVE_SCHEMA_VERSION = 1;
+
+export interface ArchiveEntry {
+  sessionId: string;
+  sourcePath: string;
+  sourceHash: string;
+  adapterName: string;
+  adapterVersion: string;
+  schemaVersion: number;
+  archivedAt: string;
+  title?: string;
+  transcripts: Transcript[];
+}
+
+/** Lightweight per-transcript summary for indexing (no message bodies). */
+export interface TranscriptSummary {
+  firstMessageTimestamp: string;
+  firstUserMessage: string;
+  metadata: Transcript["metadata"];
+}
+
+/** Entry header — full metadata but no message bodies. */
+export interface ArchiveEntryHeader {
+  sessionId: string;
+  sourcePath: string;
+  sourceHash: string;
+  title?: string;
+  segments: TranscriptSummary[];
+}
+
+export interface ArchiveResult {
+  updated: string[];
+  current: string[];
+  errors: Array<{ sessionId: string; error: string }>;
+}
+
+export function computeContentHash(content: string): string {
+  return Bun.hash(content).toString(16);
+}
+
+/** Type guard: validates that a parsed JSON value has the shape of an ArchiveEntry. */
+function isArchiveEntry(value: unknown): value is ArchiveEntry {
+  if (typeof value !== "object" || value === null) return false;
+  const v = value as Record<string, unknown>;
+  return (
+    typeof v.sessionId === "string" &&
+    typeof v.sourcePath === "string" &&
+    typeof v.sourceHash === "string" &&
+    typeof v.adapterName === "string" &&
+    typeof v.adapterVersion === "string" &&
+    typeof v.schemaVersion === "number" &&
+    typeof v.archivedAt === "string" &&
+    Array.isArray(v.transcripts)
+  );
+}
+
+export async function loadEntry(
+  archiveDir: string,
+  sessionId: string,
+): Promise<ArchiveEntry | undefined> {
+  let content: string;
+  try {
+    content = await Bun.file(join(archiveDir, `${sessionId}.json`)).text();
+  } catch (err: unknown) {
+    if (
+      err &&
+      typeof err === "object" &&
+      "code" in err &&
+      err.code === "ENOENT"
+    ) {
+      return undefined;
+    }
+    throw err;
+  }
+  const parsed: unknown = JSON.parse(content);
+  if (!isArchiveEntry(parsed)) {
+    console.error(`Warning: invalid archive entry for ${sessionId}, skipping`);
+    return undefined;
+  }
+  return parsed;
+}
+
+export async function saveEntry(
+  archiveDir: string,
+  entry: ArchiveEntry,
+): Promise<void> {
+  await mkdir(archiveDir, { recursive: true });
+
+  const filePath = join(archiveDir, `${entry.sessionId}.json`);
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  const content = JSON.stringify(entry, null, 2) + "\n";
+
+  await Bun.write(tmpPath, content);
+  try {
+    await rename(tmpPath, filePath);
+  } catch (err) {
+    try {
+      await unlink(tmpPath);
+    } catch {}
+    throw err;
+  }
+}
+
+export function isFresh(
+  entry: ArchiveEntry,
+  sourceHash: string,
+  adapter: Adapter,
+): boolean {
+  return (
+    entry.sourceHash === sourceHash &&
+    entry.adapterVersion === adapter.version &&
+    entry.schemaVersion === ARCHIVE_SCHEMA_VERSION
+  );
+}
+
+export async function archiveSession(
+  archiveDir: string,
+  session: DiscoveredSession,
+  adapter: Adapter,
+): Promise<{ entry: ArchiveEntry; updated: boolean }> {
+  const sessionId = extractSessionId(session.path);
+  const content = await Bun.file(session.path).text();
+  const sourceHash = computeContentHash(content);
+
+  const existing = await loadEntry(archiveDir, sessionId);
+  if (existing && isFresh(existing, sourceHash, adapter)) {
+    // Still update title if harness summary changed
+    if (session.summary && existing.title !== session.summary) {
+      existing.title = session.summary;
+      await saveEntry(archiveDir, existing);
+      return { entry: existing, updated: true };
+    }
+    return { entry: existing, updated: false };
+  }
+
+  const transcripts = adapter.parse(content, session.path);
+
+  const entry: ArchiveEntry = {
+    sessionId,
+    sourcePath: session.path,
+    sourceHash,
+    adapterName: adapter.name,
+    adapterVersion: adapter.version,
+    schemaVersion: ARCHIVE_SCHEMA_VERSION,
+    archivedAt: new Date().toISOString(),
+    title: session.summary ?? existing?.title,
+    transcripts,
+  };
+
+  await saveEntry(archiveDir, entry);
+  return { entry, updated: true };
+}
+
+export async function archiveAll(
+  archiveDir: string,
+  sourceDir: string,
+  adapters: Adapter[],
+  options: { quiet?: boolean } = {},
+): Promise<ArchiveResult> {
+  const result: ArchiveResult = { updated: [], current: [], errors: [] };
+
+  for (const adapter of adapters) {
+    const sessions = await adapter.discover(sourceDir);
+
+    for (const session of sessions) {
+      const sessionId = extractSessionId(session.path);
+      try {
+        const { updated } = await archiveSession(archiveDir, session, adapter);
+        if (updated) {
+          result.updated.push(sessionId);
+          if (!options.quiet) {
+            console.error(`Archived: ${sessionId}`);
+          }
+        } else {
+          result.current.push(sessionId);
+        }
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        result.errors.push({ sessionId, error: message });
+        if (!options.quiet) {
+          console.error(`Error archiving ${sessionId}: ${message}`);
+        }
+      }
+    }
+  }
+
+  return result;
+}
+
+async function readArchiveFiles<T>(
+  archiveDir: string,
+  transform: (entry: ArchiveEntry) => T,
+): Promise<T[]> {
+  let files: string[];
+  try {
+    files = await readdir(archiveDir);
+  } catch {
+    return [];
+  }
+
+  const results: T[] = [];
+  for (const file of files) {
+    if (!file.endsWith(".json")) continue;
+    try {
+      const content = await Bun.file(join(archiveDir, file)).text();
+      const parsed: unknown = JSON.parse(content);
+      if (!isArchiveEntry(parsed)) {
+        console.error(`Warning: invalid archive file ${file}, skipping`);
+        continue;
+      }
+      results.push(transform(parsed));
+    } catch {
+      // Skip corrupt/unreadable entries
+    }
+  }
+  return results;
+}
+
+export async function listEntries(archiveDir: string): Promise<ArchiveEntry[]> {
+  return readArchiveFiles(archiveDir, (entry) => entry);
+}
+
+function summarizeTranscript(t: Transcript): TranscriptSummary {
+  let firstUserMessage = "";
+  for (const msg of t.messages) {
+    if (msg.type === "user") {
+      firstUserMessage = msg.content;
+      break;
+    }
+  }
+  return {
+    firstMessageTimestamp: t.messages[0]?.timestamp ?? "",
+    firstUserMessage,
+    metadata: t.metadata,
+  };
+}
+
+/** Load entry headers only — reads each entry but discards message bodies. */
+export async function listEntryHeaders(
+  archiveDir: string,
+): Promise<ArchiveEntryHeader[]> {
+  return readArchiveFiles(archiveDir, (entry) => ({
+    sessionId: entry.sessionId,
+    sourcePath: entry.sourcePath,
+    sourceHash: entry.sourceHash,
+    title: entry.title,
+    segments: entry.transcripts.map(summarizeTranscript),
+  }));
+}

package/src/cli.ts

@@ -15,20 +15,12 @@ import {
 } from "cmd-ts";
 import { parseToTranscripts } from "./parse.ts";
 import { renderTranscript } from "./render.ts";
-import { sync, type OutputFormat } from "./sync.ts";
 import { convertToDirectory } from "./convert.ts";
 import { generateTitles } from "./title.ts";
 import { serve } from "./serve.ts";
-
-// Custom type for format option
-const formatType = {
-  async from(value: string): Promise<OutputFormat> {
-    if (value !== "md" && value !== "html") {
-      throw new Error(`Invalid format: ${value}. Must be "md" or "html".`);
-    }
-    return value;
-  },
-};
+import { archiveAll, DEFAULT_ARCHIVE_DIR } from "./archive.ts";
+import { getAdapters } from "./adapters/index.ts";
+import { ArchiveWatcher } from "./watch.ts";
 
 // Shared options
 const inputArg = positional({
@@ -57,57 +49,47 @@ const headOpt = option({
   description: "Render branch ending at this message ID (default: latest)",
 });
 
-// Sync subcommand
-const syncCmd = command({
-  name: "sync",
-  description: "Sync session files to transcripts (markdown or HTML)",
+const archiveDirOpt = option({
+  type: optional(string),
+  long: "archive-dir",
+  description: `Archive directory (default: ${DEFAULT_ARCHIVE_DIR})`,
+});
+
+// Archive subcommand
+const archiveCmd = command({
+  name: "archive",
+  description: "Archive session files from source directory",
   args: {
     source: positional({
       type: string,
       displayName: "source",
       description: "Source directory to scan for session files",
     }),
-    output: option({
-      type: string,
-      long: "output",
-      short: "o",
-      description: "Output directory for transcripts",
-    }),
-    format: option({
-      type: optional(formatType),
-      long: "format",
-      description: "Output format: md (default) or html",
-    }),
-    noTitle: flag({
-      long: "no-title",
-      description: "Skip LLM title generation (for HTML format)",
-    }),
-    force: flag({
-      long: "force",
-      short: "f",
-      description: "Re-render all sessions, ignoring mtime",
-    }),
+    archiveDir: archiveDirOpt,
     quiet: flag({
       long: "quiet",
       short: "q",
       description: "Suppress progress output",
     }),
   },
-  async handler({ source, output, format, noTitle, force, quiet }) {
-    await sync({ source, output, format, noTitle, force, quiet });
+  async handler({ source, archiveDir, quiet }) {
+    const dir = archiveDir ?? DEFAULT_ARCHIVE_DIR;
+    const result = await archiveAll(dir, source, getAdapters(), { quiet });
+
+    if (!quiet) {
+      console.error(
+        `\nArchive complete: ${result.updated.length} updated, ${result.current.length} current, ${result.errors.length} errors`,
+      );
+    }
   },
 });
 
 // Title subcommand
 const titleCmd = command({
   name: "title",
-  description: "Generate LLM titles for transcripts.json entries",
+  description: "Generate LLM titles for archive entries",
   args: {
-    output: positional({
-      type: string,
-      displayName: "output",
-      description: "Output directory containing transcripts.json",
-    }),
+    archiveDir: archiveDirOpt,
     force: flag({
       long: "force",
       short: "f",
@@ -119,21 +101,21 @@ const titleCmd = command({
       description: "Suppress progress output",
     }),
   },
-  async handler({ output, force, quiet }) {
-    await generateTitles({ outputDir: output, force, quiet });
+  async handler({ archiveDir, force, quiet }) {
+    await generateTitles({
+      archiveDir: archiveDir ?? undefined,
+      force,
+      quiet,
+    });
   },
 });
 
 // Serve subcommand
 const serveCmd = command({
   name: "serve",
-  description: "Serve transcripts via HTTP (dynamic rendering with caching)",
+  description: "Serve transcripts from archive via HTTP",
   args: {
-    source: positional({
-      type: string,
-      displayName: "source",
-      description: "Source directory to scan for session files",
-    }),
+    archiveDir: archiveDirOpt,
     port: option({
       type: optional(number),
       long: "port",
@@ -145,13 +127,66 @@ const serveCmd = command({
       short: "q",
       description: "Suppress request logging",
     }),
-    noCache: flag({
-      long: "no-cache",
-      description: "Bypass HTML cache (for development)",
+  },
+  async handler({ archiveDir, port, quiet }) {
+    await serve({
+      archiveDir: archiveDir ?? undefined,
+      port: port ?? 3000,
+      quiet,
+    });
+  },
+});
+
+// Watch subcommand
+const watchCmd = command({
+  name: "watch",
+  description: "Watch source directories and keep archive updated",
+  args: {
+    source: positional({
+      type: string,
+      displayName: "source",
+      description: "Source directory to watch for session files",
+    }),
+    archiveDir: archiveDirOpt,
+    pollInterval: option({
+      type: optional(number),
+      long: "poll-interval",
+      description: "Poll interval in milliseconds (default: 30000)",
+    }),
+    quiet: flag({
+      long: "quiet",
+      short: "q",
+      description: "Suppress progress output",
     }),
   },
-  async handler({ source, port, quiet, noCache }) {
-    await serve({ source, port: port ?? 3000, quiet, noCache });
+  async handler({ source, archiveDir, pollInterval, quiet }) {
+    const watcher = new ArchiveWatcher([source], {
+      archiveDir: archiveDir ?? undefined,
+      pollIntervalMs: pollInterval ?? undefined,
+      quiet,
+      onUpdate(result) {
+        if (!quiet && result.updated.length > 0) {
+          console.error(`Updated: ${result.updated.join(", ")}`);
+        }
+      },
+      onError(error) {
+        console.error(`Watch error: ${error.message}`);
+      },
+    });
+
+    if (!quiet) {
+      console.error(`Watching ${source}...`);
+    }
+
+    await watcher.start();
+
+    process.on("SIGINT", () => {
+      if (!quiet) {
+        console.error("\nStopping watcher...");
+      }
+      watcher.stop();
+      process.exit(0);
+    });
   },
 });
 
@@ -174,7 +209,6 @@ const convertCmd = command({
   },
   async handler({ input, output, adapter, head }) {
     if (output && isDirectoryOutput(output)) {
-      // Directory output: use provenance tracking
       await convertToDirectory({
         input,
         outputDir: output,
@@ -182,23 +216,21 @@ const convertCmd = command({
         head,
       });
     } else if (output) {
-      // Explicit file output: not supported anymore (use directory)
       console.error(
         "Error: Explicit file output not supported. Use a directory path instead.",
       );
       process.exit(1);
     } else {
-      // No output: stream to stdout
       const { transcripts } = await parseToTranscripts({ input, adapter });
       for (let i = 0; i < transcripts.length; i++) {
-        if (i > 0) console.log(); // blank line between transcripts
-        console.log(renderTranscript(transcripts[i], head));
+        if (i > 0) console.log();
+        console.log(renderTranscript(transcripts[i], { head }));
       }
     }
   },
 });
 
-const SUBCOMMANDS = ["convert", "sync", "title", "serve"] as const;
+const SUBCOMMANDS = ["convert", "archive", "title", "serve", "watch"] as const;
 
 // Main CLI with subcommands
 const cli = subcommands({
@@ -206,9 +238,10 @@ const cli = subcommands({
   description: "Transform agent session files to readable transcripts",
   cmds: {
     convert: convertCmd,
-    sync: syncCmd,
+    archive: archiveCmd,
     title: titleCmd,
     serve: serveCmd,
+    watch: watchCmd,
   },
 });
 

package/src/convert.ts

@@ -1,25 +1,15 @@
 /**
- * Convert command: full pipeline with provenance tracking.
+ * Convert command: parse source and render to markdown.
  *
- * When output is a directory, uses provenance tracking via transcripts.json
- * index to manage output files.
+ * Standalone pipeline with no archive dependency. Directory output
+ * writes markdown files with deterministic names.
  */
 
-import { join } from "path";
+import { join, resolve } from "path";
 import { mkdir } from "fs/promises";
 import { parseToTranscripts } from "./parse.ts";
 import { renderTranscript } from "./render.ts";
-import { generateOutputName, extractSessionId } from "./utils/naming.ts";
-import {
-  loadIndex,
-  saveIndex,
-  removeEntriesForSource,
-  restoreEntries,
-  deleteOutputFiles,
-  setEntry,
-  normalizeSourcePath,
-  extractFirstUserMessage,
-} from "./utils/provenance.ts";
+import { generateOutputName } from "./utils/naming.ts";
 
 export interface ConvertToDirectoryOptions {
   input: string;
@@ -28,91 +18,34 @@ export interface ConvertToDirectoryOptions {
   head?: string;
 }
 
-/**
- * Convert source file to markdown in output directory.
- * Uses provenance tracking to replace existing outputs.
- */
 export async function convertToDirectory(
   options: ConvertToDirectoryOptions,
 ): Promise<void> {
   const { input, outputDir, adapter, head } = options;
 
-  // Ensure output directory exists
   await mkdir(outputDir, { recursive: true });
 
-  // Parse input to transcripts
   const { transcripts, inputPath } = await parseToTranscripts({
     input,
     adapter,
   });
 
-  // Normalize source path for consistent index keys
-  const sourcePath = normalizeSourcePath(inputPath);
-
-  // Load index and handle existing outputs
-  const index = await loadIndex(outputDir);
-
-  // Remove old entries (save for restoration on error)
-  const removedEntries =
-    sourcePath !== "<stdin>" ? removeEntriesForSource(index, sourcePath) : [];
-
-  const sessionId = extractSessionId(inputPath);
-  const newOutputs: string[] = [];
+  const sourcePath = inputPath === "<stdin>" ? undefined : resolve(inputPath);
 
-  try {
-    // Generate fresh outputs
-    for (let i = 0; i < transcripts.length; i++) {
-      const transcript = transcripts[i];
-      const segmentIndex = transcripts.length > 1 ? i + 1 : undefined;
+  for (let i = 0; i < transcripts.length; i++) {
+    const transcript = transcripts[i];
+    const segmentIndex = transcripts.length > 1 ? i + 1 : undefined;
 
-      // Generate deterministic name
-      const baseName = generateOutputName(transcript, inputPath);
-      const suffix = segmentIndex ? `_${segmentIndex}` : "";
-      const relativePath = `${baseName}${suffix}.md`;
-      const outputPath = join(outputDir, relativePath);
+    const baseName = generateOutputName(transcript, inputPath);
+    const suffix = segmentIndex ? `_${segmentIndex}` : "";
+    const relativePath = `${baseName}${suffix}.md`;
+    const outputPath = join(outputDir, relativePath);
 
-      // Render with provenance front matter
-      const markdown = renderTranscript(transcript, {
-        head,
-        sourcePath: sourcePath !== "<stdin>" ? sourcePath : undefined,
-      });
-      await Bun.write(outputPath, markdown);
-      newOutputs.push(relativePath);
-
-      // Update index (only for non-stdin sources)
-      if (sourcePath !== "<stdin>") {
-        setEntry(index, relativePath, {
-          source: sourcePath,
-          sessionId,
-          segmentIndex,
-          syncedAt: new Date().toISOString(),
-          firstUserMessage: extractFirstUserMessage(transcript),
-          messageCount: transcript.metadata.messageCount,
-          startTime: transcript.metadata.startTime,
-          endTime: transcript.metadata.endTime,
-          cwd: transcript.metadata.cwd,
-        });
-      }
-
-      console.error(`Wrote: ${outputPath}`);
-    }
-
-    // Success: delete old output files (after new ones are written)
-    const oldFilenames = removedEntries.map((e) => e.filename);
-    const toDelete = oldFilenames.filter((f) => !newOutputs.includes(f));
-    if (toDelete.length > 0) {
-      await deleteOutputFiles(outputDir, toDelete);
-    }
-  } catch (error) {
-    // Clean up any newly written files before restoring old entries
-    if (newOutputs.length > 0) {
-      await deleteOutputFiles(outputDir, newOutputs);
-    }
-    // Restore old entries on error to preserve provenance
-    restoreEntries(index, removedEntries);
-    throw error;
+    const markdown = renderTranscript(transcript, {
+      head,
+      sourcePath,
+    });
+    await Bun.write(outputPath, markdown);
+    console.error(`Wrote: ${outputPath}`);
   }
-
-  // Save index
-  await saveIndex(outputDir, index);
 }

package/src/parse.ts

@@ -15,9 +15,6 @@ export interface ParseResult {
   inputPath: string;
 }
 
-/**
- * Read input content from file or stdin.
- */
 async function readInput(
   input: string,
 ): Promise<{ content: string; path: string }> {