@arcreflex/agent-transcripts 0.1.5 → 0.1.9

package/src/sync.ts

@@ -1,31 +1,51 @@
 /**
- * Sync command: batch export sessions to markdown transcripts.
+ * Sync command: batch export sessions to transcripts.
  *
  * Discovers session files in source directory, parses them,
- * and writes rendered markdown to output directory.
- * Uses LLM-generated descriptive names when API key is available.
- * Tracks provenance via YAML front matter to correlate updates.
+ * and writes rendered output (markdown or HTML) to output directory.
+ * Tracks provenance via transcripts.json index.
  */
 
-import { Glob } from "bun";
 import { dirname, join } from "path";
-import { mkdir, stat } from "fs/promises";
+import { mkdir } from "fs/promises";
+import { existsSync } from "fs";
 import { getAdapters } from "./adapters/index.ts";
-import type { Adapter } from "./types.ts";
+import type { Adapter, DiscoveredSession, Transcript } from "./types.ts";
 import { renderTranscript } from "./render.ts";
-import { generateOutputName, type NamingOptions } from "./utils/naming.ts";
+import { renderTranscriptHtml } from "./render-html.ts";
+import { renderIndex } from "./render-index.ts";
+import { generateOutputName, extractSessionId } from "./utils/naming.ts";
 import {
-  scanOutputDirectory,
-  deleteExistingOutputs,
-  hasStaleOutputs,
+  loadIndex,
+  saveIndex,
+  setEntry,
+  removeEntriesForSource,
+  restoreEntries,
+  deleteOutputFiles,
+  normalizeSourcePath,
+  extractFirstUserMessage,
+  getOutputsForSource,
+  type TranscriptsIndex,
 } from "./utils/provenance.ts";
+import { generateTitles } from "./title.ts";
+import {
+  computeContentHash,
+  loadCache,
+  saveCache,
+  getCachedSegments,
+  type CacheEntry,
+  type SegmentCache,
+} from "./cache.ts";
+
+export type OutputFormat = "md" | "html";
 
 export interface SyncOptions {
   source: string;
   output: string;
+  format?: OutputFormat;
+  noTitle?: boolean;
   force?: boolean;
   quiet?: boolean;
-  naming?: NamingOptions;
 }
 
 export interface SyncResult {
@@ -34,70 +54,76 @@ export interface SyncResult {
   errors: number;
 }
 
-interface SessionFile {
-  path: string;
-  relativePath: string;
-  mtime: number;
+interface SessionFile extends DiscoveredSession {
   adapter: Adapter;
 }
 
 /**
- * Discover session files for a specific adapter.
+ * Render a transcript to the specified format.
  */
-async function discoverForAdapter(
-  source: string,
-  adapter: Adapter,
-): Promise<SessionFile[]> {
-  const sessions: SessionFile[] = [];
-
-  for (const pattern of adapter.filePatterns) {
-    const glob = new Glob(`**/${pattern}`);
-
-    for await (const file of glob.scan({ cwd: source, absolute: false })) {
-      const fullPath = join(source, file);
-
-      try {
-        const fileStat = await stat(fullPath);
-        sessions.push({
-          path: fullPath,
-          relativePath: file,
-          mtime: fileStat.mtime.getTime(),
-          adapter,
-        });
-      } catch {
-        // Skip files we can't stat
-      }
-    }
+function renderToFormat(
+  transcript: Transcript,
+  format: OutputFormat,
+  options: { sourcePath?: string; title?: string },
+): Promise<string> {
+  if (format === "html") {
+    return renderTranscriptHtml(transcript, { title: options.title });
   }
+  return Promise.resolve(
+    renderTranscript(transcript, { sourcePath: options.sourcePath }),
+  );
+}
 
-  return sessions;
+/**
+ * Generate index.html for HTML output.
+ */
+async function writeIndexHtml(
+  outputDir: string,
+  index: TranscriptsIndex,
+  quiet: boolean,
+): Promise<void> {
+  const indexHtml = renderIndex(index);
+  const indexPath = join(outputDir, "index.html");
+  await Bun.write(indexPath, indexHtml);
+  if (!quiet) {
+    console.error(`Generated: ${indexPath}`);
+  }
 }
 
 /**
  * Sync session files from source to output directory.
  */
 export async function sync(options: SyncOptions): Promise<SyncResult> {
-  const { source, output, force = false, quiet = false, naming } = options;
+  const {
+    source,
+    output,
+    format = "md",
+    noTitle = false,
+    force = false,
+    quiet = false,
+  } = options;
 
+  const ext = format === "html" ? ".html" : ".md";
   const result: SyncResult = { synced: 0, skipped: 0, errors: 0 };
 
-  // Scan output directory for existing transcripts (source → output paths)
-  const existingOutputs = await scanOutputDirectory(output);
-  if (!quiet && existingOutputs.size > 0) {
-    const totalFiles = [...existingOutputs.values()].reduce(
-      (sum, paths) => sum + paths.length,
-      0,
-    );
+  // Ensure output directory exists
+  await mkdir(output, { recursive: true });
+
+  // Load index
+  const index = await loadIndex(output);
+  if (!quiet && Object.keys(index.entries).length > 0) {
     console.error(
-      `Found ${totalFiles} existing transcript(s) from ${existingOutputs.size} source(s)`,
+      `Found ${Object.keys(index.entries).length} existing transcript(s) in index`,
     );
   }
 
-  // Discover sessions for each adapter
+  // Discover sessions from all adapters
   const sessions: SessionFile[] = [];
   for (const adapter of getAdapters()) {
-    const adapterSessions = await discoverForAdapter(source, adapter);
-    sessions.push(...adapterSessions);
+    const discovered = await adapter.discover(source);
+    for (const session of discovered) {
+      sessions.push({ ...session, adapter });
+    }
   }
 
   if (!quiet) {
@@ -106,23 +132,26 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
 
   // Process each session
   for (const session of sessions) {
+    // Normalize source path for consistent index keys
+    const sourcePath = normalizeSourcePath(session.path);
+
     try {
-      // Read and parse using the adapter that discovered this file
+      // Read source and compute content hash
       const content = await Bun.file(session.path).text();
-      const transcripts = session.adapter.parse(content, session.path);
+      const contentHash = computeContentHash(content);
+
+      // Check cache
+      const cached = await loadCache(sourcePath);
+      const cachedSegments = getCachedSegments(cached, contentHash, format);
+
+      // Check if we can use cached output
+      const existingOutputs = getOutputsForSource(index, sourcePath);
+      const outputsExist =
+        existingOutputs.length > 0 &&
+        existingOutputs.every((f) => existsSync(join(output, f)));
 
-      // Get all existing outputs for this source
-      const existingPaths = existingOutputs.get(session.path) || [];
-
-      // Check if sync needed (force, count mismatch, or any stale)
-      const needsUpdate =
-        force ||
-        (await hasStaleOutputs(
-          existingPaths,
-          transcripts.length,
-          session.mtime,
-        ));
-      if (!needsUpdate) {
+      if (!force && cachedSegments && outputsExist) {
+        // Cache hit and outputs exist - skip
         if (!quiet) {
           console.error(`Skip (up to date): ${session.relativePath}`);
         }
@@ -130,39 +159,104 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
         continue;
       }
 
-      // Delete existing outputs before regenerating
-      await deleteExistingOutputs(existingPaths, quiet);
-
-      // Generate fresh outputs for all transcripts
-      for (let i = 0; i < transcripts.length; i++) {
-        const transcript = transcripts[i];
-        const suffix = transcripts.length > 1 ? `_${i + 1}` : undefined;
-
-        // Generate descriptive name, preserving directory structure
-        const baseName = await generateOutputName(
-          transcript,
-          session.path,
-          naming || {},
-        );
-        const finalName = suffix ? `${baseName}${suffix}` : baseName;
-        const relativeDir = dirname(session.relativePath);
-        const outputPath = join(output, relativeDir, `${finalName}.md`);
-
-        // Ensure output directory exists
-        await mkdir(dirname(outputPath), { recursive: true });
-
-        // Render with provenance front matter and write
-        const markdown = renderTranscript(transcript, {
-          sourcePath: session.path,
-        });
-        await Bun.write(outputPath, markdown);
+      // Need to sync: either cache miss, content changed, or force
+      // Parse the source
+      const transcripts = session.adapter.parse(content, session.path);
 
-        if (!quiet) {
-          console.error(`Synced: ${outputPath}`);
+      // Remove entries from index (save for potential restoration on error)
+      const removedEntries = removeEntriesForSource(index, sourcePath);
+
+      // Track new outputs for this session
+      const newOutputs: string[] = [];
+      const sessionId = extractSessionId(session.path);
+
+      // Build new cache entry
+      const newCache: CacheEntry = {
+        contentHash,
+        segments: [],
+      };
+
+      try {
+        // Generate fresh outputs for all transcripts
+        for (let i = 0; i < transcripts.length; i++) {
+          const transcript = transcripts[i];
+          const segmentIndex = transcripts.length > 1 ? i + 1 : undefined;
+
+          // Extract first user message
+          const firstUserMessage = extractFirstUserMessage(transcript);
+
+          // Generate deterministic name
+          const baseName = generateOutputName(transcript, session.path);
+          const suffix = segmentIndex ? `_${segmentIndex}` : "";
+          const relativeDir = dirname(session.relativePath);
+          const relativePath =
+            relativeDir === "."
+              ? `${baseName}${suffix}${ext}`
+              : join(relativeDir, `${baseName}${suffix}${ext}`);
+          const outputPath = join(output, relativePath);
+
+          // Ensure output directory exists
+          await mkdir(dirname(outputPath), { recursive: true });
+
+          // Preserve title from cache if content unchanged
+          const cachedTitle =
+            cached?.contentHash === contentHash
+              ? cached.segments[i]?.title
+              : undefined;
+
+          // Render and write
+          const rendered = await renderToFormat(transcript, format, {
+            sourcePath,
+            title: cachedTitle,
+          });
+          await Bun.write(outputPath, rendered);
+          newOutputs.push(relativePath);
+
+          // Build segment cache
+          const segmentCache: SegmentCache = { title: cachedTitle };
+          segmentCache[format] = rendered;
+          newCache.segments.push(segmentCache);
+
+          // Update index
+          setEntry(index, relativePath, {
+            source: sourcePath,
+            sessionId,
+            segmentIndex,
+            syncedAt: new Date().toISOString(),
+            firstUserMessage,
+            title: cachedTitle,
+            messageCount: transcript.metadata.messageCount,
+            startTime: transcript.metadata.startTime,
+            endTime: transcript.metadata.endTime,
+            cwd: transcript.metadata.cwd,
+          });
+
+          if (!quiet) {
+            console.error(`Synced: ${outputPath}`);
+          }
+        }
+
+        // Save cache
+        await saveCache(sourcePath, newCache);
+
+        // Success: delete old output files (after new ones are written)
+        const oldFilenames = removedEntries.map((e) => e.filename);
+        // Only delete files that aren't being reused
+        const toDelete = oldFilenames.filter((f) => !newOutputs.includes(f));
+        if (toDelete.length > 0) {
+          await deleteOutputFiles(output, toDelete, quiet);
         }
-      }
 
-      result.synced++;
+        result.synced++;
+      } catch (error) {
+        // Clean up any newly written files before restoring old entries
+        if (newOutputs.length > 0) {
+          await deleteOutputFiles(output, newOutputs, quiet);
+        }
+        // Restore old entries on error to preserve provenance
+        restoreEntries(index, removedEntries);
+        throw error;
+      }
     } catch (error) {
       const message = error instanceof Error ? error.message : String(error);
       console.error(`Error: ${session.relativePath}: ${message}`);
@@ -170,6 +264,25 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     }
   }
 
+  // Save index
+  await saveIndex(output, index);
+
+  // Generate titles for HTML format (unless --no-title)
+  if (format === "html" && !noTitle) {
+    if (!quiet) {
+      console.error("\nGenerating titles...");
+    }
+    await generateTitles({ outputDir: output, quiet });
+
+    // Reload index after title generation and regenerate index.html
+    const updatedIndex = await loadIndex(output);
+    await writeIndexHtml(output, updatedIndex, quiet);
+  } else if (format === "html") {
+    // Generate index.html without titles
+    const updatedIndex = await loadIndex(output);
+    await writeIndexHtml(output, updatedIndex, quiet);
+  }
+
   // Summary
   if (!quiet) {
     console.error(

package/src/title.ts

@@ -0,0 +1,172 @@
+/**
+ * Title generation command.
+ *
+ * Adds LLM-generated titles to transcripts.json entries that don't have them.
+ * Can be run standalone or called from sync.
+ */
+
+import { join } from "path";
+import { loadIndex, saveIndex } from "./utils/provenance.ts";
+import { getAdapters } from "./adapters/index.ts";
+import { renderTranscript } from "./render.ts";
+import { renderTranscriptHtml } from "./render-html.ts";
+import { generateTitle } from "./utils/openrouter.ts";
+import {
+  computeContentHash,
+  loadCache,
+  saveCache,
+  getCachedTitle,
+  type CacheEntry,
+} from "./cache.ts";
+
+export interface TitleOptions {
+  outputDir: string;
+  force?: boolean; // regenerate all titles, not just missing ones
+  quiet?: boolean;
+}
+
+export interface TitleResult {
+  generated: number;
+  skipped: number;
+  errors: number;
+}
+
+/**
+ * Generate titles for transcripts.json entries that don't have them.
+ */
+export async function generateTitles(
+  options: TitleOptions,
+): Promise<TitleResult> {
+  const { outputDir, force = false, quiet = false } = options;
+
+  const result: TitleResult = { generated: 0, skipped: 0, errors: 0 };
+
+  if (!process.env.OPENROUTER_API_KEY) {
+    if (!quiet) {
+      console.error("OPENROUTER_API_KEY not set, skipping title generation");
+    }
+    return result;
+  }
+
+  const index = await loadIndex(outputDir);
+  const entries = Object.entries(index.entries);
+
+  if (entries.length === 0) {
+    if (!quiet) {
+      console.error("No entries in transcripts.json");
+    }
+    return result;
+  }
+
+  const adapters = getAdapters();
+  const adapterMap = new Map(adapters.map((a) => [a.name, a]));
+
+  // Process entries that need titles
+  for (const [filename, entry] of entries) {
+    // Skip if already has title (unless force)
+    if (entry.title && !force) {
+      result.skipped++;
+      continue;
+    }
+
+    try {
+      // Read source and compute content hash
+      const content = await Bun.file(entry.source).text();
+      const contentHash = computeContentHash(content);
+
+      // Check cache for existing title
+      const cached = await loadCache(entry.source);
+      const segmentIndex = entry.segmentIndex ? entry.segmentIndex - 1 : 0;
+      const cachedTitle = getCachedTitle(cached, contentHash, segmentIndex);
+
+      if (cachedTitle && !force) {
+        entry.title = cachedTitle;
+        result.skipped++;
+        continue;
+      }
+
+      // Determine adapter from filename pattern (HTML files were synced with an adapter)
+      // We need to find which adapter was used - check the source path
+      let adapter = adapterMap.get("claude-code"); // default
+      for (const a of adapters) {
+        if (entry.source.includes(".claude/")) {
+          adapter = a;
+          break;
+        }
+      }
+
+      if (!adapter) {
+        console.error(`Warning: No adapter found for ${entry.source}`);
+        result.errors++;
+        continue;
+      }
+
+      const transcripts = adapter.parse(content, entry.source);
+
+      // Find the right transcript (by segment index if applicable)
+      const transcript = transcripts[segmentIndex];
+
+      if (!transcript) {
+        console.error(`Warning: Transcript not found for ${filename}`);
+        result.errors++;
+        continue;
+      }
+
+      const markdown = renderTranscript(transcript);
+      const title = await generateTitle(markdown);
+
+      if (title) {
+        entry.title = title;
+        result.generated++;
+        if (!quiet) {
+          console.error(`Title: ${filename} → ${title}`);
+        }
+
+        // Update cache with new title
+        // Start fresh if content changed to avoid stale md/html
+        // Deep copy segments to avoid mutating cached object
+        const newCache: CacheEntry = {
+          contentHash,
+          segments:
+            cached?.contentHash === contentHash
+              ? cached.segments.map((s) => ({ ...s }))
+              : [],
+        };
+        // Ensure segment array is long enough
+        while (newCache.segments.length <= segmentIndex) {
+          newCache.segments.push({});
+        }
+        newCache.segments[segmentIndex].title = title;
+
+        // Re-render HTML with title if this is an HTML file
+        if (filename.endsWith(".html")) {
+          const html = await renderTranscriptHtml(transcript, { title });
+          const outputPath = join(outputDir, filename);
+          await Bun.write(outputPath, html);
+          newCache.segments[segmentIndex].html = html;
+        }
+
+        await saveCache(entry.source, newCache);
+      } else {
+        result.skipped++;
+        if (!quiet) {
+          console.error(`Skip (no title generated): ${filename}`);
+        }
+      }
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.error(`Error: ${filename}: ${message}`);
+      result.errors++;
+    }
+  }
+
+  await saveIndex(outputDir, index);
+
+  if (!quiet) {
+    console.error(
+      `\nTitle generation complete: ${result.generated} generated, ${result.skipped} skipped, ${result.errors} errors`,
+    );
+  }
+
+  return result;
+}

package/src/types.ts

@@ -10,6 +10,10 @@ export interface Transcript {
   };
   metadata: {
     warnings: Warning[];
+    messageCount: number;
+    startTime: string; // ISO timestamp of first message
+    endTime: string; // ISO timestamp of last message
+    cwd?: string; // Working directory (if known)
   };
   messages: Message[];
 }
@@ -31,6 +35,7 @@ interface BaseMessage {
   sourceRef: string;
   timestamp: string;
   parentMessageRef?: string; // UUID of parent message (for tree reconstruction)
+  rawJson?: string; // Original JSON for raw view toggle
 }
 
 export interface UserMessage extends BaseMessage {
@@ -57,6 +62,8 @@ export interface ToolCallGroup extends BaseMessage {
 export interface ToolCall {
   name: string;
   summary: string;
+  input?: Record<string, unknown>;
+  result?: string;
   error?: string;
 }
 
@@ -65,13 +72,22 @@ export interface ErrorMessage extends BaseMessage {
   content: string;
 }
 
+/**
+ * A session file discovered by an adapter.
+ */
+export interface DiscoveredSession {
+  path: string;
+  relativePath: string;
+  mtime: number;
+}
+
 /**
  * Adapter interface - each source format implements this.
  */
 export interface Adapter {
   name: string;
-  /** Glob patterns for discovering session files (e.g., ["*.jsonl"]) */
-  filePatterns: string[];
+  /** Discover session files in the given directory */
+  discover(source: string): Promise<DiscoveredSession[]>;
   /** Parse source content into one or more transcripts (split by conversation) */
   parse(content: string, sourcePath: string): Transcript[];
 }

package/src/utils/html.ts

@@ -0,0 +1,12 @@
+/**
+ * HTML rendering utilities.
+ */
+
+export function escapeHtml(text: string): string {
+  return text
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#039;");
+}