@arcreflex/agent-transcripts 0.1.10 → 0.1.11

package/CLAUDE.md

@@ -4,7 +4,9 @@
 
 ## Architectural Notes
 
-- **Source paths are stable**: Session source paths (e.g., `~/.claude/projects/.../sessions/`) are standardized by the tools that create them. Don't over-engineer for path changes—use source paths as cache keys directly.
+- **Source paths are stable**: Session source paths (e.g., `~/.claude/projects/.../sessions/`) are standardized by the tools that create them. Archive entries store the absolute source path for traceability.
+- **Archive is the central store**: All derived data (titles, etc.) lives on archive entries. Rendered HTML is in-memory only (LRU in serve). No persistent cache layer.
+- **Serve is snapshot-based**: `serve` loads the archive once at startup. It does not live-reload when `watch` archives new sessions. This is a known simplification — revisit if live-updating becomes important.
 
 ## Verification
 

package/README.md

@@ -17,30 +17,35 @@ src/
   render.ts       # Intermediate format → markdown
   render-html.ts  # HTML transcript rendering
   render-index.ts # Index page rendering
-  convert.ts      # Full pipeline with provenance tracking
-  sync.ts         # Batch sync sessions → markdown
-  serve.ts        # HTTP server for dynamic transcript serving
-  cache.ts        # Content-hash-based caching (~/.cache/agent-transcripts/)
-  title.ts        # LLM title generation
+  convert.ts      # Direct pipeline (parse → render to stdout or directory)
+  archive.ts      # Persistent archive store (~/.local/share/agent-transcripts/archive/)
+  watch.ts        # Continuous archive updates via fs.watch + polling
+  serve.ts        # HTTP server serving from archive with in-memory LRU
+  title.ts        # LLM title generation (writes to archive entries)
   types.ts        # Core types (Transcript, Message, Adapter)
   adapters/       # Source format adapters (currently: claude-code)
   utils/
     naming.ts     # Deterministic output file naming
-    provenance.ts # Source tracking via transcripts.json + YAML front matter
     summary.ts    # Tool call summary extraction
     openrouter.ts # OpenRouter API client for title generation
     html.ts       # HTML escaping utility
-    tree.ts       # Tree navigation utilities
+    tree.ts       # Tree navigation and walkTranscriptTree generator
+    text.ts       # Shared text utilities (truncate)
+    theme.ts      # Shared CSS theme constants
 test/
   fixtures/       # Snapshot test inputs/outputs
   snapshots.test.ts
+  archive.test.ts
+  tree.test.ts
+  naming.test.ts
+  summary.test.ts
 ```
 
 ## Commands
 
 ```bash
 bun run check        # typecheck + prettier
-bun run test         # snapshot tests
+bun run test         # snapshot tests + archive tests
 bun run format       # auto-format
 ```
 
@@ -50,9 +55,23 @@ bun run format       # auto-format
 # Subcommands (convert is default if omitted)
 agent-transcripts convert <file>              # Parse and render to stdout
 agent-transcripts convert <file> -o <dir>     # Parse and render to directory
-agent-transcripts sync <dir> -o <out>         # Batch sync sessions
-agent-transcripts serve <dir>                 # Serve transcripts via HTTP
-agent-transcripts serve <dir> -p 8080         # Serve on custom port
+
+# Archive management
+agent-transcripts archive <source>            # Archive sessions from source dir
+agent-transcripts archive <source> --archive-dir ~/my-archive
+
+# Serving
+agent-transcripts serve                       # Serve from default archive
+agent-transcripts serve --archive-dir <dir>   # Serve from custom archive
+agent-transcripts serve -p 8080               # Custom port
+
+# Watching
+agent-transcripts watch <source>              # Keep archive updated continuously
+agent-transcripts watch <source> --poll-interval 60000
+
+# Title generation
+agent-transcripts title                       # Generate titles for archive entries
+agent-transcripts title -f                    # Force regenerate all titles
 
 # Use "-" for stdin
 cat session.jsonl | agent-transcripts -
@@ -60,74 +79,60 @@ cat session.jsonl | agent-transcripts -
 
 ## Architecture
 
-Two-stage pipeline: Parse (source → intermediate) → Render (intermediate → markdown).
+```
+Source (Claude Code sessions)
+    ↓ [archive / watch]
+Archive (~/.local/share/agent-transcripts/archive/{sessionId}.json)
+    ↓ [serve]
+HTML (rendered on demand, in-memory LRU)
+```
+
+`convert` is a standalone direct pipeline (no archive dependency).
+
+`serve` loads the archive once at startup — it won't pick up new sessions archived by a concurrent `watch` without a restart. Live-reloading could be added later (periodic re-listing or file-watch trigger) if needed.
 
 - Adapters handle source formats (see `src/adapters/index.ts` for registry)
 - Auto-detection: paths containing `.claude/` → claude-code adapter
 - Branching conversations preserved via `parentMessageRef` on messages
-- Provenance tracking via `transcripts.json` index + YAML front matter
 - Deterministic naming: `{datetime}-{sessionId}.md`
-- Sync uses sessions-index.json for discovery (claude-code), skipping subagent files
-- Sync uses content hash to skip unchanged sources (see Cache section)
 
-### Cache
+### Archive
 
-Derived content (rendered outputs, LLM-generated titles) is cached at `~/.cache/agent-transcripts/`:
+The archive is the central data store at `~/.local/share/agent-transcripts/archive/`:
 
 ```
-~/.cache/agent-transcripts/
-  {source-path-hash}.json  →  CacheEntry
+~/.local/share/agent-transcripts/archive/
+  {sessionId}.json  →  ArchiveEntry
 ```
 
 ```typescript
-interface CacheEntry {
-  contentHash: string; // hash of source content (invalidation key)
-  segments: Array<{
-    title?: string; // LLM-generated title
-    html?: string; // rendered HTML
-    md?: string; // rendered markdown
-  }>;
+interface ArchiveEntry {
+  sessionId: string;
+  sourcePath: string; // absolute source path
+  sourceHash: string; // content hash (invalidation key)
+  adapterName: string;
+  adapterVersion: string; // e.g. "claude-code:1"
+  schemaVersion: number;
+  archivedAt: string; // ISO timestamp
+  title?: string; // harness-provided or LLM-generated
+  transcripts: Transcript[];
 }
 ```
 
-Cache is keyed by source path (hashed), invalidated by content hash. When source content changes, all cached data is invalidated and regenerated.
-
-### transcripts.json
-
-The index file is a table of contents for the output directory:
-
-```typescript
-interface TranscriptsIndex {
-  version: 1;
-  entries: {
-    [outputFilename: string]: {
-      source: string; // absolute path to source
-      sessionId: string; // full session ID from filename
-      segmentIndex?: number; // for multi-transcript sources (1-indexed)
-      syncedAt: string; // ISO timestamp
-      firstUserMessage: string; // first user message content
-      title?: string; // copied from cache for convenience
-      messageCount: number;
-      startTime: string; // ISO timestamp
-      endTime: string; // ISO timestamp
-      cwd?: string; // working directory
-    };
-  };
-}
-```
+Freshness is determined by `sourceHash + adapterVersion + schemaVersion`. When any changes, the entry is re-archived.
 
 ## Key Types
 
 - `Transcript`: source info, warnings, messages array
 - `Message`: union of UserMessage | AssistantMessage | SystemMessage | ToolCallGroup | ErrorMessage
-- `Adapter`: name, discover function, parse function
+- `Adapter`: name, version, discover function, parse function
 
 ### Titles
 
 Transcripts get titles from (in priority order):
 
 1. Harness-provided summary (e.g., Claude Code's sessions-index.json `summary` field)
-2. Cached title from previous sync
+2. Existing title from previous archive entry
 3. LLM-generated title via OpenRouter (requires `OPENROUTER_API_KEY`)
 
 ## Adding an Adapter
@@ -144,4 +149,6 @@ Transcripts get titles from (in priority order):
 
 Snapshot-based: `*.input.jsonl` → parse → render → compare against `*.output.md`
 
+Archive tests: real fixture files + temp dirs to verify archiving, freshness, listing.
+
 To update snapshots: manually edit the expected `.output.md` files.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcreflex/agent-transcripts",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "description": "Transform AI coding agent session files into readable transcripts",
   "type": "module",
   "repository": {

package/src/adapters/claude-code.ts

@@ -815,6 +815,7 @@ async function discoverByGlob(source: string): Promise<DiscoveredSession[]> {
 
 export const claudeCodeAdapter: Adapter = {
   name: "claude-code",
+  version: "claude-code:1",
 
   async discover(source: string): Promise<DiscoveredSession[]> {
     // Try index-based discovery first, fall back to glob

package/src/adapters/index.ts

@@ -30,16 +30,10 @@ export function detectAdapter(filePath: string): string | undefined {
   return undefined;
 }
 
-/**
- * Get adapter by name.
- */
 export function getAdapter(name: string): Adapter | undefined {
   return adapters[name];
 }
 
-/**
- * List available adapter names.
- */
 export function listAdapters(): string[] {
   return Object.keys(adapters);
 }

package/src/archive.ts

@@ -0,0 +1,267 @@
+/**
+ * Archive module: persistent storage for parsed transcripts.
+ *
+ * Archive entries live at {archiveDir}/{sessionId}.json and contain
+ * the full parsed transcripts plus metadata for freshness checks.
+ */
+
+import { join } from "path";
+import { homedir } from "os";
+import { mkdir, readdir, rename, unlink } from "fs/promises";
+import type { Adapter, DiscoveredSession, Transcript } from "./types.ts";
+import { extractSessionId } from "./utils/naming.ts";
+
+export const DEFAULT_ARCHIVE_DIR = join(
+  homedir(),
+  ".local/share/agent-transcripts/archive",
+);
+
+const ARCHIVE_SCHEMA_VERSION = 1;
+
+export interface ArchiveEntry {
+  sessionId: string;
+  sourcePath: string;
+  sourceHash: string;
+  adapterName: string;
+  adapterVersion: string;
+  schemaVersion: number;
+  archivedAt: string;
+  title?: string;
+  transcripts: Transcript[];
+}
+
+/** Lightweight per-transcript summary for indexing (no message bodies). */
+export interface TranscriptSummary {
+  firstMessageTimestamp: string;
+  firstUserMessage: string;
+  metadata: Transcript["metadata"];
+}
+
+/** Entry header — full metadata but no message bodies. */
+export interface ArchiveEntryHeader {
+  sessionId: string;
+  sourcePath: string;
+  sourceHash: string;
+  title?: string;
+  segments: TranscriptSummary[];
+}
+
+export interface ArchiveResult {
+  updated: string[];
+  current: string[];
+  errors: Array<{ sessionId: string; error: string }>;
+}
+
+export function computeContentHash(content: string): string {
+  return Bun.hash(content).toString(16);
+}
+
+/** Type guard: validates that a parsed JSON value has the shape of an ArchiveEntry. */
+function isArchiveEntry(value: unknown): value is ArchiveEntry {
+  if (typeof value !== "object" || value === null) return false;
+  const v = value as Record<string, unknown>;
+  return (
+    typeof v.sessionId === "string" &&
+    typeof v.sourcePath === "string" &&
+    typeof v.sourceHash === "string" &&
+    typeof v.adapterName === "string" &&
+    typeof v.adapterVersion === "string" &&
+    typeof v.schemaVersion === "number" &&
+    typeof v.archivedAt === "string" &&
+    Array.isArray(v.transcripts)
+  );
+}
+
+export async function loadEntry(
+  archiveDir: string,
+  sessionId: string,
+): Promise<ArchiveEntry | undefined> {
+  let content: string;
+  try {
+    content = await Bun.file(join(archiveDir, `${sessionId}.json`)).text();
+  } catch (err: unknown) {
+    if (
+      err &&
+      typeof err === "object" &&
+      "code" in err &&
+      err.code === "ENOENT"
+    ) {
+      return undefined;
+    }
+    throw err;
+  }
+  const parsed: unknown = JSON.parse(content);
+  if (!isArchiveEntry(parsed)) {
+    console.error(`Warning: invalid archive entry for ${sessionId}, skipping`);
+    return undefined;
+  }
+  return parsed;
+}
+
+export async function saveEntry(
+  archiveDir: string,
+  entry: ArchiveEntry,
+): Promise<void> {
+  await mkdir(archiveDir, { recursive: true });
+
+  const filePath = join(archiveDir, `${entry.sessionId}.json`);
+  const tmpPath = `${filePath}.${process.pid}.${Date.now()}.tmp`;
+  const content = JSON.stringify(entry, null, 2) + "\n";
+
+  await Bun.write(tmpPath, content);
+  try {
+    await rename(tmpPath, filePath);
+  } catch (err) {
+    try {
+      await unlink(tmpPath);
+    } catch {}
+    throw err;
+  }
+}
+
+export function isFresh(
+  entry: ArchiveEntry,
+  sourceHash: string,
+  adapter: Adapter,
+): boolean {
+  return (
+    entry.sourceHash === sourceHash &&
+    entry.adapterVersion === adapter.version &&
+    entry.schemaVersion === ARCHIVE_SCHEMA_VERSION
+  );
+}
+
+export async function archiveSession(
+  archiveDir: string,
+  session: DiscoveredSession,
+  adapter: Adapter,
+): Promise<{ entry: ArchiveEntry; updated: boolean }> {
+  const sessionId = extractSessionId(session.path);
+  const content = await Bun.file(session.path).text();
+  const sourceHash = computeContentHash(content);
+
+  const existing = await loadEntry(archiveDir, sessionId);
+  if (existing && isFresh(existing, sourceHash, adapter)) {
+    // Still update title if harness summary changed
+    if (session.summary && existing.title !== session.summary) {
+      existing.title = session.summary;
+      await saveEntry(archiveDir, existing);
+      return { entry: existing, updated: true };
+    }
+    return { entry: existing, updated: false };
+  }
+
+  const transcripts = adapter.parse(content, session.path);
+
+  const entry: ArchiveEntry = {
+    sessionId,
+    sourcePath: session.path,
+    sourceHash,
+    adapterName: adapter.name,
+    adapterVersion: adapter.version,
+    schemaVersion: ARCHIVE_SCHEMA_VERSION,
+    archivedAt: new Date().toISOString(),
+    title: session.summary ?? existing?.title,
+    transcripts,
+  };
+
+  await saveEntry(archiveDir, entry);
+  return { entry, updated: true };
+}
+
+export async function archiveAll(
+  archiveDir: string,
+  sourceDir: string,
+  adapters: Adapter[],
+  options: { quiet?: boolean } = {},
+): Promise<ArchiveResult> {
+  const result: ArchiveResult = { updated: [], current: [], errors: [] };
+
+  for (const adapter of adapters) {
+    const sessions = await adapter.discover(sourceDir);
+
+    for (const session of sessions) {
+      const sessionId = extractSessionId(session.path);
+      try {
+        const { updated } = await archiveSession(archiveDir, session, adapter);
+        if (updated) {
+          result.updated.push(sessionId);
+          if (!options.quiet) {
+            console.error(`Archived: ${sessionId}`);
+          }
+        } else {
+          result.current.push(sessionId);
+        }
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        result.errors.push({ sessionId, error: message });
+        if (!options.quiet) {
+          console.error(`Error archiving ${sessionId}: ${message}`);
+        }
+      }
+    }
+  }
+
+  return result;
+}
+
+async function readArchiveFiles<T>(
+  archiveDir: string,
+  transform: (entry: ArchiveEntry) => T,
+): Promise<T[]> {
+  let files: string[];
+  try {
+    files = await readdir(archiveDir);
+  } catch {
+    return [];
+  }
+
+  const results: T[] = [];
+  for (const file of files) {
+    if (!file.endsWith(".json")) continue;
+    try {
+      const content = await Bun.file(join(archiveDir, file)).text();
+      const parsed: unknown = JSON.parse(content);
+      if (!isArchiveEntry(parsed)) {
+        console.error(`Warning: invalid archive file ${file}, skipping`);
+        continue;
+      }
+      results.push(transform(parsed));
+    } catch {
+      // Skip corrupt/unreadable entries
+    }
+  }
+  return results;
+}
+
+export async function listEntries(archiveDir: string): Promise<ArchiveEntry[]> {
+  return readArchiveFiles(archiveDir, (entry) => entry);
+}
+
+function summarizeTranscript(t: Transcript): TranscriptSummary {
+  let firstUserMessage = "";
+  for (const msg of t.messages) {
+    if (msg.type === "user") {
+      firstUserMessage = msg.content;
+      break;
+    }
+  }
+  return {
+    firstMessageTimestamp: t.messages[0]?.timestamp ?? "",
+    firstUserMessage,
+    metadata: t.metadata,
+  };
+}
+
+/** Load entry headers only — reads each entry but discards message bodies. */
+export async function listEntryHeaders(
+  archiveDir: string,
+): Promise<ArchiveEntryHeader[]> {
+  return readArchiveFiles(archiveDir, (entry) => ({
+    sessionId: entry.sessionId,
+    sourcePath: entry.sourcePath,
+    sourceHash: entry.sourceHash,
+    title: entry.title,
+    segments: entry.transcripts.map(summarizeTranscript),
+  }));
+}