codepiper 0.1.0

package/packages/daemon/src/sessions/transcriptManager.ts

@@ -0,0 +1,110 @@
+/**
+ * TranscriptManager - manages transcript tailers for multiple sessions
+ * with batched offset persistence.
+ */
+
+import { TranscriptTailer } from "./transcriptTailer";
+
+export interface TranscriptOffsetStore {
+  getOffset(sessionId: string): Promise<number>;
+  setOffset(sessionId: string, offset: number): Promise<void>;
+}
+
+export interface TranscriptManagerOptions {
+  offsetStore: TranscriptOffsetStore;
+  onLine: (sessionId: string, line: string, offset: number) => void;
+  onError?: (sessionId: string, error: Error) => void;
+}
+
+export class TranscriptManager {
+  private readonly tailers = new Map<string, TranscriptTailer>();
+  private readonly offsetStore: TranscriptOffsetStore;
+  private readonly onLine: (sessionId: string, line: string, offset: number) => void;
+  private readonly onError: ((sessionId: string, error: Error) => void) | undefined;
+  private readonly batchedOffsets = new Map<string, number>();
+  private flushInterval: Timer | null = null;
+
+  constructor(options: TranscriptManagerOptions) {
+    this.offsetStore = options.offsetStore;
+    this.onLine = options.onLine;
+    this.onError = options.onError;
+
+    this.flushInterval = setInterval(() => {
+      this.flushOffsets();
+    }, 1000);
+  }
+
+  async startTailing(sessionId: string, transcriptPath: string): Promise<void> {
+    if (this.tailers.has(sessionId)) {
+      throw new Error(`Already tailing transcript for session ${sessionId}`);
+    }
+
+    const initialOffset = await this.offsetStore.getOffset(sessionId);
+
+    const tailer = new TranscriptTailer({
+      sessionId,
+      transcriptPath,
+      initialOffset,
+      onLine: (line: string, offset: number) => {
+        this.onLine(sessionId, line, offset);
+        this.batchedOffsets.set(sessionId, offset);
+      },
+      onError: (error: Error) => {
+        this.onError?.(sessionId, error);
+      },
+    });
+
+    await tailer.start();
+    this.tailers.set(sessionId, tailer);
+  }
+
+  async stopTailing(sessionId: string): Promise<void> {
+    const tailer = this.tailers.get(sessionId);
+    if (!tailer) {
+      return;
+    }
+
+    await tailer.stop();
+    await this.offsetStore.setOffset(sessionId, tailer.getCurrentOffset());
+
+    this.tailers.delete(sessionId);
+    this.batchedOffsets.delete(sessionId);
+  }
+
+  async stopAll(): Promise<void> {
+    if (this.flushInterval) {
+      clearInterval(this.flushInterval);
+      this.flushInterval = null;
+    }
+
+    const sessionIds = Array.from(this.tailers.keys());
+    await Promise.all(sessionIds.map((id) => this.stopTailing(id)));
+  }
+
+  getCurrentOffset(sessionId: string): number | undefined {
+    return this.tailers.get(sessionId)?.getCurrentOffset();
+  }
+
+  isTailing(sessionId: string): boolean {
+    return this.tailers.has(sessionId);
+  }
+
+  getSessions(): string[] {
+    return Array.from(this.tailers.keys());
+  }
+
+  private async flushOffsets(): Promise<void> {
+    if (this.batchedOffsets.size === 0) {
+      return;
+    }
+
+    const offsets = new Map(this.batchedOffsets);
+    this.batchedOffsets.clear();
+
+    const promises: Promise<void>[] = [];
+    for (const [sessionId, offset] of offsets) {
+      promises.push(this.offsetStore.setOffset(sessionId, offset));
+    }
+    await Promise.all(promises);
+  }
+}

package/packages/daemon/src/sessions/transcriptParser.ts

@@ -0,0 +1,149 @@
+/**
+ * Transcript event parser and normalizer
+ *
+ * Parses JSONL transcript events from Claude Code and normalizes them to a common format
+ * for storage and streaming.
+ */
+
+export interface NormalizedEvent {
+  type: "message" | "tool_use" | "tool_result" | "system" | "unknown";
+  role?: "user" | "assistant" | "system";
+  content?: string;
+  toolName?: string;
+  toolInput?: any;
+  toolResult?: any;
+  timestamp?: string;
+  metadata: Record<string, any>; // Preserve all fields
+  raw: string; // Original JSON line
+}
+
+/**
+ * Parse a single JSONL line from a Claude Code transcript
+ *
+ * @param line - Raw JSONL line
+ * @returns Normalized event or null if parsing fails
+ */
+export function parseTranscriptLine(line: string): NormalizedEvent | null {
+  // Handle empty or whitespace-only lines
+  const trimmed = line.trim();
+  if (!trimmed) {
+    return null;
+  }
+
+  // Parse JSON safely
+  let parsed: any;
+  try {
+    parsed = JSON.parse(trimmed);
+  } catch {
+    return null;
+  }
+
+  // Detect event type
+  const type = detectEventType(parsed);
+
+  // Extract role (if present)
+  const role = parsed.role as "user" | "assistant" | "system" | undefined;
+
+  // Extract content
+  const content = extractContent(parsed);
+
+  // Extract tool-specific fields
+  let toolName: string | undefined;
+  let toolInput: any;
+  let toolResult: any;
+
+  if (type === "tool_use") {
+    toolName = parsed.name;
+    toolInput = parsed.input;
+  } else if (type === "tool_result") {
+    toolResult = content !== null ? content : parsed.content;
+  }
+
+  // Extract timestamp
+  const timestamp = parsed.timestamp;
+
+  // Build normalized event
+  return {
+    type,
+    role,
+    content: content !== null ? content : undefined,
+    toolName,
+    toolInput,
+    toolResult,
+    timestamp,
+    metadata: parsed, // Preserve all original fields
+    raw: trimmed,
+  };
+}
+
+/**
+ * Extract text content from an event
+ *
+ * Handles both string content and array content (with text blocks)
+ *
+ * @param event - Parsed event object
+ * @returns Extracted text content or null
+ */
+export function extractContent(event: any): string | null {
+  if (event.content === undefined || event.content === null) {
+    return null;
+  }
+
+  // Handle string content (including empty strings)
+  if (typeof event.content === "string") {
+    return event.content;
+  }
+
+  // Handle array content (extract text blocks)
+  if (Array.isArray(event.content)) {
+    const textBlocks = event.content
+      .filter((item: any) => item.type === "text" && item.text)
+      .map((item: any) => item.text);
+
+    if (textBlocks.length === 0) {
+      return null;
+    }
+
+    return textBlocks.join("\n");
+  }
+
+  return null;
+}
+
+/**
+ * Detect the type of event based on its structure
+ *
+ * @param event - Parsed event object
+ * @returns Detected event type
+ */
+export function detectEventType(event: any): NormalizedEvent["type"] {
+  // Check explicit type field first
+  if (event.type === "tool_use") {
+    return "tool_use";
+  }
+  if (event.type === "tool_result") {
+    return "tool_result";
+  }
+
+  // Check for system role
+  if (event.role === "system") {
+    return "system";
+  }
+
+  // Check for message roles
+  if (event.role === "user" || event.role === "assistant") {
+    return "message";
+  }
+
+  // Infer tool_use from structure
+  if (event.name && event.input !== undefined) {
+    return "tool_use";
+  }
+
+  // Infer tool_result from structure
+  if (event.tool_use_id) {
+    return "tool_result";
+  }
+
+  return "unknown";
+}

package/packages/daemon/src/sessions/transcriptTailer.ts

@@ -0,0 +1,214 @@
+/**
+ * TranscriptTailer - reads Claude Code JSONL transcripts line-by-line
+ * with byte offset tracking for crash-safe resumption.
+ */
+
+import * as fs from "node:fs";
+
+export interface TranscriptTailerOptions {
+  sessionId: string;
+  transcriptPath: string;
+  initialOffset: number;
+  onLine: (line: string, offset: number) => void;
+  onError?: (error: Error) => void;
+}
+
+export class TranscriptTailer {
+  private readonly transcriptPath: string;
+  private readonly onLine: (line: string, offset: number) => void;
+  private readonly onError: ((error: Error) => void) | undefined;
+
+  private watcher: fs.FSWatcher | null = null;
+  private pollInterval: Timer | null = null;
+  private isRunning = false;
+  private isReading = false;
+  private readPending = false;
+  private buffer = "";
+  private currentOffset: number;
+  private lastFileSize = 0;
+
+  constructor(options: TranscriptTailerOptions) {
+    this.transcriptPath = options.transcriptPath;
+    this.currentOffset = options.initialOffset;
+    this.onLine = options.onLine;
+    this.onError = options.onError;
+  }
+
+  async start(): Promise<void> {
+    if (this.isRunning) {
+      return;
+    }
+
+    this.isRunning = true;
+    this.buffer = "";
+
+    await this.scheduleRead();
+    this.setupWatcher();
+    this.setupPolling();
+  }
+
+  async stop(): Promise<void> {
+    if (!this.isRunning) {
+      return;
+    }
+
+    this.isRunning = false;
+    this.closeWatcher();
+    this.clearPollInterval();
+    this.buffer = "";
+  }
+
+  getCurrentOffset(): number {
+    return this.currentOffset;
+  }
+
+  private closeWatcher(): void {
+    if (!this.watcher) {
+      return;
+    }
+
+    try {
+      this.watcher.close();
+    } catch {
+      // Ignore errors during cleanup
+    }
+    this.watcher = null;
+  }
+
+  private clearPollInterval(): void {
+    if (!this.pollInterval) {
+      return;
+    }
+
+    clearInterval(this.pollInterval);
+    this.pollInterval = null;
+  }
+
+  private setupWatcher(): void {
+    try {
+      if (!fs.existsSync(this.transcriptPath)) {
+        return;
+      }
+
+      this.watcher = fs.watch(this.transcriptPath, (eventType) => {
+        if (!this.isRunning) {
+          return;
+        }
+
+        if (eventType === "change" || eventType === "rename") {
+          void this.scheduleRead();
+        }
+      });
+
+      this.watcher.on("error", () => {
+        this.closeWatcher();
+      });
+    } catch {
+      // Polling remains enabled as a fallback.
+    }
+  }
+
+  private setupPolling(): void {
+    if (this.pollInterval) {
+      return;
+    }
+
+    this.pollInterval = setInterval(async () => {
+      if (!this.isRunning) {
+        return;
+      }
+
+      void this.scheduleRead();
+    }, 100);
+  }
+
+  /**
+   * Serialize reads to avoid concurrent read/parse races between fs.watch and polling callbacks.
+   */
+  private async scheduleRead(): Promise<void> {
+    if (this.isReading) {
+      this.readPending = true;
+      return;
+    }
+
+    this.isReading = true;
+    try {
+      do {
+        this.readPending = false;
+        await this.readNewLines();
+      } while (this.readPending && this.isRunning);
+    } finally {
+      this.isReading = false;
+    }
+  }
+
+  private async readNewLines(): Promise<void> {
+    try {
+      if (!fs.existsSync(this.transcriptPath)) {
+        if (this.lastFileSize > 0 || this.currentOffset > 0) {
+          throw new Error(`Transcript file was deleted: ${this.transcriptPath}`);
+        }
+        return;
+      }
+
+      const fileSize = fs.statSync(this.transcriptPath).size;
+
+      // Detect file rotation (size decreased)
+      if (fileSize < this.lastFileSize || fileSize < this.currentOffset) {
+        this.currentOffset = 0;
+        this.buffer = "";
+      }
+
+      this.lastFileSize = fileSize;
+
+      if (fileSize === this.currentOffset) {
+        return;
+      }
+
+      const fd = fs.openSync(this.transcriptPath, "r");
+      try {
+        const bytesToRead = fileSize - this.currentOffset;
+        const readBuffer = Buffer.allocUnsafe(bytesToRead);
+        const bytesRead = fs.readSync(fd, readBuffer, 0, bytesToRead, this.currentOffset);
+
+        if (bytesRead === 0) {
+          return;
+        }
+
+        const newData = readBuffer.slice(0, bytesRead).toString("utf-8");
+        const data = this.buffer + newData;
+
+        this.parseLines(data, this.currentOffset - Buffer.byteLength(this.buffer, "utf-8"));
+        this.currentOffset += bytesRead;
+      } finally {
+        fs.closeSync(fd);
+      }
+    } catch (error) {
+      if (this.onError && this.isRunning) {
+        this.onError(error instanceof Error ? error : new Error(String(error)));
+      }
+    }
+  }
+
+  /**
+   * Parse complete lines from data, buffering any trailing incomplete line.
+   */
+  private parseLines(data: string, baseOffset: number): void {
+    const parts = data.split("\n");
+    let offset = baseOffset;
+
+    for (let i = 0; i < parts.length - 1; i++) {
+      const line = parts[i];
+      if (line === undefined) {
+        continue;
+      }
+      const lineWithNewline = `${line}\n`;
+      const lineBytes = Buffer.byteLength(lineWithNewline, "utf-8");
+      offset += lineBytes;
+      this.onLine(line, offset);
+    }
+
+    // Buffer the trailing incomplete segment for the next read
+    this.buffer = parts[parts.length - 1] ?? "";
+  }
+}

package/packages/daemon/src/tracking/tokenTracker.ts

@@ -0,0 +1,168 @@
+import type { Event } from "@codepiper/core";
+import type { Database, InsertModelSwitchParams, InsertTokenUsageParams } from "../db/db";
+
+/**
+ * Token usage data from Claude Code transcript
+ */
+export interface TokenUsageData {
+  model: string;
+  promptTokens: number;
+  completionTokens: number;
+  cacheCreationInputTokens?: number;
+  cacheReadInputTokens?: number;
+  totalTokens: number;
+  estimatedCostUsd?: number;
+  actualCostUsd?: number;
+  costDifferenceUsd?: number;
+}
+
+/**
+ * TokenTracker - Extracts and tracks token usage from transcript events
+ *
+ * Features:
+ * - Parses Claude Code transcript JSONL for token usage
+ * - Tracks cache metrics (cache_creation_input_tokens, cache_read_input_tokens)
+ * - Stores actual cost data from Claude Code
+ * - Detects model switches
+ * - Provides aggregated statistics
+ */
+export class TokenTracker {
+  private currentModels = new Map<string, string>(); // sessionId -> currentModel
+
+  constructor(private db: Database) {}
+
+  private formatError(error: unknown): string {
+    if (error instanceof Error && error.message.trim().length > 0) {
+      return error.message;
+    }
+    return String(error);
+  }
+
+  /**
+   * Process transcript event and extract token usage
+   */
+  async processTranscriptEvent(event: Event): Promise<void> {
+    if (event.type !== "transcript:line") {
+      return;
+    }
+
+    const sessionId = event.sessionId;
+    if (!sessionId) {
+      console.warn("[TokenTracker] Ignoring transcript event without sessionId");
+      return;
+    }
+
+    try {
+      const transcriptLine = event.payload as any;
+
+      // Extract token usage from assistant messages
+      if (transcriptLine.role === "assistant" && transcriptLine.usage) {
+        this.processTokenUsage(sessionId, transcriptLine);
+      }
+
+      // Detect model switches
+      if (transcriptLine.model) {
+        this.detectModelSwitch(sessionId, transcriptLine.model);
+      }
+    } catch (err) {
+      console.error(`[TokenTracker] Error processing transcript event: ${this.formatError(err)}`);
+    }
+  }
+
+  /**
+   * Process token usage from transcript line
+   */
+  private processTokenUsage(sessionId: string, transcriptLine: any): void {
+    const usage = transcriptLine.usage;
+    const model = transcriptLine.model || "unknown";
+
+    // Calculate total tokens
+    const totalTokens =
+      (usage.input_tokens || 0) +
+      (usage.output_tokens || 0) +
+      (usage.cache_creation_input_tokens || 0) +
+      (usage.cache_read_input_tokens || 0);
+
+    // Extract cost data if available
+    let estimatedCostUsd: number | undefined;
+    let actualCostUsd: number | undefined;
+    let costDifferenceUsd: number | undefined;
+
+    if (transcriptLine.cost) {
+      estimatedCostUsd = transcriptLine.cost.estimated;
+      actualCostUsd = transcriptLine.cost.actual;
+      if (estimatedCostUsd !== undefined && actualCostUsd !== undefined) {
+        costDifferenceUsd = actualCostUsd - estimatedCostUsd;
+      }
+    }
+
+    const params: InsertTokenUsageParams = {
+      sessionId,
+      model,
+      promptTokens: usage.input_tokens || 0,
+      completionTokens: usage.output_tokens || 0,
+      cacheCreationInputTokens: usage.cache_creation_input_tokens || 0,
+      cacheReadInputTokens: usage.cache_read_input_tokens || 0,
+      totalTokens,
+      estimatedCostUsd,
+      actualCostUsd,
+      costDifferenceUsd,
+    };
+
+    this.db.insertTokenUsage(params);
+  }
+
+  /**
+   * Detect and record model switches
+   */
+  private detectModelSwitch(sessionId: string, newModel: string): void {
+    const currentModel = this.currentModels.get(sessionId);
+
+    if (currentModel && currentModel !== newModel) {
+      // Model switch detected
+      const params: InsertModelSwitchParams = {
+        sessionId,
+        fromModel: currentModel,
+        toModel: newModel,
+        reason: "Model changed during session",
+      };
+
+      this.db.insertModelSwitch(params);
+    }
+
+    // Update current model
+    this.currentModels.set(sessionId, newModel);
+  }
+
+  /**
+   * Get token usage statistics for a session
+   */
+  getSessionStats(sessionId: string): {
+    totalTokens: number;
+    totalCost: number;
+    byModel: Record<string, { tokens: number; cost: number }>;
+  } {
+    return this.db.getTotalTokensBySessionId(sessionId);
+  }
+
+  /**
+   * Get recent token usage for a session
+   */
+  getRecentUsage(sessionId: string, limit: number = 10) {
+    return this.db.getTokenUsageBySessionId(sessionId, { limit });
+  }
+
+  /**
+   * Get model switch history for a session
+   */
+  getModelSwitches(sessionId: string) {
+    return this.db.getModelSwitchesBySessionId(sessionId);
+  }
+
+  /**
+   * Clear session state (call when session ends)
+   */
+  clearSessionState(sessionId: string): void {
+    this.currentModels.delete(sessionId);
+  }
+}