cognitive-core 0.1.0 → 0.2.0

package/src/session-bank/parser.ts

@@ -0,0 +1,548 @@
+import type {
+  SessionRecord,
+  SessionMetadata,
+  TranscriptMessage,
+  ContentBlock,
+  SessionSummary,
+  TokenUsage,
+  Attribution,
+  CheckpointRef,
+} from './types.js';
+
+// ─── Format Detection ──────────────────────────────────────────────
+
+export type CheckpointFormat = 'entire' | 'sessionlog' | 'unknown';
+
+/** Auto-detect whether raw metadata JSON is Entire CLI or SessionLog format. */
+export function detectFormat(raw: Record<string, unknown>): CheckpointFormat {
+  if ('checkpointID' in raw) return 'sessionlog';
+  if ('checkpoint_id' in raw) return 'entire';
+  if (raw.cliVersion === 'opentasks-sessionlog') return 'sessionlog';
+  if (typeof raw.cli_version === 'string') return 'entire';
+  return 'unknown';
+}
+
+// ─── Raw Entire JSON shapes (snake_case, matching Go structs) ───────
+
+interface RawEntireMetadata {
+  cli_version?: string;
+  checkpoint_id?: string;
+  session_id?: string;
+  strategy?: string;
+  created_at?: string;
+  branch?: string;
+  files_touched?: string[];
+  agent?: string;
+  is_task?: boolean;
+  tool_use_id?: string;
+  checkpoint_transcript_start?: number;
+  token_usage?: RawEntireTokenUsage;
+  summary?: RawEntireSummary;
+  initial_attribution?: RawEntireAttribution;
+}
+
+interface RawEntireTokenUsage {
+  input_tokens?: number;
+  cache_creation_tokens?: number;
+  cache_read_tokens?: number;
+  output_tokens?: number;
+  api_call_count?: number;
+  subagent_usage?: RawEntireTokenUsage[];
+}
+
+interface RawEntireSummary {
+  intent?: string;
+  outcome?: string;
+  learnings?: {
+    repo_patterns?: string[];
+    code_findings?: string[];
+    workflow_practices?: string[];
+  };
+  friction_points?: string[];
+  open_items?: string[];
+}
+
+interface RawEntireAttribution {
+  agent_lines_added?: number;
+  human_additions?: number;
+  human_modifications?: number;
+  human_removals?: number;
+  total_committed_lines?: number;
+  agent_contribution?: number;
+}
+
+// ─── Raw SessionLog JSON shapes (camelCase) ─────────────────────────
+
+interface RawSessionLogMetadata {
+  cliVersion?: string;
+  checkpointID?: string;
+  sessionID?: string;
+  strategy?: string;
+  createdAt?: string;
+  branch?: string;
+  filesTouched?: string[];
+  agent?: string;
+  turnID?: string;
+  isTask?: boolean;
+  toolUseID?: string;
+  checkpointTranscriptStart?: number;
+  tokenUsage?: RawSessionLogTokenUsage;
+  summary?: RawSessionLogSummary;
+  initialAttribution?: RawSessionLogAttribution;
+  tasks?: Record<string, unknown>;
+  planEntries?: unknown[];
+  skillsUsed?: unknown[];
+}
+
+interface RawSessionLogTokenUsage {
+  inputTokens?: number;
+  cacheCreationTokens?: number;
+  cacheReadTokens?: number;
+  outputTokens?: number;
+  apiCallCount?: number;
+  subagentTokens?: RawSessionLogTokenUsage;
+}
+
+interface RawSessionLogSummary {
+  intent?: string;
+  outcome?: string;
+  learnings?: {
+    repo?: string[];
+    code?: Array<{ path: string; line?: number; endLine?: number; finding: string }>;
+    workflow?: string[];
+  };
+  friction?: string[];
+  openItems?: string[];
+}
+
+interface RawSessionLogAttribution {
+  calculatedAt?: string;
+  agentLines?: number;
+  humanAdded?: number;
+  humanModified?: number;
+  humanRemoved?: number;
+  totalCommitted?: number;
+  agentPercentage?: number;
+}
+
+// ─── Shared raw types for transcript parsing ────────────────────────
+
+interface RawTranscriptLine {
+  type?: string;
+  uuid?: string;
+  message?: {
+    role?: string;
+    content?: string | RawContentBlock[];
+  };
+  // Entire CLI uses these for top-level tool_result lines
+  tool_use_id?: string;
+  content?: string | RawContentBlock[];
+  is_error?: boolean;
+  timestamp?: string;
+}
+
+interface RawContentBlock {
+  type?: string;
+  text?: string;
+  id?: string;
+  name?: string;
+  input?: Record<string, unknown>;
+  tool_use_id?: string;
+  content?: string;
+  is_error?: boolean;
+}
+
+interface RawCheckpointSummary {
+  // Entire format (snake_case)
+  sessions?: Array<{
+    metadata?: string;
+    transcript?: string;
+    context?: string;
+    content_hash?: string;
+    prompt?: string;
+  }>;
+  // SessionLog format may also use camelCase — same structure
+  checkpointID?: string;
+  checkpointsCount?: number;
+}
+
+// ─── Parser ─────────────────────────────────────────────────────────
+
+export interface ParsedMetadata {
+  sessionId: string;
+  checkpointId: string;
+  metadata: SessionMetadata;
+  summary?: SessionSummary;
+  tokenUsage: TokenUsage;
+  attribution?: Attribution;
+  format: CheckpointFormat;
+}
+
+export class CheckpointParser {
+  /**
+   * Parse a full session from its component files.
+   */
+  parseSession(
+    metadataJson: string,
+    transcriptJsonl: string,
+    checkpointPath: string,
+    sessionIndex: string,
+    sourceBranch?: string,
+  ): SessionRecord {
+    const parsed = this.parseMetadata(metadataJson);
+    const transcript = this.parseTranscript(transcriptJsonl);
+
+    return {
+      sessionId: parsed.sessionId,
+      checkpointId: parsed.checkpointId,
+      metadata: parsed.metadata,
+      transcript,
+      summary: parsed.summary,
+      tokenUsage: parsed.tokenUsage,
+      attribution: parsed.attribution,
+      source: {
+        branch: sourceBranch ?? 'sessionlog/checkpoints/v1',
+        checkpointPath,
+        sessionIndex,
+      },
+    };
+  }
+
+  /**
+   * Parse per-session metadata.json (fast path — no transcript).
+   * Auto-detects Entire (snake_case) vs SessionLog (camelCase) format.
+   */
+  parseMetadata(metadataJson: string): ParsedMetadata {
+    const raw = JSON.parse(metadataJson) as Record<string, unknown>;
+    const format = detectFormat(raw);
+
+    if (format === 'sessionlog') {
+      return this.parseSessionLogMetadata(raw as unknown as RawSessionLogMetadata);
+    }
+    return this.parseEntireMetadata(raw as unknown as RawEntireMetadata);
+  }
+
+  /**
+   * Parse a JSONL transcript into structured messages.
+   * Tolerant of malformed lines — skips them silently.
+   * Handles both Entire ("human") and SessionLog ("user") role types.
+   */
+  parseTranscript(jsonlContent: string): TranscriptMessage[] {
+    const messages: TranscriptMessage[] = [];
+    const lines = jsonlContent.split('\n');
+
+    for (let i = 0; i < lines.length; i++) {
+      const line = lines[i].trim();
+      if (!line) continue;
+
+      try {
+        const raw: RawTranscriptLine = JSON.parse(line);
+        const msg = this.parseTranscriptLine(raw, i);
+        if (msg) messages.push(msg);
+      } catch {
+        // Skip malformed lines
+      }
+    }
+
+    return messages;
+  }
+
+  /**
+   * Parse chunked transcripts (full.jsonl.001, .002, etc.)
+   */
+  parseChunkedTranscript(chunks: string[]): TranscriptMessage[] {
+    const combined = chunks.join('\n');
+    return this.parseTranscript(combined);
+  }
+
+  /**
+   * Parse the root checkpoint summary to discover session subdirectories.
+   */
+  parseCheckpointSummary(
+    metadataJson: string,
+    checkpointPath: string,
+  ): CheckpointRef {
+    const raw: RawCheckpointSummary = JSON.parse(metadataJson);
+    const sessions = raw.sessions ?? [];
+    const checkpointId = this.checkpointIdFromPath(checkpointPath);
+
+    return {
+      checkpointId,
+      sessionIds: [], // populated later from per-session metadata
+      path: checkpointPath,
+      sessionCount: sessions.length,
+    };
+  }
+
+  // ── Entire format parsing (snake_case) ────────────────────────
+
+  private parseEntireMetadata(raw: RawEntireMetadata): ParsedMetadata {
+    const tokenUsage = this.parseEntireTokenUsage(raw.token_usage);
+    const summary = raw.summary ? this.parseEntireSummary(raw.summary) : undefined;
+    const attribution = raw.initial_attribution
+      ? this.parseEntireAttribution(raw.initial_attribution)
+      : undefined;
+
+    return {
+      sessionId: raw.session_id ?? '',
+      checkpointId: raw.checkpoint_id ?? '',
+      metadata: {
+        cliVersion: raw.cli_version ?? '',
+        strategy: raw.strategy === 'auto-commit' ? 'auto-commit' : 'manual-commit',
+        createdAt: raw.created_at ? new Date(raw.created_at) : new Date(0),
+        branch: raw.branch ?? '',
+        agent: raw.agent ?? '',
+        filesTouched: raw.files_touched ?? [],
+        isTask: raw.is_task ?? false,
+        toolUseId: raw.tool_use_id,
+        checkpointTranscriptStart: raw.checkpoint_transcript_start ?? 0,
+      },
+      summary,
+      tokenUsage,
+      attribution,
+      format: 'entire',
+    };
+  }
+
+  private parseEntireTokenUsage(raw?: RawEntireTokenUsage): TokenUsage {
+    if (!raw) {
+      return {
+        inputTokens: 0,
+        cacheCreationTokens: 0,
+        cacheReadTokens: 0,
+        outputTokens: 0,
+        apiCallCount: 0,
+        total: 0,
+      };
+    }
+
+    const inputTokens = raw.input_tokens ?? 0;
+    const cacheCreationTokens = raw.cache_creation_tokens ?? 0;
+    const cacheReadTokens = raw.cache_read_tokens ?? 0;
+    const outputTokens = raw.output_tokens ?? 0;
+    const apiCallCount = raw.api_call_count ?? 0;
+
+    const subagentUsage = raw.subagent_usage?.map((s) =>
+      this.parseEntireTokenUsage(s),
+    );
+
+    return {
+      inputTokens,
+      cacheCreationTokens,
+      cacheReadTokens,
+      outputTokens,
+      apiCallCount,
+      subagentUsage,
+      total: inputTokens + cacheCreationTokens + cacheReadTokens + outputTokens,
+    };
+  }
+
+  private parseEntireSummary(raw: RawEntireSummary): SessionSummary {
+    return {
+      intent: raw.intent ?? '',
+      outcome: raw.outcome ?? '',
+      learnings: {
+        repoPatterns: raw.learnings?.repo_patterns ?? [],
+        codeFindings: raw.learnings?.code_findings ?? [],
+        workflowPractices: raw.learnings?.workflow_practices ?? [],
+      },
+      frictionPoints: raw.friction_points ?? [],
+      openItems: raw.open_items ?? [],
+    };
+  }
+
+  private parseEntireAttribution(raw: RawEntireAttribution): Attribution {
+    return {
+      agentLinesAdded: raw.agent_lines_added ?? 0,
+      humanAdditions: raw.human_additions ?? 0,
+      humanModifications: raw.human_modifications ?? 0,
+      humanRemovals: raw.human_removals ?? 0,
+      totalCommittedLines: raw.total_committed_lines ?? 0,
+      agentContribution: raw.agent_contribution ?? 0,
+    };
+  }
+
+  // ── SessionLog format parsing (camelCase) ─────────────────────
+
+  private parseSessionLogMetadata(raw: RawSessionLogMetadata): ParsedMetadata {
+    const tokenUsage = this.parseSessionLogTokenUsage(raw.tokenUsage);
+    const summary = raw.summary ? this.parseSessionLogSummary(raw.summary) : undefined;
+    const attribution = raw.initialAttribution
+      ? this.parseSessionLogAttribution(raw.initialAttribution)
+      : undefined;
+
+    return {
+      sessionId: raw.sessionID ?? '',
+      checkpointId: raw.checkpointID ?? '',
+      metadata: {
+        cliVersion: raw.cliVersion ?? '',
+        strategy: raw.strategy === 'auto-commit' ? 'auto-commit' : 'manual-commit',
+        createdAt: raw.createdAt ? new Date(raw.createdAt) : new Date(0),
+        branch: raw.branch ?? '',
+        agent: raw.agent ?? '',
+        filesTouched: raw.filesTouched ?? [],
+        isTask: raw.isTask ?? false,
+        toolUseId: raw.toolUseID,
+        checkpointTranscriptStart: raw.checkpointTranscriptStart ?? 0,
+      },
+      summary,
+      tokenUsage,
+      attribution,
+      format: 'sessionlog',
+    };
+  }
+
+  private parseSessionLogTokenUsage(raw?: RawSessionLogTokenUsage): TokenUsage {
+    if (!raw) {
+      return {
+        inputTokens: 0,
+        cacheCreationTokens: 0,
+        cacheReadTokens: 0,
+        outputTokens: 0,
+        apiCallCount: 0,
+        total: 0,
+      };
+    }
+
+    const inputTokens = raw.inputTokens ?? 0;
+    const cacheCreationTokens = raw.cacheCreationTokens ?? 0;
+    const cacheReadTokens = raw.cacheReadTokens ?? 0;
+    const outputTokens = raw.outputTokens ?? 0;
+    const apiCallCount = raw.apiCallCount ?? 0;
+
+    // SessionLog uses single subagentTokens object; wrap in array for compatibility
+    const subagentUsage = raw.subagentTokens
+      ? [this.parseSessionLogTokenUsage(raw.subagentTokens)]
+      : undefined;
+
+    return {
+      inputTokens,
+      cacheCreationTokens,
+      cacheReadTokens,
+      outputTokens,
+      apiCallCount,
+      subagentUsage,
+      total: inputTokens + cacheCreationTokens + cacheReadTokens + outputTokens,
+    };
+  }
+
+  private parseSessionLogSummary(raw: RawSessionLogSummary): SessionSummary {
+    // SessionLog uses learnings.code as CodeLearning[] — normalize to string[]
+    const codeFindings = (raw.learnings?.code ?? []).map((cl) => {
+      const loc = cl.line ? `${cl.path}:${cl.line}` : cl.path;
+      return `${loc} - ${cl.finding}`;
+    });
+
+    return {
+      intent: raw.intent ?? '',
+      outcome: raw.outcome ?? '',
+      learnings: {
+        repoPatterns: raw.learnings?.repo ?? [],
+        codeFindings,
+        workflowPractices: raw.learnings?.workflow ?? [],
+      },
+      frictionPoints: raw.friction ?? [],
+      openItems: raw.openItems ?? [],
+    };
+  }
+
+  private parseSessionLogAttribution(raw: RawSessionLogAttribution): Attribution {
+    return {
+      agentLinesAdded: raw.agentLines ?? 0,
+      humanAdditions: raw.humanAdded ?? 0,
+      humanModifications: raw.humanModified ?? 0,
+      humanRemovals: raw.humanRemoved ?? 0,
+      totalCommittedLines: raw.totalCommitted ?? 0,
+      agentContribution: raw.agentPercentage ?? 0,
+    };
+  }
+
+  // ── Transcript parsing (shared — works for both formats) ──────
+
+  private parseTranscriptLine(
+    raw: RawTranscriptLine,
+    lineIndex: number,
+  ): TranscriptMessage | null {
+    // Map Entire CLI's "human" → "user", keep "assistant" as-is.
+    // SessionLog always uses "user". Both handled here.
+    let role: 'user' | 'assistant';
+    if (raw.type === 'user' || raw.type === 'human') {
+      role = 'user';
+    } else if (raw.type === 'assistant') {
+      role = 'assistant';
+    } else if (raw.type === 'tool_result') {
+      // Entire CLI emits tool_result as a top-level JSONL line.
+      // Wrap it as a user message containing a tool_result block.
+      const toolResultBlock: ContentBlock = {
+        type: 'tool_result',
+        toolUseId: raw.tool_use_id ?? '',
+        content: typeof raw.content === 'string'
+          ? raw.content
+          : Array.isArray(raw.content)
+            ? raw.content
+                .filter((b) => b.type === 'text')
+                .map((b) => b.text ?? '')
+                .join('\n')
+            : '',
+        isError: raw.is_error,
+      };
+      return {
+        role: 'user',
+        uuid: raw.uuid ?? `line-${lineIndex}`,
+        content: [toolResultBlock],
+        lineIndex,
+      };
+    } else {
+      return null;
+    }
+
+    const uuid = raw.uuid ?? `line-${lineIndex}`;
+
+    const rawContent = raw.message?.content;
+    let content: ContentBlock[];
+
+    if (typeof rawContent === 'string') {
+      content = [{ type: 'text', text: rawContent }];
+    } else if (Array.isArray(rawContent)) {
+      content = rawContent
+        .map((block) => this.parseContentBlock(block))
+        .filter((b): b is ContentBlock => b !== null);
+    } else {
+      content = [];
+    }
+
+    return { role, uuid, content, lineIndex };
+  }
+
+  private parseContentBlock(raw: RawContentBlock): ContentBlock | null {
+    switch (raw.type) {
+      case 'text':
+        return { type: 'text', text: raw.text ?? '' };
+      case 'tool_use':
+        return {
+          type: 'tool_use',
+          id: raw.id ?? '',
+          name: raw.name ?? '',
+          input: raw.input ?? {},
+        };
+      case 'tool_result':
+        return {
+          type: 'tool_result',
+          toolUseId: raw.tool_use_id ?? '',
+          content: raw.content ?? '',
+          isError: raw.is_error,
+        };
+      default:
+        return null;
+    }
+  }
+
+  private checkpointIdFromPath(path: string): string {
+    // Path is "<shard>/<rest>" → checkpoint ID is shard + rest
+    const parts = path.split('/');
+    if (parts.length >= 2) {
+      return parts[0] + parts[1];
+    }
+    return path;
+  }
+}