@ebowwa/coder 0.7.63 → 0.7.65

package/packages/src/core/context/token-estimation.ts

@@ -0,0 +1,98 @@
+/**
+ * Token Estimation - Estimate token counts for messages
+ *
+ * Uses native Rust module for accurate, code-aware token counting.
+ * Falls back to heuristic (~4 chars/token) if native unavailable.
+ */
+
+import type { Message, ContentBlock } from "../../types/index.js";
+import { CHARS_PER_TOKEN } from "./constants.js";
+
+// Lazy-load native module to avoid circular dependencies
+let _native: { count_tokens?: (text: string) => number } | null = null;
+let _nativeLoadAttempted = false;
+
+function getNative(): { count_tokens?: (text: string) => number } | null {
+  if (_nativeLoadAttempted) return _native;
+  _nativeLoadAttempted = true;
+
+  try {
+    // Dynamic import to avoid bundling issues
+    const nativePath = require.resolve("../../../native/index.js");
+    _native = require(nativePath).getNative?.() ?? null;
+  } catch {
+    // Native module not available, use fallback
+    _native = null;
+  }
+
+  return _native;
+}
+
+/**
+ * Estimate the number of tokens in a text string.
+ * Uses native code-aware tokenizer if available, falls back to heuristic.
+ */
+export function estimateTokens(text: string): number {
+  if (!text || text.length === 0) return 0;
+
+  // Try native module first (more accurate, code-aware)
+  const native = getNative();
+  if (native?.count_tokens) {
+    return native.count_tokens(text);
+  }
+
+  // Fallback: simple heuristic
+  return Math.ceil(text.length / CHARS_PER_TOKEN);
+}
+
+/**
+ * Estimate tokens for a single content block
+ */
+export function estimateBlockTokens(block: ContentBlock): number {
+  switch (block.type) {
+    case "text":
+      return estimateTokens(block.text);
+    case "image":
+      // Images are roughly 85-110 tokens for standard sizes
+      // Use 100 as an average estimate
+      return 100;
+    case "tool_use":
+      // Tool use: name + JSON input
+      const toolInput = JSON.stringify(block.input);
+      return estimateTokens(block.name) + estimateTokens(toolInput) + 10; // overhead
+    case "tool_result":
+      // Tool result: content + metadata
+      if (typeof block.content === "string") {
+        return estimateTokens(block.content) + 10;
+      }
+      return block.content.reduce((sum, b) => sum + estimateBlockTokens(b), 0) + 10;
+    case "thinking":
+      return estimateTokens(block.thinking);
+    default:
+      return 0;
+  }
+}
+
+/**
+ * Estimate the total number of tokens in a message
+ */
+export function estimateMessageTokens(message: Message): number {
+  // Role overhead (~4 tokens)
+  const roleOverhead = 4;
+
+  // Sum all content blocks
+  const contentTokens = message.content.reduce(
+    (sum, block) => sum + estimateBlockTokens(block),
+    0
+  );
+
+  return roleOverhead + contentTokens;
+}
+
+/**
+ * Get total estimated tokens across all messages
+ */
+export function estimateMessagesTokens(messages: Message[]): number {
+  if (!messages || messages.length === 0) return 0;
+  return messages.reduce((sum, msg) => sum + estimateMessageTokens(msg), 0);
+}

package/packages/src/core/context/types.ts

@@ -0,0 +1,59 @@
+/**
+ * Context Types - Type definitions for context compaction
+ */
+
+import type { ContentBlock, Message } from "../../types/index.js";
+
+/** Options for LLM-based summarization */
+export interface LLMSummarizationOptions {
+  /** API key for the LLM */
+  apiKey?: string;
+  /** Model to use for summarization (default: haiku) */
+  model?: string;
+  /** Base URL for API (for non-Anthropic providers) */
+  baseUrl?: string;
+  /** Timeout in ms (default: 30000) */
+  timeout?: number;
+}
+
+/** Options for message compaction */
+export interface CompactionOptions {
+  /** Number of initial messages to keep unchanged */
+  keepFirst?: number;
+  /** Number of recent messages to keep unchanged */
+  keepLast?: number;
+  /** Whether to preserve tool_use/tool_result pairs */
+  preserveToolPairs?: boolean;
+  /** Use LLM for summarization (default: true if API key available) */
+  useLLMSummarization?: boolean;
+  /** API key for LLM summarization (falls back to env) */
+  apiKey?: string;
+  /** Base URL for API (for non-Anthropic providers like Z.AI) */
+  baseUrl?: string;
+}
+
+/** Result of a compaction operation */
+export interface CompactionResult {
+  /** The compacted messages */
+  messages: Message[];
+  /** Number of messages removed */
+  messagesRemoved: number;
+  /** Estimated tokens before compaction */
+  tokensBefore: number;
+  /** Estimated tokens after compaction */
+  tokensAfter: number;
+  /** Whether compaction actually occurred */
+  didCompact: boolean;
+}
+
+/** Statistics from compaction */
+export interface CompactionStats {
+  reductionPercent: number;
+  tokensSaved: number;
+}
+
+/** Tool use/result pair for preservation */
+export interface ToolPair {
+  use: { type: "tool_use"; id: string; name: string; input: unknown };
+  result?: { type: "tool_result"; tool_use_id: string; content: string | ContentBlock[]; is_error?: boolean };
+}

package/packages/src/core/git-status.ts

@@ -0,0 +1,262 @@
+/**
+ * Git Status - Retrieve repository status information
+ * Uses Bun subprocess to run git commands
+ */
+
+import type { GitStatus } from "../types/index.js";
+
+/**
+ * Run a git command and return its output
+ */
+async function runGitCommand(
+  args: string[],
+  cwd: string
+): Promise<{ stdout: string; stderr: string; exitCode: number }> {
+  const proc = Bun.spawn(["git", ...args], {
+    cwd,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+
+  const stdout = await new Response(proc.stdout).text();
+  const stderr = await new Response(proc.stderr).text();
+  const exitCode = await proc.exited;
+
+  return { stdout: stdout.trim(), stderr: stderr.trim(), exitCode };
+}
+
+/**
+ * Check if the directory is a git repository
+ */
+async function isGitRepository(workingDirectory: string): Promise<boolean> {
+  const { exitCode } = await runGitCommand(
+    ["rev-parse", "--git-dir"],
+    workingDirectory
+  );
+  return exitCode === 0;
+}
+
+/**
+ * Get the current branch name
+ */
+async function getBranch(workingDirectory: string): Promise<string> {
+  const { stdout, exitCode } = await runGitCommand(
+    ["rev-parse", "--abbrev-ref", "HEAD"],
+    workingDirectory
+  );
+
+  if (exitCode !== 0) {
+    return "HEAD";
+  }
+
+  return stdout || "HEAD";
+}
+
+/**
+ * Get ahead/behind counts compared to upstream
+ */
+async function getAheadBehind(
+  workingDirectory: string
+): Promise<{ ahead: number; behind: number }> {
+  const { stdout, exitCode } = await runGitCommand(
+    ["rev-list", "--left-right", "--count", "@{upstream}...HEAD"],
+    workingDirectory
+  );
+
+  if (exitCode !== 0 || !stdout) {
+    return { ahead: 0, behind: 0 };
+  }
+
+  const parts = stdout.split(/\s+/);
+  if (parts.length >= 2 && parts[0] !== undefined && parts[1] !== undefined) {
+    return {
+      ahead: parseInt(parts[0], 10) || 0,
+      behind: parseInt(parts[1], 10) || 0,
+    };
+  }
+
+  return { ahead: 0, behind: 0 };
+}
+
+/**
+ * Parse git status porcelain output
+ * Format: XY PATH where X is staged status, Y is unstaged status
+ *
+ * Status codes:
+ * - ' ' (space): unmodified
+ * - 'M': modified
+ * - 'A': added (staged)
+ * - 'D': deleted
+ * - 'R': renamed
+ * - 'C': copied
+ * - 'U': unmerged (conflict)
+ * - '?': untracked
+ * - '!': ignored
+ */
+async function getFileStatus(
+  workingDirectory: string
+): Promise<{
+  staged: string[];
+  unstaged: string[];
+  untracked: string[];
+  conflicted: string[];
+}> {
+  const { stdout, exitCode } = await runGitCommand(
+    ["status", "--porcelain"],
+    workingDirectory
+  );
+
+  if (exitCode !== 0 || !stdout) {
+    return {
+      staged: [],
+      unstaged: [],
+      untracked: [],
+      conflicted: [],
+    };
+  }
+
+  const staged: string[] = [];
+  const unstaged: string[] = [];
+  const untracked: string[] = [];
+  const conflicted: string[] = [];
+
+  const lines = stdout.split("\n");
+
+  for (const line of lines) {
+    if (!line) continue;
+
+    // Porcelain format: XY PATH or XY ORIG_PATH -> PATH
+    const x = line[0]; // Staged status
+    const y = line[1]; // Unstaged status
+    let path = line.substring(3);
+
+    // Handle renamed files (format: "R  old -> new")
+    if (path.includes(" -> ")) {
+      const splitPath = path.split(" -> ");
+      path = splitPath[1] ?? splitPath[0] ?? path;
+    }
+
+    // Check for conflicts (U or both A/A, D/D, etc.)
+    const isConflicted =
+      x === "U" ||
+      y === "U" ||
+      (x === "A" && y === "A") ||
+      (x === "D" && y === "D") ||
+      x === "A" && y === "U" ||
+      x === "U" && y === "A" ||
+      x === "D" && y === "U" ||
+      x === "U" && y === "D";
+
+    if (isConflicted) {
+      conflicted.push(path);
+      continue;
+    }
+
+    // Untracked files
+    if (x === "?" && y === "?") {
+      untracked.push(path);
+      continue;
+    }
+
+    // Staged changes (X is not space or ?)
+    if (x !== " " && x !== "?" && x !== "!") {
+      staged.push(path);
+    }
+
+    // Unstaged changes (Y is not space or ?)
+    if (y !== " " && y !== "?" && y !== "!") {
+      // Don't double-add if already in staged
+      if (!staged.includes(path)) {
+        unstaged.push(path);
+      }
+    }
+  }
+
+  return { staged, unstaged, untracked, conflicted };
+}
+
+/**
+ * Get comprehensive git status for a working directory
+ *
+ * @param workingDirectory - The directory to check git status for
+ * @returns GitStatus object if in a git repository, null otherwise
+ */
+export async function getGitStatus(
+  workingDirectory: string
+): Promise<GitStatus | null> {
+  try {
+    // First check if this is a git repository
+    const isRepo = await isGitRepository(workingDirectory);
+    if (!isRepo) {
+      return null;
+    }
+
+    // Run all status queries in parallel for better performance
+    const [branch, aheadBehind, fileStatus] = await Promise.all([
+      getBranch(workingDirectory),
+      getAheadBehind(workingDirectory),
+      getFileStatus(workingDirectory),
+    ]);
+
+    return {
+      branch,
+      ahead: aheadBehind.ahead,
+      behind: aheadBehind.behind,
+      staged: fileStatus.staged,
+      unstaged: fileStatus.unstaged,
+      untracked: fileStatus.untracked,
+      conflicted: fileStatus.conflicted,
+    };
+  } catch (error) {
+    // Log error for debugging but return null gracefully
+    console.error("Error getting git status:", error);
+    return null;
+  }
+}
+
+/**
+ * Check if there are any uncommitted changes
+ */
+export function hasUncommittedChanges(status: GitStatus): boolean {
+  return (
+    status.staged.length > 0 ||
+    status.unstaged.length > 0 ||
+    status.untracked.length > 0 ||
+    status.conflicted.length > 0
+  );
+}
+
+/**
+ * Check if the repository is clean (no changes and synced with upstream)
+ */
+export function isRepositoryClean(status: GitStatus): boolean {
+  return !hasUncommittedChanges(status) && status.ahead === 0 && status.behind === 0;
+}
+
+/**
+ * Get a human-readable summary of the git status
+ */
+export function getGitStatusSummary(status: GitStatus): string {
+  const parts: string[] = [];
+
+  parts.push(`branch: ${status.branch}`);
+
+  if (status.ahead > 0 || status.behind > 0) {
+    const sync: string[] = [];
+    if (status.ahead > 0) sync.push(`ahead ${status.ahead}`);
+    if (status.behind > 0) sync.push(`behind ${status.behind}`);
+    parts.push(`(${sync.join(", ")})`);
+  }
+
+  const changes: string[] = [];
+  if (status.staged.length > 0) changes.push(`${status.staged.length} staged`);
+  if (status.unstaged.length > 0) changes.push(`${status.unstaged.length} unstaged`);
+  if (status.untracked.length > 0) changes.push(`${status.untracked.length} untracked`);
+  if (status.conflicted.length > 0) changes.push(`${status.conflicted.length} conflicted`);
+
+  if (changes.length > 0) {
+    parts.push(`[${changes.join(", ")}]`);
+  }
+
+  return parts.join(" ");
+}

package/packages/src/core/image.test.ts

@@ -0,0 +1,180 @@
+import { test, expect, describe } from "bun:test";
+import {
+  IMAGE_EXTENSIONS,
+  BINARY_EXCLUSIONS,
+  detectMimeType,
+  isImageExtension,
+  isBinaryExclusion,
+  readImageFile,
+  toImageBlock,
+  formatImageResult,
+} from "./image.js";
+import type { MediaType } from "../types/index.js";
+
+describe("Image Extensions", () => {
+  test("IMAGE_EXTENSIONS contains supported formats", () => {
+    expect(IMAGE_EXTENSIONS.has("png")).toBe(true);
+    expect(IMAGE_EXTENSIONS.has("jpg")).toBe(true);
+    expect(IMAGE_EXTENSIONS.has("jpeg")).toBe(true);
+    expect(IMAGE_EXTENSIONS.has("gif")).toBe(true);
+    expect(IMAGE_EXTENSIONS.has("webp")).toBe(true);
+  });
+
+  test("IMAGE_EXTENSIONS does not contain unsupported formats", () => {
+    expect(IMAGE_EXTENSIONS.has("bmp")).toBe(false);
+    expect(IMAGE_EXTENSIONS.has("svg")).toBe(false);
+    expect(IMAGE_EXTENSIONS.has("pdf")).toBe(false);
+  });
+});
+
+describe("Binary Exclusions", () => {
+  test("BINARY_EXCLUSIONS contains common binary formats", () => {
+    expect(BINARY_EXCLUSIONS.has("exe")).toBe(true);
+    expect(BINARY_EXCLUSIONS.has("zip")).toBe(true);
+    expect(BINARY_EXCLUSIONS.has("mp4")).toBe(true);
+    expect(BINARY_EXCLUSIONS.has("doc")).toBe(true);
+  });
+
+  test("PDF is NOT in binary exclusions (handled separately)", () => {
+    // PDF files can be read with page limits, not in exclusions
+    expect(BINARY_EXCLUSIONS.has("pdf")).toBe(false);
+  });
+});
+
+describe("isImageExtension", () => {
+  test("returns true for image extensions", () => {
+    expect(isImageExtension("png")).toBe(true);
+    expect(isImageExtension("jpg")).toBe(true);
+    expect(isImageExtension(".jpeg")).toBe(true);
+    expect(isImageExtension("GIF")).toBe(true);
+  });
+
+  test("returns false for non-image extensions", () => {
+    expect(isImageExtension("txt")).toBe(false);
+    expect(isImageExtension("pdf")).toBe(false);
+    expect(isImageExtension("exe")).toBe(false);
+  });
+});
+
+describe("isBinaryExclusion", () => {
+  test("returns true for binary exclusions", () => {
+    expect(isBinaryExclusion("exe")).toBe(true);
+    expect(isBinaryExclusion("zip")).toBe(true);
+    expect(isBinaryExclusion("mp4")).toBe(true);
+  });
+
+  test("returns false for text files", () => {
+    expect(isBinaryExclusion("txt")).toBe(false);
+    expect(isBinaryExclusion("md")).toBe(false);
+  });
+});
+
+describe("detectMimeType", () => {
+  test("detects PNG magic bytes", () => {
+    // PNG magic bytes: 89 50 4E 47
+    const pngHeader = Buffer.from([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A]);
+    expect(detectMimeType(pngHeader)).toBe("image/png");
+  });
+
+  test("detects JPEG magic bytes", () => {
+    // JPEG magic bytes: FF D8 FF
+    const jpegHeader = Buffer.from([0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]);
+    expect(detectMimeType(jpegHeader)).toBe("image/jpeg");
+  });
+
+  test("detects GIF magic bytes", () => {
+    // GIF magic bytes: 47 49 46 38
+    const gifHeader = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]);
+    expect(detectMimeType(gifHeader)).toBe("image/gif");
+  });
+
+  test("detects WEBP magic bytes", () => {
+    // WEBP: RIFF....WEBP
+    const webpHeader = Buffer.from([
+      0x52, 0x49, 0x46, 0x46, // RIFF
+      0x00, 0x00, 0x00, 0x00, // size
+      0x57, 0x45, 0x42, 0x50, // WEBP
+    ]);
+    expect(detectMimeType(webpHeader)).toBe("image/webp");
+  });
+
+  test("returns null for unknown data", () => {
+    const unknownData = Buffer.from([0x00, 0x01, 0x02, 0x03]);
+    expect(detectMimeType(unknownData)).toBe(null);
+  });
+
+  test("returns null for empty buffer", () => {
+    expect(detectMimeType(Buffer.alloc(0))).toBe(null);
+  });
+});
+
+describe("toImageBlock", () => {
+  test("converts ImageFileResult to ImageBlock", () => {
+    const result = {
+      type: "image" as const,
+      base64: "dGVzdCBkYXRh", // "test data" in base64
+      mediaType: "image/png" as MediaType,
+      originalSize: 1000,
+      dimensions: {
+        originalWidth: 100,
+        originalHeight: 100,
+        displayWidth: 100,
+        displayHeight: 100,
+      },
+    };
+
+    const block = toImageBlock(result);
+
+    expect(block.type).toBe("image");
+    expect(block.source.type).toBe("base64");
+    expect(block.source.data).toBe("dGVzdCBkYXRh");
+    expect(block.source.media_type).toBe("image/png");
+  });
+});
+
+describe("formatImageResult", () => {
+  test("formats image result with dimensions", () => {
+    const result = {
+      type: "image" as const,
+      base64: "dGVzdA==",
+      mediaType: "image/png" as MediaType,
+      originalSize: 1024,
+      dimensions: {
+        originalWidth: 100,
+        originalHeight: 200,
+        displayWidth: 100,
+        displayHeight: 200,
+      },
+    };
+
+    const formatted = formatImageResult(result);
+    expect(formatted).toContain("image/png");
+    expect(formatted).toContain("100x200");
+    expect(formatted).toContain("1.0KB");
+  });
+
+  test("formats image result without dimensions", () => {
+    const result = {
+      type: "image" as const,
+      base64: "dGVzdA==",
+      mediaType: "image/jpeg" as MediaType,
+      originalSize: 2048,
+      dimensions: {},
+    };
+
+    const formatted = formatImageResult(result);
+    expect(formatted).toContain("image/jpeg");
+    expect(formatted).toContain("2.0KB");
+  });
+});
+
+describe("readImageFile", () => {
+  test("throws error for non-existent file", async () => {
+    try {
+      await readImageFile("/non/existent/file.png");
+      expect(true).toBe(false); // Should not reach here
+    } catch (error) {
+      expect(error).toBeDefined();
+    }
+  });
+});