aegiscode 3.1.8 → 3.1.10

package/src/context/TokenCounter.ts

@@ -0,0 +1,265 @@
+/**
+ * Token 
+ * 
+ * 
+ */
+
+import { encodingForModel, type Tiktoken } from 'js-tiktoken';
+import type { Message } from '../agent/types.js';
+
+export class TokenCounter {
+  private static encodingCache = new Map<string, Tiktoken>();
+
+  /**
+   * 
+   */
+  static countTokens(messages: Message[], modelName: string): number {
+    const encoding = this.getEncoding(modelName);
+    let totalTokens = 0;
+
+    for (const msg of messages) {
+      // 
+      totalTokens += 4;
+
+      // Role 
+      if (msg.role) {
+        totalTokens += encoding.encode(msg.role).length;
+      }
+
+      // Content 
+      if (msg.content) {
+        if (typeof msg.content === 'string') {
+          totalTokens += encoding.encode(msg.content).length;
+        } else {
+          totalTokens += encoding.encode(JSON.stringify(msg.content)).length;
+        }
+      }
+
+      // 
+      if (msg.tool_calls && Array.isArray(msg.tool_calls)) {
+        totalTokens += this.countToolCallTokens(msg.tool_calls, encoding);
+      }
+    }
+
+    return totalTokens;
+  }
+
+  /**
+   * 
+   */
+  static countMessageTokens(message: Message, modelName: string): number {
+    return this.countTokens([message], modelName);
+  }
+
+  /**
+   * 
+   */
+  static countTextTokens(text: string, modelName: string): number {
+    const encoding = this.getEncoding(modelName);
+    return encoding.encode(text).length;
+  }
+
+  /**
+   * 
+   */
+  private static countToolCallTokens(
+    toolCalls: Array<{ id?: string; function?: { name?: string; arguments?: string } }>,
+    encoding: Tiktoken
+  ): number {
+    let tokens = 0;
+
+    for (const call of toolCalls) {
+      // 
+      tokens += 4;
+
+      if (call.id) {
+        tokens += encoding.encode(call.id).length;
+      }
+
+      if (call.function) {
+        if (call.function.name) {
+          tokens += encoding.encode(call.function.name).length;
+        }
+        if (call.function.arguments) {
+          tokens += encoding.encode(call.function.arguments).length;
+        }
+      }
+    }
+
+    return tokens;
+  }
+
+  /**
+   * 
+   */
+  static shouldCompact(
+    messages: Message[],
+    modelName: string,
+    maxTokens: number,
+    thresholdPercent: number = 0.8
+  ): boolean {
+    const currentTokens = this.countTokens(messages, modelName);
+    const threshold = Math.floor(maxTokens * thresholdPercent);
+    return currentTokens >= threshold;
+  }
+
+  /**
+   * 
+   */
+  private static getEncoding(modelName: string): Tiktoken {
+    // 
+    const normalizedName = this.normalizeModelName(modelName);
+
+    if (!this.encodingCache.has(normalizedName)) {
+      try {
+        const encoding = encodingForModel(normalizedName as Parameters<typeof encodingForModel>[0]);
+        this.encodingCache.set(normalizedName, encoding);
+      } catch {
+        // Fallback to gpt-4o tokenizer
+        try {
+          const encoding = encodingForModel('gpt-4o');
+          this.encodingCache.set(normalizedName, encoding);
+        } catch {
+          // 
+          console.warn(` ${modelName}  encoding`);
+          const fallbackEncoding = {
+            encode: (text: string) => new Array(Math.ceil(text.length / 4)),
+            decode: () => '',
+            free: () => {},
+          } as unknown as Tiktoken;
+          this.encodingCache.set(normalizedName, fallbackEncoding);
+        }
+      }
+    }
+    return this.encodingCache.get(normalizedName)!;
+  }
+
+  /**
+   * 
+   */
+  private static normalizeModelName(modelName: string): string {
+    // Map model names to valid js-tiktoken encodings
+    const lower = modelName.toLowerCase();
+
+    // Anthropic Claude models
+    if (lower.includes('claude-sonnet-4') || lower.includes('claude-3.5') || lower.includes('claude-3')) {
+      return 'gpt-4o';
+    }
+    if (lower.includes('claude')) {
+      return 'gpt-4o';
+    }
+    // GPT-4o variants
+    if (lower.includes('gpt-4o') || lower.includes('gpt4o')) {
+      return 'gpt-4o';
+    }
+    // GPT-4 variants
+    if (lower.includes('gpt-4') || lower.includes('gpt4')) {
+      return 'gpt-4';
+    }
+    // GPT-3.5 variants -> gpt-4o-mini (current replacement)
+    if (lower.includes('gpt-3.5') || lower.includes('gpt35') || lower.includes('gpt-3')) {
+      return 'gpt-4o-mini';
+    }
+    // Chinese GLM models
+    if (lower.includes('glm')) {
+      return 'gpt-4o';
+    }
+    // DeepSeek models
+    if (lower.includes('deepseek')) {
+      return 'gpt-4o';
+    }
+    // Llama/Mixtral/Groq models
+    if (lower.includes('llama') || lower.includes('mixtral') || lower.includes('groq')) {
+      return 'gpt-4o-mini';
+    }
+
+    return modelName;
+  }
+
+  /**
+   * 
+   * 
+   * 
+   */
+  static estimateTokens(text: string): number {
+    // 1 token ≈ 1.5 
+    // 1 token ≈ 4 
+    const chineseChars = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
+    const otherChars = text.length - chineseChars;
+    return Math.ceil(chineseChars / 1.5 + otherChars / 4);
+  }
+
+  /**
+   * 
+   */
+  static estimateMessagesTokens(messages: Message[]): number {
+    let total = 0;
+
+    for (const msg of messages) {
+      // 
+      total += 4;
+
+      if (msg.content) {
+        const content = typeof msg.content === 'string' 
+          ? msg.content 
+          : JSON.stringify(msg.content);
+        total += this.estimateTokens(content);
+      }
+
+      if (msg.tool_calls) {
+        total += this.estimateTokens(JSON.stringify(msg.tool_calls));
+      }
+    }
+
+    return total;
+  }
+
+  /**
+   * 
+   */
+  static getRemainingTokens(
+    messages: Message[],
+    modelName: string,
+    maxContextTokens: number,
+    maxOutputTokens: number = 4096
+  ): number {
+    const currentTokens = this.countTokens(messages, modelName);
+    const availableForInput = maxContextTokens - maxOutputTokens;
+    return Math.max(0, availableForInput - currentTokens);
+  }
+
+  /**
+   * 
+   */
+  static truncateMessages(
+    messages: Message[],
+    modelName: string,
+    maxTokens: number
+  ): Message[] {
+    const result: Message[] = [];
+    let currentTokens = 0;
+
+    // 
+    for (let i = messages.length - 1; i >= 0; i--) {
+      const msg = messages[i];
+      const msgTokens = this.countMessageTokens(msg, modelName);
+
+      if (currentTokens + msgTokens > maxTokens) {
+        break;
+      }
+
+      result.unshift(msg);
+      currentTokens += msgTokens;
+    }
+
+    return result;
+  }
+
+  /**
+   * 
+   */
+  static clearCache(): void {
+    // js-tiktoken  Tiktoken 
+    this.encodingCache.clear();
+  }
+}

package/src/context/index.ts

@@ -0,0 +1,27 @@
+/**
+ * 
+ */
+
+// 类型导
+export * from './types.js';
+
+// 核心类导
+export { ContextManager } from './ContextManager.js';
+export { TokenCounter } from './TokenCounter.js';
+export { CompactionService } from './CompactionService.js';
+export { FileAnalyzer } from './FileAnalyzer.js';
+
+// 存储层导
+export {
+  MemoryStore,
+  PersistentStore,
+  CacheStore,
+  JSONLStore,
+  getStorageRoot,
+  getProjectStoragePath,
+  getSessionFilePath,
+  escapeProjectPath,
+  detectGitBranch,
+  detectGitRemote,
+  getLatestSessionFile,
+} from './storage/index.js';

package/src/context/storage/CacheStore.ts

@@ -0,0 +1,176 @@
+/**
+ * 
+ * 
+ * 
+ */
+
+interface CacheEntry<T> {
+  value: T;
+  createdAt: number;
+  lastAccessedAt: number;
+  ttl: number;
+}
+
+export class CacheStore {
+  private readonly cache: Map<string, CacheEntry<unknown>> = new Map();
+  private readonly maxSize: number;
+  private readonly defaultTtl: number;
+
+  constructor(maxSize: number = 100, defaultTtl: number = 5 * 60 * 1000) {
+    this.maxSize = maxSize;
+    this.defaultTtl = defaultTtl;
+  }
+
+  /**
+   * 
+   */
+  set<T>(key: string, value: T, ttl?: number): void {
+    // 检查是否需要淘
+    if (this.cache.size >= this.maxSize && !this.cache.has(key)) {
+      this.evictLRU();
+    }
+
+    const now = Date.now();
+    this.cache.set(key, {
+      value,
+      createdAt: now,
+      lastAccessedAt: now,
+      ttl: ttl ?? this.defaultTtl,
+    });
+  }
+
+  /**
+   * 
+   */
+  get<T>(key: string): T | undefined {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return undefined;
+    }
+
+    // 检查是否过
+    const now = Date.now();
+    if (now - entry.createdAt > entry.ttl) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    // 更新访问时
+    entry.lastAccessedAt = now;
+    return entry.value as T;
+  }
+
+  /**
+   * 
+   */
+  has(key: string): boolean {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return false;
+    }
+
+    const now = Date.now();
+    if (now - entry.createdAt > entry.ttl) {
+      this.cache.delete(key);
+      return false;
+    }
+
+    return true;
+  }
+
+  /**
+   * 
+   */
+  delete(key: string): boolean {
+    return this.cache.delete(key);
+  }
+
+  /**
+   * 
+   */
+  async getOrSet<T>(
+    key: string,
+    factory: () => T | Promise<T>,
+    ttl?: number
+  ): Promise<T> {
+    const existing = this.get<T>(key);
+    if (existing !== undefined) {
+      return existing;
+    }
+
+    const value = await factory();
+    this.set(key, value, ttl);
+    return value;
+  }
+
+  /**
+   * 
+   */
+  private evictLRU(): void {
+    let oldestKey: string | null = null;
+    let oldestTime = Infinity;
+
+    for (const [key, entry] of this.cache) {
+      if (entry.lastAccessedAt < oldestTime) {
+        oldestTime = entry.lastAccessedAt;
+        oldestKey = key;
+      }
+    }
+
+    if (oldestKey) {
+      this.cache.delete(oldestKey);
+    }
+  }
+
+  /**
+   * 
+   */
+  cleanup(): number {
+    const now = Date.now();
+    let cleaned = 0;
+
+    for (const [key, entry] of this.cache) {
+      if (now - entry.createdAt > entry.ttl) {
+        this.cache.delete(key);
+        cleaned++;
+      }
+    }
+
+    return cleaned;
+  }
+
+  /**
+   * 
+   */
+  clear(): void {
+    this.cache.clear();
+  }
+
+  /**
+   * 
+   */
+  get size(): number {
+    return this.cache.size;
+  }
+
+  /**
+   * 
+   */
+  keys(): string[] {
+    return Array.from(this.cache.keys());
+  }
+
+  /**
+   * 
+   */
+  getStats(): {
+    size: number;
+    maxSize: number;
+    hitRate?: number;
+  } {
+    return {
+      size: this.cache.size,
+      maxSize: this.maxSize,
+    };
+  }
+}

package/src/context/storage/JSONLStore.ts

@@ -0,0 +1,201 @@
+/**
+ * JSONL 文件存储
+ * 
+ * 
+ */
+
+import * as fs from 'node:fs/promises';
+import { existsSync, createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import * as path from 'node:path';
+import type { JSONLEntry } from '../types.js';
+
+export class JSONLStore {
+  private readonly filePath: string;
+
+  constructor(filePath: string) {
+    this.filePath = filePath;
+  }
+
+  /**
+   * 
+   */
+  getFilePath(): string {
+    return this.filePath;
+  }
+
+  /**
+   * 
+   */
+  exists(): boolean {
+    return existsSync(this.filePath);
+  }
+
+  /**
+   * 
+   */
+  async append(entry: JSONLEntry): Promise<void> {
+    // 确保父目录存
+    await fs.mkdir(path.dirname(this.filePath), { recursive: true });
+
+    // 序列化并追
+    const line = JSON.stringify(entry) + '\n';
+    await fs.appendFile(this.filePath, line, 'utf-8');
+  }
+
+  /**
+   * 
+   */
+  async appendBatch(entries: JSONLEntry[]): Promise<void> {
+    if (entries.length === 0) return;
+
+    await fs.mkdir(path.dirname(this.filePath), { recursive: true });
+
+    const lines = entries.map(entry => JSON.stringify(entry)).join('\n') + '\n';
+    await fs.appendFile(this.filePath, lines, 'utf-8');
+  }
+
+  /**
+   * 
+   */
+  async readAll(): Promise<JSONLEntry[]> {
+    if (!this.exists()) {
+      return [];
+    }
+
+    const content = await fs.readFile(this.filePath, 'utf-8');
+    const lines = content.split('\n').filter(line => line.trim().length > 0);
+
+    const entries: JSONLEntry[] = [];
+    for (const line of lines) {
+      try {
+        entries.push(JSON.parse(line) as JSONLEntry);
+      } catch (parseError) {
+        console.warn(`[JSONLStore] 解析 JSON 行失败: ${line.substring(0, 100)}...`);
+      }
+    }
+    return entries;
+  }
+
+  /**
+   * 
+   */
+  async readStream(callback: (entry: JSONLEntry) => void | Promise<void>): Promise<void> {
+    return new Promise((resolve, reject) => {
+      if (!this.exists()) {
+        resolve();
+        return;
+      }
+
+      const fileStream = createReadStream(this.filePath, { encoding: 'utf-8' });
+      const rl = createInterface({
+        input: fileStream,
+        crlfDelay: Number.POSITIVE_INFINITY,
+      });
+
+      const processLine = async (line: string) => {
+        const trimmed = line.trim();
+        if (trimmed.length === 0) return;
+
+        try {
+          const entry = JSON.parse(trimmed) as JSONLEntry;
+          await callback(entry);
+        } catch (error) {
+          console.warn(`[JSONLStore] 解析失败: ${trimmed.substring(0, 50)}...`);
+        }
+      };
+
+      rl.on('line', (line) => {
+        processLine(line).catch(reject);
+      });
+
+      rl.on('close', () => resolve());
+      rl.on('error', reject);
+      fileStream.on('error', reject);
+    });
+  }
+
+  /**
+   * 
+   */
+  async readLast(count: number): Promise<JSONLEntry[]> {
+    const all = await this.readAll();
+    return all.slice(-count);
+  }
+
+  /**
+   * 
+   */
+  async filter(predicate: (entry: JSONLEntry) => boolean): Promise<JSONLEntry[]> {
+    const all = await this.readAll();
+    return all.filter(predicate);
+  }
+
+  /**
+   * 
+   */
+  async count(): Promise<number> {
+    if (!this.exists()) {
+      return 0;
+    }
+
+    let count = 0;
+    await this.readStream(() => {
+      count++;
+    });
+    return count;
+  }
+
+  /**
+   * 
+   */
+  async readAfterCompaction(): Promise<JSONLEntry[]> {
+    const all = await this.readAll();
+    
+    // 找到最后一个压缩边
+    let boundaryIndex = -1;
+    for (let i = all.length - 1; i >= 0; i--) {
+      if (all[i].subtype === 'compact_boundary') {
+        boundaryIndex = i;
+        break;
+      }
+    }
+
+    // 如果没有压缩边界，返回所有记
+    if (boundaryIndex === -1) {
+      return all;
+    }
+
+    // 返回压缩边界之后的记录（包括压缩总
+    return all.slice(boundaryIndex);
+  }
+
+  /**
+   * 
+   */
+  async clear(): Promise<void> {
+    if (this.exists()) {
+      await fs.writeFile(this.filePath, '', 'utf-8');
+    }
+  }
+
+  /**
+   * 
+   */
+  async delete(): Promise<void> {
+    if (this.exists()) {
+      await fs.unlink(this.filePath);
+    }
+  }
+
+  /**
+   * 
+   */
+  async getFileSize(): Promise<number> {
+    if (!this.exists()) {
+      return 0;
+    }
+    const stats = await fs.stat(this.filePath);
+    return stats.size;
+  }
+}