@dommaker/harness 0.19.0 → 1.1.0

package/src/context/file-budget.ts

@@ -1,142 +0,0 @@
-/**
- * 文件读取预算
- *
- * 按行数、字节数、Token 数三重限制读取文件
- * 超限时生成 continuationHint 提示 offset
- */
-
-import * as fs from 'fs';
-import { TokenEstimator } from './token-budget';
-import type { FileBudgetConfig } from './types';
-import { DEFAULT_FILE_BUDGET } from './types';
-
-export interface FileCheckResult {
-  allowed: boolean;
-  reason?: string;
-  lineCount?: number;
-  byteCount?: number;
-}
-
-export interface FileReadResult {
-  content: string;
-  truncated: boolean;
-  linesRead: number;
-  bytesRead: number;
-  tokensEstimate: number;
-  continuationHint?: string;
-}
-
-export class FileBudget {
-  private config: FileBudgetConfig;
-
-  constructor(config?: Partial<FileBudgetConfig>) {
-    this.config = { ...DEFAULT_FILE_BUDGET, ...config };
-  }
-
-  /**
-   * 检查文件是否在预算内（stat 检查）
-   */
-  checkFile(filePath: string): FileCheckResult {
-    try {
-      const stat = fs.statSync(filePath);
-
-      if (stat.size > this.config.maxBytes) {
-        return {
-          allowed: false,
-          reason: `文件大小 ${(stat.size / 1024).toFixed(1)}KB 超过限制 ${(this.config.maxBytes / 1024).toFixed(1)}KB`,
-          byteCount: stat.size,
-        };
-      }
-
-      return { allowed: true, byteCount: stat.size };
-    } catch (error) {
-      return {
-        allowed: false,
-        reason: `无法读取文件: ${(error as Error).message}`,
-      };
-    }
-  }
-
-  /**
-   * 按预算读取文件
-   *
-   * 三重限制：行数 → 字节数 → Token 数
-   */
-  readWithBudget(filePath: string, config?: Partial<FileBudgetConfig>): FileReadResult {
-    const effectiveConfig = { ...this.config, ...config };
-
-    try {
-      const content = fs.readFileSync(filePath, 'utf-8');
-      const lines = content.split('\n');
-
-      let truncated = false;
-      let selectedLines = lines;
-      let continuationHint: string | undefined;
-
-      // 限制 1: 行数
-      if (lines.length > effectiveConfig.maxLines) {
-        selectedLines = lines.slice(0, effectiveConfig.maxLines);
-        truncated = true;
-
-        if (effectiveConfig.continuationHint) {
-          continuationHint = `文件被截断。已读取 ${effectiveConfig.maxLines} 行，共 ${lines.length} 行。使用 offset=${effectiveConfig.maxLines} 继续读取。`;
-        }
-      }
-
-      let result = selectedLines.join('\n');
-
-      // 限制 2: 字节数
-      const byteCount = Buffer.byteLength(result, 'utf-8');
-      if (byteCount > effectiveConfig.maxBytes) {
-        const encoder = new TextEncoder();
-        const bytes = encoder.encode(result);
-        const truncatedBytes = bytes.slice(0, effectiveConfig.maxBytes);
-        result = new TextDecoder().decode(truncatedBytes);
-        truncated = true;
-
-        if (!continuationHint && effectiveConfig.continuationHint) {
-          continuationHint = `文件内容被截断。已读取 ${effectiveConfig.maxBytes} 字节。`;
-        }
-      }
-
-      // 限制 3: Token 数
-      const tokensEstimate = TokenEstimator.estimateText(result);
-      if (tokensEstimate > effectiveConfig.maxTokenEstimate) {
-        // 按比例截断
-        const ratio = effectiveConfig.maxTokenEstimate / tokensEstimate;
-        const charLimit = Math.floor(result.length * ratio);
-        result = result.slice(0, charLimit);
-        truncated = true;
-
-        if (!continuationHint && effectiveConfig.continuationHint) {
-          continuationHint = `文件内容被截断。已读取约 ${effectiveConfig.maxTokenEstimate} tokens。`;
-        }
-      }
-
-      return {
-        content: result,
-        truncated,
-        linesRead: selectedLines.length,
-        bytesRead: Buffer.byteLength(result, 'utf-8'),
-        tokensEstimate: TokenEstimator.estimateText(result),
-        continuationHint,
-      };
-    } catch (error) {
-      return {
-        content: '',
-        truncated: false,
-        linesRead: 0,
-        bytesRead: 0,
-        tokensEstimate: 0,
-        continuationHint: `无法读取文件: ${(error as Error).message}`,
-      };
-    }
-  }
-
-  /**
-   * 获取当前配置
-   */
-  getConfig(): FileBudgetConfig {
-    return { ...this.config };
-  }
-}

package/src/context/progressive-loader.ts

@@ -1,260 +0,0 @@
-import { delay } from '../utils/exec';
-
-/**
- * @spec HZ-002
- * @implements HZ-002-C1
- * @acceptance AC-001-1, AC-001-4
- * 
- * Progressive Content Loader
- * 
- * 通用内容分片加载能力
- * - 支持大数据集分片加载
- * - 支持 Token 预算管理
- * - 零业务逻辑，可被任意系统使用
- */
-
-export interface ChunkLoadOptions<T = any> {
-  /** 每片大小 */
-  chunkSize: number;
-  /** 每片加载回调 */
-  onChunk?: (chunk: T[], index: number) => Promise<void> | void;
-  /** 进度回调 (loaded, total) */
-  onProgress?: (loaded: number, total: number) => void;
-  /** 是否并行加载 */
-  parallel?: boolean;
-  /** 延迟（毫秒），用于节流 */
-  delay?: number;
-}
-
-export interface TokenBudgetOptions<T = any> {
-  /** Token 预算 */
-  budget: number;
-  /** Token 估算函数 */
-  estimator: (item: T) => number;
-  /** 预算超限策略 */
-  onBudgetExceeded?: 'truncate' | 'error' | 'skip';
-  /** 最小保留项数（即使超预算） */
-  minItems?: number;
-}
-
-export interface LoadResult<T> {
-  /** 加载的项 */
-  items: T[];
-  /** 实际消耗的 Token */
-  tokensUsed: number;
-  /** 是否被截断 */
-  truncated: boolean;
-  /** 跳过的项数 */
-  skipped: number;
-}
-
-/**
- * 渐进式内容加载器
- * 
- * 提供通用的内容分片加载和 Token 预算管理能力
- */
-export class ProgressiveLoader {
-  /**
-   * 分片加载
-   * 
-   * 将大数据集分片加载，避免一次性加载全部内容
-   * 
-   * @example
-   * ```typescript
-   * const loader = new ProgressiveLoader();
-   * const largeData = Array.from({ length: 1000 }, (_, i) => ({ id: i }));
-   * 
-   * await loader.loadInChunks(largeData, {
-   *   chunkSize: 100,
-   *   onChunk: async (chunk, index) => {
-   *     console.log(`Loaded chunk ${index}: ${chunk.length} items`);
-   *   },
-   *   onProgress: (loaded, total) => {
-   *     console.log(`Progress: ${loaded}/${total}`);
-   *   }
-   * });
-   * ```
-   */
-  async loadInChunks<T>(
-    items: T[],
-    options: ChunkLoadOptions<T>
-  ): Promise<void> {
-    const { chunkSize, onChunk, onProgress, parallel = false, delay: throttleDelay = 0 } = options;
-    const total = items.length;
-    const chunks: T[][] = [];
-
-    // 分片
-    for (let i = 0; i < total; i += chunkSize) {
-      chunks.push(items.slice(i, i + chunkSize));
-    }
-
-    // 加载
-    if (parallel) {
-      // 并行加载
-      await Promise.all(
-        chunks.map(async (chunk, index) => {
-          if (throttleDelay > 0) {
-            await delay(throttleDelay * index);
-          }
-          await this.processChunk(chunk, index, onChunk, onProgress, total);
-        })
-      );
-    } else {
-      // 串行加载
-      for (let i = 0; i < chunks.length; i++) {
-        if (throttleDelay > 0 && i > 0) {
-          await delay(throttleDelay);
-        }
-        await this.processChunk(chunks[i], i, onChunk, onProgress, total);
-      }
-    }
-  }
-
-  /**
-   * Token 预算加载
-   * 
-   * 在 Token 预算范围内加载内容，超预算时按策略处理
-   * 
-   * @example
-   * ```typescript
-   * const loader = new ProgressiveLoader();
-   * const items = [
-   *   { content: 'short text' },
-   *   { content: 'long text '.repeat(100) }
-   * ];
-   * 
-   * const result = await loader.loadWithBudget(items, {
-   *   budget: 1000,
-   *   estimator: (item) => item.content.length,
-   *   onBudgetExceeded: 'truncate'
-   * });
-   * 
-   * console.log(`Loaded ${result.items.length} items, used ${result.tokensUsed} tokens`);
-   * ```
-   */
-  async loadWithBudget<T>(
-    items: T[],
-    options: TokenBudgetOptions<T>
-  ): Promise<LoadResult<T>> {
-    const {
-      budget,
-      estimator,
-      onBudgetExceeded = 'truncate',
-      minItems = 1
-    } = options;
-
-    const result: LoadResult<T> = {
-      items: [],
-      tokensUsed: 0,
-      truncated: false,
-      skipped: 0
-    };
-
-    let currentBudget = budget;
-
-    for (const item of items) {
-      const itemTokens = estimator(item);
-
-      // 检查预算
-      if (itemTokens > currentBudget) {
-        // 预算超限
-        if (result.items.length >= minItems) {
-          // 已达到最小项数，按策略处理
-          switch (onBudgetExceeded) {
-            case 'truncate':
-              result.truncated = true;
-              return result;
-            case 'error':
-              throw new Error(
-                `Token budget exceeded: need ${itemTokens}, remaining ${currentBudget}`
-              );
-            case 'skip':
-              result.skipped++;
-              continue;
-          }
-        }
-        // 未达到最小项数，强制添加（可能超预算）
-      }
-
-      // 添加项
-      result.items.push(item);
-      result.tokensUsed += itemTokens;
-      currentBudget -= itemTokens;
-    }
-
-    return result;
-  }
-
-  /**
-   * 创建流式加载器
-   * 
-   * 支持大文件/大内容的流式分片加载
-   */
-  async *createStream<T>(
-    source: AsyncIterable<T>,
-    chunkSize: number
-  ): AsyncIterable<T[]> {
-    let chunk: T[] = [];
-
-    for await (const item of source) {
-      chunk.push(item);
-
-      if (chunk.length >= chunkSize) {
-        yield chunk;
-        chunk = [];
-      }
-    }
-
-    //  yield remaining items
-    if (chunk.length > 0) {
-      yield chunk;
-    }
-  }
-
-  /**
-   * 批量处理（带并发控制）
-   */
-  async processBatch<T, R>(
-    items: T[],
-    processor: (item: T) => Promise<R>,
-    concurrency: number = 5
-  ): Promise<R[]> {
-    const results = new Array<R>(items.length);
-    let nextIndex = 0;
-
-    const worker = async (): Promise<void> => {
-      while (nextIndex < items.length) {
-        const index = nextIndex++;
-        results[index] = await processor(items[index]);
-      }
-    };
-
-    const workers = Array.from(
-      { length: Math.min(concurrency, items.length) },
-      () => worker()
-    );
-    await Promise.all(workers);
-    return results;
-  }
-
-  private async processChunk<T>(
-    chunk: T[],
-    index: number,
-    onChunk: ((chunk: T[], index: number) => Promise<void> | void) | undefined,
-    onProgress: ((loaded: number, total: number) => void) | undefined,
-    total: number
-  ): Promise<void> {
-    if (onChunk) {
-      await onChunk(chunk, index);
-    }
-
-    if (onProgress) {
-      const loaded = Math.min((index + 1) * chunk.length, total);
-      onProgress(loaded, total);
-    }
-  }
-
-}
-
-// 默认实例
-export const progressiveLoader = new ProgressiveLoader();

package/src/context/token-pipeline.ts

@@ -1,265 +0,0 @@
-/**
- * Token 流水线
- *
- * 五步：收集 → 排序 → 压缩 → 预算 → 组装
- * 每次 LLM 调用前运行
- */
-
-import { TokenEstimator, TokenBudget } from './token-budget';
-import type {
-  ContextSource,
-  TokenBudgetAllocation,
-  PipelineInput,
-  PipelineOutput,
-  ContextUsageSnapshot,
-} from './types';
-
-interface CollectedGroup {
-  priority: number;
-  sources: ContextSource[];
-}
-
-export class TokenPipeline {
-  /**
-   * 完整流水线：收集 → 排序 → 压缩 → 预算 → 组装
-   */
-  run(input: PipelineInput): PipelineOutput {
-    const { sources, budget } = input;
-
-    // Step 1: 收集 — 按 priority 分组
-    const collected = this.collect(sources);
-
-    // Step 2: 排序 — 按 P1→P6 排序
-    const sorted = this.sort(collected);
-
-    // Step 3: 压缩 — 对 P3-P5 应用预算裁剪
-    const { compressed, dropped } = this.compress(sorted, budget);
-
-    // Step 4: 预算 — 按分配裁剪
-    const budgeted = this.applyBudget(compressed, budget);
-
-    // Step 5: 组装 — 拼装最终 prompt
-    const prompt = this.assemble(budgeted);
-
-    // 生成快照
-    const snapshot = this.buildSnapshot(budgeted, dropped);
-
-    return { prompt, snapshot, dropped };
-  }
-
-  /**
-   * Step 1: 收集 — 从 sources 按 priority 分组
-   */
-  collect(sources: ContextSource[]): CollectedGroup[] {
-    const groups = new Map<number, ContextSource[]>();
-
-    for (const source of sources) {
-      const existing = groups.get(source.priority) || [];
-      existing.push(source);
-      groups.set(source.priority, existing);
-    }
-
-    return Array.from(groups.entries())
-      .map(([priority, sources]) => ({ priority, sources }))
-      .sort((a, b) => a.priority - b.priority);
-  }
-
-  /**
-   * Step 2: 排序 — 按 P1→P6，组内保持原序
-   */
-  sort(groups: CollectedGroup[]): ContextSource[] {
-    const sorted: ContextSource[] = [];
-
-    for (const group of groups) {
-      sorted.push(...group.sources);
-    }
-
-    return sorted;
-  }
-
-  /**
-   * Step 3: 压缩 — 对 P3-P5 应用 token 预算裁剪
-   *
-   * P1（系统提示词）和 P6（用户消息）不压缩
-   * P2（工具定义）按需懒加载，这里不处理
-   */
-  compress(sorted: ContextSource[], budget: TokenBudgetAllocation): {
-    compressed: ContextSource[];
-    dropped: Array<{ type: string; id: string; reason: string }>;
-  } {
-    const compressed: ContextSource[] = [];
-    const dropped: Array<{ type: string; id: string; reason: string }> = [];
-
-    // 各段预算
-    const budgetMap: Record<number, number> = {
-      1: budget.systemPrompt,
-      2: budget.toolDefinitions,
-      3: budget.knowledge,
-      4: budget.notes,
-      5: budget.history,
-      6: Infinity, // 用户消息不限
-    };
-
-    // 按优先级分组处理
-    const byPriority = new Map<number, ContextSource[]>();
-    for (const item of sorted) {
-      const existing = byPriority.get(item.priority) || [];
-      existing.push(item);
-      byPriority.set(item.priority, existing);
-    }
-
-    for (const [priority, items] of byPriority) {
-      const sectionBudget = budgetMap[priority] || budget.history;
-
-      if (priority === 1 || priority === 6) {
-        // P1/P6 不压缩
-        compressed.push(...items);
-        continue;
-      }
-
-      // 按 token 预算裁剪
-      let usedTokens = 0;
-      for (const item of items) {
-        const tokens = TokenEstimator.estimateText(item.content);
-
-        if (usedTokens + tokens <= sectionBudget) {
-          compressed.push(item);
-          usedTokens += tokens;
-        } else {
-          // 尝试截断
-          const remaining = sectionBudget - usedTokens;
-          if (remaining > 50) {
-            const ratio = remaining / tokens;
-            const truncatedContent = item.content.slice(0, Math.floor(item.content.length * ratio));
-            compressed.push({ ...item, content: truncatedContent });
-            dropped.push({
-              type: item.type,
-              id: item.id,
-              reason: `截断: ${tokens} → ${Math.floor(tokens * ratio)} tokens`,
-            });
-          } else {
-            dropped.push({
-              type: item.type,
-              id: item.id,
-              reason: `超出预算: 需要 ${tokens} tokens，剩余 ${remaining}`,
-            });
-          }
-          usedTokens += tokens; // 仍然计入已用
-        }
-      }
-    }
-
-    return { compressed, dropped };
-  }
-
-  /**
-   * Step 4: 预算 — 按 TokenBudgetAllocation 分配各段
-   */
-  applyBudget(compressed: ContextSource[], budget: TokenBudgetAllocation): ContextSource[] {
-    const tokenBudget = new TokenBudget(budget.total);
-    const result: ContextSource[] = [];
-
-    for (const item of compressed) {
-      const tokens = TokenEstimator.estimateText(item.content);
-
-      if (tokenBudget.canAfford(tokens)) {
-        tokenBudget.consume(tokens);
-        result.push(item);
-      } else if (item.priority === 1 || item.priority === 6) {
-        // P1/P6 强制保留
-        tokenBudget.forceConsume(tokens);
-        result.push(item);
-      }
-      // 其他优先级的超预算项被丢弃
-    }
-
-    return result;
-  }
-
-  /**
-   * Step 5: 组装 — 拼装最终 prompt
-   *
-   * P1 开头 + P6 结尾，中间 P2-P5
-   */
-  assemble(budgeted: ContextSource[]): string {
-    const sections: Record<number, string[]> = {
-      1: [], 2: [], 3: [], 4: [], 5: [], 6: [],
-    };
-
-    for (const item of budgeted) {
-      sections[item.priority] = sections[item.priority] || [];
-      sections[item.priority].push(item.content);
-    }
-
-    const parts: string[] = [];
-
-    // P1: 系统提示词（开头）
-    if (sections[1].length > 0) {
-      parts.push(sections[1].join('\n'));
-    }
-
-    // P2-P5: 中间部分
-    for (const priority of [2, 3, 4, 5]) {
-      if (sections[priority].length > 0) {
-        parts.push(sections[priority].join('\n'));
-      }
-    }
-
-    // P6: 用户消息（结尾）
-    if (sections[6].length > 0) {
-      parts.push(sections[6].join('\n'));
-    }
-
-    return parts.join('\n\n');
-  }
-
-  /**
-   * 构建上下文使用快照
-   */
-  private buildSnapshot(budgeted: ContextSource[], dropped: Array<{ type: string; id: string; reason: string }>): ContextUsageSnapshot {
-    const breakdown = {
-      systemPrompt: 0,
-      messages: 0,
-      toolOutputs: 0,
-      knowledge: 0,
-      other: 0,
-    };
-
-    for (const item of budgeted) {
-      const tokens = TokenEstimator.estimateText(item.content);
-
-      switch (item.type) {
-        case 'system_prompt':
-          breakdown.systemPrompt += tokens;
-          break;
-        case 'user_message':
-        case 'session_event':
-          breakdown.messages += tokens;
-          break;
-        case 'tool_output':
-        case 'tool_definition':
-          breakdown.toolOutputs += tokens;
-          break;
-        case 'knowledge':
-          breakdown.knowledge += tokens;
-          break;
-        default:
-          breakdown.other += tokens;
-      }
-    }
-
-    return {
-      timestamp: new Date().toISOString(),
-      totalTokens: breakdown.systemPrompt + breakdown.messages + breakdown.toolOutputs + breakdown.knowledge + breakdown.other,
-      breakdown,
-      truncatedItems: dropped.filter(d => d.reason.startsWith('截断')).map(d => ({
-        type: d.type,
-        id: d.id,
-        originalTokens: 0,
-        keptTokens: 0,
-      })),
-      offloadedItems: [],
-      compactionTriggered: false,
-    };
-  }
-}