@dommaker/harness 0.13.0 → 0.13.2

package/src/llm/adapter.ts

@@ -0,0 +1,149 @@
+/**
+ * 默认 LLM 适配器
+ *
+ * 提供基础实现，使用者可以替换为自己的实现
+ */
+
+import type {
+  LLMAdapter,
+  LLMAdapterConfig,
+  LLMOptions,
+  Message,
+  SummarizeConfig,
+  LLMProvider,
+} from './types';
+
+/**
+ * 创建 LLM 适配器
+ */
+export function createLLMAdapter(config: LLMAdapterConfig): LLMAdapter {
+  if (config.customAdapter) {
+    return config.customAdapter;
+  }
+
+  return new DefaultLLMAdapter(config);
+}
+
+/**
+ * 默认 LLM 适配器
+ *
+ * 使用零 token 成本的结构化方法：
+ * - summarize: 基于模板提取关键信息
+ * - extract: 基于正则提取结构化数据
+ * - complete/chat: 需要注入具体实现
+ */
+export class DefaultLLMAdapter implements LLMAdapter {
+  private config: LLMAdapterConfig;
+
+  constructor(config: LLMAdapterConfig) {
+    this.config = config;
+  }
+
+  async complete(prompt: string, options?: LLMOptions): Promise<string> {
+    throw new Error(
+      `LLM complete 未实现。请通过 createLLMAdapter({ customAdapter }) 注入具体实现。` +
+      `\n提供商: ${this.config.provider}, 模型: ${options?.model ?? this.config.model ?? '未指定'}`,
+    );
+  }
+
+  async chat(messages: Message[], options?: LLMOptions): Promise<string> {
+    throw new Error(
+      `LLM chat 未实现。请通过 createLLMAdapter({ customAdapter }) 注入具体实现。` +
+      `\n提供商: ${this.config.provider}, 消息数: ${messages.length}`,
+    );
+  }
+
+  async *streamChat(messages: Message[], options?: LLMOptions): AsyncIterable<string> {
+    throw new Error(
+      `LLM streamChat 未实现。请通过 createLLMAdapter({ customAdapter }) 注入具体实现。`,
+    );
+  }
+
+  /**
+   * 结构化总结（零 token 成本，基于模板）
+   */
+  async summarize(messages: Message[], config: SummarizeConfig): Promise<string> {
+    const parts: string[] = [];
+
+    // 提取用户目标
+    if (config.preserve.includes('user_goals')) {
+      const userMessages = messages.filter(m => m.role === 'user');
+      if (userMessages.length > 0) {
+        parts.push('## 用户目标');
+        for (const msg of userMessages.slice(-3)) {
+          parts.push(`- ${msg.content.slice(0, 200)}`);
+        }
+      }
+    }
+
+    // 提取工具调用
+    if (config.preserve.includes('tool_calls')) {
+      const toolMessages = messages.filter(m => m.role === 'tool');
+      if (toolMessages.length > 0) {
+        parts.push('\n## 工具调用');
+        parts.push(`共 ${toolMessages.length} 次工具调用`);
+      }
+    }
+
+    // 提取错误
+    if (config.preserve.includes('errors')) {
+      const errorMessages = messages.filter(m =>
+        m.content.toLowerCase().includes('error') ||
+        m.content.toLowerCase().includes('failed'),
+      );
+      if (errorMessages.length > 0) {
+        parts.push('\n## 错误');
+        for (const msg of errorMessages.slice(-3)) {
+          parts.push(`- ${msg.content.slice(0, 200)}`);
+        }
+      }
+    }
+
+    // 提取决策
+    if (config.preserve.includes('decisions')) {
+      const decisionMessages = messages.filter(m =>
+        m.content.includes('决策') ||
+        m.content.includes('decision') ||
+        m.content.includes('选择') ||
+        m.content.includes('方案'),
+      );
+      if (decisionMessages.length > 0) {
+        parts.push('\n## 决策');
+        for (const msg of decisionMessages.slice(-3)) {
+          parts.push(`- ${msg.content.slice(0, 200)}`);
+        }
+      }
+    }
+
+    const summary = parts.join('\n');
+    return summary.slice(0, config.maxTokens * 4); // 粗略字符/token 比
+  }
+
+  /**
+   * 结构化提取（基于正则，零 token 成本）
+   */
+  async extract(content: string, schema: Record<string, unknown>): Promise<unknown> {
+    const result: Record<string, unknown> = {};
+
+    for (const [key, type] of Object.entries(schema)) {
+      if (typeof type === 'string') {
+        // 简单类型提取
+        const pattern = new RegExp(`${key}[:\\s]+(.+)`, 'i');
+        const match = content.match(pattern);
+        if (match) {
+          result[key] = match[1].trim();
+        }
+      }
+    }
+
+    return result;
+  }
+
+  getProvider(): LLMProvider {
+    return this.config.provider;
+  }
+
+  getModel(): string | undefined {
+    return this.config.model;
+  }
+}

package/src/llm/index.ts

@@ -0,0 +1,2 @@
+export * from './types';
+export * from './adapter';

package/src/llm/types.ts

@@ -0,0 +1,81 @@
+/**
+ * LLM 适配器类型定义
+ *
+ * harness 核心不硬编码 LLM 依赖，通过接口注入
+ */
+
+// ── 消息类型 ─────────────────────────────────────────────
+
+export interface Message {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content: string;
+  toolCallId?: string;
+  name?: string;
+}
+
+// ── LLM 选项 ─────────────────────────────────────────────
+
+export interface LLMOptions {
+  /** 模型名称 */
+  model?: string;
+  /** 温度 */
+  temperature?: number;
+  /** 最大 token 数 */
+  maxTokens?: number;
+  /** 停止序列 */
+  stop?: string[];
+  /** 工具定义（function calling） */
+  tools?: LLMToolSpec[];
+}
+
+export interface LLMToolSpec {
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+}
+
+// ── 总结配置 ─────────────────────────────────────────────
+
+export interface SummarizeConfig {
+  /** 目标 token 数 */
+  maxTokens: number;
+  /** 保留的关键信息 */
+  preserve: Array<'user_goals' | 'tool_calls' | 'errors' | 'decisions'>;
+}
+
+// ── 提取配置 ─────────────────────────────────────────────
+
+export interface ExtractConfig {
+  /** 目标 schema */
+  schema: Record<string, unknown>;
+  /** 是否严格模式 */
+  strict?: boolean;
+}
+
+// ── LLM 适配器接口 ───────────────────────────────────────
+
+export interface LLMAdapter {
+  /** 对话补全 */
+  complete(prompt: string, options?: LLMOptions): Promise<string>;
+  /** 多轮对话 */
+  chat(messages: Message[], options?: LLMOptions): Promise<string>;
+  /** 流式对话 */
+  streamChat(messages: Message[], options?: LLMOptions): AsyncIterable<string>;
+  /** 总结 */
+  summarize(messages: Message[], config: SummarizeConfig): Promise<string>;
+  /** 结构化提取 */
+  extract(content: string, schema: Record<string, unknown>): Promise<unknown>;
+}
+
+// ── 适配器配置 ───────────────────────────────────────────
+
+export type LLMProvider = 'openai' | 'anthropic' | 'local' | 'custom';
+
+export interface LLMAdapterConfig {
+  provider: LLMProvider;
+  model?: string;
+  apiKey?: string;
+  baseUrl?: string;
+  /** 自定义适配器实现 */
+  customAdapter?: LLMAdapter;
+}

package/src/monitoring/CONTEXT.md

@@ -0,0 +1,29 @@
+# monitoring/
+
+## 职责
+运行时监控：Execution Trace(收集/分析/诊断)、Performance Trace(收集/分析)、约束进化提案、知识进化、上下文追踪。
+
+## 核心导出
+- `TraceCollector` — 执行追踪收集(append-only JSONL)
+- `TraceAnalyzer` — 追踪统计分析 + 异常检测
+- `ConstraintDoctor` — 约束诊断接口(Agent 消费)
+- `ConstraintEvolver` — 约束进化提案流程
+- `PerformanceCollector` — 性能日志收集
+- `PerformanceAnalyzer` — 性能统计分析
+- `ContextTracker` — 上下文追踪
+- `KnowledgeDoctor` — 知识健康诊断
+- `KnowledgeEvolver` — 知识进化引擎
+
+## 依赖关系
+- 依赖 `src/core/constraints/` 约束定义
+- 依赖 `src/types/trace` ExecutionTrace 类型
+- 被 `src/cli/` 命令消费
+
+## 约定
+- Trace 文件存储在 .harness/ 目录(不提交)
+- 追加写入，单行 JSON，自动滚动
+- 零 Token 成本：不调用 LLM
+
+## 注意事项
+- 追踪数据用于约束退化分析(拦截率 < 阈值 → 自动降级)
+- PerformanceCollector 和 TraceCollector 独立运作

package/src/monitoring/__tests__/context-tracker.test.ts

@@ -0,0 +1,172 @@
+/**
+ * ContextTracker 测试
+ */
+
+import { ContextTracker } from '../context-tracker';
+import * as fs from 'fs';
+import type { ContextUsageSnapshot } from '../../context/types';
+
+jest.mock('fs', () => ({
+  existsSync: jest.fn().mockReturnValue(true),
+  mkdirSync: jest.fn(),
+  readFileSync: jest.fn().mockReturnValue(''),
+  appendFileSync: jest.fn(),
+  statSync: jest.fn().mockReturnValue({ size: 0 }),
+  renameSync: jest.fn(),
+}));
+
+const mockFs = fs as jest.Mocked<typeof fs>;
+
+function makeSnapshot(overrides?: Partial<ContextUsageSnapshot>): ContextUsageSnapshot {
+  return {
+    timestamp: new Date().toISOString(),
+    totalTokens: 1000,
+    breakdown: {
+      systemPrompt: 200,
+      messages: 300,
+      toolOutputs: 200,
+      knowledge: 200,
+      other: 100,
+    },
+    truncatedItems: [],
+    offloadedItems: [],
+    compactionTriggered: false,
+    ...overrides,
+  };
+}
+
+describe('ContextTracker', () => {
+  let tracker: ContextTracker;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    // Re-setup default mock return values after clearAllMocks
+    (mockFs.existsSync as jest.Mock).mockReturnValue(true);
+    (mockFs.readFileSync as jest.Mock).mockReturnValue('');
+    (mockFs.statSync as jest.Mock).mockReturnValue({ size: 0 });
+    tracker = new ContextTracker('/test');
+  });
+
+  describe('record', () => {
+    it('应该追加快照到日志文件', () => {
+      tracker.record(makeSnapshot());
+      expect(mockFs.appendFileSync).toHaveBeenCalled();
+    });
+
+    it('应该创建目录当不存在', () => {
+      (mockFs.existsSync as jest.Mock).mockReturnValue(false);
+      tracker.record(makeSnapshot());
+      expect(mockFs.mkdirSync).toHaveBeenCalled();
+      expect(mockFs.appendFileSync).toHaveBeenCalled();
+    });
+
+    it('应该轮转文件当超过大小限制', () => {
+      (mockFs.existsSync as jest.Mock)
+        .mockReturnValueOnce(true)   // dir exists
+        .mockReturnValueOnce(true);  // log file exists
+      (mockFs.statSync as jest.Mock).mockReturnValue({ size: 20 * 1024 * 1024 });
+      tracker.record(makeSnapshot());
+      expect(mockFs.renameSync).toHaveBeenCalled();
+    });
+  });
+
+  describe('getRecent', () => {
+    it('应该返回最近 N 条记录', () => {
+      const snapshots = [
+        makeSnapshot({ totalTokens: 100 }),
+        makeSnapshot({ totalTokens: 200 }),
+        makeSnapshot({ totalTokens: 300 }),
+      ];
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        snapshots.map(s => JSON.stringify(s)).join('\n')
+      );
+
+      const recent = tracker.getRecent(2);
+      expect(recent.length).toBe(2);
+      expect(recent[0].totalTokens).toBe(200);
+      expect(recent[1].totalTokens).toBe(300);
+    });
+
+    it('应该返回空数组当文件不存在', () => {
+      (mockFs.existsSync as jest.Mock).mockReturnValue(false);
+      const recent = tracker.getRecent(10);
+      expect(recent).toEqual([]);
+    });
+
+    it('应该跳过无效 JSON 行', () => {
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        'invalid\n' + JSON.stringify(makeSnapshot()) + '\n'
+      );
+      const recent = tracker.getRecent(10);
+      expect(recent.length).toBe(1);
+    });
+  });
+
+  describe('getAverages', () => {
+    it('应该计算均值', () => {
+      const snapshots = [
+        makeSnapshot({ totalTokens: 1000, compactionTriggered: false, breakdown: { systemPrompt: 200, messages: 300, toolOutputs: 200, knowledge: 200, other: 100 } }),
+        makeSnapshot({ totalTokens: 2000, compactionTriggered: true, breakdown: { systemPrompt: 400, messages: 600, toolOutputs: 400, knowledge: 400, other: 200 } }),
+      ];
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        snapshots.map(s => JSON.stringify(s)).join('\n')
+      );
+
+      const avg = tracker.getAverages();
+      expect(avg.avgTokens).toBe(1500);
+      expect(avg.avgCompactionRate).toBe(0.5);
+    });
+
+    it('应该返回零当无记录', () => {
+      (mockFs.readFileSync as jest.Mock).mockReturnValue('');
+      const avg = tracker.getAverages();
+      expect(avg.avgTokens).toBe(0);
+    });
+  });
+
+  describe('detectIssues', () => {
+    it('应该检测工具输出占比过高', () => {
+      const snapshots = Array(10).fill(null).map(() =>
+        makeSnapshot({
+          totalTokens: 1000,
+          breakdown: { systemPrompt: 100, messages: 100, toolOutputs: 600, knowledge: 100, other: 100 },
+        })
+      );
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        snapshots.map(s => JSON.stringify(s)).join('\n')
+      );
+
+      const issues = tracker.detectIssues();
+      expect(issues.some(i => i.includes('工具输出占比'))).toBe(true);
+    });
+
+    it('应该检测压缩频率过高', () => {
+      const snapshots = Array(10).fill(null).map(() =>
+        makeSnapshot({ compactionTriggered: true })
+      );
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        snapshots.map(s => JSON.stringify(s)).join('\n')
+      );
+
+      const issues = tracker.detectIssues();
+      expect(issues.some(i => i.includes('压缩触发频率'))).toBe(true);
+    });
+
+    it('应该返回空数组当无问题', () => {
+      const snapshots = Array(10).fill(null).map(() =>
+        makeSnapshot({
+          totalTokens: 1000,
+          compactionTriggered: false,
+          breakdown: { systemPrompt: 200, messages: 300, toolOutputs: 100, knowledge: 200, other: 200 },
+          truncatedItems: [],
+        })
+      );
+      (mockFs.readFileSync as jest.Mock).mockReturnValue(
+        snapshots.map(s => JSON.stringify(s)).join('\n')
+      );
+
+      const issues = tracker.detectIssues();
+      expect(issues.length).toBe(0);
+    });
+  });
+});