@dommaker/harness 0.13.0 → 0.13.2

package/src/context/knowledge-injector.ts

@@ -0,0 +1,168 @@
+/**
+ * 知识注入器
+ *
+ * 将知识条目转换为 ContextSource，注入到 TokenPipeline 的 P3 位置
+ * 支持按阶段预算、类型过滤、去重
+ */
+
+import { KnowledgeQuery } from '../knowledge/query';
+import { TokenEstimator } from './token-budget';
+import type { KnowledgeEntry, QueryResult, KnowledgeType } from '../knowledge/types';
+import type { ContextSource } from './types';
+
+export interface InjectionConfig {
+  /** Token 预算（默认 800） */
+  budget: number;
+  /** 关注的知识类型 */
+  focusTypes?: KnowledgeType[];
+  /** 当前阶段 */
+  phase?: string;
+  /** 已注入的条目 ID（去重） */
+  exclude?: string[];
+  /** 是否注入摘要版本（已注入的条目） */
+  injectSummaryForExcluded?: boolean;
+}
+
+export interface InjectionResult {
+  sources: ContextSource[];
+  tokensUsed: number;
+  entriesIncluded: number;
+  entriesExcluded: number;
+  entriesSummarized: number;
+}
+
+export class KnowledgeInjector {
+  private query: KnowledgeQuery;
+
+  constructor(query: KnowledgeQuery) {
+    this.query = query;
+  }
+
+  /**
+   * 注入知识到上下文
+   */
+  inject(config: InjectionConfig): InjectionResult {
+    const {
+      budget,
+      focusTypes,
+      phase,
+      exclude = [],
+      injectSummaryForExcluded = true,
+    } = config;
+
+    // 查询知识
+    const queryResult = this.query.query({
+      phase: phase || 'default',
+      maxTokens: budget,
+      maxEntries: 20,
+      focusTypes: focusTypes || [],
+    });
+
+    const sources: ContextSource[] = [];
+    let tokensUsed = 0;
+    let entriesIncluded = 0;
+    let entriesExcluded = 0;
+    let entriesSummarized = 0;
+
+    const excludeSet = new Set(exclude);
+
+    for (const entry of queryResult.entries) {
+      if (excludeSet.has(entry.id)) {
+        entriesExcluded++;
+
+        // 注入摘要版本
+        if (injectSummaryForExcluded) {
+          const summary = this.formatEntrySummary(entry);
+          const summaryTokens = TokenEstimator.estimateText(summary);
+
+          if (tokensUsed + summaryTokens <= budget) {
+            sources.push({
+              type: 'knowledge',
+              id: `knowledge-summary-${entry.id}`,
+              content: summary,
+              priority: 3,
+              metadata: { entryId: entry.id, isSummary: true },
+            });
+            tokensUsed += summaryTokens;
+            entriesSummarized++;
+          }
+        }
+
+        continue;
+      }
+
+      // 注入完整版本
+      const formatted = this.formatEntry(entry);
+      const entryTokens = TokenEstimator.estimateText(formatted);
+
+      if (tokensUsed + entryTokens <= budget) {
+        sources.push({
+          type: 'knowledge',
+          id: `knowledge-${entry.id}`,
+          content: formatted,
+          priority: 3,
+          metadata: { entryId: entry.id, maturity: entry.maturity },
+        });
+        tokensUsed += entryTokens;
+        entriesIncluded++;
+      } else {
+        // 预算不足，尝试注入摘要
+        const summary = this.formatEntrySummary(entry);
+        const summaryTokens = TokenEstimator.estimateText(summary);
+
+        if (tokensUsed + summaryTokens <= budget) {
+          sources.push({
+            type: 'knowledge',
+            id: `knowledge-summary-${entry.id}`,
+            content: summary,
+            priority: 3,
+            metadata: { entryId: entry.id, isSummary: true },
+          });
+          tokensUsed += summaryTokens;
+          entriesSummarized++;
+        }
+      }
+    }
+
+    return {
+      sources,
+      tokensUsed,
+      entriesIncluded,
+      entriesExcluded,
+      entriesSummarized,
+    };
+  }
+
+  /**
+   * 将知识条目格式化为上下文内容
+   */
+  formatEntry(entry: KnowledgeEntry): string {
+    const parts: string[] = [];
+
+    parts.push(`## [${entry.type.toUpperCase()}] ${entry.title}`);
+    parts.push(`ID: ${entry.id} | 成熟度: ${entry.maturity} | 层级: ${entry.layer}`);
+
+    if (entry.tags.length > 0) {
+      parts.push(`标签: ${entry.tags.join(', ')}`);
+    }
+
+    parts.push('');
+    parts.push(entry.content);
+
+    return parts.join('\n');
+  }
+
+  /**
+   * 将知识条目格式化为摘要（一行）
+   */
+  formatEntrySummary(entry: KnowledgeEntry): string {
+    return `[${entry.id}] ${entry.title} (${entry.maturity}) — ${entry.content.slice(0, 100)}${entry.content.length > 100 ? '...' : ''}`;
+  }
+
+  /**
+   * 获取查询引擎
+   */
+  getQuery(): KnowledgeQuery {
+    return this.query;
+  }
+}

package/src/context/progressive-loader.ts

@@ -0,0 +1,260 @@
+import { delay } from '../utils/exec';
+
+/**
+ * @spec HZ-002
+ * @implements HZ-002-C1
+ * @acceptance AC-001-1, AC-001-4
+ * 
+ * Progressive Content Loader
+ * 
+ * 通用内容分片加载能力
+ * - 支持大数据集分片加载
+ * - 支持 Token 预算管理
+ * - 零业务逻辑，可被任意系统使用
+ */
+
+export interface ChunkLoadOptions<T = any> {
+  /** 每片大小 */
+  chunkSize: number;
+  /** 每片加载回调 */
+  onChunk?: (chunk: T[], index: number) => Promise<void> | void;
+  /** 进度回调 (loaded, total) */
+  onProgress?: (loaded: number, total: number) => void;
+  /** 是否并行加载 */
+  parallel?: boolean;
+  /** 延迟（毫秒），用于节流 */
+  delay?: number;
+}
+
+export interface TokenBudgetOptions<T = any> {
+  /** Token 预算 */
+  budget: number;
+  /** Token 估算函数 */
+  estimator: (item: T) => number;
+  /** 预算超限策略 */
+  onBudgetExceeded?: 'truncate' | 'error' | 'skip';
+  /** 最小保留项数（即使超预算） */
+  minItems?: number;
+}
+
+export interface LoadResult<T> {
+  /** 加载的项 */
+  items: T[];
+  /** 实际消耗的 Token */
+  tokensUsed: number;
+  /** 是否被截断 */
+  truncated: boolean;
+  /** 跳过的项数 */
+  skipped: number;
+}
+
+/**
+ * 渐进式内容加载器
+ * 
+ * 提供通用的内容分片加载和 Token 预算管理能力
+ */
+export class ProgressiveLoader {
+  /**
+   * 分片加载
+   * 
+   * 将大数据集分片加载，避免一次性加载全部内容
+   * 
+   * @example
+   * ```typescript
+   * const loader = new ProgressiveLoader();
+   * const largeData = Array.from({ length: 1000 }, (_, i) => ({ id: i }));
+   * 
+   * await loader.loadInChunks(largeData, {
+   *   chunkSize: 100,
+   *   onChunk: async (chunk, index) => {
+   *     console.log(`Loaded chunk ${index}: ${chunk.length} items`);
+   *   },
+   *   onProgress: (loaded, total) => {
+   *     console.log(`Progress: ${loaded}/${total}`);
+   *   }
+   * });
+   * ```
+   */
+  async loadInChunks<T>(
+    items: T[],
+    options: ChunkLoadOptions<T>
+  ): Promise<void> {
+    const { chunkSize, onChunk, onProgress, parallel = false, delay: throttleDelay = 0 } = options;
+    const total = items.length;
+    const chunks: T[][] = [];
+
+    // 分片
+    for (let i = 0; i < total; i += chunkSize) {
+      chunks.push(items.slice(i, i + chunkSize));
+    }
+
+    // 加载
+    if (parallel) {
+      // 并行加载
+      await Promise.all(
+        chunks.map(async (chunk, index) => {
+          if (throttleDelay > 0) {
+            await delay(throttleDelay * index);
+          }
+          await this.processChunk(chunk, index, onChunk, onProgress, total);
+        })
+      );
+    } else {
+      // 串行加载
+      for (let i = 0; i < chunks.length; i++) {
+        if (throttleDelay > 0 && i > 0) {
+          await delay(throttleDelay);
+        }
+        await this.processChunk(chunks[i], i, onChunk, onProgress, total);
+      }
+    }
+  }
+
+  /**
+   * Token 预算加载
+   * 
+   * 在 Token 预算范围内加载内容，超预算时按策略处理
+   * 
+   * @example
+   * ```typescript
+   * const loader = new ProgressiveLoader();
+   * const items = [
+   *   { content: 'short text' },
+   *   { content: 'long text '.repeat(100) }
+   * ];
+   * 
+   * const result = await loader.loadWithBudget(items, {
+   *   budget: 1000,
+   *   estimator: (item) => item.content.length,
+   *   onBudgetExceeded: 'truncate'
+   * });
+   * 
+   * console.log(`Loaded ${result.items.length} items, used ${result.tokensUsed} tokens`);
+   * ```
+   */
+  async loadWithBudget<T>(
+    items: T[],
+    options: TokenBudgetOptions<T>
+  ): Promise<LoadResult<T>> {
+    const {
+      budget,
+      estimator,
+      onBudgetExceeded = 'truncate',
+      minItems = 1
+    } = options;
+
+    const result: LoadResult<T> = {
+      items: [],
+      tokensUsed: 0,
+      truncated: false,
+      skipped: 0
+    };
+
+    let currentBudget = budget;
+
+    for (const item of items) {
+      const itemTokens = estimator(item);
+
+      // 检查预算
+      if (itemTokens > currentBudget) {
+        // 预算超限
+        if (result.items.length >= minItems) {
+          // 已达到最小项数，按策略处理
+          switch (onBudgetExceeded) {
+            case 'truncate':
+              result.truncated = true;
+              return result;
+            case 'error':
+              throw new Error(
+                `Token budget exceeded: need ${itemTokens}, remaining ${currentBudget}`
+              );
+            case 'skip':
+              result.skipped++;
+              continue;
+          }
+        }
+        // 未达到最小项数，强制添加（可能超预算）
+      }
+
+      // 添加项
+      result.items.push(item);
+      result.tokensUsed += itemTokens;
+      currentBudget -= itemTokens;
+    }
+
+    return result;
+  }
+
+  /**
+   * 创建流式加载器
+   * 
+   * 支持大文件/大内容的流式分片加载
+   */
+  async *createStream<T>(
+    source: AsyncIterable<T>,
+    chunkSize: number
+  ): AsyncIterable<T[]> {
+    let chunk: T[] = [];
+
+    for await (const item of source) {
+      chunk.push(item);
+
+      if (chunk.length >= chunkSize) {
+        yield chunk;
+        chunk = [];
+      }
+    }
+
+    //  yield remaining items
+    if (chunk.length > 0) {
+      yield chunk;
+    }
+  }
+
+  /**
+   * 批量处理（带并发控制）
+   */
+  async processBatch<T, R>(
+    items: T[],
+    processor: (item: T) => Promise<R>,
+    concurrency: number = 5
+  ): Promise<R[]> {
+    const results = new Array<R>(items.length);
+    let nextIndex = 0;
+
+    const worker = async (): Promise<void> => {
+      while (nextIndex < items.length) {
+        const index = nextIndex++;
+        results[index] = await processor(items[index]);
+      }
+    };
+
+    const workers = Array.from(
+      { length: Math.min(concurrency, items.length) },
+      () => worker()
+    );
+    await Promise.all(workers);
+    return results;
+  }
+
+  private async processChunk<T>(
+    chunk: T[],
+    index: number,
+    onChunk: ((chunk: T[], index: number) => Promise<void> | void) | undefined,
+    onProgress: ((loaded: number, total: number) => void) | undefined,
+    total: number
+  ): Promise<void> {
+    if (onChunk) {
+      await onChunk(chunk, index);
+    }
+
+    if (onProgress) {
+      const loaded = Math.min((index + 1) * chunk.length, total);
+      onProgress(loaded, total);
+    }
+  }
+
+}
+
+// 默认实例
+export const progressiveLoader = new ProgressiveLoader();