@holoscript/framework 6.0.3

package/src/self-improve/prompt-optimizer.ts

@@ -0,0 +1,212 @@
+// STATUS: Scaffold — requires absorb service connection and LLM provider for production use
+/**
+ * PromptOptimizer — A/B test prompts to find the most effective phrasing.
+ *
+ * @experimental
+ *
+ * Part of FW-1.0 self-evolution: the framework optimizes its own prompts
+ * by running controlled experiments and measuring quality metrics.
+ */
+
+import { callLLM } from '../llm/llm-adapter';
+import type { ModelConfig } from '../types';
+
+export interface ABTestConfig {
+  /** LLM model configuration for running the test */
+  model: ModelConfig;
+  /** Number of runs per prompt variant (default: 3) */
+  runs?: number;
+  /** Evaluation criteria (default: quality + relevance) */
+  criteria?: EvaluationCriteria[];
+  /** Maximum tokens per response */
+  maxTokens?: number;
+  /** Temperature for test runs */
+  temperature?: number;
+}
+
+export interface EvaluationCriteria {
+  name: string;
+  description: string;
+  weight: number;
+}
+
+export interface PromptVariantResult {
+  prompt: string;
+  responses: string[];
+  avgScore: number;
+  scores: number[];
+  avgTokens: number;
+  avgLatencyMs: number;
+}
+
+export interface ABTestResult {
+  /** The task/question both prompts were tested against */
+  task: string;
+  /** Results for prompt A */
+  variantA: PromptVariantResult;
+  /** Results for prompt B */
+  variantB: PromptVariantResult;
+  /** Which variant won ('A' | 'B' | 'tie') */
+  winner: 'A' | 'B' | 'tie';
+  /** Confidence in the result (0-1, based on score delta and consistency) */
+  confidence: number;
+  /** Total tokens consumed across all runs */
+  totalTokens: number;
+}
+
+const DEFAULT_CRITERIA: EvaluationCriteria[] = [
+  { name: 'relevance', description: 'How relevant is the response to the task?', weight: 0.4 },
+  { name: 'quality', description: 'How well-structured and clear is the response?', weight: 0.3 },
+  { name: 'completeness', description: 'Does the response fully address the task?', weight: 0.3 },
+];
+
+const JUDGE_SYSTEM = `You are a prompt quality evaluator. Score the following response on a scale of 1-10 for each criterion. Return ONLY a JSON object with scores, e.g.: {"relevance": 8, "quality": 7, "completeness": 9}`;
+
+/**
+ * PromptOptimizer — A/B tests prompt variants to find the best one.
+ *
+ * @experimental
+ *
+ * Usage:
+ * ```ts
+ * const optimizer = new PromptOptimizer({
+ *   model: { provider: 'anthropic', model: 'claude-sonnet-4-20250514' }
+ * });
+ * const result = await optimizer.abTest(
+ *   'Explain HoloScript in one paragraph.',
+ *   'You are a HoloScript expert. Explain HoloScript concisely.',
+ *   'What is HoloScript?'
+ * );
+ * console.log(`Winner: ${result.winner} (confidence: ${result.confidence})`);
+ * ```
+ */
+export class PromptOptimizer {
+  private readonly config: ABTestConfig;
+
+  constructor(config: ABTestConfig) {
+    this.config = {
+      runs: 3,
+      criteria: DEFAULT_CRITERIA,
+      maxTokens: 1024,
+      temperature: 0.7,
+      ...config,
+    };
+  }
+
+  /**
+   * Run an A/B test comparing two prompt variants on a given task.
+   */
+  async abTest(promptA: string, promptB: string, task: string): Promise<ABTestResult> {
+    const runs = this.config.runs!;
+
+    const [variantA, variantB] = await Promise.all([
+      this.runVariant(promptA, task, runs),
+      this.runVariant(promptB, task, runs),
+    ]);
+
+    const delta = variantA.avgScore - variantB.avgScore;
+    const winner: 'A' | 'B' | 'tie' = Math.abs(delta) < 0.5 ? 'tie' : delta > 0 ? 'A' : 'B';
+
+    // Confidence: based on score delta magnitude and low variance
+    const varianceA = this.variance(variantA.scores);
+    const varianceB = this.variance(variantB.scores);
+    const avgVariance = (varianceA + varianceB) / 2;
+    const confidence = Math.min(1, Math.abs(delta) / 3) * Math.max(0.3, 1 - avgVariance / 10);
+
+    return {
+      task,
+      variantA,
+      variantB,
+      winner,
+      confidence: Math.round(confidence * 100) / 100,
+      totalTokens: variantA.avgTokens * runs + variantB.avgTokens * runs,
+    };
+  }
+
+  /**
+   * Run a single variant N times and evaluate.
+   */
+  private async runVariant(prompt: string, task: string, runs: number): Promise<PromptVariantResult> {
+    const responses: string[] = [];
+    const scores: number[] = [];
+    let totalTokens = 0;
+    let totalLatency = 0;
+
+    for (let i = 0; i < runs; i++) {
+      const start = Date.now();
+      const response = await callLLM(this.config.model, [
+        { role: 'system', content: prompt },
+        { role: 'user', content: task },
+      ], {
+        maxTokens: this.config.maxTokens,
+        temperature: this.config.temperature,
+      });
+      const latency = Date.now() - start;
+
+      responses.push(response.content);
+      totalTokens += response.tokensUsed ?? 0;
+      totalLatency += latency;
+
+      // Judge the response
+      const score = await this.judge(response.content, task);
+      scores.push(score);
+    }
+
+    return {
+      prompt,
+      responses,
+      avgScore: scores.reduce((a, b) => a + b, 0) / scores.length,
+      scores,
+      avgTokens: totalTokens / runs,
+      avgLatencyMs: totalLatency / runs,
+    };
+  }
+
+  /**
+   * Judge a response using the LLM as evaluator.
+   * Returns a weighted score (0-10).
+   */
+  private async judge(response: string, task: string): Promise<number> {
+    const criteria = this.config.criteria!;
+
+    try {
+      const judgeResponse = await callLLM(this.config.model, [
+        { role: 'system', content: JUDGE_SYSTEM },
+        {
+          role: 'user',
+          content: [
+            `Task: ${task}`,
+            `Response: ${response}`,
+            `Criteria: ${criteria.map(c => `${c.name} (${c.description})`).join(', ')}`,
+          ].join('\n'),
+        },
+      ], { maxTokens: 200, temperature: 0 });
+
+      const parsed = JSON.parse(judgeResponse.content) as Record<string, number>;
+      let weightedSum = 0;
+      let totalWeight = 0;
+
+      for (const c of criteria) {
+        const score = parsed[c.name];
+        if (typeof score === 'number' && score >= 1 && score <= 10) {
+          weightedSum += score * c.weight;
+          totalWeight += c.weight;
+        }
+      }
+
+      return totalWeight > 0 ? weightedSum / totalWeight : 5;
+    } catch {
+      // If judge fails, return neutral score
+      return 5;
+    }
+  }
+
+  /**
+   * Compute variance of a number array.
+   */
+  private variance(values: number[]): number {
+    if (values.length < 2) return 0;
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    return values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (values.length - 1);
+  }
+}

package/src/self-improve/test-generator.ts

@@ -0,0 +1,175 @@
+// STATUS: Scaffold — requires absorb service connection and LLM provider for production use
+/**
+ * TestGenerator — Auto-generate tests for framework source files.
+ *
+ * @experimental
+ *
+ * Uses the LLM adapter to generate vitest test files from source code.
+ * Part of FW-1.0 self-evolution: the framework writes its own tests.
+ */
+
+import { callLLM } from '../llm/llm-adapter';
+import type { ModelConfig } from '../types';
+import { readFileSync, existsSync } from 'fs';
+import { basename, dirname, resolve } from 'path';
+
+export interface TestGeneratorConfig {
+  /** LLM model configuration */
+  model: ModelConfig;
+  /** Maximum tokens for test generation */
+  maxTokens?: number;
+  /** Temperature (lower = more deterministic tests) */
+  temperature?: number;
+  /** Test framework (default: vitest) */
+  testFramework?: 'vitest' | 'jest';
+  /** Additional context to include in the prompt */
+  additionalContext?: string;
+}
+
+export interface GeneratedTest {
+  /** Source file path */
+  sourceFile: string;
+  /** Generated test file content */
+  testContent: string;
+  /** Suggested test file path */
+  testFilePath: string;
+  /** Number of test cases generated */
+  testCount: number;
+  /** LLM tokens used */
+  tokensUsed: number;
+}
+
+const SYSTEM_PROMPT = `You are a test generation expert for TypeScript projects using vitest.
+Given a source file, generate comprehensive test cases that:
+1. Test all exported functions and classes
+2. Cover happy paths and edge cases
+3. Use vi.mock() for external dependencies
+4. Follow the AAA pattern (Arrange, Act, Assert)
+5. Use descriptive test names
+
+Output ONLY the test file content — no explanations, no markdown fences.
+Use import { describe, it, expect, vi } from 'vitest';`;
+
+const JEST_SYSTEM_PROMPT = SYSTEM_PROMPT.replace(
+  "import { describe, it, expect, vi } from 'vitest';",
+  "Use jest globals (describe, it, expect, jest)."
+);
+
+/**
+ * TestGenerator — generates test files from source code via LLM.
+ *
+ * @experimental
+ *
+ * Usage:
+ * ```ts
+ * const gen = new TestGenerator({ model: { provider: 'anthropic', model: 'claude-sonnet-4-20250514' } });
+ * const result = await gen.generateTests('src/self-improve/absorb-scanner.ts');
+ * ```
+ */
+export class TestGenerator {
+  private readonly config: TestGeneratorConfig;
+
+  constructor(config: TestGeneratorConfig) {
+    this.config = {
+      maxTokens: 4096,
+      temperature: 0.3,
+      testFramework: 'vitest',
+      ...config,
+    };
+  }
+
+  /**
+   * Generate a test file for the given source file.
+   *
+   * @param filePath - Path to the TypeScript source file
+   * @returns Generated test content and metadata
+   */
+  async generateTests(filePath: string): Promise<GeneratedTest> {
+    const absPath = resolve(filePath);
+
+    if (!existsSync(absPath)) {
+      throw new Error(`Source file not found: ${absPath}`);
+    }
+
+    const source = readFileSync(absPath, 'utf-8');
+    const fileName = basename(absPath, '.ts');
+    const dir = dirname(absPath);
+
+    const systemPrompt = this.config.testFramework === 'jest'
+      ? JEST_SYSTEM_PROMPT
+      : SYSTEM_PROMPT;
+
+    const userPrompt = [
+      `Generate tests for this TypeScript file: ${basename(absPath)}`,
+      '',
+      '```typescript',
+      source,
+      '```',
+      this.config.additionalContext ? `\nAdditional context:\n${this.config.additionalContext}` : '',
+    ].join('\n');
+
+    const response = await callLLM(this.config.model, [
+      { role: 'system', content: systemPrompt },
+      { role: 'user', content: userPrompt },
+    ], {
+      maxTokens: this.config.maxTokens,
+      temperature: this.config.temperature,
+    });
+
+    const testContent = cleanTestOutput(response.content);
+    const testCount = countTestCases(testContent);
+
+    // Determine test file path
+    const testFilePath = resolve(dir, '__tests__', `${fileName}.test.ts`);
+
+    return {
+      sourceFile: absPath,
+      testContent,
+      testFilePath,
+      testCount,
+      tokensUsed: response.tokensUsed ?? 0,
+    };
+  }
+
+  /**
+   * Generate tests for multiple files.
+   *
+   * @param filePaths - Array of source file paths
+   * @returns Array of generated tests
+   */
+  async generateBatch(filePaths: string[]): Promise<GeneratedTest[]> {
+    const results: GeneratedTest[] = [];
+    for (const filePath of filePaths) {
+      try {
+        const result = await this.generateTests(filePath);
+        results.push(result);
+      } catch {
+        // Skip files that fail — don't block the batch
+      }
+    }
+    return results;
+  }
+}
+
+/** Strip markdown fences if the LLM included them. */
+function cleanTestOutput(content: string): string {
+  let cleaned = content.trim();
+  // Remove leading ```typescript or ```ts
+  if (cleaned.startsWith('```')) {
+    const firstNewline = cleaned.indexOf('\n');
+    if (firstNewline !== -1) {
+      cleaned = cleaned.slice(firstNewline + 1);
+    }
+  }
+  // Remove trailing ```
+  if (cleaned.endsWith('```')) {
+    cleaned = cleaned.slice(0, -3).trimEnd();
+  }
+  return cleaned;
+}
+
+/** Count the number of it() or test() calls in the content. */
+function countTestCases(content: string): number {
+  const matches = content.match(/\b(?:it|test)\s*\(/g);
+  return matches ? matches.length : 0;
+}

package/src/skill-router.ts

@@ -0,0 +1,186 @@
+/**
+ * SkillRouter — Skill-based task routing via agent capabilities.
+ *
+ * Matches task requirements to agent capabilities, scores by overlap,
+ * and returns the best-fit agent. Supports required vs preferred skills.
+ *
+ * FW-0.6 — Skill-based routing via agent capabilities.
+ *
+ * @module skill-router
+ */
+
+import type { AgentConfig, TaskDef, SlotRole } from './types';
+
+// =============================================================================
+// TYPES
+// =============================================================================
+
+export interface RoutingResult {
+  /** The selected agent, or null if no match */
+  agent: AgentConfig | null;
+  /** Score of the selected agent (0-100) */
+  score: number;
+  /** All candidates with their scores, sorted descending */
+  candidates: ScoredCandidate[];
+  /** Reason for selection or rejection */
+  reason: string;
+}
+
+export interface ScoredCandidate {
+  agent: AgentConfig;
+  score: number;
+  matchedCapabilities: string[];
+  roleMatch: boolean;
+}
+
+export interface RoutingPolicy {
+  /** Minimum score threshold (0-100) to be eligible (default 10) */
+  minScore?: number;
+  /** Weight for capability matches (default 15) */
+  capabilityWeight?: number;
+  /** Weight for exact role match (default 25) */
+  roleWeight?: number;
+  /** Weight for priority alignment (default 10) */
+  priorityWeight?: number;
+  /** Required capabilities — agent MUST have all of these */
+  requiredCapabilities?: string[];
+}
+
+// =============================================================================
+// SKILL ROUTER
+// =============================================================================
+
+export class SkillRouter {
+  private defaultPolicy: Required<RoutingPolicy>;
+
+  constructor(policy: RoutingPolicy = {}) {
+    this.defaultPolicy = {
+      minScore: policy.minScore ?? 10,
+      capabilityWeight: policy.capabilityWeight ?? 15,
+      roleWeight: policy.roleWeight ?? 25,
+      priorityWeight: policy.priorityWeight ?? 10,
+      requiredCapabilities: policy.requiredCapabilities ?? [],
+    };
+  }
+
+  /**
+   * Route a task to the best-fit agent.
+   *
+   * Scoring:
+   * - Each capability keyword match in task title/description: +capabilityWeight
+   * - Exact role match (task.role === agent role mapping): +roleWeight
+   * - Priority alignment (agent.claimFilter.maxPriority >= task.priority): +priorityWeight
+   * - Required capabilities must ALL be present or agent is filtered out
+   */
+  route(task: TaskDef, agents: AgentConfig[], policy?: RoutingPolicy): RoutingResult {
+    const p = { ...this.defaultPolicy, ...policy };
+
+    if (agents.length === 0) {
+      return { agent: null, score: 0, candidates: [], reason: 'No agents available' };
+    }
+
+    const taskText = `${task.title} ${task.description}`.toLowerCase();
+    const candidates: ScoredCandidate[] = [];
+
+    for (const agent of agents) {
+      // Check required capabilities
+      const agentCaps = new Set(agent.capabilities.map(c => c.toLowerCase()));
+      if (p.requiredCapabilities.length > 0) {
+        const hasAll = p.requiredCapabilities.every(rc => agentCaps.has(rc.toLowerCase()));
+        if (!hasAll) continue;
+      }
+
+      // Check priority alignment
+      if (task.priority > agent.claimFilter.maxPriority) continue;
+
+      let score = 0;
+      const matchedCapabilities: string[] = [];
+
+      // Capability matching — check if agent capabilities appear in task text
+      for (const cap of agent.capabilities) {
+        const capLower = cap.toLowerCase();
+        if (taskText.includes(capLower)) {
+          score += p.capabilityWeight;
+          matchedCapabilities.push(cap);
+        }
+      }
+
+      // Role matching
+      const roleMatch = task.role ? this.agentMatchesRole(agent, task.role) : false;
+      if (roleMatch) {
+        score += p.roleWeight;
+      }
+
+      // Priority alignment bonus (closer to task priority = better fit)
+      const priorityDelta = agent.claimFilter.maxPriority - task.priority;
+      if (priorityDelta >= 0) {
+        // Agents whose maxPriority closely matches get a bonus
+        score += Math.max(0, p.priorityWeight - priorityDelta);
+      }
+
+      // Base score for being eligible at all
+      score += 5;
+
+      candidates.push({ agent, score, matchedCapabilities, roleMatch });
+    }
+
+    // Sort by score descending
+    candidates.sort((a, b) => b.score - a.score);
+
+    // Apply minimum threshold
+    const eligible = candidates.filter(c => c.score >= p.minScore);
+
+    if (eligible.length === 0) {
+      return {
+        agent: null,
+        score: 0,
+        candidates,
+        reason: candidates.length > 0
+          ? `${candidates.length} candidates scored below minimum threshold (${p.minScore})`
+          : 'No candidates matched task requirements',
+      };
+    }
+
+    const winner = eligible[0];
+    return {
+      agent: winner.agent,
+      score: winner.score,
+      candidates,
+      reason: `Best fit: ${winner.agent.name} (score ${winner.score}, ${winner.matchedCapabilities.length} capability matches${winner.roleMatch ? ', role match' : ''})`,
+    };
+  }
+
+  /**
+   * Route a task to multiple agents (for parallel execution or fallback chains).
+   */
+  routeMultiple(
+    task: TaskDef,
+    agents: AgentConfig[],
+    count: number,
+    policy?: RoutingPolicy
+  ): RoutingResult {
+    const result = this.route(task, agents, policy);
+    // Already sorted — just return top N candidates
+    const topN = result.candidates.slice(0, count);
+    return {
+      ...result,
+      agent: topN[0]?.agent ?? null,
+      candidates: topN,
+      reason: `Top ${Math.min(count, topN.length)} candidates selected`,
+    };
+  }
+
+  // ── Private ──
+
+  private agentMatchesRole(agent: AgentConfig, taskRole: SlotRole): boolean {
+    // Map agent roles to slot roles
+    const roleMapping: Record<string, SlotRole[]> = {
+      architect: ['researcher', 'reviewer'],
+      coder: ['coder', 'flex'],
+      researcher: ['researcher'],
+      reviewer: ['reviewer', 'tester'],
+    };
+    const mappedRoles = roleMapping[agent.role] ?? [];
+    return mappedRoles.includes(taskRole) || agent.claimFilter.roles.includes(taskRole);
+  }
+}

package/src/skills/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Skills module barrel
+ * Re-exports all skills functionality from @holoscript/framework
+ */
+export * from './skill-md-bridge';