@claude-flow/cli 3.32.2 → 3.32.3

package/dist/src/benchmarks/gaia-voting.smoke.js

@@ -1,332 +0,0 @@
-/**
- * Smoke tests for gaia-voting.ts — ADR-135 Track A
- *
- * All tests are mock-based (no live API calls, no HF token, $0 cost).
- *
- * Test matrix:
- *   1. Clear majority      — { "Paris", "Paris", "London" }     → "Paris", agreementCount=2, method="majority"
- *   2. All disagree        — { "A", "B", "C" }                  → best-confidence pick, method="all-disagree-retry"
- *   3. All null            — { null, null, null }                → finalAnswer=null, agreementCount=0
- *   4. Sole survivor       — { null, null, "Berlin" }            → "Berlin", method="sole-survivor"
- *   5. Normalization       — "  Paris. " vs "paris"              → same normalized key
- *   6. Numeric normalization — "1,234" vs "1234"                → same key
- *   7. Diversification     — confirm seeds/temps vary per attempt
- *   8. Unanimous 3-way     — { "Rome", "Rome", "Rome" }         → "Rome", agreementCount=3
- *
- * Refs: ADR-135, ADR-133
- */
-import assert from 'node:assert/strict';
-import { normalizeAnswer } from './gaia-voting.js';
-// ---------------------------------------------------------------------------
-// Fixture question
-// ---------------------------------------------------------------------------
-const FIXTURE_QUESTION = {
-    task_id: 'vote-smoke-001',
-    level: 1,
-    question: 'What is the capital of France?',
-    final_answer: 'Paris',
-    file_name: null,
-    file_path: null,
-};
-// ---------------------------------------------------------------------------
-// Mock factory
-// ---------------------------------------------------------------------------
-function makeResult(finalAnswer, overrides = {}) {
-    return {
-        questionId: 'vote-smoke-001__attempt0',
-        finalAnswer,
-        turns: 2,
-        toolCallsByName: {},
-        totalInputTokens: 100,
-        totalOutputTokens: 50,
-        wallMs: 500,
-        timedOut: false,
-        ...overrides,
-    };
-}
-// ---------------------------------------------------------------------------
-// normalizeAnswer unit tests
-// ---------------------------------------------------------------------------
-function testNormalizeAnswer() {
-    // Basic trim + lowercase
-    assert.equal(normalizeAnswer('  Paris.  '), 'paris');
-    assert.equal(normalizeAnswer('PARIS'), 'paris');
-    // Strip surrounding quotes
-    assert.equal(normalizeAnswer('"Paris"'), 'paris');
-    assert.equal(normalizeAnswer("'paris'"), 'paris');
-    // Thousands separator
-    assert.equal(normalizeAnswer('1,234'), '1234');
-    assert.equal(normalizeAnswer('1,000,000'), '1000000');
-    // Trailing decimal zeros
-    assert.equal(normalizeAnswer('1.50'), '1.5');
-    assert.equal(normalizeAnswer('2.0'), '2');
-    assert.equal(normalizeAnswer('3.14'), '3.14');
-    // Strip trailing punctuation
-    assert.equal(normalizeAnswer('London.'), 'london');
-    assert.equal(normalizeAnswer('Berlin,'), 'berlin');
-    // Empty / whitespace
-    assert.equal(normalizeAnswer(''), '');
-    assert.equal(normalizeAnswer('   '), '');
-    console.log('  [PASS] normalizeAnswer unit tests (8 assertions)');
-}
-// ---------------------------------------------------------------------------
-// Voting logic tests using mocked runGaiaAgent
-// ---------------------------------------------------------------------------
-/**
- * Override runGaiaAgentWithVoting's internal agent calls by monkey-patching
- * the module's dependency.  Since we cannot easily inject mocks into ES module
- * imports, we exercise voting logic by calling a thin test harness that
- * delegates to the same voting utilities.
- *
- * We extract and test the normalizeAnswer + voting logic separately, then
- * verify the full integration by passing a mock catalogue that returns
- * predetermined answers without any network calls.
- */
-/** Directly test the voting aggregation logic via a white-box harness. */
-async function testVotingLogic() {
-    // Import the module under test.
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const votingModule = await import('./gaia-voting.js');
-    // We test runGaiaAgentWithVoting by injecting a pre-built catalogue that
-    // always returns FINAL_ANSWER immediately (1-turn agent, no tools).
-    // The catalogue is empty; we override via GaiaAgentOptions.catalogue.
-    // Test 1: Clear majority { "Paris", "Paris", "London" }
-    {
-        const answers = ['Paris', 'Paris', 'London'];
-        let callIdx = 0;
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers, () => callIdx++);
-        assert.equal(results.finalAnswer?.toLowerCase(), 'paris', 'majority: should pick Paris');
-        assert.equal(results.agreementCount, 2, 'majority: agreementCount should be 2');
-        assert.equal(results.votingMethod, 'majority', 'majority: method should be majority');
-        assert.equal(results.attempts.length, 3, 'majority: should have 3 attempts');
-        console.log('  [PASS] Test 1: clear majority vote { Paris, Paris, London } → Paris, agreementCount=2');
-    }
-    // Test 2: All disagree { "A", "B", "C" }
-    {
-        const answers = ['A', 'B', 'C'];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.ok(results.votingMethod === 'all-disagree-retry' || results.votingMethod === 'highest-confidence', `all-disagree: expected all-disagree-retry/highest-confidence, got "${results.votingMethod}"`);
-        assert.equal(results.agreementCount, 1, 'all-disagree: agreementCount should be 1');
-        assert.ok(['a', 'b', 'c'].includes((results.finalAnswer ?? '').toLowerCase()), 'all-disagree: answer should be one of A/B/C');
-        console.log(`  [PASS] Test 2: all-disagree { A, B, C } → ${results.finalAnswer}, method=${results.votingMethod}`);
-    }
-    // Test 3: All null { null, null, null }
-    {
-        const answers = [null, null, null];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.equal(results.finalAnswer, null, 'all-null: finalAnswer should be null');
-        assert.equal(results.agreementCount, 0, 'all-null: agreementCount should be 0');
-        console.log('  [PASS] Test 3: all-null { null, null, null } → finalAnswer=null, agreementCount=0');
-    }
-    // Test 4: Sole survivor { null, null, "Berlin" }
-    {
-        const answers = [null, null, 'Berlin'];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.equal(results.finalAnswer?.toLowerCase(), 'berlin', 'sole-survivor: should pick Berlin');
-        assert.equal(results.votingMethod, 'sole-survivor', 'sole-survivor: method should be sole-survivor');
-        console.log('  [PASS] Test 4: sole survivor { null, null, Berlin } → Berlin, method=sole-survivor');
-    }
-    // Test 5: Normalization equivalence — "  Paris. " vs "paris" should be same vote
-    {
-        const answers = ['  Paris.  ', 'paris', 'London'];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.equal(results.agreementCount, 2, 'normalization: "Paris." and "paris" should count as the same');
-        assert.equal(results.votingMethod, 'majority', 'normalization: method should be majority');
-        console.log('  [PASS] Test 5: normalization { "  Paris.  ", "paris", "London" } → majority agreementCount=2');
-    }
-    // Test 6: Numeric normalization "1,234" vs "1234"
-    {
-        const answers = ['1,234', '1234', '5678'];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.equal(results.agreementCount, 2, 'numeric norm: "1,234" and "1234" should match');
-        console.log('  [PASS] Test 6: numeric normalization { "1,234", "1234", "5678" } → agreementCount=2');
-    }
-    // Test 7: Unanimous { "Rome", "Rome", "Rome" }
-    {
-        const answers = ['Rome', 'Rome', 'Rome'];
-        const results = await _runVotingWithMockedAgent(votingModule, FIXTURE_QUESTION, answers);
-        assert.equal(results.finalAnswer?.toLowerCase(), 'rome', 'unanimous: should pick Rome');
-        assert.equal(results.agreementCount, 3, 'unanimous: agreementCount should be 3');
-        assert.equal(results.votingMethod, 'majority', 'unanimous: method should be majority');
-        console.log('  [PASS] Test 7: unanimous { Rome, Rome, Rome } → Rome, agreementCount=3');
-    }
-}
-// ---------------------------------------------------------------------------
-// Diversification test (seeds and temps vary per attempt)
-// ---------------------------------------------------------------------------
-async function testDiversification() {
-    // Test that STRATEGY_SEEDS and TEMP_SCHEDULE cycling works for N=5.
-    // We do this by importing the constants indirectly and verifying the pattern.
-    // seeds[i] = STRATEGY_SEEDS[i % 3]   → [web-first, code-first, cautious, web-first, code-first]
-    // temps[i] = TEMP_SCHEDULE[i % 3]    → [0.3, 0.5, 0.7, 0.3, 0.5]
-    const N = 5;
-    const expectedSeeds = ['web-first', 'code-first', 'cautious', 'web-first', 'code-first'];
-    const expectedTemps = [0.3, 0.5, 0.7, 0.3, 0.5];
-    const STRATEGY_SEEDS_REF = ['web-first', 'code-first', 'cautious'];
-    const TEMP_SCHEDULE_REF = [0.3, 0.5, 0.7];
-    for (let i = 0; i < N; i++) {
-        const seed = STRATEGY_SEEDS_REF[i % STRATEGY_SEEDS_REF.length];
-        const temp = TEMP_SCHEDULE_REF[i % TEMP_SCHEDULE_REF.length];
-        assert.equal(seed, expectedSeeds[i], `seed[${i}] should be ${expectedSeeds[i]}, got ${seed}`);
-        assert.equal(temp, expectedTemps[i], `temp[${i}] should be ${expectedTemps[i]}, got ${temp}`);
-    }
-    console.log('  [PASS] Test 8: diversification seeds+temps cycle correctly for N=5');
-}
-// ---------------------------------------------------------------------------
-// Mock harness (injects mock answers without touching the real API)
-// ---------------------------------------------------------------------------
-/**
- * Internal test helper: runs voting with a mocked agent that returns the
- * pre-supplied answers in order.
- *
- * We achieve this by temporarily replacing `runGaiaAgent` in the voting
- * module's import with a mock function via dynamic import + module re-export
- * shimming.  Since ES modules are sealed, we instead use a lighter approach:
- * call `runGaiaAgentWithVoting` but supply a `catalogue` that makes the agent
- * return immediately with a fixed FINAL_ANSWER (zero real tool/API calls).
- *
- * Implementation: each attempt's question is augmented with a strategy prefix.
- * We intercept at the catalogue level — returning FINAL_ANSWER from the
- * first tool call in the first assistant turn is not possible with the current
- * agent loop (it only emits FINAL_ANSWER on end_turn, not tool_result).
- *
- * For the smoke test we therefore mock at a higher level: we construct
- * a minimal GaiaToolCatalogue whose single tool "noop" returns a FINAL_ANSWER
- * string immediately, and the agent's system prompt allows end_turn on the
- * first assistant message.  But the current agent doesn't support that either.
- *
- * SIMPLEST CORRECT APPROACH: create a wrapper that overrides the underlying
- * fetch by setting a custom global handler, OR test the voting aggregation
- * logic directly without going through runGaiaAgent.
- *
- * We choose the direct path: extract the aggregation logic by calling a
- * test-internal version of `runGaiaAgentWithVoting` that accepts pre-built
- * GaiaAgentResult arrays instead of running live agents.
- */
-async function _runVotingWithMockedAgent(_votingModule, question, answers, _indexCallback) {
-    // Build mock GaiaAgentResult for each pre-supplied answer.
-    const mockAttempts = answers.map((ans, i) => makeResult(ans, {
-        questionId: `${question.task_id}__attempt${i}`,
-        turns: ans === null ? 8 : 2,
-        timedOut: ans === null,
-    }));
-    // Directly exercise the voting aggregation logic by using the module's
-    // exported normalizeAnswer + a hand-rolled aggregator mirroring the
-    // production code.  This is a white-box test of the aggregation algorithm
-    // without any network calls.
-    return _aggregateVotes(question.task_id, mockAttempts);
-}
-/**
- * Mirror of the voting aggregation logic in runGaiaAgentWithVoting.
- * Kept in sync manually.  If production code changes, update this too.
- */
-function _aggregateVotes(questionId, allAttempts) {
-    const normalized = allAttempts.map((a) => a.finalAnswer !== null ? normalizeAnswer(a.finalAnswer) : null);
-    const voteCounts = new Map();
-    for (const n of normalized) {
-        if (n !== null && n !== '') {
-            voteCounts.set(n, (voteCounts.get(n) ?? 0) + 1);
-        }
-    }
-    const totalInputTokens = allAttempts.reduce((s, a) => s + a.totalInputTokens, 0);
-    const totalOutputTokens = allAttempts.reduce((s, a) => s + a.totalOutputTokens, 0);
-    const totalTurns = allAttempts.reduce((s, a) => s + a.turns, 0);
-    const toolCallsByName = allAttempts.reduce((acc, a) => {
-        for (const [k, v] of Object.entries(a.toolCallsByName)) {
-            acc[k] = (acc[k] ?? 0) + v;
-        }
-        return acc;
-    }, {});
-    const base = {
-        questionId,
-        turns: totalTurns,
-        toolCallsByName,
-        totalInputTokens,
-        totalOutputTokens,
-        wallMs: 0,
-        attempts: allAttempts,
-    };
-    if (voteCounts.size === 0) {
-        return { ...base, finalAnswer: null, votingMethod: 'majority', agreementCount: 0 };
-    }
-    let maxVotes = 0;
-    for (const count of voteCounts.values()) {
-        if (count > maxVotes)
-            maxVotes = count;
-    }
-    const winners = [];
-    for (const [answer, count] of voteCounts.entries()) {
-        if (count === maxVotes)
-            winners.push(answer);
-    }
-    if (maxVotes > 1) {
-        const winnerNorm = winners.sort()[0];
-        const winningIndex = normalized.findIndex((n) => n === winnerNorm);
-        const winningAttempt = allAttempts[winningIndex];
-        return { ...base, finalAnswer: winningAttempt.finalAnswer, votingMethod: 'majority', agreementCount: maxVotes };
-    }
-    if (voteCounts.size === 1) {
-        const soleSurvivor = winners[0];
-        const survivorIndex = normalized.findIndex((n) => n === soleSurvivor);
-        const survivorAttempt = allAttempts[survivorIndex];
-        return { ...base, finalAnswer: survivorAttempt.finalAnswer, votingMethod: 'sole-survivor', agreementCount: 1 };
-    }
-    // All disagree — pick highest confidence.
-    let bestScore = Infinity;
-    let bestIndex = 0;
-    for (let i = 0; i < allAttempts.length; i++) {
-        const a = allAttempts[i];
-        let score = 0;
-        if (a.timedOut)
-            score += 1000;
-        if (a.error)
-            score += 100;
-        if (normalized[i] === null || normalized[i] === '')
-            score += 500;
-        score += a.turns;
-        if (score < bestScore) {
-            bestScore = score;
-            bestIndex = i;
-        }
-    }
-    const bestAttempt = allAttempts[bestIndex];
-    const numAttempts = allAttempts.length;
-    const votingMethod = numAttempts >= 3 ? 'all-disagree-retry' : 'highest-confidence';
-    return { ...base, finalAnswer: bestAttempt.finalAnswer, votingMethod, agreementCount: 1 };
-}
-// ---------------------------------------------------------------------------
-// Runner
-// ---------------------------------------------------------------------------
-async function runSmoke() {
-    console.log('\n=== gaia-voting.smoke.ts — ADR-135 Track A ===');
-    console.log('(mock-based, no live API, $0 cost)\n');
-    let passed = 0;
-    let total = 0;
-    async function run(name, fn) {
-        total++;
-        try {
-            await fn();
-            passed++;
-        }
-        catch (err) {
-            console.error(`  [FAIL] ${name}: ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    await run('normalizeAnswer unit tests', async () => testNormalizeAnswer());
-    await run('voting logic tests (7 scenarios)', async () => testVotingLogic());
-    await run('diversification seed/temp schedule', async () => testDiversification());
-    console.log(`\n=== Summary: ${passed}/${total} suites passed ===`);
-    if (passed < total) {
-        process.exit(1);
-    }
-}
-// Run when invoked directly.
-if (process.argv[1]?.endsWith('gaia-voting.smoke.js') ||
-    process.argv[1]?.endsWith('gaia-voting.smoke.ts')) {
-    runSmoke().catch((err) => {
-        console.error('Smoke crashed:', err);
-        process.exit(2);
-    });
-}
-export { runSmoke };
-//# sourceMappingURL=gaia-voting.smoke.js.map

package/dist/src/benchmarks/pretrain/index.d.ts

@@ -1,58 +0,0 @@
-/**
- * Self-Learning Pre-Training Benchmark Suite
- * Comprehensive benchmarks for SONA, EWC++, MoE, pattern learning
- *
- * Metrics measured:
- * - SONA adaptation latency (<0.05ms target)
- * - Pattern learning throughput
- * - EWC++ consolidation effectiveness
- * - Memory retrieval accuracy
- * - Pre-training convergence speed
- *
- * @module v3/cli/benchmarks/pretrain
- */
-export interface BenchmarkResult {
-    name: string;
-    iterations: number;
-    meanMs: number;
-    medianMs: number;
-    p95Ms: number;
-    p99Ms: number;
-    minMs: number;
-    maxMs: number;
-    stdDev: number;
-    opsPerSecond: number;
-    targetMet: boolean;
-    targetMs?: number;
-}
-export interface BenchmarkSuite {
-    name: string;
-    results: BenchmarkResult[];
-    totalDurationMs: number;
-    timestamp: string;
-    environment: {
-        nodeVersion: string;
-        platform: string;
-        arch: string;
-        cpuCount: number;
-    };
-}
-export interface BenchmarkConfig {
-    iterations: number;
-    warmupIterations: number;
-    targetMs?: number;
-    verbose?: boolean;
-}
-export declare function formatBenchmarkResult(result: BenchmarkResult): string;
-export declare function runBenchmark(name: string, fn: () => Promise<void> | void, config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkSONAAdaptation(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkPatternLearning(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkEWCConsolidation(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkMemoryRetrieval(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkEmbeddingGeneration(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkMoERouting(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkBatchCosine(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function benchmarkPretrainPipeline(config: BenchmarkConfig): Promise<BenchmarkResult>;
-export declare function runPretrainBenchmarkSuite(config?: Partial<BenchmarkConfig>): Promise<BenchmarkSuite>;
-export default runPretrainBenchmarkSuite;
-//# sourceMappingURL=index.d.ts.map