claude-flow-novice 2.2.4 → 2.2.5

package/scripts/test-routing-telemetry.cjs

@@ -0,0 +1,147 @@
+/**
+ * Test Provider Routing Telemetry
+ *
+ * Tests tiered routing with telemetry to measure API calls to:
+ * - Z.ai (Tier 2 default)
+ * - Anthropic subscription (Tier 1)
+ * - Anthropic explicit (Tier 3)
+ */
+
+const { TelemetrySystem } = require('../.claude-flow-novice/dist/src/observability/telemetry.js');
+const { TieredProviderRouter } = require('../.claude-flow-novice/dist/src/providers/tiered-router.js');
+
+async function testRoutingTelemetry() {
+  console.log('🔬 Testing Provider Routing Telemetry\n');
+
+  // Initialize telemetry
+  const telemetry = new TelemetrySystem({
+    enableTracing: true,
+    enableMetrics: true,
+    enableStructuredLogging: true,
+  });
+
+  telemetry.initialize();
+
+  // Listen to metric events
+  const metricEvents = [];
+  telemetry.on('metric:counter', (metric) => {
+    if (metric.name === 'provider.request') {
+      metricEvents.push(metric);
+      console.log(`📊 Metric: ${metric.tags.provider} via ${metric.tags.tier} (agent: ${metric.tags.agentType})`);
+    }
+  });
+
+  telemetry.on('metric:gauge', (metric) => {
+    if (metric.name === 'subscription.usage') {
+      console.log(`📈 Subscription: ${metric.value}/${metric.tags.limit} (${metric.tags.remaining} remaining)`);
+    }
+  });
+
+  // Create tiered router with injected telemetry instance
+  const router = new TieredProviderRouter(
+    undefined, // Use default tier config
+    { used: 0, limit: 5 }, // Mock subscription: 5 requests limit
+    undefined, // Use default agents directory
+    telemetry // Inject telemetry instance to capture metrics
+  );
+
+  console.log('🎯 Test Scenario: Simulate 10 agent requests\n');
+
+  // Test 1: Default agents (should use Z.ai)
+  console.log('--- Test 1: Default agents (Z.ai expected) ---');
+  for (let i = 0; i < 3; i++) {
+    await router.selectProvider('coder');
+  }
+
+  // Test 2: Coordinator agents (should use subscription until exhausted)
+  console.log('\n--- Test 2: Coordinator agents (subscription expected) ---');
+  for (let i = 0; i < 4; i++) {
+    await router.selectProvider('coordinator');
+  }
+
+  // Test 3: More coordinator agents (subscription exhausted, fallback to Z.ai)
+  console.log('\n--- Test 3: More coordinators (fallback to Z.ai expected) ---');
+  for (let i = 0; i < 3; i++) {
+    await router.selectProvider('coordinator');
+  }
+
+  // Wait for metrics flush
+  await new Promise(resolve => setTimeout(resolve, 100));
+
+  // Analyze results
+  console.log('\n📊 === TELEMETRY RESULTS ===\n');
+
+  const providerBreakdown = {};
+  const tierBreakdown = {};
+  const sourceBreakdown = {};
+
+  metricEvents.forEach(metric => {
+    const provider = metric.tags.provider;
+    const tier = metric.tags.tier;
+    const source = metric.tags.source;
+
+    providerBreakdown[provider] = (providerBreakdown[provider] || 0) + metric.value;
+    tierBreakdown[tier] = (tierBreakdown[tier] || 0) + metric.value;
+    sourceBreakdown[source] = (sourceBreakdown[source] || 0) + metric.value;
+  });
+
+  console.log('By Provider:');
+  Object.entries(providerBreakdown).forEach(([provider, count]) => {
+    console.log(`  ${provider}: ${count} requests`);
+  });
+
+  console.log('\nBy Tier:');
+  Object.entries(tierBreakdown).forEach(([tier, count]) => {
+    console.log(`  ${tier}: ${count} requests`);
+  });
+
+  console.log('\nBy Source:');
+  Object.entries(sourceBreakdown).forEach(([source, count]) => {
+    console.log(`  ${source}: ${count} requests`);
+  });
+
+  // Get subscription usage
+  const subUsage = router.getSubscriptionUsage();
+  console.log('\n📊 Subscription Usage:');
+  console.log(`  Used: ${subUsage.used}/${subUsage.limit}`);
+  console.log(`  Remaining: ${subUsage.limit - subUsage.used}`);
+
+  // Validate expectations
+  console.log('\n✅ === VALIDATION ===\n');
+
+  const zaiCount = providerBreakdown['custom'] || 0;
+  const anthropicCount = providerBreakdown['anthropic'] || 0;
+
+  console.log(`Z.ai requests: ${zaiCount} (expected: 5 - 3 coder + 2 fallback)`);
+  console.log(`Anthropic requests: ${anthropicCount} (expected: 5 - subscription limit reached)`);
+
+  if (zaiCount === 5 && anthropicCount === 5) {
+    console.log('\n✅ SUCCESS: Telemetry correctly tracked routing decisions!');
+    console.log('   - Z.ai received 3 coder requests via profile override');
+    console.log('   - Subscription tier consumed all 5 slots for coordinators');
+    console.log('   - 2 coordinator requests fell back to Z.ai after limit');
+    console.log('   - All 10 requests were measured correctly');
+  } else {
+    console.log('\n❌ UNEXPECTED: Metric counts differ from expected');
+  }
+
+  // Query telemetry directly
+  console.log('\n📈 === TELEMETRY QUERY ===\n');
+  const providerMetrics = telemetry.getMetrics('provider.request');
+  console.log(`Total provider.request metrics: ${providerMetrics.length}`);
+
+  const throughput = telemetry.getThroughputMetrics('provider.request', 60000);
+  console.log(`Throughput: ${throughput.requestsPerSecond.toFixed(2)} req/s`);
+  console.log(`Success rate: ${(throughput.successRate * 100).toFixed(1)}%`);
+
+  // Cleanup
+  telemetry.shutdown();
+
+  console.log('\n🎉 Test complete!\n');
+}
+
+// Run test
+testRoutingTelemetry().catch(err => {
+  console.error('❌ Test failed:', err);
+  process.exit(1);
+});

package/scripts/test-runner.cjs

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+
+/**
+ * Intelligent Test Runner
+ *
+ * Categorizes and runs tests based on type:
+ * - unit: Fast, isolated tests (run first)
+ * - integration: Component integration tests
+ * - e2e: End-to-end workflow tests
+ * - performance: Benchmark tests (skip in CI)
+ * - chaos: Chaos engineering tests (skip in quick runs)
+ */
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const CATEGORIES = {
+  unit: {
+    pattern: 'tests/unit/**/*.test.{ts,js}',
+    timeout: 30000,
+    priority: 1,
+    description: 'Unit tests (fast, isolated)'
+  },
+  integration: {
+    pattern: 'tests/integration/**/*.test.{ts,js}',
+    timeout: 60000,
+    priority: 2,
+    description: 'Integration tests'
+  },
+  e2e: {
+    pattern: 'tests/**/e2e/**/*.test.{ts,js}',
+    timeout: 120000,
+    priority: 3,
+    description: 'End-to-end tests'
+  },
+  uncategorized: {
+    pattern: 'tests/*.test.{ts,js}',
+    timeout: 30000,
+    priority: 4,
+    description: 'Uncategorized tests'
+  }
+};
+
+const SKIP_CATEGORIES = {
+  chaos: 'tests/chaos/**/*.test.{ts,js}',
+  benchmark: 'tests/benchmarks/**/*.test.{ts,js}',
+  archived: 'tests/archived/**/*.test.{ts,js}',
+  performance: 'tests/performance/**/*.test.{ts,js}'
+};
+
+function parseArgs() {
+  const args = process.argv.slice(2);
+  const options = {
+    category: 'all',
+    coverage: false,
+    watch: false,
+    verbose: false,
+    quick: false
+  };
+
+  for (const arg of args) {
+    if (arg === '--coverage') options.coverage = true;
+    else if (arg === '--watch') options.watch = true;
+    else if (arg === '--verbose') options.verbose = true;
+    else if (arg === '--quick') options.quick = true;
+    else if (arg.startsWith('--category=')) {
+      options.category = arg.split('=')[1];
+    }
+  }
+
+  return options;
+}
+
+function runTests(category, pattern, timeout, options) {
+  console.log(`\n🧪 Running ${category} tests...`);
+
+  try {
+    // Convert glob pattern to regex for Jest
+    const regexPattern = pattern
+      .replace(/\*\*/g, '.*')
+      .replace(/\*/g, '[^/]*')
+      .replace(/\{ts,js\}/g, '(ts|js)');
+
+    const jestArgs = [
+      `--config=config/jest/jest.config.js`,
+      `--testPathPatterns="${regexPattern}"`,
+      `--testTimeout=${timeout}`,
+      `--maxWorkers=2`,
+      '--bail',
+      options.coverage ? '--coverage' : '',
+      options.watch ? '--watch' : '',
+      options.verbose ? '--verbose' : ''
+    ].filter(Boolean).join(' ');
+
+    const cmd = `NODE_OPTIONS='--experimental-vm-modules --max-old-space-size=8192' jest ${jestArgs}`;
+
+    execSync(cmd, {
+      stdio: 'inherit',
+      cwd: path.resolve(__dirname, '..')
+    });
+
+    console.log(`✅ ${category} tests passed\n`);
+    return true;
+  } catch (error) {
+    console.error(`❌ ${category} tests failed\n`);
+    return false;
+  }
+}
+
+function main() {
+  const options = parseArgs();
+
+  console.log('🎯 Claude Flow Test Runner');
+  console.log('━'.repeat(50));
+
+  if (options.quick) {
+    console.log('⚡ Quick mode: Running unit tests only');
+    const success = runTests('unit', CATEGORIES.unit.pattern, CATEGORIES.unit.timeout, options);
+    process.exit(success ? 0 : 1);
+  }
+
+  if (options.category !== 'all') {
+    const cat = CATEGORIES[options.category];
+    if (!cat) {
+      console.error(`❌ Unknown category: ${options.category}`);
+      console.log('Available categories:', Object.keys(CATEGORIES).join(', '));
+      process.exit(1);
+    }
+
+    const success = runTests(options.category, cat.pattern, cat.timeout, options);
+    process.exit(success ? 0 : 1);
+  }
+
+  // Run all categories in order of priority
+  const sortedCategories = Object.entries(CATEGORIES)
+    .sort(([, a], [, b]) => a.priority - b.priority);
+
+  let allPassed = true;
+  for (const [name, config] of sortedCategories) {
+    const success = runTests(name, config.pattern, config.timeout, options);
+    if (!success) allPassed = false;
+  }
+
+  if (allPassed) {
+    console.log('\n✅ All test categories passed!');
+  } else {
+    console.log('\n❌ Some tests failed');
+  }
+
+  process.exit(allPassed ? 0 : 1);
+}
+
+main();

package/scripts/test-zai-10k.cjs

@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+
+const https = require('https');
+const fs = require('fs');
+const path = require('path');
+
+// Load .env
+const envPath = path.join(process.cwd(), '.env');
+if (fs.existsSync(envPath)) {
+  fs.readFileSync(envPath, 'utf8').split('\n').forEach(line => {
+    const match = line.trim().match(/^([^=]+)=(.*)$/);
+    if (match) {
+      const key = match[1].trim();
+      let value = match[2].trim().replace(/^["']|["']$/g, '');
+      process.env[key] = value;
+    }
+  });
+}
+
+const API_KEY = process.env.ZAI_API_KEY;
+
+console.log('\n🚀 Z.ai API Test - 10K Max Tokens\n');
+console.log('='.repeat(80) + '\n');
+
+const payload = {
+  model: 'claude-3-5-sonnet-20241022',
+  max_tokens: 10000,
+  messages: [
+    { role: 'user', content: 'Write a haiku about AI and coding. Then explain why you chose those words.' }
+  ]
+};
+
+const url = new URL('https://api.z.ai/api/anthropic/v1/messages');
+
+const options = {
+  hostname: url.hostname,
+  port: 443,
+  path: url.pathname,
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'x-api-key': API_KEY,
+    'anthropic-version': '2023-06-01'
+  }
+};
+
+const req = https.request(options, (res) => {
+  let body = '';
+  res.on('data', chunk => body += chunk);
+  res.on('end', () => {
+    const result = JSON.parse(body);
+
+    console.log('✅ Response received!\n');
+    console.log('Status:', res.statusCode);
+    console.log('Model:', result.model);
+    console.log('\nContent:');
+    console.log(result.content[0].text);
+    console.log('\nToken usage:');
+    console.log('  Input:', result.usage.input_tokens);
+    console.log('  Output:', result.usage.output_tokens);
+    console.log('  Total:', result.usage.input_tokens + result.usage.output_tokens);
+
+    const cost = (result.usage.input_tokens / 1000 * 0.003) +
+                 (result.usage.output_tokens / 1000 * 0.015);
+    console.log('\nEstimated cost: $' + cost.toFixed(6));
+    console.log('\n' + '='.repeat(80) + '\n');
+    console.log('✅ Z.ai with 10K max_tokens working correctly!\n');
+  });
+});
+
+req.on('error', (error) => {
+  console.error('❌ Error:', error.message);
+});
+
+req.setTimeout(45000, () => {
+  req.destroy();
+  console.error('❌ Request timeout');
+});
+
+req.write(JSON.stringify(payload));
+req.end();

package/scripts/test-zai-api.cjs

@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+
+/**
+ * Direct Z.ai API Test
+ * Tests Z.ai API connectivity and response
+ */
+
+const https = require('https');
+const http = require('http');
+
+const API_KEY = process.env.Z_AI_API_KEY || process.env.ZAI_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN;
+const BASE_URL = 'https://api.z.ai/api/anthropic';
+const DEFAULT_MODEL = 'claude-3-5-sonnet-20241022';
+
+function makeRequest(options, data) {
+  return new Promise((resolve, reject) => {
+    const protocol = options.protocol === 'https:' ? https : http;
+
+    const req = protocol.request(options, (res) => {
+      let body = '';
+
+      res.on('data', (chunk) => {
+        body += chunk;
+      });
+
+      res.on('end', () => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          try {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              body: JSON.parse(body)
+            });
+          } catch (e) {
+            resolve({
+              statusCode: res.statusCode,
+              headers: res.headers,
+              body: body
+            });
+          }
+        } else {
+          reject(new Error(`HTTP ${res.statusCode}: ${body}`));
+        }
+      });
+    });
+
+    req.on('error', (error) => {
+      reject(error);
+    });
+
+    req.setTimeout(30000, () => {
+      req.destroy();
+      reject(new Error('Request timeout after 30s'));
+    });
+
+    if (data) {
+      req.write(data);
+    }
+
+    req.end();
+  });
+}
+
+async function testZaiAPI() {
+  console.log('\n🧪 Z.ai API Direct Test\n');
+  console.log('='.repeat(80) + '\n');
+
+  // Check API key
+  if (!API_KEY) {
+    console.error('❌ ERROR: Z_AI_API_KEY or ZAI_API_KEY environment variable not set');
+    console.log('\nPlease set your Z.ai API key:');
+    console.log('  export ZAI_API_KEY=$YOUR_KEY');
+    console.log('  or');
+    console.log('  ZAI_API_KEY=$YOUR_KEY node scripts/test-zai-api.cjs\n');
+    process.exit(1);
+  }
+
+  console.log('✓ API Key found:', API_KEY.substring(0, 10) + '...' + API_KEY.substring(API_KEY.length - 4));
+  console.log('✓ Base URL:', BASE_URL);
+  console.log('✓ Model:', DEFAULT_MODEL);
+  console.log('\n' + '-'.repeat(80) + '\n');
+
+  // Test payload (Anthropic Messages API format)
+  const payload = {
+    model: DEFAULT_MODEL,
+    max_tokens: 50,
+    messages: [
+      {
+        role: 'user',
+        content: 'Say "Hello from Z.ai!" and nothing else.'
+      }
+    ]
+  };
+
+  console.log('📤 Sending request...\n');
+  console.log('Request payload:');
+  console.log(JSON.stringify(payload, null, 2));
+  console.log('\n' + '-'.repeat(80) + '\n');
+
+  try {
+    const url = new URL(`${BASE_URL}/v1/messages`);
+
+    const options = {
+      hostname: url.hostname,
+      port: url.port || 443,
+      path: url.pathname,
+      method: 'POST',
+      protocol: url.protocol,
+      headers: {
+        'Content-Type': 'application/json',
+        'x-api-key': API_KEY,
+        'anthropic-version': '2023-06-01'
+      }
+    };
+
+    const startTime = Date.now();
+    const response = await makeRequest(options, JSON.stringify(payload));
+    const duration = Date.now() - startTime;
+
+    console.log('📥 Response received!\n');
+    console.log(`⏱️  Duration: ${duration}ms`);
+    console.log(`📊 Status: ${response.statusCode}`);
+    console.log('\n' + '-'.repeat(80) + '\n');
+
+    if (response.body.content && response.body.content.length > 0) {
+      console.log('✅ SUCCESS!\n');
+      console.log('Response content:');
+      console.log('  ' + response.body.content[0].text);
+      console.log('\nModel:', response.body.model);
+      console.log('Stop reason:', response.body.stop_reason);
+
+      if (response.body.usage) {
+        console.log('\nToken usage:');
+        console.log('  Input tokens:  ', response.body.usage.input_tokens);
+        console.log('  Output tokens: ', response.body.usage.output_tokens);
+
+        // Calculate cost (Z.ai pricing: $0.003 per 1K prompt, $0.015 per 1K completion)
+        const promptCost = (response.body.usage.input_tokens / 1000) * 0.003;
+        const completionCost = (response.body.usage.output_tokens / 1000) * 0.015;
+        const totalCost = promptCost + completionCost;
+
+        console.log('\nEstimated cost:');
+        console.log(`  Input:  $${promptCost.toFixed(6)}`);
+        console.log(`  Output: $${completionCost.toFixed(6)}`);
+        console.log(`  Total:  $${totalCost.toFixed(6)}`);
+      }
+
+      console.log('\n' + '='.repeat(80));
+      console.log('\n✅ Z.ai API is working correctly!\n');
+
+      return {
+        success: true,
+        response: response.body,
+        duration
+      };
+    } else {
+      console.log('⚠️  Warning: Unexpected response format');
+      console.log('\nFull response:');
+      console.log(JSON.stringify(response.body, null, 2));
+      console.log('\n' + '='.repeat(80) + '\n');
+
+      return {
+        success: false,
+        error: 'Unexpected response format',
+        response: response.body
+      };
+    }
+  } catch (error) {
+    console.error('❌ ERROR:\n');
+    console.error(error.message);
+
+    if (error.message.includes('401')) {
+      console.log('\n💡 Tip: Check that your Z_AI_API_KEY is valid');
+    } else if (error.message.includes('429')) {
+      console.log('\n💡 Tip: Rate limit exceeded. Wait a moment and try again');
+    } else if (error.message.includes('ENOTFOUND') || error.message.includes('ECONNREFUSED')) {
+      console.log('\n💡 Tip: Check your internet connection');
+    }
+
+    console.log('\n' + '='.repeat(80) + '\n');
+    process.exit(1);
+  }
+}
+
+// Run test
+testZaiAPI()
+  .then(() => process.exit(0))
+  .catch((error) => {
+    console.error('\n❌ Unhandled error:', error);
+    process.exit(1);
+  });