gsd-trae 1.0.0 → 1.0.2

package/refs/vbenchmark/packages/cli/src/lib/task-variation.ts

@@ -1,153 +0,0 @@
-/**
- * Task Variation System
- *
- * Generates dynamic parameters for each benchmark run to ensure
- * test diversity and prevent overfitting to specific patterns.
- */
-
-export interface TaskVariation {
-  seed: number;
-  entityNames: EntityNames;
-  apiEndpoints: ApiEndpoints;
-  dataSchemas: DataSchemas;
-  businessRules: BusinessRules;
-}
-
-export interface EntityNames {
-  user: string;
-  product: string;
-  order: string;
-  payment: string;
-  settings: string;
-}
-
-export interface ApiEndpoints {
-  prefix: string;
-  version: string;
-}
-
-export interface DataSchemas {
-  idType: 'uuid' | 'nanoid' | 'cuid' | 'incremental';
-  timestampFormat: 'iso' | 'unix' | 'epoch_ms';
-  currencyFormat: 'cents' | 'decimal';
-}
-
-export interface BusinessRules {
-  maxItemsPerPage: number;
-  sessionTimeout: number;
-  passwordMinLength: number;
-  otpExpiry: number;
-}
-
-// Name pools for entity variation
-const USER_NAMES = ['User', 'Member', 'Account', 'Profile', 'Customer'];
-const PRODUCT_NAMES = ['Product', 'Item', 'Listing', 'Article', 'Goods'];
-const ORDER_NAMES = ['Order', 'Purchase', 'Transaction', 'Booking', 'Request'];
-const PAYMENT_NAMES = ['Payment', 'Charge', 'Invoice', 'Bill', 'Receipt'];
-const SETTINGS_NAMES = ['Settings', 'Preferences', 'Config', 'Options', 'Profile'];
-
-const API_PREFIXES = ['api', 'v1', 'rest', 'service', 'backend'];
-const API_VERSIONS = ['v1', 'v2', 'v3', '2024-01', '2025-01'];
-
-/**
- * Seeded random number generator for reproducibility
- */
-class SeededRandom {
-  private seed: number;
-
-  constructor(seed: number) {
-    this.seed = seed;
-  }
-
-  next(): number {
-    this.seed = (this.seed * 1103515245 + 12345) & 0x7fffffff;
-    return this.seed / 0x7fffffff;
-  }
-
-  pick<T>(array: T[]): T {
-    return array[Math.floor(this.next() * array.length)];
-  }
-
-  range(min: number, max: number): number {
-    return Math.floor(this.next() * (max - min + 1)) + min;
-  }
-}
-
-/**
- * Generate a task variation based on a seed
- */
-export function generateVariation(seed?: number): TaskVariation {
-  const actualSeed = seed ?? Date.now();
-  const rng = new SeededRandom(actualSeed);
-
-  return {
-    seed: actualSeed,
-    entityNames: {
-      user: rng.pick(USER_NAMES),
-      product: rng.pick(PRODUCT_NAMES),
-      order: rng.pick(ORDER_NAMES),
-      payment: rng.pick(PAYMENT_NAMES),
-      settings: rng.pick(SETTINGS_NAMES),
-    },
-    apiEndpoints: {
-      prefix: rng.pick(API_PREFIXES),
-      version: rng.pick(API_VERSIONS),
-    },
-    dataSchemas: {
-      idType: rng.pick(['uuid', 'nanoid', 'cuid', 'incremental'] as const),
-      timestampFormat: rng.pick(['iso', 'unix', 'epoch_ms'] as const),
-      currencyFormat: rng.pick(['cents', 'decimal'] as const),
-    },
-    businessRules: {
-      maxItemsPerPage: rng.pick([10, 20, 25, 50, 100]),
-      sessionTimeout: rng.pick([900, 1800, 3600, 7200]), // seconds
-      passwordMinLength: rng.pick([8, 10, 12, 14]),
-      otpExpiry: rng.pick([60, 120, 180, 300]), // seconds
-    },
-  };
-}
-
-/**
- * Apply variation to a task prompt
- */
-export function applyVariation(prompt: string, variation: TaskVariation): string {
-  let result = prompt;
-
-  // Replace entity names
-  result = result.replace(/\bUser\b/g, variation.entityNames.user);
-  result = result.replace(/\bProduct\b/g, variation.entityNames.product);
-  result = result.replace(/\bOrder\b/g, variation.entityNames.order);
-  result = result.replace(/\bPayment\b/g, variation.entityNames.payment);
-  result = result.replace(/\bSettings\b/g, variation.entityNames.settings);
-
-  // Replace API endpoints
-  result = result.replace(/\/api\//g, `/${variation.apiEndpoints.prefix}/`);
-
-  // Add variation context header
-  const variationHeader = `
-<!-- Task Variation (Seed: ${variation.seed}) -->
-<!-- Entity Naming: ${variation.entityNames.user}, ${variation.entityNames.product} -->
-<!-- API: /${variation.apiEndpoints.prefix}/${variation.apiEndpoints.version} -->
-<!-- ID Type: ${variation.dataSchemas.idType} -->
-<!-- Pagination: ${variation.businessRules.maxItemsPerPage} items/page -->
-
-`;
-
-  return variationHeader + result;
-}
-
-/**
- * Generate variation metadata for result tracking
- */
-export function variationMetadata(variation: TaskVariation): Record<string, string | number> {
-  return {
-    seed: variation.seed,
-    userEntity: variation.entityNames.user,
-    productEntity: variation.entityNames.product,
-    apiPrefix: variation.apiEndpoints.prefix,
-    idType: variation.dataSchemas.idType,
-    pageSize: variation.businessRules.maxItemsPerPage,
-  };
-}
-
-export default { generateVariation, applyVariation, variationMetadata };

package/refs/vbenchmark/packages/cli/src/loader.ts

@@ -1,258 +0,0 @@
-import * as fs from 'fs/promises';
-import * as path from 'path';
-import { watch } from 'fs';
-import { parse as parseYaml } from 'yaml';
-import Ajv from 'ajv';
-// eslint-disable-next-line @typescript-eslint/no-require-imports
-const taskSchema = require('./schema/task.schema.json');
-import { EventEmitter } from 'events';
-
-export interface Task {
-  id: string;
-  name: string;
-  category: string;
-  subcategory?: string;
-  description: string;
-  difficulty: 'easy' | 'medium' | 'hard';
-  stack?: string;
-  timeout: number;
-  tokenLimit: number;
-  template?: string;
-  promptFile: string;
-  tests: {
-    functional?: string;
-    visual?: string;
-    security?: string;
-  };
-  docker?: {
-    compose?: string;
-    image?: string;
-  };
-  weight: number;
-  tags: string[];
-}
-
-const ajv = new Ajv();
-const validateTask = ajv.compile(taskSchema);
-
-const TASKS_DIR = path.resolve(process.cwd(), 'tasks');
-
-/**
- * Load a single task by ID (e.g., "saas-core/auth/supabase-oauth")
- */
-export async function loadTask(taskId: string): Promise<Task | null> {
-  const taskPath = path.join(TASKS_DIR, taskId, 'task.yaml');
-
-  try {
-    const content = await fs.readFile(taskPath, 'utf-8');
-    const raw = parseYaml(content);
-
-    if (!validateTask(raw)) {
-      console.error(`Invalid task.yaml at ${taskPath}:`, validateTask.errors);
-      return null;
-    }
-
-    const taskData = raw as Record<string, unknown>;
-    return {
-      id: taskId,
-      name: taskData.name as string,
-      category: taskData.category as string,
-      subcategory: taskData.subcategory as string | undefined,
-      description: taskData.description as string,
-      difficulty: taskData.difficulty as 'easy' | 'medium' | 'hard',
-      stack: taskData.stack as string | undefined,
-      timeout: (taskData.timeout as number) || 300,
-      tokenLimit: (taskData.tokenLimit as number) || 100000,
-      template: taskData.template as string | undefined,
-      promptFile: path.join(TASKS_DIR, taskId, (taskData.promptFile as string) || 'PROMPT.md'),
-      tests: {
-        functional: (taskData.tests as Record<string, string>)?.functional
-          ? path.join(TASKS_DIR, taskId, (taskData.tests as Record<string, string>).functional)
-          : undefined,
-        visual: (taskData.tests as Record<string, string>)?.visual
-          ? path.join(TASKS_DIR, taskId, (taskData.tests as Record<string, string>).visual)
-          : undefined,
-        security: (taskData.tests as Record<string, string>)?.security
-          ? path.join(TASKS_DIR, taskId, (taskData.tests as Record<string, string>).security)
-          : undefined,
-      },
-      docker: taskData.docker as { compose?: string; image?: string } | undefined,
-      weight: (taskData.weight as number) || 1.0,
-      tags: (taskData.tags as string[]) || []
-    };
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
-      return null;
-    }
-    throw error;
-  }
-}
-
-/**
- * Discover and load all tasks from the tasks directory.
- * Tasks are modular - just add a new folder with task.yaml to register.
- */
-export async function loadAllTasks(): Promise<Task[]> {
-  const tasks: Task[] = [];
-
-  async function scanDirectory(dir: string, prefix: string = ''): Promise<void> {
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-
-    for (const entry of entries) {
-      if (!entry.isDirectory()) continue;
-
-      const fullPath = path.join(dir, entry.name);
-      const taskId = prefix ? `${prefix}/${entry.name}` : entry.name;
-      const taskYaml = path.join(fullPath, 'task.yaml');
-
-      try {
-        await fs.access(taskYaml);
-        const task = await loadTask(taskId);
-        if (task) tasks.push(task);
-      } catch {
-        // No task.yaml, recurse into subdirectory
-        await scanDirectory(fullPath, taskId);
-      }
-    }
-  }
-
-  try {
-    await fs.access(TASKS_DIR);
-    await scanDirectory(TASKS_DIR);
-  } catch {
-    // tasks directory doesn't exist yet
-  }
-
-  return tasks.sort((a, b) => {
-    // Sort by category, then by difficulty
-    if (a.category !== b.category) return a.category.localeCompare(b.category);
-    const diffOrder = { easy: 0, medium: 1, hard: 2 };
-    return diffOrder[a.difficulty] - diffOrder[b.difficulty];
-  });
-}
-
-/**
- * Get all unique categories from loaded tasks
- */
-export async function getCategories(): Promise<string[]> {
-  const tasks = await loadAllTasks();
-  return [...new Set(tasks.map(t => t.category))].sort();
-}
-
-/**
- * Load tasks filtered by options (for composable benchmark subsets)
- */
-export async function loadTasksFiltered(options: {
-  categories?: string[];
-  difficulties?: ('easy' | 'medium' | 'hard')[];
-  tags?: string[];
-  limit?: number;
-}): Promise<Task[]> {
-  let tasks = await loadAllTasks();
-
-  if (options.categories?.length) {
-    tasks = tasks.filter(t => options.categories!.includes(t.category));
-  }
-
-  if (options.difficulties?.length) {
-    tasks = tasks.filter(t => options.difficulties!.includes(t.difficulty));
-  }
-
-  if (options.tags?.length) {
-    tasks = tasks.filter(t => options.tags!.some(tag => t.tags.includes(tag)));
-  }
-
-  if (options.limit) {
-    tasks = tasks.slice(0, options.limit);
-  }
-
-  return tasks;
-}
-
-/**
- * Hot-reload task watcher for development.
- * Watches the tasks directory and emits events when tasks change.
- */
-export class TaskWatcher extends EventEmitter {
-  private watchers: ReturnType<typeof watch>[] = [];
-  private taskCache = new Map<string, Task>();
-  private debounceTimers = new Map<string, NodeJS.Timeout>();
-
-  async start(): Promise<void> {
-    try {
-      await fs.access(TASKS_DIR);
-    } catch {
-      await fs.mkdir(TASKS_DIR, { recursive: true });
-    }
-
-    // Initial load
-    const tasks = await loadAllTasks();
-    for (const task of tasks) {
-      this.taskCache.set(task.id, task);
-    }
-
-    // Watch for changes
-    this.watchDirectory(TASKS_DIR);
-    this.emit('ready', { taskCount: tasks.length });
-  }
-
-  private watchDirectory(dir: string): void {
-    const watcher = watch(dir, { recursive: true }, async (eventType, filename) => {
-      if (!filename) return;
-
-      // Debounce rapid changes
-      const key = filename;
-      const existing = this.debounceTimers.get(key);
-      if (existing) clearTimeout(existing);
-
-      this.debounceTimers.set(key, setTimeout(async () => {
-        await this.handleChange(filename);
-        this.debounceTimers.delete(key);
-      }, 100));
-    });
-
-    this.watchers.push(watcher);
-  }
-
-  private async handleChange(filename: string): Promise<void> {
-    // Extract task ID from path
-    const parts = filename.split(path.sep);
-    if (!parts.includes('task.yaml') && !filename.endsWith('task.yaml')) {
-      return;
-    }
-
-    const taskId = parts.slice(0, -1).join('/');
-
-    try {
-      const task = await loadTask(taskId);
-      if (task) {
-        const isNew = !this.taskCache.has(taskId);
-        this.taskCache.set(taskId, task);
-        this.emit(isNew ? 'added' : 'changed', task);
-      }
-    } catch {
-      if (this.taskCache.has(taskId)) {
-        const task = this.taskCache.get(taskId);
-        this.taskCache.delete(taskId);
-        this.emit('removed', task);
-      }
-    }
-  }
-
-  getTasks(): Task[] {
-    return [...this.taskCache.values()];
-  }
-
-  getTask(id: string): Task | undefined {
-    return this.taskCache.get(id);
-  }
-
-  stop(): void {
-    for (const watcher of this.watchers) {
-      watcher.close();
-    }
-    this.watchers = [];
-    this.debounceTimers.forEach(timer => clearTimeout(timer));
-    this.debounceTimers.clear();
-  }
-}

package/refs/vbenchmark/packages/cli/src/reporter.ts

@@ -1,222 +0,0 @@
-/**
- * Reporter module for sending live updates to the leaderboard service.
- * Streams progress metrics during task execution.
- */
-
-export interface ReporterConfig {
-  apiUrl: string;
-  enabled: boolean;
-  updateInterval?: number;
-}
-
-export interface RunMetrics {
-  tokensUsed: number;
-  filesRead: number;
-  filesWritten: number;
-  testsPass: number;
-  testsFail: number;
-  elapsedMs: number;
-}
-
-export interface RunUpdate {
-  status?: 'initializing' | 'running' | 'evaluating' | 'completed' | 'failed';
-  progress?: number;
-  currentStep?: string;
-  metrics?: Partial<RunMetrics>;
-  logs?: Array<{ timestamp: string; message: string }>;
-}
-
-export class LiveReporter {
-  private runId: string | null = null;
-  private intervalId: ReturnType<typeof setInterval> | null = null;
-  private metrics: RunMetrics = {
-    tokensUsed: 0,
-    filesRead: 0,
-    filesWritten: 0,
-    testsPass: 0,
-    testsFail: 0,
-    elapsedMs: 0,
-  };
-  private startTime: number = 0;
-  private logs: Array<{ timestamp: string; message: string }> = [];
-
-  constructor(private config: ReporterConfig) {
-    if (!config.updateInterval) {
-      this.config.updateInterval = 5000; // Default 5 seconds
-    }
-  }
-
-  async start(agentName: string, taskId: string): Promise<string | null> {
-    if (!this.config.enabled) {
-      return null;
-    }
-
-    try {
-      const response = await fetch(`${this.config.apiUrl}/runs`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ agentName, taskId }),
-      });
-
-      if (!response.ok) {
-        console.error('Failed to start live reporting');
-        return null;
-      }
-
-      const data = await response.json() as { id: string };
-      this.runId = data.id;
-      this.startTime = Date.now();
-      this.metrics = {
-        tokensUsed: 0,
-        filesRead: 0,
-        filesWritten: 0,
-        testsPass: 0,
-        testsFail: 0,
-        elapsedMs: 0,
-      };
-
-      // Start periodic updates
-      this.intervalId = setInterval(() => {
-        this.sendUpdate({
-          metrics: {
-            ...this.metrics,
-            elapsedMs: Date.now() - this.startTime,
-          },
-        });
-      }, this.config.updateInterval);
-
-      return this.runId;
-    } catch (error) {
-      console.error('Error starting live reporter:', error);
-      return null;
-    }
-  }
-
-  async sendUpdate(update: RunUpdate): Promise<void> {
-    if (!this.config.enabled || !this.runId) {
-      return;
-    }
-
-    try {
-      // Send any pending logs
-      if (this.logs.length > 0) {
-        update.logs = [...this.logs];
-        this.logs = [];
-      }
-
-      await fetch(`${this.config.apiUrl}/runs/${this.runId}`, {
-        method: 'PATCH',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify(update),
-      });
-    } catch (error) {
-      // Silently fail - don't interrupt task execution
-      console.error('Error sending update:', error);
-    }
-  }
-
-  log(message: string): void {
-    this.logs.push({
-      timestamp: new Date().toISOString(),
-      message,
-    });
-  }
-
-  updateMetrics(updates: Partial<RunMetrics>): void {
-    this.metrics = { ...this.metrics, ...updates };
-  }
-
-  incrementTokens(count: number): void {
-    this.metrics.tokensUsed += count;
-  }
-
-  incrementFilesRead(): void {
-    this.metrics.filesRead++;
-  }
-
-  incrementFilesWritten(): void {
-    this.metrics.filesWritten++;
-  }
-
-  setTestResults(pass: number, fail: number): void {
-    this.metrics.testsPass = pass;
-    this.metrics.testsFail = fail;
-  }
-
-  async setStatus(
-    status: 'initializing' | 'running' | 'evaluating' | 'completed' | 'failed',
-    progress?: number,
-    currentStep?: string
-  ): Promise<void> {
-    await this.sendUpdate({ status, progress, currentStep });
-  }
-
-  async complete(finalMetrics?: Partial<RunMetrics>): Promise<void> {
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
-      this.intervalId = null;
-    }
-
-    if (finalMetrics) {
-      this.metrics = { ...this.metrics, ...finalMetrics };
-    }
-
-    await this.sendUpdate({
-      status: 'completed',
-      progress: 100,
-      currentStep: 'Complete',
-      metrics: {
-        ...this.metrics,
-        elapsedMs: Date.now() - this.startTime,
-      },
-    });
-  }
-
-  async fail(error: string): Promise<void> {
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
-      this.intervalId = null;
-    }
-
-    this.log(`Error: ${error}`);
-
-    await this.sendUpdate({
-      status: 'failed',
-      currentStep: 'Failed',
-      metrics: {
-        ...this.metrics,
-        elapsedMs: Date.now() - this.startTime,
-      },
-    });
-  }
-
-  async cleanup(): Promise<void> {
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
-      this.intervalId = null;
-    }
-
-    if (this.runId && this.config.enabled) {
-      try {
-        await fetch(`${this.config.apiUrl}/runs/${this.runId}`, {
-          method: 'DELETE',
-        });
-      } catch {
-        // Ignore cleanup errors
-      }
-    }
-
-    this.runId = null;
-  }
-}
-
-// Factory function for creating reporter
-export function createReporter(options?: {
-  apiUrl?: string;
-  enabled?: boolean;
-}): LiveReporter {
-  return new LiveReporter({
-    apiUrl: options?.apiUrl || process.env.LEADERBOARD_API_URL || 'http://localhost:3001/api/live',
-    enabled: options?.enabled ?? (process.env.LIVE_REPORTING === 'true'),
-  });
-}