fraim-framework 2.0.64 → 2.0.66

package/registry/scripts/validate-openapi-limits.ts

@@ -1,366 +0,0 @@
-#!/usr/bin/env node
-
-/**
- * FRAIM Generic OpenAPI Limits Validator
- * This script validates OpenAPI file sizes against specified limits.
- * 
- * Validates ChatGPT limitations for any project:
- * - instructions.txt: 8,000 character limit
- * - OpenAPI endpoints: 35 endpoint limit
- * - OpenAPI descriptions: 300 character limit per operation
- * - Missing operationIds
- * - Missing schema definitions
- * - x-openai-isConsequential on non-GET endpoints
- * 
- * Usage:
- *   npx tsx scripts/validate-openapi-limits.ts [instructions-file] [openapi-file]
- * 
- * Defaults:
- *   instructions-file: src/openapi/instructions.txt
- *   openapi-file: src/openapi/openapi.json
- */
-
-import { readFileSync, existsSync } from 'fs';
-import { join } from 'path';
-
-const INSTRUCTIONS_LIMIT = 8000;
-const OPENAPI_ENDPOINT_LIMIT = 35;
-const OPENAPI_DESC_LIMIT = 300;
-
-interface ValidationResult {
-  passed: boolean;
-  errors: string[];
-  warnings: string[];
-}
-
-/**
- * Collect all schema references from an object recursively
- */
-function collectSchemaRefs(obj: any, refs: Set<string> = new Set()): Set<string> {
-  if (typeof obj !== 'object' || obj === null) {
-    return refs;
-  }
-
-  if (Array.isArray(obj)) {
-    obj.forEach(item => collectSchemaRefs(item, refs));
-    return refs;
-  }
-
-  if (obj.$ref && typeof obj.$ref === 'string') {
-    // Extract schema name from #/components/schemas/SchemaName
-    const match = obj.$ref.match(/#\/components\/schemas\/(.+)$/);
-    if (match) {
-      refs.add(match[1]);
-    }
-  }
-
-  Object.values(obj).forEach(value => collectSchemaRefs(value, refs));
-
-  return refs;
-}
-
-/**
- * Validate instructions.txt length
- */
-function validateInstructions(instructionsPath: string): ValidationResult {
-  const result: ValidationResult = {
-    passed: true,
-    errors: [],
-    warnings: []
-  };
-
-  if (!existsSync(instructionsPath)) {
-    result.warnings.push(`instructions.txt not found at ${instructionsPath}`);
-    return result;
-  }
-
-  try {
-    const instructionsContent = readFileSync(instructionsPath, 'utf-8');
-    const instructionsLength = instructionsContent.length;
-
-    console.log(`📄 instructions.txt: ${instructionsLength} characters (limit: ${INSTRUCTIONS_LIMIT})`);
-
-    if (instructionsLength > INSTRUCTIONS_LIMIT) {
-      result.passed = false;
-      result.errors.push(
-        `instructions.txt exceeds ${INSTRUCTIONS_LIMIT} character limit (${instructionsLength} chars). Move detailed content to KB files.`
-      );
-      console.log(`   ❌ FAILED: Exceeds limit`);
-    } else {
-      console.log(`   ✅ PASSED: Within limit\n`);
-    }
-  } catch (error: any) {
-    result.passed = false;
-    result.errors.push(`Failed to read instructions.txt: ${error.message}`);
-  }
-
-  return result;
-}
-
-/**
- * Validate OpenAPI specification
- */
-function validateOpenAPI(openapiPath: string): ValidationResult {
-  const result: ValidationResult = {
-    passed: true,
-    errors: [],
-    warnings: []
-  };
-
-  if (!existsSync(openapiPath)) {
-    result.warnings.push(`OpenAPI file not found at ${openapiPath}`);
-    return result;
-  }
-
-  try {
-    const openapiContent = readFileSync(openapiPath, 'utf-8');
-    const openapiSpec = JSON.parse(openapiContent);
-
-    // Count endpoints (operations) and validate
-    let endpointCount = 0;
-    const endpointList: string[] = [];
-    const longDescriptions: Array<{ path: string; method: string; length: number }> = [];
-    const missingOperationIds: Array<{ path: string; method: string }> = [];
-    const allSchemaRefs = new Set<string>();
-    const missingConsequential: Array<{ path: string; method: string }> = [];
-    const wrongConsequential: Array<{ path: string; method: string; value: any }> = [];
-
-    if (openapiSpec.paths) {
-      for (const [path, methods] of Object.entries(openapiSpec.paths)) {
-        for (const [method, details] of Object.entries(methods as any)) {
-          if (typeof details === 'object' && details !== null) {
-            // Skip if it's a parameter definition or response definition
-            if (['parameters', 'responses', 'requestBody'].includes(method)) {
-              continue;
-            }
-
-            endpointCount++;
-            endpointList.push(`${method.toUpperCase()} ${path}`);
-
-            // Check for operationId
-            if (!(details as any).operationId) {
-              missingOperationIds.push({
-                path,
-                method: method.toUpperCase()
-              });
-            }
-
-            // Check description length
-            if ((details as any).description) {
-              const descLength = (details as any).description.length;
-              if (descLength > OPENAPI_DESC_LIMIT) {
-                longDescriptions.push({
-                  path,
-                  method: method.toUpperCase(),
-                  length: descLength
-                });
-              }
-            }
-
-            // Collect all schema references from this operation
-            collectSchemaRefs(details, allSchemaRefs);
-
-            // Check for x-openai-isConsequential on non-GET endpoints
-            const nonGetMethods = ['post', 'put', 'patch', 'delete'];
-            if (nonGetMethods.includes(method.toLowerCase())) {
-              const hasConsequential = details.hasOwnProperty('x-openai-isConsequential');
-              const consequentialValue = (details as any)['x-openai-isConsequential'];
-
-              // ALL non-GET endpoints should have x-openai-isConsequential: false
-              if (!hasConsequential) {
-                missingConsequential.push({
-                  path,
-                  method: method.toUpperCase()
-                });
-              } else if (consequentialValue !== false) {
-                wrongConsequential.push({
-                  path,
-                  method: method.toUpperCase(),
-                  value: consequentialValue
-                });
-              }
-            }
-          }
-        }
-      }
-    }
-
-    // Also collect schema refs from components/responses and components/parameters if they exist
-    if (openapiSpec.components) {
-      if (openapiSpec.components.responses) {
-        Object.values(openapiSpec.components.responses).forEach((response: any) => {
-          collectSchemaRefs(response, allSchemaRefs);
-        });
-      }
-      if (openapiSpec.components.parameters) {
-        Object.values(openapiSpec.components.parameters).forEach((param: any) => {
-          collectSchemaRefs(param, allSchemaRefs);
-        });
-      }
-    }
-
-    // Check for missing schema definitions
-    const definedSchemas = new Set<string>();
-    if (openapiSpec.components && openapiSpec.components.schemas) {
-      Object.keys(openapiSpec.components.schemas).forEach(schemaName => {
-        definedSchemas.add(schemaName);
-      });
-    }
-
-    const missingSchemas: string[] = [];
-    allSchemaRefs.forEach(refName => {
-      if (!definedSchemas.has(refName)) {
-        missingSchemas.push(refName);
-      }
-    });
-
-    // Validate endpoint count
-    console.log(`📋 OpenAPI endpoints: ${endpointCount} (limit: ${OPENAPI_ENDPOINT_LIMIT})`);
-
-    if (endpointCount > OPENAPI_ENDPOINT_LIMIT) {
-      result.passed = false;
-      result.errors.push(
-        `OpenAPI spec exceeds ${OPENAPI_ENDPOINT_LIMIT} endpoint limit (${endpointCount} endpoints). ChatGPT cannot handle more than ${OPENAPI_ENDPOINT_LIMIT} endpoints.`
-      );
-      console.log(`   ❌ FAILED: Too many endpoints`);
-      console.log(`   Endpoints:`);
-      endpointList.slice(0, 10).forEach(ep => console.log(`     - ${ep}`));
-      if (endpointCount > 10) {
-        console.log(`     ... and ${endpointCount - 10} more`);
-      }
-    } else {
-      console.log(`   ✅ PASSED: Within limit`);
-    }
-
-    // Check description lengths
-    if (longDescriptions.length > 0) {
-      console.log(`\n📝 OpenAPI descriptions: ${longDescriptions.length} exceed ${OPENAPI_DESC_LIMIT} char limit`);
-      longDescriptions.forEach(({ path, method, length }) => {
-        result.passed = false;
-        result.errors.push(
-          `OpenAPI operation ${method} ${path} description exceeds ${OPENAPI_DESC_LIMIT} character limit (${length} chars). Shorten description or move details to KB files.`
-        );
-        console.log(`   ❌ ${method} ${path}: ${length} chars`);
-      });
-    } else {
-      console.log(`\n📝 OpenAPI descriptions: All within ${OPENAPI_DESC_LIMIT} char limit`);
-      console.log(`   ✅ PASSED: All descriptions within limit`);
-    }
-
-    // Check for missing operationIds
-    if (missingOperationIds.length > 0) {
-      console.log(`\n🔑 OpenAPI operationIds: ${missingOperationIds.length} missing`);
-      missingOperationIds.forEach(({ path, method }) => {
-        result.passed = false;
-        result.errors.push(
-          `OpenAPI operation ${method} ${path} is missing required 'operationId' field. Add a unique operationId (e.g., "getTask", "createTask").`
-        );
-        console.log(`   ❌ ${method} ${path}: Missing operationId`);
-      });
-    } else {
-      console.log(`\n🔑 OpenAPI operationIds: All operations have operationId`);
-      console.log(`   ✅ PASSED: All operations have operationId`);
-    }
-
-    // Check for missing schema definitions
-    if (missingSchemas.length > 0) {
-      console.log(`\n📚 OpenAPI schemas: ${missingSchemas.length} referenced but not defined`);
-      missingSchemas.forEach(schemaName => {
-        result.passed = false;
-        result.errors.push(
-          `OpenAPI schema '${schemaName}' is referenced but not defined in components.schemas. Add the schema definition to components.schemas.`
-        );
-        console.log(`   ❌ Missing schema: ${schemaName}`);
-      });
-    } else {
-      console.log(`\n📚 OpenAPI schemas: All referenced schemas are defined`);
-      console.log(`   ✅ PASSED: All schema references resolve`);
-    }
-
-    // Check for missing or incorrect x-openai-isConsequential on non-GET endpoints
-    const allConsequentialIssues = [...missingConsequential, ...wrongConsequential];
-    if (allConsequentialIssues.length > 0) {
-      console.log(`\n🔐 OpenAPI x-openai-isConsequential: ${allConsequentialIssues.length} non-GET endpoint(s) missing or incorrectly set`);
-      missingConsequential.forEach(({ path, method }) => {
-        result.passed = false;
-        result.errors.push(
-          `OpenAPI operation ${method} ${path} is missing required 'x-openai-isConsequential: false' field. Add this field to bypass ChatGPT confirmation prompts.`
-        );
-        console.log(`   ❌ ${method} ${path}: Missing x-openai-isConsequential`);
-      });
-      wrongConsequential.forEach(({ path, method, value }) => {
-        result.passed = false;
-        result.errors.push(
-          `OpenAPI operation ${method} ${path} has x-openai-isConsequential set to ${JSON.stringify(value)} but should be false. Non-GET endpoints should have x-openai-isConsequential: false to bypass ChatGPT confirmation prompts.`
-        );
-        console.log(`   ❌ ${method} ${path}: x-openai-isConsequential is ${JSON.stringify(value)}, should be false`);
-      });
-    } else {
-      console.log(`\n🔐 OpenAPI x-openai-isConsequential: All non-GET endpoints have x-openai-isConsequential: false`);
-      console.log(`   ✅ PASSED: All non-GET endpoints configured to bypass confirmation`);
-    }
-
-  } catch (error: any) {
-    result.passed = false;
-    result.errors.push(`Failed to parse OpenAPI JSON: ${error.message}`);
-    console.log(`   ❌ ERROR: ${error.message}`);
-  }
-
-  return result;
-}
-
-/**
- * Main validation function
- */
-export function validateAll(
-  instructionsPath: string = 'src/openapi/instructions.txt',
-  openapiPath: string = 'src/openapi/openapi.json'
-): boolean {
-  console.log('🔍 Validating OpenAPI and instructions limits...\n');
-
-  const instructionsResult = validateInstructions(instructionsPath);
-  const openapiResult = validateOpenAPI(openapiPath);
-
-  // Combine results
-  const allErrors = [...instructionsResult.errors, ...openapiResult.errors];
-  const allWarnings = [...instructionsResult.warnings, ...openapiResult.warnings];
-  const passed = instructionsResult.passed && openapiResult.passed;
-
-  // Summary
-  console.log('\n' + '='.repeat(50));
-  console.log('📊 VALIDATION SUMMARY');
-  console.log('='.repeat(50));
-
-  if (allWarnings.length > 0) {
-    console.log('\n⚠️  WARNINGS:');
-    allWarnings.forEach(w => console.log(`   - ${w}`));
-  }
-
-  if (allErrors.length > 0) {
-    console.log('\n❌ ERRORS:');
-    allErrors.forEach(e => console.log(`   - ${e}`));
-    console.log('\n💡 Solutions:');
-    console.log('   - Move detailed content from instructions.txt to KB files (KB-*.txt)');
-    console.log('   - Reduce number of OpenAPI endpoints (merge or remove unused endpoints)');
-    console.log('   - Shorten OpenAPI operation descriptions, reference KB files for details');
-    console.log('   - Add missing operationId fields to all operations (e.g., "getTask", "createTask")');
-    console.log('   - Define missing schemas in components.schemas section');
-    console.log('   - Add x-openai-isConsequential: false to all non-GET endpoints (POST, PUT, PATCH, DELETE)');
-    console.log('\n❌ BUILD FAILED: Fix errors above before proceeding\n');
-    return false;
-  } else {
-    console.log('\n✅ ALL CHECKS PASSED');
-    console.log('   Instructions and OpenAPI spec are within ChatGPT limits\n');
-    return true;
-  }
-}
-
-// Run if executed directly
-// @ts-ignore
-if (import.meta.url === `file://${process.argv[1]}`) {
-  const instructionsPath = process.argv[2] || 'src/openapi/instructions.txt';
-  const openapiPath = process.argv[3] || 'src/openapi/openapi.json';
-
-  const passed = validateAll(instructionsPath, openapiPath);
-  process.exit(passed ? 0 : 1);
-}

package/registry/scripts/validate-test-coverage.ts

@@ -1,280 +0,0 @@
-#!/usr/bin/env node
-/**
- * Validation Plan Coverage Checker
- * 
- * Extracts validation plan from spec/design documents and verifies evidence coverage.
- * Used by Gate 9 in code-quality-check.sh
- * 
- * Usage: npx tsx <this-script> <issue-number>
- */
-
-import * as fs from 'fs';
-import * as path from 'path';
-
-interface ValidationScenario {
-  userScenario: string;
-  expectedOutcome: string;
-  validationMethod: string;
-}
-
-interface CoverageResult {
-  complete: boolean;
-  totalScenarios: number;
-  coveredScenarios: number;
-  missingScenarios: ValidationScenario[];
-  coverage: string; // "X/Y scenarios (Z%)"
-}
-
-/**
- * Extract issue number from branch name
- */
-function extractIssueNumberFromBranch(): number | null {
-  const branchName = process.env.GIT_BRANCH || '';
-  const match = branchName.match(/feature\/(\d+)/);
-  return match ? parseInt(match[1]) : null;
-}
-
-/**
- * Find spec or design file for an issue
- */
-function findSpecOrDesignFile(issueNumber: number): string | null {
-  const specDir = path.join(process.cwd(), 'docs', 'feature specs');
-  const rfcDir = path.join(process.cwd(), 'docs', 'rfcs');
-
-  // Try feature specs first
-  if (fs.existsSync(specDir)) {
-    const files = fs.readdirSync(specDir);
-    const specFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
-    if (specFile) {
-      return path.join(specDir, specFile);
-    }
-  }
-
-  // Then try RFCs
-  if (fs.existsSync(rfcDir)) {
-    const files = fs.readdirSync(rfcDir);
-    const rfcFile = files.find(f => f.startsWith(`${issueNumber}-`) && f.endsWith('.md'));
-    if (rfcFile) {
-      return path.join(rfcDir, rfcFile);
-    }
-  }
-
-  return null;
-}
-
-/**
- * Extract validation plan table from markdown file
- */
-function extractValidationPlan(filePath: string): ValidationScenario[] {
-  const content = fs.readFileSync(filePath, 'utf-8');
-  const scenarios: ValidationScenario[] = [];
-
-  // Find "## Validation Plan" section (not "Validation Plan Coverage")
-  const validationPlanMatch = content.match(/##\s+Validation Plan\s*\n[\s\S]*?(?=\n##|$)/i);
-  if (!validationPlanMatch) {
-    return scenarios;
-  }
-
-  const validationSection = validationPlanMatch[0];
-
-  // Look for markdown table - find table header row with "User Scenario"
-  // Pattern: | User Scenario | Expected outcome | Validation method |
-  const tablePattern = /\|[\s\*]*User Scenario[\s\*]*\|[\s\*]*Expected outcome[\s\*]*\|[\s\*]*Validation method[\s\*]*\|[\s]*\n\|[\s:\-|]+\|[\s]*\n((?:\|.*?\|[\s]*\n)+)/i;
-  const tableMatch = validationSection.match(tablePattern);
-
-  if (!tableMatch) {
-    return scenarios;
-  }
-
-  const tableRows = tableMatch[1].trim().split('\n').filter(row => row.trim().startsWith('|'));
-
-  for (const row of tableRows) {
-    // Skip separator rows (all dashes/pipes)
-    if (row.match(/^\|[\s:\-|]+\|$/)) {
-      continue;
-    }
-
-    // Parse table row: | Scenario | Outcome | Method |
-    // Handle both formats: | **bold** | text | and | text | text |
-    const cells = row.split('|').map(c => c.trim()).filter(c => c && c !== '');
-
-    if (cells.length >= 2) {
-      // Remove markdown bold (**text**) from cells
-      const cleanCell = (cell: string): string => {
-        return cell
-          .replace(/^\*\*/, '')
-          .replace(/\*\*$/, '')
-          .replace(/^\*\*/g, '')
-          .replace(/\*\*/g, '')
-          .trim();
-      };
-
-      const userScenario = cleanCell(cells[0] || '');
-      const expectedOutcome = cleanCell(cells[1] || '');
-      const validationMethod = cleanCell(cells[2] || 'Manual validation');
-
-      // Skip header rows
-      const scenarioLower = userScenario.toLowerCase();
-      const outcomeLower = expectedOutcome.toLowerCase();
-      if (scenarioLower.includes('user scenario') ||
-        (scenarioLower.includes('scenario') && outcomeLower.includes('expected')) ||
-        scenarioLower === 'user scenario' ||
-        outcomeLower === 'expected outcome') {
-        continue;
-      }
-
-      // Skip empty rows or separator rows
-      if (!userScenario || userScenario === '-' || userScenario.length === 0) {
-        continue;
-      }
-
-      scenarios.push({
-        userScenario,
-        expectedOutcome,
-        validationMethod
-      });
-    }
-  }
-
-  return scenarios;
-}
-
-/**
- * Check if evidence exists for a scenario
- * Evidence can be in PR comments, test.log, or other sources
- */
-function checkEvidenceForScenario(scenario: ValidationScenario, issueNumber: number): boolean {
-  // Extract key terms from scenario for searching
-  const scenarioKeywords = scenario.userScenario
-    .toLowerCase()
-    .replace(/\*\*/g, '')
-    .split(/\s+/)
-    .filter(w => w.length > 3 && !['agent', 'the', 'with', 'without'].includes(w));
-
-  const outcomeKeywords = scenario.expectedOutcome
-    .toLowerCase()
-    .replace(/\*\*/g, '')
-    .split(/\s+/)
-    .filter(w => w.length > 3);
-
-  // Method 1: Check test.log for scenario keywords
-  const testLogPath = path.join(process.cwd(), 'test.log');
-  if (fs.existsSync(testLogPath)) {
-    const testLogContent = fs.readFileSync(testLogPath, 'utf-8').toLowerCase();
-    // Check if multiple keywords from scenario appear
-    const matchingKeywords = scenarioKeywords.filter(kw => testLogContent.includes(kw));
-    if (matchingKeywords.length >= 2) {
-      return true;
-    }
-  }
-
-  // Method 2: Check for evidence file (created by agent with scenario validation)
-  const evidenceFiles = ['test-evidence.md', 'validation-evidence.md', 'implementation-evidence.md'];
-  for (const evidenceFile of evidenceFiles) {
-    const evidencePath = path.join(process.cwd(), evidenceFile);
-    if (fs.existsSync(evidencePath)) {
-      const evidenceContent = fs.readFileSync(evidencePath, 'utf-8').toLowerCase();
-      const matchingKeywords = scenarioKeywords.filter(kw => evidenceContent.includes(kw));
-      if (matchingKeywords.length >= 2) {
-        return true;
-      }
-    }
-  }
-
-  // Method 3: Check server.log for scenario-related activity
-  const serverLogPath = path.join(process.cwd(), 'server.log');
-  if (fs.existsSync(serverLogPath)) {
-    const serverLogContent = fs.readFileSync(serverLogPath, 'utf-8').toLowerCase();
-    // Look for validation-related keywords
-    if (scenarioKeywords.some(kw => serverLogContent.includes(kw)) ||
-      outcomeKeywords.some(kw => serverLogContent.includes('gate') && serverLogContent.includes(kw))) {
-      return true;
-    }
-  }
-
-  // Note: PR comment checking is done in the shell script (Gate 9)
-  // This TypeScript tool focuses on file-based evidence
-
-  return false; // Conservative: require evidence
-}
-
-/**
- * Main function
- */
-function main(): void {
-  const issueNumberArg = process.argv[2];
-  const issueNumber = issueNumberArg ? parseInt(issueNumberArg) : extractIssueNumberFromBranch();
-
-  if (!issueNumber) {
-    console.error('❌ ERROR: Could not determine issue number');
-    console.error('   Usage: npx tsx validate-coverage.ts <issue-number>');
-    console.error('   Or set GIT_BRANCH environment variable with feature/N-* format');
-    process.exit(1);
-  }
-
-  const specFile = findSpecOrDesignFile(issueNumber);
-
-  if (!specFile) {
-    // No spec/design found - Gate 9 warns but doesn't block
-    console.log('ℹ️  No spec/design found for issue #' + issueNumber);
-    console.log('   Skipping validation plan coverage check');
-    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no spec/design)');
-    process.exit(0);
-  }
-
-  console.log(`Found spec/design: ${specFile}`);
-
-  const scenarios = extractValidationPlan(specFile);
-
-  if (scenarios.length === 0) {
-    console.log('ℹ️  No validation plan table found in spec/design');
-    console.log('   Gate 9: Validation Plan Coverage - SKIPPED (no validation plan)');
-    process.exit(0);
-  }
-
-  console.log(`Found ${scenarios.length} validation scenario(s)`);
-
-  // Check evidence for each scenario
-  const covered: ValidationScenario[] = [];
-  const missing: ValidationScenario[] = [];
-
-  for (const scenario of scenarios) {
-    if (checkEvidenceForScenario(scenario, issueNumber)) {
-      covered.push(scenario);
-    } else {
-      missing.push(scenario);
-    }
-  }
-
-  const coverage: CoverageResult = {
-    complete: missing.length === 0,
-    totalScenarios: scenarios.length,
-    coveredScenarios: covered.length,
-    missingScenarios: missing,
-    coverage: `${covered.length}/${scenarios.length} scenarios (${Math.round(covered.length / scenarios.length * 100)}%)`
-  };
-
-  // Output results
-  console.log('\n📊 Validation Plan Coverage:');
-  console.log(`   ${coverage.coverage}`);
-
-  if (coverage.complete) {
-    console.log('✅ PASSED: All validation scenarios have evidence');
-    console.log('   Gate 9: Validation Plan Coverage - PASSED');
-    process.exit(0);
-  } else {
-    console.log('❌ FAILED: Missing evidence for validation scenarios:');
-    for (const scenario of missing) {
-      console.log(`   - "${scenario.userScenario}"`);
-      console.log(`     Expected: ${scenario.expectedOutcome}`);
-      console.log(`     Method: ${scenario.validationMethod}`);
-    }
-    console.log('\n   Required: Provide evidence for all validation scenarios before marking PR ready');
-    console.log('   Evidence can be: PR comment, test output, manual verification log');
-    console.log('   Gate 9: Validation Plan Coverage - FAILED');
-    process.exit(1);
-  }
-}
-
-// Run main function
-main();