@cogitator-ai/self-modifying 19.2.1 → 19.3.1

package/src/tool-generation/gap-analyzer.ts

@@ -14,27 +14,38 @@ export interface GapAnalyzerOptions {
   config: ToolSelfGenerationConfig;
   /** Model the LLM calls use, without the provider prefix of the backend */
   model: string;
+  /** Upper bound on the output tokens of the analysis call. Unset leaves the backend default. */
+  maxTokens?: number;
+}
+
+export interface GapAnalysisContext {
+  failedAttempts?: string[];
+  previousGaps?: CapabilityGap[];
+  /**
+   * The agent's instructions. Shown to the analyzer, so a capability the instructions require
+   * a tool for counts as a gap even when the model could answer on its own.
+   */
+  instructions?: string;
 }
 
 export class GapAnalyzer {
   private readonly llm: LLMBackend;
   private readonly config: ToolSelfGenerationConfig;
   private readonly model: string;
+  private readonly maxTokens?: number;
   private readonly analysisCache = new Map<string, GapAnalysisResult>();
 
   constructor(options: GapAnalyzerOptions) {
     this.llm = options.llm;
     this.config = options.config;
     this.model = options.model;
+    this.maxTokens = options.maxTokens;
   }
 
   async analyze(
     userIntent: string,
     availableTools: Tool[],
-    context?: {
-      failedAttempts?: string[];
-      previousGaps?: CapabilityGap[];
-    }
+    context?: GapAnalysisContext
   ): Promise<GapAnalysisResult> {
     const cacheKey = this.buildCacheKey(userIntent, availableTools, context);
     const cached = this.analysisCache.get(cacheKey);
@@ -47,7 +58,12 @@ export class GapAnalyzer {
       description: t.description,
     }));
 
-    const prompt = buildGapAnalysisPrompt(userIntent, toolSummaries, context?.failedAttempts);
+    const prompt = buildGapAnalysisPrompt(
+      userIntent,
+      toolSummaries,
+      context?.failedAttempts,
+      context?.instructions
+    );
 
     const content = await llmChat(
       this.llm,
@@ -61,7 +77,7 @@ Consider tool composition before suggesting new tools.`,
         },
         { role: 'user', content: prompt },
       ],
-      { model: this.model, temperature: 0.3 }
+      { model: this.model, temperature: 0.3, maxTokens: this.maxTokens }
     );
 
     const parsed = parseGapAnalysisResponse(content);
@@ -220,11 +236,7 @@ Consider tool composition before suggesting new tools.`,
     return `Identified ${gaps.length} capability gap(s):\n${gapDescriptions}`;
   }
 
-  private buildCacheKey(
-    userIntent: string,
-    tools: Tool[],
-    context?: { failedAttempts?: string[]; previousGaps?: CapabilityGap[] }
-  ): string {
+  private buildCacheKey(userIntent: string, tools: Tool[], context?: GapAnalysisContext): string {
     const toolSignature = tools
       .map((t) => `${t.name}:${t.description}`)
       .sort()
@@ -234,6 +246,7 @@ Consider tool composition before suggesting new tools.`,
       toolSignature,
       context?.failedAttempts ?? [],
       (context?.previousGaps ?? []).map((g) => g.id).sort(),
+      context?.instructions ?? '',
     ]);
     return createHash('sha256').update(raw).digest('hex');
   }

package/src/tool-generation/index.ts

@@ -1,4 +1,4 @@
-export { GapAnalyzer, type GapAnalyzerOptions } from './gap-analyzer';
+export { GapAnalyzer, type GapAnalyzerOptions, type GapAnalysisContext } from './gap-analyzer';
 export {
   ToolGenerator,
   type ToolGeneratorOptions,
@@ -26,4 +26,5 @@ export {
   parseToolGenerationResponse,
   parseValidationResponse,
   TOOL_GENERATION_SYSTEM_PROMPT,
+  type ToolReview,
 } from './prompts';

package/src/tool-generation/prompts.ts

@@ -1,4 +1,5 @@
 import type { CapabilityGap, GeneratedTool, ToolValidationResult } from '@cogitator-ai/types';
+import { Script } from 'node:vm';
 import { extractJson } from '../utils';
 
 export const TOOL_GENERATION_SYSTEM_PROMPT = `You generate JavaScript tools. Respond with ONLY a JSON object, no other text.
@@ -35,9 +36,16 @@ function sanitizeToolName(name: string): string {
 export function buildGapAnalysisPrompt(
   userIntent: string,
   availableTools: Array<{ name: string; description: string }>,
-  failedAttempts?: string[]
+  failedAttempts?: string[],
+  agentInstructions?: string
 ): string {
-  const toolList = availableTools.map((t) => `- ${t.name}: ${t.description}`).join('\n');
+  const toolList = availableTools.length
+    ? availableTools.map((t) => `- ${t.name}: ${t.description}`).join('\n')
+    : '(none)';
+
+  const instructionsSection = agentInstructions?.trim()
+    ? `\nAGENT INSTRUCTIONS (the agent must follow these, so a capability they require a tool for is a gap when no tool provides it, even if the model could answer on its own):\n${agentInstructions.trim()}\n`
+    : '';
 
   const failureContext = failedAttempts?.length
     ? `\n\nPrevious failed attempts:\n${failedAttempts.map((f) => `- ${f}`).join('\n')}`
@@ -47,8 +55,8 @@ export function buildGapAnalysisPrompt(
 
 USER INTENT:
 ${userIntent}
-
-AVAILABLE TOOLS:
+${instructionsSection}
+AVAILABLE TOOLS (each one works as described, including tools generated earlier; a capability a listed tool already provides is not a gap, whatever the tool is called):
 ${toolList}
 ${failureContext}
 
@@ -82,7 +90,7 @@ export function buildToolGenerationPrompt(
   }
 ): string {
   const toolName = sanitizeToolName(gap.suggestedToolName);
-  const description = sanitizeForPrompt(gap.description);
+  const task = sanitizeForPrompt(gap.description);
   const capability = sanitizeForPrompt(gap.requiredCapability);
 
   const existingToolsSection =
@@ -121,11 +129,14 @@ export function buildToolGenerationPrompt(
     : '';
 
   return `Create a tool named "${toolName}".
-Task: ${description}
+Task: ${task}
 Formula/Logic: ${capability}
 ${existingToolsSection}${constraintsText}${parametersText}
+"description" tells an agent what the tool does: one sentence naming its input and its output, not the task above.
+"examples" lists 2 or 3 realistic valid inputs the tool must accept without throwing.
+
 Respond with ONLY this JSON (no other text):
-{"name":"${toolName}","description":"${description}","implementation":"async function execute(params) { /* YOUR CODE HERE — throw on invalid input */ }","parameters":${parametersJson},"reasoning":"explanation"}`;
+{"name":"${toolName}","description":"WHAT THE TOOL DOES","implementation":"async function execute(params) { /* YOUR CODE HERE - throw on invalid input */ }","parameters":${parametersJson},"examples":[{"input":{/* VALID PARAMS */}}],"reasoning":"explanation"}`;
 }
 
 export function buildToolValidationPrompt(
@@ -199,9 +210,10 @@ IMPORTANT: Use plain JavaScript only (NO TypeScript, NO type annotations). The e
 Respond with the same JSON format as before:
 {
   "name": "${tool.name}",
-  "description": "Updated description if needed",
+  "description": "What the tool does, its input and its output",
   "implementation": "async function execute(params) { ... }",
   "parameters": { ... },
+  "examples": [{ "input": { ... } }],
   "reasoning": "What was changed and why"
 }`;
 }
@@ -315,23 +327,72 @@ function normalizeParameters(value: unknown): Record<string, unknown> {
   return normalized;
 }
 
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/** Example inputs from a generation response, as `{ input }` entries. Bare input objects are accepted. */
+function parseExamples(value: unknown): Array<{ input: Record<string, unknown> }> {
+  if (!Array.isArray(value)) return [];
+  return value.flatMap((entry: unknown) => {
+    if (!isPlainObject(entry)) return [];
+    if ('input' in entry) return isPlainObject(entry.input) ? [{ input: entry.input }] : [];
+    return [{ input: entry }];
+  });
+}
+
+function compiles(code: string): boolean {
+  try {
+    new Script(`(async function () {\n${code}\n})`);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Some models escape the code twice, so it arrives on one line with literal `\n` and `\"`
+ * sequences that do not compile. Such code is unescaped once, and kept as is when that does not
+ * make it compile.
+ */
+function repairEscapedImplementation(code: string): string {
+  if (code.includes('\n') || !/\\[nrt"]/.test(code) || compiles(code)) return code;
+  try {
+    const unescaped: unknown = JSON.parse(`"${code}"`);
+    return typeof unescaped === 'string' && compiles(unescaped) ? unescaped : code;
+  } catch {
+    return code;
+  }
+}
+
 function buildToolFromParsed(parsed: Record<string, unknown>): GeneratedTool {
+  const examples = parseExamples(parsed.examples);
+  const metadata: Record<string, unknown> = {};
+  if (parsed.reasoning) metadata.reasoning = String(parsed.reasoning);
+  if (examples.length > 0) metadata.examples = examples;
+
   return {
     id: `gen_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
     name: sanitizeToolName(String(parsed.name)),
-    description: String(parsed.description || ''),
-    implementation: String(parsed.implementation),
+    description: typeof parsed.description === 'string' ? parsed.description.trim() : '',
+    implementation: repairEscapedImplementation(String(parsed.implementation)),
     parameters: normalizeParameters(parsed.parameters),
     createdAt: new Date(),
     version: 1,
     status: 'pending_validation',
-    metadata: {
-      reasoning: parsed.reasoning ? String(parsed.reasoning) : undefined,
-    },
+    metadata,
   };
 }
 
-export function parseValidationResponse(response: string): ToolValidationResult | null {
+/** A reviewer's verdict on a generated tool. */
+export interface ToolReview extends ToolValidationResult {
+  /** What the reviewer wants done with the tool, when it said so */
+  recommendation?: 'approve' | 'revise' | 'reject';
+}
+
+const RECOMMENDATIONS = new Set(['approve', 'revise', 'reject']);
+
+export function parseValidationResponse(response: string): ToolReview | null {
   const json = extractJson(response);
   if (!json) {
     return null;
@@ -355,6 +416,9 @@ export function parseValidationResponse(response: string): ToolValidationResult
           }))
         : [],
       overallScore: typeof parsed.overallScore === 'number' ? parsed.overallScore : 0,
+      ...(RECOMMENDATIONS.has(parsed.recommendation) && {
+        recommendation: parsed.recommendation as ToolReview['recommendation'],
+      }),
     };
   } catch {
     return null;

package/src/tool-generation/tool-generator.ts

@@ -23,6 +23,11 @@ export interface ToolGeneratorOptions {
   config: ToolSelfGenerationConfig;
   /** Model the LLM calls use, without the provider prefix of the backend */
   model: string;
+  /**
+   * Upper bound on the output tokens of each generation, improvement and review call.
+   * Unset leaves the backend default.
+   */
+  maxTokens?: number;
 }
 
 export interface GenerateOptions {
@@ -41,6 +46,7 @@ export class ToolGenerator {
   private readonly llm: LLMBackend;
   private readonly config: ToolSelfGenerationConfig;
   private readonly model: string;
+  private readonly maxTokens?: number;
   private readonly validator: ToolValidator;
   private readonly sandbox: ToolSandbox;
 
@@ -48,10 +54,12 @@ export class ToolGenerator {
     this.llm = options.llm;
     this.config = options.config;
     this.model = options.model;
+    this.maxTokens = options.maxTokens;
     this.validator = new ToolValidator({
       llm: options.llm,
       config: options.config,
       model: this.model,
+      maxTokens: options.maxTokens,
     });
     this.sandbox = new ToolSandbox(options.config.sandboxConfig);
   }
@@ -86,6 +94,8 @@ export class ToolGenerator {
           continue;
         }
 
+        currentTool.description = describeTool(currentTool.description, gap);
+
         if (options.parameters) {
           currentTool.parameters = options.parameters;
         }
@@ -183,7 +193,7 @@ export class ToolGenerator {
         { role: 'system', content: TOOL_GENERATION_SYSTEM_PROMPT },
         { role: 'user', content: prompt },
       ],
-      { model: this.model, temperature: 0.4 }
+      { model: this.model, temperature: 0.4, maxTokens: this.maxTokens }
     );
 
     const tool = parseToolGenerationResponse(content);
@@ -192,6 +202,7 @@ export class ToolGenerator {
       tool.metadata = {
         ...tool.metadata,
         gapId: gap.id,
+        capability: gap.requiredCapability,
         complexity: gap.complexity,
       };
     }
@@ -212,12 +223,13 @@ export class ToolGenerator {
         { role: 'system', content: TOOL_GENERATION_SYSTEM_PROMPT },
         { role: 'user', content: prompt },
       ],
-      { model: this.model, temperature: 0.3 }
+      { model: this.model, temperature: 0.3, maxTokens: this.maxTokens }
     );
 
     const improved = parseToolGenerationResponse(content);
 
     if (improved) {
+      improved.description ||= tool.description;
       improved.id = tool.id;
       improved.name = tool.name;
       improved.createdAt = tool.createdAt;
@@ -268,6 +280,25 @@ export class ToolGenerator {
   }
 }
 
+/**
+ * What an agent is told the tool does. The model writes it, but a copy of the gap text
+ * ("no tool can ...") or an empty one would make the next gap analysis think the
+ * capability is still missing, so those fall back to the capability the tool provides.
+ */
+function describeTool(description: string, gap: CapabilityGap): string {
+  const written = description.trim();
+  const echoesGap = normalize(written) === normalize(gap.description);
+  if (written && !echoesGap) return written;
+  return gap.requiredCapability.trim() || gap.description.trim() || written;
+}
+
+function normalize(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim();
+}
+
 function toToolJsonSchema(parameters: Record<string, unknown>): ToolSchema['parameters'] {
   const properties =
     parameters.type === 'object' &&

package/src/tool-generation/tool-sandbox.ts

@@ -85,10 +85,22 @@ const context = vm.createContext(Object.create(null), {
   codeGeneration: { strings: false, wasm: false },
 });
 
-let started = false;
+let script;
 try {
+  script = new vm.Script(runner);
+} catch (err) {
+  parentPort.postMessage({
+    success: false,
+    thrown: false,
+    error: 'Tool code does not compile: ' + (err && typeof err.message === 'string' ? err.message : String(err)),
+    logs: [],
+  });
+}
+
+let started = false;
+if (script) try {
   vm.runInContext(BOOTSTRAP, context);
-  vm.runInContext(runner, context, { timeout });
+  script.runInContext(context, { timeout });
   started = true;
 } catch (err) {
   const timedOut = err && err.code === 'ERR_SCRIPT_EXECUTION_TIMEOUT';

package/src/tool-generation/tool-validator.ts

@@ -5,17 +5,19 @@ import type {
   ToolSelfGenerationConfig,
 } from '@cogitator-ai/types';
 import { ToolSandbox, type SandboxTestCase } from './tool-sandbox';
-import { buildToolValidationPrompt, parseValidationResponse } from './prompts';
+import { buildToolValidationPrompt, parseValidationResponse, type ToolReview } from './prompts';
 import { llmChat, requireModelForLLM } from '../utils/llm-helper';
 
 export interface ToolValidatorOptions {
   llm?: LLMBackend;
   config: ToolSelfGenerationConfig;
   model?: string;
+  /** Upper bound on the output tokens of the review call. Unset leaves the backend default. */
+  maxTokens?: number;
 }
 
 interface JsonPropertySchema {
-  type?: string;
+  type?: string | string[];
   default?: unknown;
   enum?: unknown[];
   examples?: unknown[];
@@ -147,20 +149,89 @@ const STATIC_VALIDATION_RULES: ValidationRule[] = [
   },
 ];
 
+/** Edge cases and approved review notes lower the score, but never below a passing grade on their own. */
+const MAX_ADVISORY_PENALTY = 0.3;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function primaryType(type: JsonPropertySchema['type']): string | undefined {
+  return Array.isArray(type) ? type.find((t) => t !== 'null') : type;
+}
+
+function matchesType(value: unknown, type: JsonPropertySchema['type']): boolean {
+  if (type === undefined) return true;
+  const types = Array.isArray(type) ? type : [type];
+  return types.some((t) => {
+    switch (t) {
+      case 'string':
+        return typeof value === 'string';
+      case 'number':
+        return typeof value === 'number' && Number.isFinite(value);
+      case 'integer':
+        return Number.isInteger(value);
+      case 'boolean':
+        return typeof value === 'boolean';
+      case 'array':
+        return Array.isArray(value);
+      case 'object':
+        return isRecord(value);
+      case 'null':
+        return value === null;
+      default:
+        return true;
+    }
+  });
+}
+
+/** Whether an input has every required parameter, no unknown one, and values of the declared types. */
+function fitsSchema(input: Record<string, unknown>, parameters: Record<string, unknown>): boolean {
+  const properties = isRecord(parameters.properties)
+    ? (parameters.properties as Record<string, JsonPropertySchema>)
+    : {};
+  const required = Array.isArray(parameters.required) ? parameters.required : [];
+  if (required.some((key) => typeof key === 'string' && !(key in input))) return false;
+  if (Object.keys(properties).length === 0) return true;
+
+  return Object.entries(input).every(([key, value]) => {
+    const schema = properties[key];
+    return schema !== undefined && matchesType(value, schema.type);
+  });
+}
+
+/** A review approves when it calls the tool valid and does not ask for a revision or a rejection. */
+function approves(review: ToolReview): boolean {
+  return review.isValid && (review.recommendation ?? 'approve') === 'approve';
+}
+
 export class ToolValidator {
   private readonly llm?: LLMBackend;
   private readonly config: ToolSelfGenerationConfig;
   private readonly model: string;
+  private readonly maxTokens?: number;
   private readonly sandbox: ToolSandbox;
   private readonly customRules: ValidationRule[] = [];
 
   constructor(options: ToolValidatorOptions) {
     this.llm = options.llm;
+    this.maxTokens = options.maxTokens;
     this.config = options.config;
     this.model = requireModelForLLM(options.llm, options.model, 'ToolValidator');
     this.sandbox = new ToolSandbox(options.config.sandboxConfig);
   }
 
+  /**
+   * Grades a tool. Static security checks and sandbox test failures are authoritative: either
+   * makes the tool invalid whatever a review says. When `requireLLMValidation` is on, the
+   * reviewer's verdict decides the rest: an approving review keeps its security and logic
+   * notes as suggestions, while a review that recommends a revision or rejects the tool makes
+   * it invalid and its findings become the issues to fix.
+   *
+   * Without `testCases`, the tool runs on the example inputs it was generated with (they must
+   * succeed) and on inputs synthesized from its parameters schema (they only prove the tool
+   * runs: a descriptive error is accepted).
+   */
   async validate(
     tool: GeneratedTool,
     testCases?: SandboxTestCase[]
@@ -186,7 +257,10 @@ export class ToolValidator {
       };
     }
 
-    const effectiveTestCases = testCases || this.generateBasicTestCases(tool);
+    const effectiveTestCases = testCases ?? [
+      ...this.exampleTestCases(tool),
+      ...this.generateBasicTestCases(tool),
+    ];
     const sandboxResult = await this.sandbox.testWithCases(tool, effectiveTestCases);
 
     const testResults = sandboxResult.results.map((r) => ({
@@ -206,26 +280,42 @@ export class ToolValidator {
       );
     }
 
+    let advisoryNotes = 0;
     if (this.llm && this.config.requireLLMValidation) {
-      const llmResult = await this.runLLMValidation(tool, effectiveTestCases);
-      if (llmResult) {
-        securityIssues.push(...llmResult.securityIssues);
-        logicIssues.push(...llmResult.logicIssues);
-        edgeCases.push(...llmResult.edgeCases);
-        suggestions.push(...llmResult.suggestions);
+      const review = await this.runLLMValidation(tool, effectiveTestCases);
+      if (review) {
+        edgeCases.push(...review.edgeCases);
+        suggestions.push(...review.suggestions);
+
+        if (approves(review)) {
+          const notes = [
+            ...review.securityIssues.map((issue) => `Review note (security): ${issue}`),
+            ...review.logicIssues.map((issue) => `Review note (logic): ${issue}`),
+          ];
+          advisoryNotes = notes.length;
+          suggestions.push(...notes);
+        } else {
+          securityIssues.push(...review.securityIssues);
+          logicIssues.push(...review.logicIssues);
+          if (review.securityIssues.length === 0 && review.logicIssues.length === 0) {
+            logicIssues.push(
+              `The review recommends to ${review.recommendation ?? 'reject'} the tool without naming an issue: check it against its description`
+            );
+          }
+        }
       }
     }
 
     const score = this.calculateScore(
       securityIssues.length,
       logicIssues.length,
-      edgeCases.length,
+      edgeCases.length + advisoryNotes,
       sandboxResult.passed,
       sandboxResult.failed
     );
 
     return {
-      isValid: securityIssues.length === 0 && logicIssues.length === 0 && score >= 0.7,
+      isValid: securityIssues.length === 0 && logicIssues.length === 0,
       securityIssues: [...new Set(securityIssues)],
       logicIssues: [...new Set(logicIssues)],
       edgeCases: [...new Set(edgeCases)],
@@ -264,7 +354,7 @@ export class ToolValidator {
   private async runLLMValidation(
     tool: GeneratedTool,
     testCases: SandboxTestCase[]
-  ): Promise<ToolValidationResult | null> {
+  ): Promise<ToolReview | null> {
     if (!this.llm) return null;
 
     try {
@@ -293,7 +383,7 @@ Be thorough but practical - focus on real issues.`,
           },
           { role: 'user', content: prompt },
         ],
-        { model: this.model, temperature: 0.2 }
+        { model: this.model, temperature: 0.2, maxTokens: this.maxTokens }
       );
 
       return parseValidationResponse(content);
@@ -302,6 +392,17 @@ Be thorough but practical - focus on real issues.`,
     }
   }
 
+  /** The example inputs the tool was generated with that fit its schema, as cases that must succeed. */
+  private exampleTestCases(tool: GeneratedTool): SandboxTestCase[] {
+    const examples = tool.metadata?.examples;
+    if (!Array.isArray(examples)) return [];
+
+    return examples.flatMap((example: unknown) => {
+      if (!isRecord(example) || !isRecord(example.input)) return [];
+      return fitsSchema(example.input, tool.parameters) ? [{ input: example.input }] : [];
+    });
+  }
+
   private generateBasicTestCases(tool: GeneratedTool): SandboxTestCase[] {
     const params = tool.parameters;
 
@@ -318,11 +419,11 @@ Be thorough but practical - focus on real issues.`,
     const entries = Object.entries(properties);
     const testCases: SandboxTestCase[] = [];
 
-    const validInput: Record<string, unknown> = {};
+    const sampleInput: Record<string, unknown> = {};
     for (const [key, schema] of entries) {
-      validInput[key] = this.generateSampleValue(schema);
+      sampleInput[key] = this.generateSampleValue(schema);
     }
-    testCases.push({ input: validInput });
+    testCases.push({ input: sampleInput, allowThrow: true });
 
     const hasOptional = entries.some(([key]) => !required.has(key));
     if (required.size > 0 && hasOptional) {
@@ -332,12 +433,12 @@ Be thorough but practical - focus on real issues.`,
           requiredOnlyInput[key] = this.generateSampleValue(schema);
         }
       }
-      testCases.push({ input: requiredOnlyInput });
+      testCases.push({ input: requiredOnlyInput, allowThrow: true });
     }
 
     const edgeInput: Record<string, unknown> = {};
     for (const [key, schema] of entries) {
-      edgeInput[key] = this.generateEdgeValue(schema.type);
+      edgeInput[key] = this.generateEdgeValue(primaryType(schema.type));
     }
     testCases.push({ input: edgeInput, allowThrow: true });
 
@@ -359,7 +460,7 @@ Be thorough but practical - focus on real issues.`,
     if (Array.isArray(schema.examples) && schema.examples.length > 0) return schema.examples[0];
     if (Array.isArray(schema.enum) && schema.enum.length > 0) return schema.enum[0];
 
-    switch (schema.type) {
+    switch (primaryType(schema.type)) {
       case 'string':
         return 'test';
       case 'number':
@@ -407,7 +508,7 @@ Be thorough but practical - focus on real issues.`,
     let score = 1.0;
 
     score -= logicCount * 0.2;
-    score -= edgeCaseCount * 0.05;
+    score -= Math.min(MAX_ADVISORY_PENALTY, edgeCaseCount * 0.05);
 
     const totalTests = testsPassed + testsFailed;
     if (totalTests > 0) {