@holoscript/framework 6.0.3

package/src/ai/__tests__/AIOutputValidator.prod.test.ts

@@ -0,0 +1,226 @@
+/**
+ * AIOutputValidator Production Tests
+ * Sprint CLXVI — validateAIOutput, isAISafe, confidence scoring
+ */
+import { describe, it, expect } from 'vitest';
+import { validateAIOutput, isAISafe } from '../AIOutputValidator';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function lines(n: number): string {
+  return Array.from({ length: n }, (_, i) => `const x${i} = ${i};`).join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// validateAIOutput
+// ---------------------------------------------------------------------------
+
+describe('validateAIOutput', () => {
+  // --- Basic valid code ---
+  describe('valid code', () => {
+    it('returns valid=true for empty string', () => {
+      const r = validateAIOutput('');
+      expect(r.valid).toBe(true);
+      expect(r.issues).toHaveLength(0);
+    });
+
+    it('returns valid=true for simple safe code', () => {
+      const r = validateAIOutput('const x = 1;\nconst y = 2;');
+      expect(r.valid).toBe(true);
+      expect(r.confidence).toBeGreaterThan(0);
+    });
+
+    it('confidence is 1.0 for clean code', () => {
+      const r = validateAIOutput('const a = 42;');
+      expect(r.confidence).toBe(1.0);
+    });
+
+    it('stats.lineCount reflects actual lines', () => {
+      const r = validateAIOutput('a\nb\nc');
+      expect(r.stats.lineCount).toBe(3);
+    });
+
+    it('stats.traitCount counts @decorators', () => {
+      const r = validateAIOutput('@Physics\n@Renderable\nconst x = 1;');
+      expect(r.stats.traitCount).toBe(2);
+    });
+
+    it('stats.dangerousPatternCount is 0 for safe code', () => {
+      const r = validateAIOutput('const x = 1;');
+      expect(r.stats.dangerousPatternCount).toBe(0);
+    });
+
+    it('stats.maxNesting tracks brace depth', () => {
+      const r = validateAIOutput('{ { { } } }');
+      expect(r.stats.maxNesting).toBe(3);
+    });
+  });
+
+  // --- Dangerous patterns ---
+  describe('dangerous patterns', () => {
+    it('flags eval()', () => {
+      const r = validateAIOutput('eval("bad code")');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-eval')).toBe(true);
+    });
+
+    it('flags Function constructor', () => {
+      const r = validateAIOutput('const f = Function("return 1")');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-function-constructor')).toBe(true);
+    });
+
+    it('flags require()', () => {
+      const r = validateAIOutput('const fs = require("fs")');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-require')).toBe(true);
+    });
+
+    it('flags __proto__ access', () => {
+      const r = validateAIOutput('obj.__proto__');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-proto')).toBe(true);
+    });
+
+    it('flags process.* access', () => {
+      const r = validateAIOutput('process.exit(0)');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-process')).toBe(true);
+    });
+
+    it('flags fs.* access', () => {
+      const r = validateAIOutput('fs.readFileSync("x")');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-fs')).toBe(true);
+    });
+
+    it('flags dynamic import()', () => {
+      const r = validateAIOutput('import("./foo")');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-dynamic-import')).toBe(true);
+    });
+
+    it('flags globalThis', () => {
+      const r = validateAIOutput('globalThis.x = 1');
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'no-globalThis')).toBe(true);
+    });
+
+    it('includes line number in issue', () => {
+      const r = validateAIOutput('const x = 1;\neval("x")');
+      const issue = r.issues.find((i) => i.rule === 'no-eval');
+      expect(issue?.line).toBe(2);
+    });
+
+    it('does NOT flag if blockDangerousPatterns=false', () => {
+      const r = validateAIOutput('eval("x")', { blockDangerousPatterns: false });
+      expect(r.valid).toBe(true);
+    });
+
+    it('reduces confidence for each dangerous pattern', () => {
+      const clean = validateAIOutput('const x = 1;');
+      const dirty = validateAIOutput('eval("x")');
+      expect(dirty.confidence).toBeLessThan(clean.confidence);
+    });
+  });
+
+  // --- Max lines ---
+  describe('max lines', () => {
+    it('flags code exceeding maxLines', () => {
+      const r = validateAIOutput(lines(2001));
+      expect(r.valid).toBe(false);
+      expect(r.issues.some((i) => i.rule === 'max-lines')).toBe(true);
+    });
+
+    it('does NOT flag code at maxLines', () => {
+      const r = validateAIOutput(lines(2000));
+      expect(r.issues.some((i) => i.rule === 'max-lines')).toBe(false);
+    });
+
+    it('custom maxLines config applies', () => {
+      const r = validateAIOutput(lines(6), { maxLines: 5 });
+      expect(r.issues.some((i) => i.rule === 'max-lines')).toBe(true);
+    });
+  });
+
+  // --- Nesting ---
+  describe('nesting depth', () => {
+    it('warns for excessive nesting', () => {
+      const nested = '{'.repeat(16) + '}'.repeat(16);
+      const r = validateAIOutput(nested);
+      expect(r.issues.some((i) => i.rule === 'max-nesting')).toBe(true);
+      expect(r.issues.find((i) => i.rule === 'max-nesting')?.severity).toBe('warning');
+    });
+
+    it('errors for unbalanced braces', () => {
+      const r = validateAIOutput('{ {');
+      expect(r.issues.some((i) => i.rule === 'balanced-braces')).toBe(true);
+      expect(r.issues.find((i) => i.rule === 'balanced-braces')?.severity).toBe('error');
+    });
+
+    it('does not warn at exactly maxNesting', () => {
+      const nested = '{'.repeat(15) + '}'.repeat(15);
+      const r = validateAIOutput(nested);
+      expect(r.issues.some((i) => i.rule === 'max-nesting')).toBe(false);
+    });
+  });
+
+  // --- Allowed traits ---
+  describe('allowed traits', () => {
+    it('warns for unknown trait when allowedTraits is set', () => {
+      const r = validateAIOutput('@Physics @Renderable', { allowedTraits: ['@Renderable'] });
+      const issue = r.issues.find((i) => i.rule === 'allowed-traits');
+      expect(issue).toBeTruthy();
+      expect(issue?.message).toContain('@Physics');
+    });
+
+    it('does not warn when trait is in allowed list', () => {
+      const r = validateAIOutput('@Renderable', { allowedTraits: ['Renderable'] });
+      expect(r.issues.some((i) => i.rule === 'allowed-traits')).toBe(false);
+    });
+
+    it('does not check traits if allowedTraits is empty', () => {
+      const r = validateAIOutput('@Anything @Custom', { allowedTraits: [] });
+      expect(r.issues.some((i) => i.rule === 'allowed-traits')).toBe(false);
+    });
+  });
+
+  // --- Confidence ---
+  describe('confidence scoring', () => {
+    it('confidence is clamped to 0 for many errors', () => {
+      const code = 'eval("x")\neval("y")\neval("z")\neval("w")';
+      const r = validateAIOutput(code);
+      expect(r.confidence).toBeGreaterThanOrEqual(0);
+    });
+
+    it('warnings reduce confidence less than errors', () => {
+      const withWarning = validateAIOutput('{'.repeat(16) + '}'.repeat(16)); // max-nesting warning
+      const withError = validateAIOutput('eval("x")'); // dangerous pattern error
+      expect(withError.confidence).toBeLessThan(withWarning.confidence);
+    });
+  });
+});
+
+// ---------------------------------------------------------------------------
+// isAISafe
+// ---------------------------------------------------------------------------
+
+describe('isAISafe', () => {
+  it('returns true for valid code', () => {
+    expect(isAISafe('const x = 1;')).toBe(true);
+  });
+
+  it('returns false for code with eval', () => {
+    expect(isAISafe('eval("x")')).toBe(false);
+  });
+
+  it('returns false for unbalanced braces', () => {
+    expect(isAISafe('{')).toBe(false);
+  });
+
+  it('respects custom config', () => {
+    expect(isAISafe('eval("x")', { blockDangerousPatterns: false })).toBe(true);
+  });
+});

package/src/ai/__tests__/AIOutputValidator.test.ts

@@ -0,0 +1,138 @@
+import { describe, it, expect } from 'vitest';
+import { validateAIOutput, isAISafe } from '../AIOutputValidator';
+
+describe('validateAIOutput', () => {
+  // Clean code
+  it('valid code passes', () => {
+    const result = validateAIOutput('const x = 1;\nconst y = 2;\n');
+    expect(result.valid).toBe(true);
+    expect(result.issues.length).toBe(0);
+    expect(result.confidence).toBeGreaterThan(0.9);
+  });
+
+  // Line count
+  it('detects excessive line count', () => {
+    const code = Array(3000).fill('const x = 1;').join('\n');
+    const result = validateAIOutput(code);
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'max-lines')).toBe(true);
+  });
+
+  it('custom maxLines overrides default', () => {
+    const code = Array(50).fill('x').join('\n');
+    const result = validateAIOutput(code, { maxLines: 10 });
+    expect(result.valid).toBe(false);
+  });
+
+  // Dangerous patterns
+  it('detects eval()', () => {
+    const result = validateAIOutput('eval("alert(1)")');
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'no-eval')).toBe(true);
+  });
+
+  it('detects Function constructor', () => {
+    const result = validateAIOutput('new Function("return 1")');
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'no-function-constructor')).toBe(true);
+  });
+
+  it('detects require()', () => {
+    const result = validateAIOutput('const fs = require("fs")');
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'no-require')).toBe(true);
+  });
+
+  it('detects __proto__', () => {
+    const result = validateAIOutput('obj.__proto__.x = 1');
+    expect(result.valid).toBe(false);
+  });
+
+  it('detects process.', () => {
+    const result = validateAIOutput('process.exit(1)');
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'no-process')).toBe(true);
+  });
+
+  it('detects dynamic import()', () => {
+    const result = validateAIOutput('const m = import("evil")');
+    expect(result.valid).toBe(false);
+  });
+
+  it('detects globalThis', () => {
+    const result = validateAIOutput('globalThis.x = 1');
+    expect(result.valid).toBe(false);
+  });
+
+  it('blockDangerousPatterns=false skips checks', () => {
+    const result = validateAIOutput('eval("1")', { blockDangerousPatterns: false });
+    expect(result.valid).toBe(true);
+  });
+
+  // Nesting
+  it('warns on excessive nesting', () => {
+    const code = '{'.repeat(20) + '}'.repeat(20);
+    const result = validateAIOutput(code);
+    expect(result.issues.some((i) => i.rule === 'max-nesting')).toBe(true);
+  });
+
+  it('custom maxNesting overrides default', () => {
+    const code = '{ { { } } }';
+    const result = validateAIOutput(code, { maxNesting: 2 });
+    expect(result.issues.some((i) => i.rule === 'max-nesting')).toBe(true);
+  });
+
+  // Unbalanced braces
+  it('detects unbalanced braces', () => {
+    const result = validateAIOutput('{ { }');
+    expect(result.valid).toBe(false);
+    expect(result.issues.some((i) => i.rule === 'balanced-braces')).toBe(true);
+  });
+
+  // Traits
+  it('counts traits', () => {
+    const result = validateAIOutput('@animate @physics @grabbable');
+    expect(result.stats.traitCount).toBe(3);
+  });
+
+  it('warns on too many traits', () => {
+    const code = Array(200).fill('@trait').join(' ');
+    const result = validateAIOutput(code);
+    expect(result.issues.some((i) => i.rule === 'max-traits')).toBe(true);
+  });
+
+  it('warns on disallowed traits', () => {
+    const result = validateAIOutput('@physics @animate', { allowedTraits: ['physics'] });
+    expect(result.issues.some((i) => i.rule === 'allowed-traits')).toBe(true);
+  });
+
+  // Confidence scoring
+  it('confidence drops with errors', () => {
+    const clean = validateAIOutput('const x = 1;');
+    const dirty = validateAIOutput('eval("1")');
+    expect(clean.confidence).toBeGreaterThan(dirty.confidence);
+  });
+
+  // Stats
+  it('stats include line count and nesting', () => {
+    const result = validateAIOutput('a\nb\nc');
+    expect(result.stats.lineCount).toBe(3);
+    expect(result.stats.maxNesting).toBe(0);
+  });
+
+  it('reports line numbers for dangerous patterns', () => {
+    const result = validateAIOutput('const x = 1;\neval("2");\nconst y = 3;');
+    const evalIssue = result.issues.find((i) => i.rule === 'no-eval')!;
+    expect(evalIssue.line).toBe(2);
+  });
+});
+
+describe('isAISafe', () => {
+  it('returns true for safe code', () => {
+    expect(isAISafe('const x = 1;')).toBe(true);
+  });
+
+  it('returns false for dangerous code', () => {
+    expect(isAISafe('eval("1")')).toBe(false);
+  });
+});