circle-ir-ai 1.1.0

package/CHANGELOG.md

@@ -0,0 +1,105 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.1.0] - 2026-03-05
+
+### Added
+
+- **AI Skills Analysis**: Complete security analysis framework for AI agent skills (OpenClaw, MCP servers)
+  - `analyze-skill` CLI command for skill directory analysis
+  - Multi-artifact analysis: SKILL.md (natural language) + code files + MCP config
+  - LLM-based capability mismatch detection using semantic comparison
+  - Trust score calculation (0-100%) based on severity-weighted findings
+  - Test file exclusion to eliminate false positives (74% FP reduction overall)
+  - Validation utility recognition (normalizePath, isPathWithin, etc.)
+  - Safe utility methods whitelist for external_taint_escape detection
+  - JSON and markdown output formats
+  - Real-world validated on official MCP servers (fetch: 84% trust score)
+
+### Changed
+
+- **LLM Timeout Configuration**: Increased enrichment phase timeout from 15s to 60s to support large SKILL.md files (10KB+)
+- **Sanitizer Patterns**: Added path validation and normalization methods (normalizePath, isPathWithinAllowedDirectories, relative, join)
+- **Bundle Loader**: Added comprehensive test file exclusion (\__tests__/, *.test.ts, *.spec.js, vitest.config.ts, etc.)
+
+### Fixed
+
+- **external_taint_escape False Positives**: Eliminated 10 false positives by excluding validation utilities (tailFile, headFile, readFileContent, minimatch, etc.)
+- **SKILL.md Extraction Timeouts**: Large documentation files now extract successfully without timing out
+- **Test Contamination**: Test files no longer analyzed as production code (61% FP reduction on filesystem server)
+
+### Performance
+
+- **Analysis Speed**: 50% faster on typical skills due to timeout fixes (50.44s → 25.20s on filesystem server)
+- **False Positive Reduction**: 74% overall (91 → 24 findings on filesystem server)
+
+## [3.4.0] - 2026-02-23
+
+### Changed
+
+- **Discovery Mode Validated**: Tested 8 LLM models in discovery mode - 7 of 8 improved 31-92% over static baseline
+- **Default Model Updated**: Changed default from `gemini-pro-latest` to `gpt-oss-120b` (FREE, 73/120, beats CodeQL and IRIS+GPT-4)
+- **Documentation Consolidated**: Comprehensive discovery mode guide added to MODEL_SCORECARD.md with usage examples and troubleshooting
+
+### Removed
+
+- **Verification Mode Removed**: ALL models failed in verification mode (≤51/120, at or below static baseline)
+  - Removed `--llm` flag from benchmark runner
+  - Removed `verifyWithLLM()` function and all verification code paths
+  - Verification mode asked LLMs to judge sanitizer effectiveness (requires deep security expertise) - models pattern-matched instead
+
+### Added
+
+- **Research Documentation**: "Why Discovery Works" section explaining task difficulty differences
+- **Model Rankings**: Complete performance comparison of 8 models across 120 real-world CVEs
+
+### Discovery Mode Results
+
+| Model | Score | vs Static | Tier |
+|-------|-------|-----------|------|
+| Claude Opus 4.6 | 98/120 (81.7%) | +92% | Premium |
+| Grok-Code | 94/120 (78.3%) | +84% | Budget |
+| DeepSeek V3.2 | 91/120 (75.8%) | +78% | Budget |
+| Claude Haiku 4.5 | 85/120 (70.8%) | +67% | Budget |
+| GPT-5.2 | 82/120 (68.3%) | +61% | Premium |
+| **gpt-oss-120b (FREE)** | **73/120 (60.8%)** | **+43%** | **FREE** |
+| Gemini Pro | 67/120 (55.8%) | +31% | Premium |
+
+**Key Finding**: Discovery mode (trace taint flows) works universally better than verification mode (judge sanitizers) because it matches LLM strengths (code flow analysis) rather than weaknesses (security domain expertise).
+
+### Breaking Changes
+
+- `--llm` flag removed (use `--llm-discovery` instead)
+- Verification mode API removed from codebase
+
+## [1.0.0] - 2025-02-19
+
+### Added
+
+- **LLM Integration**: ax-llm client with DSPy-style typed signatures for enrichment, verification, and discovery
+- **Mastra Agents**: AI agent orchestration (enrichment, verification, cross-file, orchestrator)
+- **Swarm Analysis**: Parallel multi-file analysis with configurable concurrency
+- **Security Scan**: Continuous scanning with OWASP Top 10 mapping and trend tracking
+- **Dead Code Detection**: Entry point analysis, call graph construction, reachability checks
+- **Secret Scanning**: Pattern-based detection (AWS, GitHub, Stripe, etc.) in code and Git history
+- **Health Score**: Weighted codebase health scoring (security, maintainability, quality, performance)
+- **Action Queue**: Aggregate findings into prioritized action items with sprint planning and export (CSV, Jira, Markdown)
+- **CLI**: Command-line interface for file/directory analysis, benchmarks, scanning, and reporting
+- **Two-Phase Project Analyzer**: Per-file LLM enrichment + cross-file LLM analysis
+- **File Cache**: Caching layer for analysis results
+
+### Benchmark Results
+
+- **OWASP Benchmark**: +100% (1415/1415 test cases)
+- **Juliet Test Suite**: +100% (156/156 test cases)
+- **SecuriBench Micro**: 97.7% TPR, 6.7% FPR (105/108 vulns detected)
+- **CWE-Bench-Java**: 42.5% static, 80.8% with Claude hybrid (vs CodeQL 22.5%, IRIS+GPT-4 45.8%)
+- **NodeJS Synthetic**: 100% TPR, 94.1% Precision (25 test cases)
+- **OWASP NodeGoat**: +100% (14 tests)
+- **CWE-Bench-Rust**: +100% (30 test cases)
+
+[1.0.0]: https://github.com/cogniumhq/circle-ir/releases/tag/circle-ir-ai-v1.0.0

package/LICENSE

@@ -0,0 +1,15 @@
+ISC License
+
+Copyright (c) 2025 Cognium Labs
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

package/README.md

@@ -0,0 +1,336 @@
+# circle-ir-ai
+
+LLM-enhanced static analysis built on [circle-ir](https://github.com/cognitim/circle-ir). Provides CLI tools, AI-powered vulnerability verification, and comprehensive security analysis.
+
+## Features
+
+- **Multi-Language Support**: Java, JavaScript/TypeScript, Python, Rust
+- **LLM Verification**: Verify exploitability of detected vulnerabilities
+- **Pattern Discovery**: Discover sources/sinks beyond YAML patterns
+- **Cross-File Analysis**: Track taint across file boundaries
+- **AI Skills Analysis**: Security analysis for AI agent skills (OpenClaw, MCP servers)
+- **Security Scanning**: OWASP Top 10 mapping with trend tracking
+- **Dead Code Detection**: Find unreachable code via call graph analysis
+- **Secret Scanning**: Detect secrets in code and Git history
+- **Health Scoring**: Calculate overall codebase health
+- **Benchmark Suite**: OWASP, Juliet, SecuriBench, CWE-Bench test runners
+
+## Installation
+
+```bash
+npm install circle-ir-ai circle-ir
+```
+
+## Quick Start
+
+### CLI
+
+```bash
+# Analyze a single file
+circle-ir-ai src/App.java
+
+# Analyze with LLM verification
+circle-ir-ai src/App.java --llm
+
+# Analyze a directory (auto-parallelized)
+circle-ir-ai ./src --threads 20
+
+# Security scan with OWASP Top 10 mapping
+circle-ir-ai scan ./src
+
+# Detect dead/unreachable code
+circle-ir-ai dead-code ./src
+
+# Scan for secrets
+circle-ir-ai secrets ./src
+
+# Calculate codebase health score
+circle-ir-ai health ./src
+
+# Run OWASP Benchmark
+circle-ir-ai benchmark ./testcode --expected ./expected.csv
+
+# Analyze AI Skills (OpenClaw, MCP servers)
+circle-ir-ai analyze-skill ./my-skill
+circle-ir-ai analyze-skill ./my-skill --format json -o report.json
+```
+
+### AI Skills Analysis
+
+Analyze AI agent skills for security issues and capability mismatches. Supports OpenClaw skills and MCP servers.
+
+**CLI Usage:**
+
+```bash
+# Analyze a skill directory
+circle-ir-ai analyze-skill ./my-skill
+
+# With JSON output
+circle-ir-ai analyze-skill ./my-skill --format json -o report.json
+
+# With markdown report
+circle-ir-ai analyze-skill ./my-skill --format markdown -o SECURITY.md
+
+# Filter by severity and confidence
+circle-ir-ai analyze-skill ./my-skill --min-severity high --min-confidence 0.8
+```
+
+**What it analyzes:**
+
+- `SKILL.md` - Natural language instructions (extracts declared capabilities)
+- Code files (`.ts`, `.js`, `.py`, `.java`, `.rs`) - Actual implementation
+- `mcp-server.json` - MCP server configuration (if present)
+- Cross-artifact analysis - Detects capability mismatches between documentation and code
+
+**Output:**
+
+```
+================================================================================
+SKILL ANALYSIS REPORT: github-reader
+================================================================================
+
+Skill ID:       github-reader-001
+Version:        1.0.0
+Trust Score:    22.0% 🚨
+Findings:       13
+Analysis Time:  21.84s
+
+CRITICAL FINDINGS (1)
+────────────────────────────────────────────────────────────────────────────────
+
+1. Server-Side Request Forgery (CWE-918) not declared
+   Type:       capability_mismatch
+   Artifact:   src/index.ts
+   Confidence: 90%
+   Description: The code performs outbound HTTP requests via `https.request()`
+                but SKILL.md does not list any such sink.
+
+RECOMMENDATIONS
+────────────────────────────────────────────────────────────────────────────────
+
+1. Update SKILL.md to accurately describe all capabilities
+2. Add input validation and sanitization
+```
+
+**Trust Score Interpretation:**
+
+- `90-100%` ✅ - High trust, minimal issues
+- `70-89%` ⚠️ - Medium trust, some concerns
+- `50-69%` ❌ - Low trust, significant issues
+- `<50%` 🚨 - Critical issues, do not deploy
+
+**Programmatic API:**
+
+```typescript
+import { analyzeSkillBundle } from 'circle-ir-ai/skills';
+
+const result = await analyzeSkillBundle('./my-skill', {
+  enableCrossArtifact: true,    // Detect capability mismatches
+  enableVerification: true,      // LLM verification
+  minConfidence: 0.7,
+  minSeverity: 'medium',
+  onProgress: (status) => console.log(status.message),
+});
+
+console.log(`Trust Score: ${result.score}`);
+console.log(`Findings: ${result.findings.length}`);
+console.log(`Recommendations: ${result.recommendations.join('\n')}`);
+```
+
+**Key Features:**
+
+- **Multi-Artifact Analysis**: Analyzes SKILL.md + code + MCP config together
+- **Capability Mismatch Detection**: LLM-based semantic comparison between declared and actual behavior
+- **Vulnerability Detection**: Standard taint analysis on code files
+- **Trust Score**: 0.0-1.0 score based on severity-weighted findings
+- **Actionable Recommendations**: Specific guidance for remediation
+
+**Production Quality:**
+
+- ✅ **74% false positive reduction** through test file exclusion, validation utility recognition, and timeout fixes
+- ✅ **Real-world validated** on official MCP servers (fetch: 84% trust, filesystem: accurate findings)
+- ✅ **Fast analysis** (~25s for typical skills)
+- ⚠️ Large SKILL.md files (>10KB) may take longer to extract
+
+**Known Limitations:**
+
+- Capability mismatch detection quality depends on LLM model used
+- Trust score may be conservative for complex codebases with many filesystem operations
+
+### Programmatic API
+
+```typescript
+import { analyze, initAnalyzer } from 'circle-ir';
+import { runHybridAnalysis, applyLLMAnalysis } from 'circle-ir-ai';
+
+// Initialize
+await initAnalyzer();
+
+// Run hybrid analysis with LLM
+const result = await runHybridAnalysis(
+  'MyClass.java',
+  code,
+  sources,
+  sinks,
+  types,
+  imports,
+  {
+    enableEnrichment: true,
+    enableVerification: true,
+  }
+);
+
+console.log('Verified vulnerabilities:', result.vulnerabilities);
+```
+
+### Project-Level Analysis
+
+```typescript
+import { analyzeProjectTwoPhase } from 'circle-ir-ai';
+
+const files = [
+  { path: 'src/Controller.java', content: controllerCode },
+  { path: 'src/Service.java', content: serviceCode },
+];
+
+const result = await analyzeProjectTwoPhase(files, 'java', {
+  enableEnrichment: true,
+  parallelPhase1: true,
+  maxConcurrency: 10,
+  enablePhase2: true,
+});
+
+console.log('Cross-file taint flows:', result.crossFileFlows);
+```
+
+### Security Scanning
+
+```typescript
+import { scanDirectory, formatScanReport } from 'circle-ir-ai';
+
+const result = await scanDirectory('/path/to/project');
+console.log(formatScanReport(result));
+// Shows vulnerabilities mapped to OWASP Top 10 categories
+```
+
+### Dead Code Detection
+
+```typescript
+import { detectDeadCode, formatDeadCodeReport } from 'circle-ir-ai';
+
+const result = await detectDeadCode({
+  target: '/path/to/project',
+  languages: ['javascript', 'typescript'],
+});
+
+console.log(formatDeadCodeReport(result));
+// Shows: dead methods, entry points, and LOC with reachability info
+```
+
+### Secret Scanning
+
+```typescript
+import { scanForSecrets, formatSecretReport } from 'circle-ir-ai';
+
+const result = await scanForSecrets('/path/to/project', {
+  scanHistory: true,      // Scan Git history
+  maxCommits: 100,
+  minSeverity: 'medium',
+});
+
+console.log(formatSecretReport(result));
+```
+
+### Health Scoring
+
+```typescript
+import { calculateHealthScore, formatHealthReport } from 'circle-ir-ai';
+
+const result = await calculateHealthScore('/path/to/project', {
+  includeSecurity: true,
+  includeSecrets: true,
+  includeDeadCode: true,
+});
+
+console.log(formatHealthReport(result));
+// Scores: Security 35%, Maintainability 25%, Quality 25%, Performance 15%
+```
+
+## LLM Configuration
+
+Set environment variables or pass options:
+
+```bash
+export LLM_API_KEY=your-api-key
+export LLM_BASE_URL=http://localhost:4000/v1
+```
+
+Or in code:
+
+```typescript
+import { createLLMConfig } from 'circle-ir-ai';
+
+const config = createLLMConfig({
+  baseUrl: 'http://localhost:4000/v1',
+  apiKey: 'your-api-key',
+});
+```
+
+## CLI Options
+
+```
+circle-ir-ai <file|dir> [options]
+
+Commands:
+  <file|dir>           Analyze file or directory
+  scan <dir>           Security scan with OWASP mapping
+  dead-code <dir>      Detect unreachable code
+  secrets <dir>        Scan for secrets
+  health <dir>         Calculate health score
+  benchmark <dir>      Run benchmark suite
+
+Options:
+  -f, --format <fmt>       Output format: json, summary, sarif
+  -o, --output <file>      Write output to file
+  --llm                    Enable LLM enrichment and verification
+  --llm-enrich             Enable LLM enrichment only
+  --llm-verify             Enable LLM verification only
+  --llm-base-url <url>     LLM proxy URL
+  --llm-api-key <key>      LLM API key
+  --threads <n>            Parallel analysis threads (default: 10)
+  -d, --discover-patterns  Enable dynamic pattern discovery
+  -q, --quiet              Suppress progress messages
+  -h, --help               Show help
+```
+
+## Benchmark Results
+
+| Benchmark | Score | Details |
+|-----------|-------|---------|
+| **OWASP Benchmark** (Java) | +100% | 1415/1415 perfect |
+| **OWASP BenchmarkPython** | 25.2% | 1230 test cases |
+| **Juliet Test Suite** | +100% | 156/156 perfect |
+| **SecuriBench Micro** | 97.7% TPR | 105/108 vulns, 6.7% FPR |
+| **CWE-Bench-Java** | 81.7% (with LLM) | 98/120 projects (vs CodeQL 22.5%, IRIS+GPT-4 45.8%) |
+| **NodeJS Synthetic** | 100% TPR | 25 test cases, 94.1% Precision |
+| **OWASP NodeGoat** | +100% | 14 tests, 5 vulns detected |
+| **CWE-Bench-Rust** | +100% | 30 test cases (6 CWEs) |
+
+## Supported Languages
+
+| Language | Extensions | Frameworks |
+|----------|-----------|------------|
+| **Java** | .java | Spring, JAX-RS, Servlet API |
+| **JavaScript** | .js, .jsx, .mjs, .cjs | Express, Fastify, Node.js |
+| **TypeScript** | .ts, .tsx, .mts, .cts | Express, Fastify, Node.js |
+| **Python** | .py | Flask, Django, FastAPI |
+| **Rust** | .rs | Actix-web, Rocket, Axum |
+
+## Related Packages
+
+- **[circle-ir](https://github.com/cognitim/circle-ir)**: Core SAST library
+- **[circle-pack](https://github.com/cognitim/circle-pack)**: Cloudflare Workers API deployment
+
+## License
+
+ISC

package/dist/action-queue/aggregator.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Action Aggregator
+ *
+ * Collects findings from all analysis dimensions and converts them
+ * to unified action items.
+ */
+import type { ScanResult } from '../security-scan/scanner.js';
+import type { DeadCodeResult } from '../dead-code/detector.js';
+import type { SecretScanResult } from '../secret-scan/scanner.js';
+import type { HealthScoreResult } from '../health-score/calculator.js';
+import { type ActionItem, type PriorityWeights } from './types.js';
+/**
+ * Aggregation input - results from various scans
+ */
+export interface AggregationInput {
+    /** Security scan result */
+    securityScan?: ScanResult;
+    /** Dead code detection result */
+    deadCode?: DeadCodeResult;
+    /** Secret scan result */
+    secretScan?: SecretScanResult;
+    /** Health score result */
+    healthScore?: HealthScoreResult;
+}
+/**
+ * Aggregator options
+ */
+export interface AggregatorOptions {
+    /** Priority weights */
+    weights?: PriorityWeights;
+    /** Include info-level items */
+    includeInfo?: boolean;
+    /** Base path to strip from file paths */
+    basePath?: string;
+}
+/**
+ * Aggregate findings from all sources into unified action items
+ */
+export declare function aggregateFindings(input: AggregationInput, options?: AggregatorOptions): ActionItem[];
+//# sourceMappingURL=aggregator.d.ts.map

package/dist/action-queue/aggregator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aggregator.d.ts","sourceRoot":"","sources":["../../src/action-queue/aggregator.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAe,MAAM,6BAA6B,CAAC;AAC3E,OAAO,KAAK,EAAE,cAAc,EAAgB,MAAM,0BAA0B,CAAC;AAC7E,OAAO,KAAK,EAAE,gBAAgB,EAAkB,MAAM,2BAA2B,CAAC;AAClF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,KAAK,UAAU,EAQf,KAAK,eAAe,EACrB,MAAM,YAAY,CAAC;AAEpB;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,2BAA2B;IAC3B,YAAY,CAAC,EAAE,UAAU,CAAC;IAC1B,iCAAiC;IACjC,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,yBAAyB;IACzB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,0BAA0B;IAC1B,WAAW,CAAC,EAAE,iBAAiB,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,uBAAuB;IACvB,OAAO,CAAC,EAAE,eAAe,CAAC;IAC1B,+BAA+B;IAC/B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,gBAAgB,EACvB,OAAO,GAAE,iBAAsB,GAC9B,UAAU,EAAE,CAqDd"}