@kevinrabun/judges 3.4.0 → 3.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +189 -4
- package/dist/api.d.ts +17 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +18 -0
- package/dist/api.js.map +1 -1
- package/dist/calibration.d.ts +58 -0
- package/dist/calibration.d.ts.map +1 -0
- package/dist/calibration.js +125 -0
- package/dist/calibration.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +334 -12
- package/dist/cli.js.map +1 -1
- package/dist/commands/baseline.d.ts +2 -0
- package/dist/commands/baseline.d.ts.map +1 -0
- package/dist/commands/baseline.js +126 -0
- package/dist/commands/baseline.js.map +1 -0
- package/dist/commands/benchmark.d.ts +100 -0
- package/dist/commands/benchmark.d.ts.map +1 -0
- package/dist/commands/benchmark.js +663 -0
- package/dist/commands/benchmark.js.map +1 -0
- package/dist/commands/completions.d.ts +2 -0
- package/dist/commands/completions.d.ts.map +1 -0
- package/dist/commands/completions.js +226 -0
- package/dist/commands/completions.js.map +1 -0
- package/dist/commands/config-share.d.ts +49 -0
- package/dist/commands/config-share.d.ts.map +1 -0
- package/dist/commands/config-share.js +181 -0
- package/dist/commands/config-share.js.map +1 -0
- package/dist/commands/deps.d.ts +6 -0
- package/dist/commands/deps.d.ts.map +1 -0
- package/dist/commands/deps.js +123 -0
- package/dist/commands/deps.js.map +1 -0
- package/dist/commands/diff.d.ts +7 -0
- package/dist/commands/diff.d.ts.map +1 -0
- package/dist/commands/diff.js +209 -0
- package/dist/commands/diff.js.map +1 -0
- package/dist/commands/docs.d.ts +2 -0
- package/dist/commands/docs.d.ts.map +1 -0
- package/dist/commands/docs.js +157 -0
- package/dist/commands/docs.js.map +1 -0
- package/dist/commands/feedback.d.ts +87 -0
- package/dist/commands/feedback.d.ts.map +1 -0
- package/dist/commands/feedback.js +320 -0
- package/dist/commands/feedback.js.map +1 -0
- package/dist/commands/language-packs.d.ts +43 -0
- package/dist/commands/language-packs.d.ts.map +1 -0
- package/dist/commands/language-packs.js +151 -0
- package/dist/commands/language-packs.js.map +1 -0
- package/dist/commands/rule.d.ts +50 -0
- package/dist/commands/rule.d.ts.map +1 -0
- package/dist/commands/rule.js +202 -0
- package/dist/commands/rule.js.map +1 -0
- package/dist/commands/smart-output.d.ts +39 -0
- package/dist/commands/smart-output.d.ts.map +1 -0
- package/dist/commands/smart-output.js +176 -0
- package/dist/commands/smart-output.js.map +1 -0
- package/dist/commands/watch.js.map +1 -1
- package/dist/comparison.d.ts +68 -0
- package/dist/comparison.d.ts.map +1 -0
- package/dist/comparison.js +254 -0
- package/dist/comparison.js.map +1 -0
- package/dist/fingerprint.d.ts +40 -0
- package/dist/fingerprint.d.ts.map +1 -0
- package/dist/fingerprint.js +180 -0
- package/dist/fingerprint.js.map +1 -0
- package/dist/fix-history.d.ts +72 -0
- package/dist/fix-history.d.ts.map +1 -0
- package/dist/fix-history.js +127 -0
- package/dist/fix-history.js.map +1 -0
- package/dist/formatters/badge.d.ts +17 -0
- package/dist/formatters/badge.d.ts.map +1 -0
- package/dist/formatters/badge.js +79 -0
- package/dist/formatters/badge.js.map +1 -0
- package/dist/formatters/codeclimate.d.ts +25 -0
- package/dist/formatters/codeclimate.d.ts.map +1 -0
- package/dist/formatters/codeclimate.js +81 -0
- package/dist/formatters/codeclimate.js.map +1 -0
- package/dist/formatters/diagnostics.d.ts +82 -0
- package/dist/formatters/diagnostics.d.ts.map +1 -0
- package/dist/formatters/diagnostics.js +153 -0
- package/dist/formatters/diagnostics.js.map +1 -0
- package/dist/formatters/junit.d.ts +7 -0
- package/dist/formatters/junit.d.ts.map +1 -0
- package/dist/formatters/junit.js +69 -0
- package/dist/formatters/junit.js.map +1 -0
- package/dist/index.js +23 -2
- package/dist/index.js.map +1 -1
- package/dist/plugins.d.ts +103 -0
- package/dist/plugins.d.ts.map +1 -0
- package/dist/plugins.js +187 -0
- package/dist/plugins.js.map +1 -0
- package/dist/presets.d.ts +22 -0
- package/dist/presets.d.ts.map +1 -0
- package/dist/presets.js +115 -0
- package/dist/presets.js.map +1 -0
- package/judgesrc.schema.json +74 -0
- package/package.json +30 -1
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ An MCP (Model Context Protocol) server that provides a panel of **35 specialized
|
|
|
11
11
|
[](https://www.npmjs.com/package/@kevinrabun/judges)
|
|
12
12
|
[](https://www.npmjs.com/package/@kevinrabun/judges)
|
|
13
13
|
[](https://opensource.org/licenses/MIT)
|
|
14
|
-
[](https://github.com/KevinRabun/judges/actions)
|
|
15
15
|
|
|
16
16
|
---
|
|
17
17
|
|
|
@@ -63,6 +63,15 @@ judges eval --fail-on-findings src/api.ts
|
|
|
63
63
|
# Suppress known findings via baseline
|
|
64
64
|
judges eval --baseline baseline.json src/api.ts
|
|
65
65
|
|
|
66
|
+
# Use a named preset
|
|
67
|
+
judges eval --preset security-only src/api.ts
|
|
68
|
+
|
|
69
|
+
# Use a config file
|
|
70
|
+
judges eval --config .judgesrc.json src/api.ts
|
|
71
|
+
|
|
72
|
+
# Set a minimum score threshold (exit 1 if below)
|
|
73
|
+
judges eval --min-score 80 src/api.ts
|
|
74
|
+
|
|
66
75
|
# One-line summary for scripts
|
|
67
76
|
judges eval --summary src/api.ts
|
|
68
77
|
|
|
@@ -88,6 +97,32 @@ judges watch src/
|
|
|
88
97
|
# Project-level report (local directory)
|
|
89
98
|
judges report . --format html --output report.html
|
|
90
99
|
|
|
100
|
+
# Evaluate a unified diff (pipe from git diff)
|
|
101
|
+
git diff HEAD~1 | judges diff
|
|
102
|
+
|
|
103
|
+
# Analyze dependencies for supply-chain risks
|
|
104
|
+
judges deps --path . --format json
|
|
105
|
+
|
|
106
|
+
# Create a baseline file to suppress known findings
|
|
107
|
+
judges baseline create --file src/api.ts -o baseline.json
|
|
108
|
+
|
|
109
|
+
# Generate CI template files
|
|
110
|
+
judges ci-templates --provider github
|
|
111
|
+
judges ci-templates --provider gitlab
|
|
112
|
+
judges ci-templates --provider azure
|
|
113
|
+
judges ci-templates --provider bitbucket
|
|
114
|
+
|
|
115
|
+
# Generate per-judge rule documentation
|
|
116
|
+
judges docs
|
|
117
|
+
judges docs --judge cybersecurity
|
|
118
|
+
judges docs --output docs/
|
|
119
|
+
|
|
120
|
+
# Install shell completions
|
|
121
|
+
judges completions bash # eval "$(judges completions bash)"
|
|
122
|
+
judges completions zsh
|
|
123
|
+
judges completions fish
|
|
124
|
+
judges completions powershell
|
|
125
|
+
|
|
91
126
|
# Install pre-commit hook
|
|
92
127
|
judges hook install
|
|
93
128
|
|
|
@@ -371,11 +406,17 @@ Evaluate a file with all 35 judges or a single judge.
|
|
|
371
406
|
| `--file <path>` / positional | File to evaluate |
|
|
372
407
|
| `--judge <id>` / `-j <id>` | Single judge mode |
|
|
373
408
|
| `--language <lang>` / `-l <lang>` | Language hint (auto-detected from extension) |
|
|
374
|
-
| `--format <fmt>` / `-f <fmt>` | Output format: `text`, `json`, `sarif`, `markdown`, `html` |
|
|
409
|
+
| `--format <fmt>` / `-f <fmt>` | Output format: `text`, `json`, `sarif`, `markdown`, `html`, `junit`, `codeclimate` |
|
|
375
410
|
| `--output <path>` / `-o <path>` | Write output to file |
|
|
376
411
|
| `--fail-on-findings` | Exit with code 1 if verdict is FAIL |
|
|
377
412
|
| `--baseline <path>` / `-b <path>` | JSON baseline file — suppress known findings |
|
|
378
413
|
| `--summary` | Print a single summary line (ideal for scripts) |
|
|
414
|
+
| `--config <path>` | Load a `.judgesrc` / `.judgesrc.json` config file |
|
|
415
|
+
| `--preset <name>` | Use a named preset: `strict`, `lenient`, `security-only`, `startup`, `compliance`, `performance` |
|
|
416
|
+
| `--min-score <n>` | Exit with code 1 if overall score is below this threshold |
|
|
417
|
+
| `--verbose` | Print timing and debug information |
|
|
418
|
+
| `--quiet` | Suppress non-essential output |
|
|
419
|
+
| `--no-color` | Disable ANSI colors |
|
|
379
420
|
|
|
380
421
|
### `judges init`
|
|
381
422
|
|
|
@@ -428,6 +469,131 @@ judges hook uninstall # remove pre-commit hook
|
|
|
428
469
|
|
|
429
470
|
Detects Husky (`.husky/pre-commit`) and falls back to `.git/hooks/pre-commit`. Uses marker-based injection so it won't clobber existing hooks.
|
|
430
471
|
|
|
472
|
+
### `judges diff`
|
|
473
|
+
|
|
474
|
+
Evaluate only the changed lines from a unified diff (e.g., `git diff` output).
|
|
475
|
+
|
|
476
|
+
| Flag | Description |
|
|
477
|
+
|------|-------------|
|
|
478
|
+
| `--file <path>` | Read diff from file instead of stdin |
|
|
479
|
+
| `--format <fmt>` | Output format: `text`, `json`, `sarif`, `junit`, `codeclimate` |
|
|
480
|
+
| `--output <path>` | Write output to file |
|
|
481
|
+
|
|
482
|
+
```bash
|
|
483
|
+
git diff HEAD~1 | judges diff
|
|
484
|
+
judges diff --file changes.patch --format sarif
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
### `judges deps`
|
|
488
|
+
|
|
489
|
+
Analyze project dependencies for supply-chain risks.
|
|
490
|
+
|
|
491
|
+
| Flag | Description |
|
|
492
|
+
|------|-------------|
|
|
493
|
+
| `--path <dir>` | Project root to scan (default: `.`) |
|
|
494
|
+
| `--format <fmt>` | Output format: `text`, `json` |
|
|
495
|
+
|
|
496
|
+
```bash
|
|
497
|
+
judges deps --path .
|
|
498
|
+
judges deps --path ./backend --format json
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
### `judges baseline`
|
|
502
|
+
|
|
503
|
+
Create a baseline file to suppress known findings in future evaluations.
|
|
504
|
+
|
|
505
|
+
```bash
|
|
506
|
+
judges baseline create --file src/api.ts
|
|
507
|
+
judges baseline create --file src/api.ts -o .judges-baseline.json
|
|
508
|
+
```
|
|
509
|
+
|
|
510
|
+
### `judges ci-templates`
|
|
511
|
+
|
|
512
|
+
Generate CI/CD configuration templates for popular providers.
|
|
513
|
+
|
|
514
|
+
```bash
|
|
515
|
+
judges ci-templates --provider github # .github/workflows/judges.yml
|
|
516
|
+
judges ci-templates --provider gitlab # .gitlab-ci.judges.yml
|
|
517
|
+
judges ci-templates --provider azure # azure-pipelines.judges.yml
|
|
518
|
+
judges ci-templates --provider bitbucket # bitbucket-pipelines.yml (snippet)
|
|
519
|
+
```
|
|
520
|
+
|
|
521
|
+
### `judges docs`
|
|
522
|
+
|
|
523
|
+
Generate per-judge rule documentation in Markdown.
|
|
524
|
+
|
|
525
|
+
| Flag | Description |
|
|
526
|
+
|------|-------------|
|
|
527
|
+
| `--judge <id>` | Generate docs for a single judge |
|
|
528
|
+
| `--output <dir>` | Write individual `.md` files per judge |
|
|
529
|
+
|
|
530
|
+
```bash
|
|
531
|
+
judges docs # all judges to stdout
|
|
532
|
+
judges docs --judge cybersecurity # single judge
|
|
533
|
+
judges docs --output docs/judges/ # write files to directory
|
|
534
|
+
```
|
|
535
|
+
|
|
536
|
+
### `judges completions`
|
|
537
|
+
|
|
538
|
+
Generate shell completion scripts.
|
|
539
|
+
|
|
540
|
+
```bash
|
|
541
|
+
eval "$(judges completions bash)" # Bash
|
|
542
|
+
eval "$(judges completions zsh)" # Zsh
|
|
543
|
+
judges completions fish | source # Fish
|
|
544
|
+
judges completions powershell # PowerShell (Register-ArgumentCompleter)
|
|
545
|
+
```
|
|
546
|
+
|
|
547
|
+
### Named Presets
|
|
548
|
+
|
|
549
|
+
Use `--preset` to apply pre-configured evaluation settings:
|
|
550
|
+
|
|
551
|
+
| Preset | Description |
|
|
552
|
+
|--------|-------------|
|
|
553
|
+
| `strict` | All severities, all judges — maximum thoroughness |
|
|
554
|
+
| `lenient` | Only high and critical findings — fast and focused |
|
|
555
|
+
| `security-only` | Security judges only — cybersecurity, data-security, authentication, logging-privacy |
|
|
556
|
+
| `startup` | Skip compliance, sovereignty, i18n judges — move fast |
|
|
557
|
+
| `compliance` | Only compliance, data-sovereignty, authentication — regulatory focus |
|
|
558
|
+
| `performance` | Only performance, scalability, caching, cost-effectiveness |
|
|
559
|
+
|
|
560
|
+
```bash
|
|
561
|
+
judges eval --preset security-only src/api.ts
|
|
562
|
+
judges eval --preset strict --format sarif src/app.ts > results.sarif
|
|
563
|
+
```
|
|
564
|
+
|
|
565
|
+
### CI Output Formats
|
|
566
|
+
|
|
567
|
+
#### JUnit XML
|
|
568
|
+
|
|
569
|
+
Generate JUnit XML for Jenkins, Azure DevOps, GitHub Actions, or GitLab test result viewers:
|
|
570
|
+
|
|
571
|
+
```bash
|
|
572
|
+
judges eval --format junit src/api.ts > results.xml
|
|
573
|
+
```
|
|
574
|
+
|
|
575
|
+
Each judge maps to a `<testsuite>`, each finding becomes a `<testcase>` with `<failure>` for critical/high severity.
|
|
576
|
+
|
|
577
|
+
#### CodeClimate / GitLab Code Quality
|
|
578
|
+
|
|
579
|
+
Generate CodeClimate JSON for GitLab Code Quality or similar tools:
|
|
580
|
+
|
|
581
|
+
```bash
|
|
582
|
+
judges eval --format codeclimate src/api.ts > codequality.json
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
#### Score Badges
|
|
586
|
+
|
|
587
|
+
Generate SVG or text badges for your README:
|
|
588
|
+
|
|
589
|
+
```typescript
|
|
590
|
+
import { generateBadgeSvg, generateBadgeText } from "@kevinrabun/judges/badge";
|
|
591
|
+
|
|
592
|
+
const svg = generateBadgeSvg(85); // shields.io-style SVG
|
|
593
|
+
const text = generateBadgeText(85); // "✓ judges 85/100"
|
|
594
|
+
const svg2 = generateBadgeSvg(75, "quality"); // custom label
|
|
595
|
+
```
|
|
596
|
+
|
|
431
597
|
---
|
|
432
598
|
|
|
433
599
|
## The Judge Panel
|
|
@@ -926,14 +1092,23 @@ judges/
|
|
|
926
1092
|
│ │ └── *.ts # One analyzer per judge (35 files)
|
|
927
1093
|
│ ├── formatters/ # Output formatters
|
|
928
1094
|
│ │ ├── sarif.ts # SARIF 2.1.0 output
|
|
929
|
-
│ │
|
|
1095
|
+
│ │ ├── html.ts # Self-contained HTML report (dark/light theme, filters)
|
|
1096
|
+
│ │ ├── junit.ts # JUnit XML output (Jenkins, Azure DevOps, GitHub Actions)
|
|
1097
|
+
│ │ ├── codeclimate.ts # CodeClimate/GitLab Code Quality JSON
|
|
1098
|
+
│ │ └── badge.ts # SVG and text badge generator
|
|
930
1099
|
│ ├── commands/ # CLI subcommands
|
|
931
1100
|
│ │ ├── init.ts # Interactive project setup wizard
|
|
932
1101
|
│ │ ├── fix.ts # Auto-fix patch preview and application
|
|
933
1102
|
│ │ ├── watch.ts # Watch mode — re-evaluate on save
|
|
934
1103
|
│ │ ├── report.ts # Project-level local report
|
|
935
1104
|
│ │ ├── hook.ts # Pre-commit hook install/uninstall
|
|
936
|
-
│ │
|
|
1105
|
+
│ │ ├── ci-templates.ts # GitLab, Azure, Bitbucket CI templates
|
|
1106
|
+
│ │ ├── diff.ts # Evaluate unified diff (git diff)
|
|
1107
|
+
│ │ ├── deps.ts # Dependency supply-chain analysis
|
|
1108
|
+
│ │ ├── baseline.ts # Create baseline for finding suppression
|
|
1109
|
+
│ │ ├── completions.ts # Shell completions (bash/zsh/fish/PowerShell)
|
|
1110
|
+
│ │ └── docs.ts # Per-judge rule documentation generator
|
|
1111
|
+
│ ├── presets.ts # Named evaluation presets (strict, lenient, security-only, …)
|
|
937
1112
|
│ ├── reports/
|
|
938
1113
|
│ │ └── public-repo-report.ts # Public repo clone + full tribunal report generation
|
|
939
1114
|
│ └── judges/ # Judge definitions (id, name, domain, system prompt)
|
|
@@ -955,6 +1130,7 @@ judges/
|
|
|
955
1130
|
│ ├── tree-sitter-rust.wasm
|
|
956
1131
|
│ ├── tree-sitter-java.wasm
|
|
957
1132
|
│ └── tree-sitter-c_sharp.wasm
|
|
1133
|
+
├── judgesrc.schema.json # JSON Schema for .judgesrc config files
|
|
958
1134
|
├── server.json # MCP Registry manifest
|
|
959
1135
|
├── package.json
|
|
960
1136
|
├── tsconfig.json
|
|
@@ -981,6 +1157,12 @@ judges/
|
|
|
981
1157
|
| `judges watch <dir>` | Watch mode — re-evaluate on file save |
|
|
982
1158
|
| `judges report <dir>` | Full tribunal report on a local directory |
|
|
983
1159
|
| `judges hook install` | Install a Git pre-commit hook |
|
|
1160
|
+
| `judges diff` | Evaluate changed lines from unified diff |
|
|
1161
|
+
| `judges deps` | Analyze dependencies for supply-chain risks |
|
|
1162
|
+
| `judges baseline create` | Create baseline for finding suppression |
|
|
1163
|
+
| `judges ci-templates` | Generate CI pipeline templates |
|
|
1164
|
+
| `judges docs` | Generate per-judge rule documentation |
|
|
1165
|
+
| `judges completions <shell>` | Shell completion scripts |
|
|
984
1166
|
|
|
985
1167
|
---
|
|
986
1168
|
|
|
@@ -1051,6 +1233,9 @@ const sarif = findingsToSarif(verdict.evaluations.flatMap(e => e.findings));
|
|
|
1051
1233
|
| `@kevinrabun/judges/api` | Programmatic API (default) |
|
|
1052
1234
|
| `@kevinrabun/judges/server` | MCP server entry point |
|
|
1053
1235
|
| `@kevinrabun/judges/sarif` | SARIF 2.1.0 formatter |
|
|
1236
|
+
| `@kevinrabun/judges/junit` | JUnit XML formatter |
|
|
1237
|
+
| `@kevinrabun/judges/codeclimate` | CodeClimate/GitLab Code Quality JSON |
|
|
1238
|
+
| `@kevinrabun/judges/badge` | SVG and text badge generator |
|
|
1054
1239
|
|
|
1055
1240
|
### SARIF Output
|
|
1056
1241
|
|
package/dist/api.d.ts
CHANGED
|
@@ -20,6 +20,23 @@ export { clearProjectCache } from "./evaluators/project.js";
|
|
|
20
20
|
export { findingsToSarif, evaluationToSarif, verdictToSarif, validateSarifLog } from "./formatters/sarif.js";
|
|
21
21
|
export type { SarifValidationError } from "./formatters/sarif.js";
|
|
22
22
|
export { runCli } from "./cli.js";
|
|
23
|
+
export { registerPlugin, unregisterPlugin, getRegisteredPlugins, getCustomRules, getPluginJudges, evaluateCustomRules, runBeforeHooks, runAfterHooks, clearPlugins, } from "./plugins.js";
|
|
24
|
+
export type { CustomRule, JudgesPlugin, PluginRegistration } from "./plugins.js";
|
|
25
|
+
export { fingerprintCode, fingerprintToFindings } from "./fingerprint.js";
|
|
26
|
+
export type { AiFingerprint, AiSignal } from "./fingerprint.js";
|
|
27
|
+
export { buildCalibrationProfile, calibrateFindings, autoCalibrateFindings } from "./calibration.js";
|
|
28
|
+
export type { CalibrationProfile } from "./calibration.js";
|
|
29
|
+
export { loadFeedbackStore, saveFeedbackStore, computeFeedbackStats, getFpRateByRule } from "./commands/feedback.js";
|
|
30
|
+
export type { FeedbackEntry, FeedbackStore, FeedbackVerdict, FeedbackStats } from "./commands/feedback.js";
|
|
31
|
+
export { loadFixHistory, saveFixHistory, computeFixStats, recordFixAccepted, recordFixRejected, getFixAcceptanceRate, getLowAcceptanceRules, } from "./fix-history.js";
|
|
32
|
+
export type { FixOutcome, FixHistory, FixStats } from "./fix-history.js";
|
|
33
|
+
export { findingToDiagnostic, findingsToDiagnostics, findingsToCodeActions, formatForProblemMatcher, formatAsJsonRpc, } from "./formatters/diagnostics.js";
|
|
34
|
+
export type { Diagnostic, DiagnosticSeverity, Position, Range, CodeAction, PublishDiagnosticsParams, } from "./formatters/diagnostics.js";
|
|
35
|
+
export { compareCapabilities, formatComparisonReport, formatFullComparisonMatrix, TOOL_PROFILES, CAPABILITY_MATRIX, } from "./comparison.js";
|
|
36
|
+
export type { ToolProfile, ToolCapability, ComparisonResult } from "./comparison.js";
|
|
37
|
+
export { getLanguagePack, listLanguagePacks, suggestPack, LANGUAGE_PACKS } from "./commands/language-packs.js";
|
|
38
|
+
export { formatSmartOutput, formatSmartSingleJudge } from "./commands/smart-output.js";
|
|
39
|
+
export type { SmartOutputOptions } from "./commands/smart-output.js";
|
|
23
40
|
import type { EvaluationOptions } from "./evaluators/index.js";
|
|
24
41
|
import type { JudgeEvaluation, TribunalVerdict } from "./types.js";
|
|
25
42
|
/**
|
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,YAAY,EACV,QAAQ,EACR,OAAO,EACP,OAAO,EACP,KAAK,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,WAAW,EACX,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,YAAY,EACZ,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGpF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAIxE,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAGnG,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAGlE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,YAAY,EACV,QAAQ,EACR,OAAO,EACP,OAAO,EACP,KAAK,EACL,UAAU,EACV,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,WAAW,EACX,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,wBAAwB,EACxB,oBAAoB,EACpB,YAAY,EACZ,aAAa,GACd,MAAM,YAAY,CAAC;AAGpB,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGpF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAIxE,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAGnG,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAGlE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAG5D,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAGlE,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGlC,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,UAAU,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAGjF,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAC1E,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAGhE,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACrG,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAG3D,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACrH,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAG3G,OAAO,EACL,cAAc,EACd,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAGzE,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,eAAe,GAChB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EACV,UAAU,EACV,kBAAkB,EAClB,QAAQ,EACR,KAAK,EACL,UAAU,EACV,wBAAwB,GACzB,MAAM,6BAA6B,CAAC;AAGrC,OAAO,EACL,mBAAmB,EACnB,sBAAsB,EACtB,0BAA0B,EAC1B,aAAa,EACb,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AAGrF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAG/G,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AACvF,YAAY,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAKrE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC/D,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAInE;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,eAAe,CAEzG;AAED;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,eAAe,CAMjB"}
|
package/dist/api.js
CHANGED
|
@@ -27,6 +27,24 @@ export { clearProjectCache } from "./evaluators/project.js";
|
|
|
27
27
|
export { findingsToSarif, evaluationToSarif, verdictToSarif, validateSarifLog } from "./formatters/sarif.js";
|
|
28
28
|
// ─── CLI ─────────────────────────────────────────────────────────────────────
|
|
29
29
|
export { runCli } from "./cli.js";
|
|
30
|
+
// ─── Plugin API ──────────────────────────────────────────────────────────────
|
|
31
|
+
export { registerPlugin, unregisterPlugin, getRegisteredPlugins, getCustomRules, getPluginJudges, evaluateCustomRules, runBeforeHooks, runAfterHooks, clearPlugins, } from "./plugins.js";
|
|
32
|
+
// ─── AI Code Fingerprinting ─────────────────────────────────────────────────
|
|
33
|
+
export { fingerprintCode, fingerprintToFindings } from "./fingerprint.js";
|
|
34
|
+
// ─── Confidence Calibration ─────────────────────────────────────────────────
|
|
35
|
+
export { buildCalibrationProfile, calibrateFindings, autoCalibrateFindings } from "./calibration.js";
|
|
36
|
+
// ─── Feedback ────────────────────────────────────────────────────────────────
|
|
37
|
+
export { loadFeedbackStore, saveFeedbackStore, computeFeedbackStats, getFpRateByRule } from "./commands/feedback.js";
|
|
38
|
+
// ─── Fix History / Learning ──────────────────────────────────────────────────
|
|
39
|
+
export { loadFixHistory, saveFixHistory, computeFixStats, recordFixAccepted, recordFixRejected, getFixAcceptanceRate, getLowAcceptanceRules, } from "./fix-history.js";
|
|
40
|
+
// ─── IDE Diagnostics ─────────────────────────────────────────────────────────
|
|
41
|
+
export { findingToDiagnostic, findingsToDiagnostics, findingsToCodeActions, formatForProblemMatcher, formatAsJsonRpc, } from "./formatters/diagnostics.js";
|
|
42
|
+
// ─── Comparison Benchmarks ───────────────────────────────────────────────────
|
|
43
|
+
export { compareCapabilities, formatComparisonReport, formatFullComparisonMatrix, TOOL_PROFILES, CAPABILITY_MATRIX, } from "./comparison.js";
|
|
44
|
+
// ─── Language Packs ──────────────────────────────────────────────────────────
|
|
45
|
+
export { getLanguagePack, listLanguagePacks, suggestPack, LANGUAGE_PACKS } from "./commands/language-packs.js";
|
|
46
|
+
// ─── Smart Output ────────────────────────────────────────────────────────────
|
|
47
|
+
export { formatSmartOutput, formatSmartSingleJudge } from "./commands/smart-output.js";
|
|
30
48
|
// ─── Convenience Aliases ─────────────────────────────────────────────────────
|
|
31
49
|
import { evaluateWithTribunal, evaluateWithJudge } from "./evaluators/index.js";
|
|
32
50
|
import { getJudge } from "./judges/index.js";
|
package/dist/api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA+BH,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEpF,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEzD,gFAAgF;AAChF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAExE,gFAAgF;AAEhF,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAE/B,+EAA+E;AAC/E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAEnG,gFAAgF;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAElE,gFAAgF;AAChF,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,gFAAgF;AAChF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAG7G,gFAAgF;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,gFAAgF;AAEhF,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGhF,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB,EAAE,OAA2B;IACtF,OAAO,oBAAoB,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,IAAY,EACZ,QAAgB,EAChB,OAA2B;IAE3B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,eAAe,CAAC,mBAAmB,OAAO,GAAG,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC"}
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AA+BH,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEpF,gFAAgF;AAChF,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAEzD,gFAAgF;AAChF,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAExE,gFAAgF;AAEhF,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EACjB,mBAAmB,EACnB,uBAAuB,EACvB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,qBAAqB,GACtB,MAAM,uBAAuB,CAAC;AAE/B,+EAA+E;AAC/E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAEnG,gFAAgF;AAChF,OAAO,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAElE,gFAAgF;AAChF,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAE5D,gFAAgF;AAChF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAG7G,gFAAgF;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAElC,gFAAgF;AAChF,OAAO,EACL,cAAc,EACd,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,EACd,eAAe,EACf,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,YAAY,GACb,MAAM,cAAc,CAAC;AAGtB,+EAA+E;AAC/E,OAAO,EAAE,eAAe,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAG1E,+EAA+E;AAC/E,OAAO,EAAE,uBAAuB,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGrG,gFAAgF;AAChF,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAGrH,gFAAgF;AAChF,OAAO,EACL,cAAc,EACd,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAG1B,gFAAgF;AAChF,OAAO,EACL,mBAAmB,EACnB,qBAAqB,EACrB,qBAAqB,EACrB,uBAAuB,EACvB,eAAe,GAChB,MAAM,6BAA6B,CAAC;AAUrC,gFAAgF;AAChF,OAAO,EACL,mBAAmB,EACnB,sBAAsB,EACtB,0BAA0B,EAC1B,aAAa,EACb,iBAAiB,GAClB,MAAM,iBAAiB,CAAC;AAGzB,gFAAgF;AAChF,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAE/G,gFAAgF;AAChF,OAAO,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,4BAA4B,CAAC;AAGvF,gFAAgF;AAEhF,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGhF,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,QAAgB,EAAE,OAA2B;IACtF,OAAO,oBAAoB,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AAClE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAAe,EACf,IAAY,EACZ,QAAgB,EAChB,OAA2B;IAE3B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,eAAe,CAAC,mBAAmB,OAAO,GAAG,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,iBAAiB,CAAC,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;AACtE,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Confidence Calibration System
|
|
3
|
+
*
|
|
4
|
+
* Uses historical feedback data (from `judges feedback`) to dynamically
|
|
5
|
+
* adjust finding confidence scores based on observed false-positive rates.
|
|
6
|
+
*
|
|
7
|
+
* When a rule has a known high FP rate from user feedback, its confidence
|
|
8
|
+
* is reduced proportionally. When a rule has a proven high TP rate, its
|
|
9
|
+
* confidence is boosted.
|
|
10
|
+
*/
|
|
11
|
+
import type { Finding } from "./types.js";
|
|
12
|
+
import { type FeedbackStore } from "./commands/feedback.js";
|
|
13
|
+
export interface CalibrationProfile {
|
|
14
|
+
/** Name of the calibration profile */
|
|
15
|
+
name: string;
|
|
16
|
+
/** FP rate lookup by rule ID */
|
|
17
|
+
fpRateByRule: Map<string, number>;
|
|
18
|
+
/** FP rate lookup by rule prefix (judge-level) */
|
|
19
|
+
fpRateByPrefix: Map<string, number>;
|
|
20
|
+
/** Whether calibration is active (has enough data) */
|
|
21
|
+
isActive: boolean;
|
|
22
|
+
/** Total feedback entries used for calibration */
|
|
23
|
+
feedbackCount: number;
|
|
24
|
+
}
|
|
25
|
+
export interface CalibrationOptions {
|
|
26
|
+
/** Path to feedback store file */
|
|
27
|
+
feedbackPath?: string;
|
|
28
|
+
/** Minimum feedback entries for a rule before calibration applies */
|
|
29
|
+
minSamples?: number;
|
|
30
|
+
/** Maximum confidence reduction from calibration (default: 0.3) */
|
|
31
|
+
maxReduction?: number;
|
|
32
|
+
/** Maximum confidence boost from calibration (default: 0.15) */
|
|
33
|
+
maxBoost?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Load a calibration profile from the feedback store.
|
|
37
|
+
*/
|
|
38
|
+
export declare function loadCalibrationProfile(options?: CalibrationOptions): CalibrationProfile;
|
|
39
|
+
/**
|
|
40
|
+
* Build a calibration profile from a feedback store.
|
|
41
|
+
*/
|
|
42
|
+
export declare function buildCalibrationProfile(store: FeedbackStore, options?: CalibrationOptions): CalibrationProfile;
|
|
43
|
+
/**
|
|
44
|
+
* Apply confidence calibration to a list of findings based on
|
|
45
|
+
* historical feedback data.
|
|
46
|
+
*
|
|
47
|
+
* - High FP rate → confidence is reduced
|
|
48
|
+
* - Low FP rate (high TP rate) → confidence is boosted
|
|
49
|
+
* - Neutral (FP rate ~50%) → no change
|
|
50
|
+
*
|
|
51
|
+
* Calibration threshold: FP rate > 0.5 → reduce, FP rate < 0.2 → boost
|
|
52
|
+
*/
|
|
53
|
+
export declare function calibrateFindings(findings: Finding[], profile: CalibrationProfile, options?: CalibrationOptions): Finding[];
|
|
54
|
+
/**
|
|
55
|
+
* Convenience: load feedback, build profile, and calibrate findings in one call.
|
|
56
|
+
*/
|
|
57
|
+
export declare function autoCalibrateFindings(findings: Finding[], options?: CalibrationOptions): Finding[];
|
|
58
|
+
//# sourceMappingURL=calibration.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibration.d.ts","sourceRoot":"","sources":["../src/calibration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAsC,KAAK,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAIhG,MAAM,WAAW,kBAAkB;IACjC,sCAAsC;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,gCAAgC;IAChC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,kDAAkD;IAClD,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,sDAAsD;IACtD,QAAQ,EAAE,OAAO,CAAC;IAClB,kDAAkD;IAClD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IACjC,kCAAkC;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAQD;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,kBAAkB,CAGvF;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,kBAAkB,CAgD9G;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,OAAO,EAAE,EACnB,OAAO,EAAE,kBAAkB,EAC3B,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,EAAE,CAsCX;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,EAAE,CAGlG"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Confidence Calibration System
|
|
3
|
+
*
|
|
4
|
+
* Uses historical feedback data (from `judges feedback`) to dynamically
|
|
5
|
+
* adjust finding confidence scores based on observed false-positive rates.
|
|
6
|
+
*
|
|
7
|
+
* When a rule has a known high FP rate from user feedback, its confidence
|
|
8
|
+
* is reduced proportionally. When a rule has a proven high TP rate, its
|
|
9
|
+
* confidence is boosted.
|
|
10
|
+
*/
|
|
11
|
+
import { loadFeedbackStore } from "./commands/feedback.js";
|
|
12
|
+
// ─── Calibration Engine ─────────────────────────────────────────────────────
|
|
13
|
+
const DEFAULT_MIN_SAMPLES = 3;
|
|
14
|
+
const DEFAULT_MAX_REDUCTION = 0.3;
|
|
15
|
+
const DEFAULT_MAX_BOOST = 0.15;
|
|
16
|
+
/**
|
|
17
|
+
* Load a calibration profile from the feedback store.
|
|
18
|
+
*/
|
|
19
|
+
export function loadCalibrationProfile(options) {
|
|
20
|
+
const store = loadFeedbackStore(options?.feedbackPath);
|
|
21
|
+
return buildCalibrationProfile(store, options);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Build a calibration profile from a feedback store.
|
|
25
|
+
*/
|
|
26
|
+
export function buildCalibrationProfile(store, options) {
|
|
27
|
+
const minSamples = options?.minSamples ?? DEFAULT_MIN_SAMPLES;
|
|
28
|
+
const fpRateByRule = new Map();
|
|
29
|
+
const fpRateByPrefix = new Map();
|
|
30
|
+
// Group entries by rule ID
|
|
31
|
+
const byRule = new Map();
|
|
32
|
+
const byPrefix = new Map();
|
|
33
|
+
for (const entry of store.entries) {
|
|
34
|
+
// Per-rule aggregation
|
|
35
|
+
const ruleStats = byRule.get(entry.ruleId) || { tp: 0, fp: 0, total: 0 };
|
|
36
|
+
ruleStats.total++;
|
|
37
|
+
if (entry.verdict === "tp")
|
|
38
|
+
ruleStats.tp++;
|
|
39
|
+
else if (entry.verdict === "fp")
|
|
40
|
+
ruleStats.fp++;
|
|
41
|
+
byRule.set(entry.ruleId, ruleStats);
|
|
42
|
+
// Per-prefix aggregation
|
|
43
|
+
const prefix = entry.ruleId.split("-")[0];
|
|
44
|
+
if (prefix) {
|
|
45
|
+
const prefixStats = byPrefix.get(prefix) || { tp: 0, fp: 0, total: 0 };
|
|
46
|
+
prefixStats.total++;
|
|
47
|
+
if (entry.verdict === "tp")
|
|
48
|
+
prefixStats.tp++;
|
|
49
|
+
else if (entry.verdict === "fp")
|
|
50
|
+
prefixStats.fp++;
|
|
51
|
+
byPrefix.set(prefix, prefixStats);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
// Compute FP rates for rules with enough data
|
|
55
|
+
for (const [ruleId, stats] of byRule) {
|
|
56
|
+
if (stats.total >= minSamples) {
|
|
57
|
+
fpRateByRule.set(ruleId, stats.fp / stats.total);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
for (const [prefix, stats] of byPrefix) {
|
|
61
|
+
if (stats.total >= minSamples) {
|
|
62
|
+
fpRateByPrefix.set(prefix, stats.fp / stats.total);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
name: "feedback-calibrated",
|
|
67
|
+
fpRateByRule,
|
|
68
|
+
fpRateByPrefix,
|
|
69
|
+
isActive: fpRateByRule.size > 0 || fpRateByPrefix.size > 0,
|
|
70
|
+
feedbackCount: store.entries.length,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Apply confidence calibration to a list of findings based on
|
|
75
|
+
* historical feedback data.
|
|
76
|
+
*
|
|
77
|
+
* - High FP rate → confidence is reduced
|
|
78
|
+
* - Low FP rate (high TP rate) → confidence is boosted
|
|
79
|
+
* - Neutral (FP rate ~50%) → no change
|
|
80
|
+
*
|
|
81
|
+
* Calibration threshold: FP rate > 0.5 → reduce, FP rate < 0.2 → boost
|
|
82
|
+
*/
|
|
83
|
+
export function calibrateFindings(findings, profile, options) {
|
|
84
|
+
if (!profile.isActive)
|
|
85
|
+
return findings;
|
|
86
|
+
const maxReduction = options?.maxReduction ?? DEFAULT_MAX_REDUCTION;
|
|
87
|
+
const maxBoost = options?.maxBoost ?? DEFAULT_MAX_BOOST;
|
|
88
|
+
return findings.map((f) => {
|
|
89
|
+
const currentConf = f.confidence ?? 0.5;
|
|
90
|
+
// Look up FP rate: prefer rule-specific, fall back to prefix
|
|
91
|
+
const ruleFpRate = profile.fpRateByRule.get(f.ruleId);
|
|
92
|
+
const prefix = f.ruleId.split("-")[0];
|
|
93
|
+
const prefixFpRate = profile.fpRateByPrefix.get(prefix);
|
|
94
|
+
const fpRate = ruleFpRate ?? prefixFpRate;
|
|
95
|
+
if (fpRate === undefined)
|
|
96
|
+
return f;
|
|
97
|
+
let adjustment = 0;
|
|
98
|
+
if (fpRate > 0.5) {
|
|
99
|
+
// High FP rate: reduce confidence proportionally
|
|
100
|
+
// FP rate 0.5 → 0% reduction, FP rate 1.0 → maxReduction
|
|
101
|
+
adjustment = -maxReduction * ((fpRate - 0.5) / 0.5);
|
|
102
|
+
}
|
|
103
|
+
else if (fpRate < 0.2) {
|
|
104
|
+
// Low FP rate: boost confidence
|
|
105
|
+
// FP rate 0.2 → 0% boost, FP rate 0.0 → maxBoost
|
|
106
|
+
adjustment = maxBoost * ((0.2 - fpRate) / 0.2);
|
|
107
|
+
}
|
|
108
|
+
if (adjustment === 0)
|
|
109
|
+
return f;
|
|
110
|
+
const calibratedConf = Math.max(0.05, Math.min(1.0, currentConf + adjustment));
|
|
111
|
+
return {
|
|
112
|
+
...f,
|
|
113
|
+
confidence: calibratedConf,
|
|
114
|
+
provenance: f.provenance ? `${f.provenance}, confidence-calibrated` : "confidence-calibrated",
|
|
115
|
+
};
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Convenience: load feedback, build profile, and calibrate findings in one call.
|
|
120
|
+
*/
|
|
121
|
+
export function autoCalibrateFindings(findings, options) {
|
|
122
|
+
const profile = loadCalibrationProfile(options);
|
|
123
|
+
return calibrateFindings(findings, profile, options);
|
|
124
|
+
}
|
|
125
|
+
//# sourceMappingURL=calibration.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"calibration.js","sourceRoot":"","sources":["../src/calibration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,EAAE,iBAAiB,EAAuC,MAAM,wBAAwB,CAAC;AA4BhG,+EAA+E;AAE/E,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B;;GAEG;AACH,MAAM,UAAU,sBAAsB,CAAC,OAA4B;IACjE,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACvD,OAAO,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AACjD,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAoB,EAAE,OAA4B;IACxF,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,mBAAmB,CAAC;IAC9D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEjD,2BAA2B;IAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqD,CAAC;IAC5E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAqD,CAAC;IAE9E,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAClC,uBAAuB;QACvB,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACzE,SAAS,CAAC,KAAK,EAAE,CAAC;QAClB,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI;YAAE,SAAS,CAAC,EAAE,EAAE,CAAC;aACtC,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI;YAAE,SAAS,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAEpC,yBAAyB;QACzB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACvE,WAAW,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI;gBAAE,WAAW,CAAC,EAAE,EAAE,CAAC;iBACxC,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI;gBAAE,WAAW,CAAC,EAAE,EAAE,CAAC;YAClD,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,8CAA8C;IAC9C,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9B,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvC,IAAI,KAAK,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;YAC9B,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,qBAAqB;QAC3B,YAAY;QACZ,cAAc;QACd,QAAQ,EAAE,YAAY,CAAC,IAAI,GAAG,CAAC,IAAI,cAAc,CAAC,IAAI,GAAG,CAAC;QAC1D,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;KACpC,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAC/B,QAAmB,EACnB,OAA2B,EAC3B,OAA4B;IAE5B,IAAI,CAAC,OAAO,CAAC,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAEvC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,qBAAqB,CAAC;IACpE,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,iBAAiB,CAAC;IAExD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,WAAW,GAAG,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC;QAExC,6DAA6D;QAC7D,MAAM,UAAU,GAAG,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,YAAY,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,MAAM,GAAG,UAAU,IAAI,YAAY,CAAC;QAE1C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,CAAC,CAAC;QAEnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;YACjB,iDAAiD;YACjD,yDAAyD;YACzD,UAAU,GAAG,CAAC,YAAY,GAAG,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QACtD,CAAC;aAAM,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;YACxB,gCAAgC;YAChC,iDAAiD;YACjD,UAAU,GAAG,QAAQ,GAAG,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC;QACjD,CAAC;QAED,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAE/B,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,WAAW,GAAG,UAAU,CAAC,CAAC,CAAC;QAC/E,OAAO;YACL,GAAG,CAAC;YACJ,UAAU,EAAE,cAAc;YAC1B,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,yBAAyB,CAAC,CAAC,CAAC,uBAAuB;SAC9F,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,QAAmB,EAAE,OAA4B;IACrF,MAAM,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,CAAC;IAChD,OAAO,iBAAiB,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AACvD,CAAC"}
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;;;;GAmBG;
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA;;;;;;;;;;;;;;;;;;;GAmBG;AA6bH,wBAAsB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAmQ1D"}
|