ownkey 1.1.2 → 1.3.0

package/README.md

@@ -102,16 +102,33 @@ ownkey undo --list                    # recent applies
 
 For a whole-project audit of any folder, git or not, use `ownkey suggest .`
 
+## Interactive session
+
+Run `ownkey` with no arguments in any project folder and it opens a session for that folder. The header shows the repository, branch, how many files changed, and which model is reviewing. Then type any command without the `ownkey` prefix:
+
+```text
+› review --staged
+› apply 3 --verify "npm test"
+› undo --last
+› bench --case off-by-one
+```
+
+Tab completes commands and their flags, the arrow keys walk your history (kept in `~/.ownkey/history`), `help` lists every command, and Ctrl+D leaves. Commands run exactly as they do from the shell, including interactive prompts like the `config` wizard. Ctrl+C stops the running command without closing the session.
+
+Outside an interactive terminal (pipes, scripts, CI), or with `OWNKEY_NO_UI=1` set, a bare `ownkey` prints help as before. Every subcommand behaves the same with or without the session.
+
 ## Commands
 
 | Command | Purpose |
 |---|---|
+| `ownkey` | Open the interactive session for the current folder. |
 | `ownkey config` | Set the AI provider, model, endpoint, and optional database. Run with no flags for an interactive wizard. |
 | `ownkey review [path]` | Review git changes: staged, uncommitted, or a branch against its base. |
 | `ownkey suggest [path]` | Audit a whole folder and store suggestions. |
 | `ownkey scan [path]` | List the files OwnKey would analyze, honoring `.gitignore` and `.ownkeyignore`. |
 | `ownkey apply [id]` | Apply one suggestion, or all pending ones with `--all`, optionally verified. |
 | `ownkey undo [backup-id]` | Restore files from a backup created by `apply`. |
+| `ownkey bench` | Measure review quality on seeded-bug cases with your model. |
 
 <details>
 <summary><strong>Full option reference</strong></summary>
@@ -147,6 +164,17 @@ For a whole-project audit of any folder, git or not, use `ownkey suggest .`
 
 Exit codes: `0` success, `1` the `--fail-on` gate tripped, `2` the review could not run.
 
+#### `ownkey bench`
+
+| Option | Description |
+|---|---|
+| `--list` | List the cases and exit. |
+| `--case <ids>` | Comma-separated case ids to run. Default: all. |
+| `--model <model>` | Use this model for the run instead of the configured one. |
+| `--format <format>` | `text` (default), `json`, or `markdown`. |
+| `--output <file>` | Write the report to a file. |
+| `--min-recall <ratio>` | Exit with code 1 when recall is below this ratio, from `0` to `1`. |
+
 #### `ownkey scan [path]`
 
 | Option | Description |
@@ -190,6 +218,21 @@ Exit codes: `0` success, `1` the `--fail-on` gate tripped, `2` the review could
 
 </details>
 
+## Measuring review quality
+
+`ownkey bench` shows how well a model reviews before you trust it with your pull requests. It runs a fixed set of small cases, each a throwaway git repository with one staged change:
+
+- **Bug cases** plant one real defect: an off-by-one, SQL injection, a missing `await`, a null dereference, a hardcoded secret, path traversal, command injection, an inverted auth check, a mutable default argument, a numeric sort with the string comparator, a switch fallthrough, and `is` used to compare strings. They cover JavaScript, TypeScript, and Python.
+- **Control cases** are correct changes. Anything reported there at medium severity or above is a false alarm.
+
+```bash
+ownkey bench                                  # all cases with your configured model
+ownkey bench --model claude-opus-5-5          # compare another model on the same key
+ownkey bench --format markdown --output bench.md
+```
+
+It reports **recall** (planted bugs reported within 3 lines of the defect with a matching type), **precision** (matching findings over all findings at medium or above), and how many controls stayed clean. Low and info findings never count against a model. Bench runs in temporary directories and stores nothing, but it does send every case to your provider, so a full run costs about 15 small review calls.
+
 ## Providers
 
 | Provider | Setup | Default model |

package/dist/ai/factory.d.ts

@@ -6,7 +6,9 @@ export declare class AIProviderFactory {
     /**
      * Create an AI provider instance based on configuration
      */
-    static createProvider(): Promise<AIProvider>;
+    static createProvider(overrides?: {
+        model?: string;
+    }): Promise<AIProvider>;
     /**
      * Test provider connection
      */

package/dist/ai/factory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C;;GAEG;AACH,qBAAa,iBAAiB;IAC1B;;OAEG;WACU,cAAc,IAAI,OAAO,CAAC,UAAU,CAAC;IA6ClD;;OAEG;WACU,YAAY,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBjE;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAepD;;OAEG;WACU,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAmBvF"}
+{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C;;GAEG;AACH,qBAAa,iBAAiB;IAC1B;;OAEG;WACU,cAAc,CAAC,SAAS,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,UAAU,CAAC;IA6CpF;;OAEG;WACU,YAAY,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBjE;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAepD;;OAEG;WACU,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAmBvF"}

package/dist/ai/factory.js

@@ -13,7 +13,7 @@ export class AIProviderFactory {
     /**
      * Create an AI provider instance based on configuration
      */
-    static async createProvider() {
+    static async createProvider(overrides = {}) {
         const config = await configManager.load();
         if (!config.activeProvider) {
             throw new ConfigError('No active AI provider configured. Run: ownkey config');
@@ -26,23 +26,23 @@ export class AIProviderFactory {
                 if (!apiKey) {
                     throw new ConfigError('Gemini API key not found. Run: ownkey config');
                 }
-                const geminiModel = config.providers.gemini?.model || GeminiProvider.getRecommendedModel();
+                const geminiModel = overrides.model || config.providers.gemini?.model || GeminiProvider.getRecommendedModel();
                 return new GeminiProvider(apiKey, geminiModel);
             case 'openai':
                 if (!apiKey) {
                     throw new ConfigError('OpenAI API key not found. Run: ownkey config');
                 }
-                const openaiModel = config.providers.openai?.model || DEFAULT_MODELS.openai;
+                const openaiModel = overrides.model || config.providers.openai?.model || DEFAULT_MODELS.openai;
                 return new OpenAIProvider(apiKey, openaiModel, config.providers.openai?.endpoint);
             case 'anthropic':
                 if (!apiKey) {
                     throw new ConfigError('Anthropic API key not found. Run: ownkey config');
                 }
-                const anthropicModel = config.providers.anthropic?.model || DEFAULT_MODELS.anthropic;
+                const anthropicModel = overrides.model || config.providers.anthropic?.model || DEFAULT_MODELS.anthropic;
                 return new AnthropicProvider(apiKey, anthropicModel);
             case 'ollama':
                 // Ollama doesn't require API key - runs locally
-                const ollamaModel = config.providers.ollama?.model || DEFAULT_MODELS.ollama;
+                const ollamaModel = overrides.model || config.providers.ollama?.model || DEFAULT_MODELS.ollama;
                 const ollamaBaseUrl = config.providers.ollama?.endpoint || 'http://localhost:11434';
                 return new OllamaProvider(undefined, ollamaModel, ollamaBaseUrl);
             default:

package/dist/ai/factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAC1B;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc;QACvB,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,WAAW,CAAC,sDAAsD,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,cAAc,CAAC,CAAC;QAEnD,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,mBAAmB,EAAE,CAAC;gBAC3F,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAEnD,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC5E,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEtF,KAAK,WAAW;gBACZ,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,iDAAiD,CAAC,CAAC;gBAC7E,CAAC;gBACD,MAAM,cAAc,GAAG,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC;gBACrF,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEzD,KAAK,QAAQ;gBACT,gDAAgD;gBAChD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC5E,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,IAAI,wBAAwB,CAAC;gBACpF,OAAO,IAAI,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAErE;gBACI,MAAM,IAAI,WAAW,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,QAAoB;QAC1C,IAAI,CAAC;YACD,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAgB;QACvC,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAChD,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC,KAAK,WAAW;gBACZ,OAAO,cAAc,CAAC,SAAS,CAAC;YACpC,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC;gBACI,OAAO,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAgB,EAAE,MAAc;QAC5D,IAAI,CAAC;YACD,QAAQ,QAAQ,EAAE,CAAC;gBACf,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,WAAW;oBACZ,OAAO,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC9D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;gBACxE;oBACI,OAAO,EAAE,CAAC;YAClB,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,EAAE,KAAc,CAAC,CAAC;YACxE,OAAO,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;CACJ"}
+{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAC1B;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,YAAgC,EAAE;QAC1D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,WAAW,CAAC,sDAAsD,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,cAAc,CAAC,CAAC;QAEnD,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,mBAAmB,EAAE,CAAC;gBAC9G,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAEnD,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC/F,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEtF,KAAK,WAAW;gBACZ,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,iDAAiD,CAAC,CAAC;gBAC7E,CAAC;gBACD,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC;gBACxG,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEzD,KAAK,QAAQ;gBACT,gDAAgD;gBAChD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,IAAI,wBAAwB,CAAC;gBACpF,OAAO,IAAI,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAErE;gBACI,MAAM,IAAI,WAAW,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,QAAoB;QAC1C,IAAI,CAAC;YACD,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAgB;QACvC,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAChD,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC,KAAK,WAAW;gBACZ,OAAO,cAAc,CAAC,SAAS,CAAC;YACpC,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC;gBACI,OAAO,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAgB,EAAE,MAAc;QAC5D,IAAI,CAAC;YACD,QAAQ,QAAQ,EAAE,CAAC;gBACf,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,WAAW;oBACZ,OAAO,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC9D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;gBACxE;oBACI,OAAO,EAAE,CAAC;YAClB,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,EAAE,KAAc,CAAC,CAAC;YACxE,OAAO,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;CACJ"}

package/dist/bench/cases.d.ts

@@ -0,0 +1,25 @@
+import type { SuggestionType } from '../types/index.js';
+/**
+ * Seeded-bug benchmark cases.
+ *
+ * Each case is a tiny repository: `base` is committed, then `changed` is staged and reviewed.
+ * A bug case plants exactly one defect in the change; `expect.needle` is a substring of the
+ * line that carries it. A control case is a correct change: anything the reviewer reports
+ * there at medium severity or above counts as a false alarm.
+ */
+export interface BenchCase {
+    id: string;
+    title: string;
+    language: string;
+    file: string;
+    base: string;
+    changed: string;
+    /** Absent for control cases */
+    expect?: {
+        needle: string;
+        /** Finding types that count as a catch */
+        types: SuggestionType[];
+    };
+}
+export declare const BENCH_CASES: BenchCase[];
+//# sourceMappingURL=cases.d.ts.map

package/dist/bench/cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cases.d.ts","sourceRoot":"","sources":["../../src/bench/cases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,0CAA0C;QAC1C,KAAK,EAAE,cAAc,EAAE,CAAC;KAC3B,CAAC;CACL;AAOD,eAAO,MAAM,WAAW,EAAE,SAAS,EA4VlC,CAAC"}

package/dist/bench/cases.js

@@ -0,0 +1,142 @@
+const lines = (...l) => l.join('\n') + '\n';
+/** Fake, non-working payment key for the hardcoded-secret case. */
+const FAKE_LIVE_KEY = ['sk', 'live', '51Hx9aQ2eZvKYlo2C8b7fW3mN0pR4tUv'].join('_');
+export const BENCH_CASES = [
+    {
+        id: 'off-by-one',
+        title: 'Loop reads one past the end of the array',
+        language: 'javascript',
+        file: 'src/cart.js',
+        base: lines('export function subtotal(items) {', '    return 0;', '}'),
+        changed: lines('export function subtotal(items) {', '    let total = 0;', '    for (let i = 0; i <= items.length; i++) {', '        total += items[i].price * items[i].qty;', '    }', '    return total;', '}'),
+        expect: { needle: 'i <= items.length', types: ['bug'] },
+    },
+    {
+        id: 'sql-injection',
+        title: 'User input concatenated into a SQL query',
+        language: 'typescript',
+        file: 'src/users.ts',
+        base: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', "    return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}'),
+        changed: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', "    return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}', '', 'export async function searchUsers(name: string) {', "    return db.query(`SELECT id, email FROM users WHERE name LIKE '%${name}%'`);", '}'),
+        expect: { needle: "LIKE '%${name}%'", types: ['security'] },
+    },
+    {
+        id: 'missing-await',
+        title: 'Async call not awaited inside try/catch',
+        language: 'typescript',
+        file: 'src/orders.ts',
+        base: lines("import { saveOrder } from './store';", '', 'export async function placeOrder(order: { id: string }) {', '    await saveOrder(order);', '    return { ok: true };', '}'),
+        changed: lines("import { saveOrder } from './store';", "import { log } from './log';", '', 'export async function placeOrder(order: { id: string }) {', '    try {', '        saveOrder(order);', '        return { ok: true };', '    } catch (err) {', "        log.error('order failed', err);", '        return { ok: false };', '    }', '}'),
+        expect: { needle: '        saveOrder(order);', types: ['bug'] },
+    },
+    {
+        id: 'null-deref',
+        title: 'Optional lookup result used without a check',
+        language: 'typescript',
+        file: 'src/profile.ts',
+        base: lines('interface User { id: string; name: string; address?: { city: string } }', '', 'export function displayName(user: User): string {', '    return user.name;', '}'),
+        changed: lines('interface User { id: string; name: string; address?: { city: string } }', '', 'export function displayName(user: User): string {', '    return user.name;', '}', '', 'export function cityLabel(users: User[], id: string): string {', '    const user = users.find((u) => u.id === id);', '    return `${user.name} (${user.address.city})`;', '}'),
+        expect: { needle: '${user.name} (${user.address.city})', types: ['bug'] },
+    },
+    {
+        id: 'hardcoded-secret',
+        title: 'Live API key committed in source',
+        language: 'python',
+        file: 'app/payments.py',
+        base: lines('import os', 'import requests', '', 'def charge(amount_cents, token):', "    key = os.environ['PAYMENTS_KEY']", "    return requests.post('https://api.payments.example/charge', auth=(key, ''), data={'amount': amount_cents, 'source': token})"),
+        changed: lines('import requests', '', 
+        // Assembled at runtime so this fake key never appears whole in OwnKey's own source
+        // (secret scanners would flag it); the case file on disk gets the full value.
+        `PAYMENTS_KEY = '${FAKE_LIVE_KEY}'`, '', 'def charge(amount_cents, token):', "    return requests.post('https://api.payments.example/charge', auth=(PAYMENTS_KEY, ''), data={'amount': amount_cents, 'source': token})"),
+        expect: { needle: "PAYMENTS_KEY = '", types: ['security'] },
+    },
+    {
+        id: 'path-traversal',
+        title: 'Request parameter joined into a file path',
+        language: 'javascript',
+        file: 'src/files.js',
+        base: lines("import express from 'express';", '', 'export const router = express.Router();'),
+        changed: lines("import express from 'express';", "import path from 'path';", '', 'export const router = express.Router();', "const UPLOADS = path.resolve('uploads');", '', "router.get('/download', (req, res) => {", '    const file = path.join(UPLOADS, req.query.name);', '    res.sendFile(file);', '});'),
+        expect: { needle: 'path.join(UPLOADS, req.query.name)', types: ['security'] },
+    },
+    {
+        id: 'command-injection',
+        title: 'Shell command built from user input',
+        language: 'javascript',
+        file: 'src/thumbs.js',
+        base: lines("import { execFile } from 'child_process';", '', 'export function thumbnail(src, out, cb) {', "    execFile('convert', [src, '-resize', '200x200', out], cb);", '}'),
+        changed: lines("import { exec } from 'child_process';", '', 'export function thumbnail(src, out, size, cb) {', '    exec(`convert ${src} -resize ${size} ${out}`, cb);', '}'),
+        expect: { needle: 'exec(`convert ${src}', types: ['security'] },
+    },
+    {
+        id: 'inverted-auth',
+        title: 'Authorization check inverted',
+        language: 'typescript',
+        file: 'src/admin.ts',
+        base: lines("import type { Request, Response, NextFunction } from 'express';", '', 'export function requireAdmin(req: Request, res: Response, next: NextFunction) {', "    res.status(403).end();", '}'),
+        changed: lines("import type { Request, Response, NextFunction } from 'express';", '', 'export function requireAdmin(req: Request, res: Response, next: NextFunction) {', '    const user = (req as any).user;', '    if (!user || !user.isAdmin) {', '        return next();', '    }', "    res.status(403).json({ error: 'forbidden' });", '}'),
+        expect: { needle: 'if (!user || !user.isAdmin)', types: ['security', 'bug'] },
+    },
+    {
+        id: 'mutable-default',
+        title: 'Mutable default argument shared across calls',
+        language: 'python',
+        file: 'app/tags.py',
+        base: lines('def normalize(tag):', '    return tag.strip().lower()'),
+        changed: lines('def normalize(tag):', '    return tag.strip().lower()', '', 'def add_tag(tag, tags=[]):', '    tags.append(normalize(tag))', '    return tags'),
+        expect: { needle: 'def add_tag(tag, tags=[])', types: ['bug'] },
+    },
+    {
+        id: 'numeric-sort',
+        title: 'Numbers sorted with the default string comparator',
+        language: 'javascript',
+        file: 'src/stats.js',
+        base: lines('export function mean(values) {', '    return values.reduce((a, b) => a + b, 0) / values.length;', '}'),
+        changed: lines('export function mean(values) {', '    return values.reduce((a, b) => a + b, 0) / values.length;', '}', '', 'export function median(values) {', '    const sorted = [...values].sort();', '    const mid = Math.floor(sorted.length / 2);', '    return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;', '}'),
+        expect: { needle: '[...values].sort()', types: ['bug'] },
+    },
+    {
+        id: 'switch-fallthrough',
+        title: 'Missing break makes one case fall into the next',
+        language: 'javascript',
+        file: 'src/pricing.js',
+        base: lines('export function discount(tier) {', '    return 0;', '}'),
+        changed: lines('export function discount(tier) {', '    let pct = 0;', '    switch (tier) {', "        case 'gold':", '            pct = 20;', "        case 'silver':", '            pct = 10;', '            break;', '        default:', '            pct = 0;', '    }', '    return pct;', '}'),
+        expect: { needle: 'pct = 20;', types: ['bug'] },
+    },
+    {
+        id: 'string-identity',
+        title: 'String compared with `is` instead of `==`',
+        language: 'python',
+        file: 'app/roles.py',
+        base: lines('def can_edit(user):', '    return False'),
+        changed: lines('def can_edit(user):', "    role = user.get('role', '').lower()", "    return role is 'editor' or role is 'admin'"),
+        expect: { needle: "role is 'editor'", types: ['bug'] },
+    },
+    // Controls: correct changes. Reports here at medium or above are false alarms.
+    {
+        id: 'clean-refactor',
+        title: 'Control: loop rewritten as reduce',
+        language: 'javascript',
+        file: 'src/cart.js',
+        base: lines('export function subtotal(items) {', '    let total = 0;', '    for (let i = 0; i < items.length; i++) {', '        total += items[i].price * items[i].qty;', '    }', '    return total;', '}'),
+        changed: lines('export function subtotal(items) {', '    return items.reduce((total, item) => total + item.price * item.qty, 0);', '}'),
+    },
+    {
+        id: 'clean-feature',
+        title: 'Control: parameterized query added',
+        language: 'typescript',
+        file: 'src/users.ts',
+        base: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', "    return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}'),
+        changed: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', "    return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}', '', 'export async function findByEmail(email: string) {', "    return db.query('SELECT id, email FROM users WHERE email = $1', [email.trim().toLowerCase()]);", '}'),
+    },
+    {
+        id: 'clean-guard',
+        title: 'Control: null check added',
+        language: 'typescript',
+        file: 'src/profile.ts',
+        base: lines('interface User { id: string; name: string }', '', 'export function greet(user: User): string {', '    return `Hello, ${user.name}`;', '}'),
+        changed: lines('interface User { id: string; name: string }', '', 'export function greet(user: User | undefined): string {', "    if (!user) return 'Hello, guest';", '    return `Hello, ${user.name}`;', '}'),
+    },
+];
+//# sourceMappingURL=cases.js.map

package/dist/bench/cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cases.js","sourceRoot":"","sources":["../../src/bench/cases.ts"],"names":[],"mappings":"AAyBA,MAAM,KAAK,GAAG,CAAC,GAAG,CAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAEtD,mEAAmE;AACnE,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,kCAAkC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEnF,MAAM,CAAC,MAAM,WAAW,GAAgB;IACpC;QACI,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,mCAAmC,EACnC,eAAe,EACf,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,mCAAmC,EACnC,oBAAoB,EACpB,+CAA+C,EAC/C,iDAAiD,EACjD,OAAO,EACP,mBAAmB,EACnB,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC1D;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,EACH,EAAE,EACF,mDAAmD,EACnD,iFAAiF,EACjF,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAC9D;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,yCAAyC;QAChD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,KAAK,CACP,sCAAsC,EACtC,EAAE,EACF,2DAA2D,EAC3D,6BAA6B,EAC7B,0BAA0B,EAC1B,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,sCAAsC,EACtC,8BAA8B,EAC9B,EAAE,EACF,2DAA2D,EAC3D,WAAW,EACX,2BAA2B,EAC3B,8BAA8B,EAC9B,qBAAqB,EACrB,yCAAyC,EACzC,+BAA+B,EAC/B,OAAO,EACP,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,6CAA6C;QACpD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,yEAAyE,EACzE,EAAE,EACF,mDAAmD,EACnD,uBAAuB,EACvB,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,yEAAyE,EACzE,EAAE,EACF,mDAAmD,EACnD,uBAAuB,EACvB,GAAG,EACH,EAAE,EACF,gEAAgE,EAChE,kDAAkD,EAClD,mDAAmD,EACnD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,qCAAqC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC5E;IACD;QACI,EAAE,EAAE,kBAAkB;QACtB,KAAK,EAAE,kCAAkC;QACzC,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,KAAK,CACP,WAAW,EACX,iBAAiB,EACjB,EAAE,EACF,kCAAkC,EAClC,sCAAsC,EACtC,iIAAiI,CACpI;QACD,OAAO,EAAE,KAAK,CACV,iBAAiB,EACjB,EAAE;QACF,mFAAmF;QACnF,8EAA8E;QAC9E,mBAAmB,aAAa,GAAG,EACnC,EAAE,EACF,kCAAkC,EAClC,0IAA0I,CAC7I;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAC9D;IACD;QACI,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,gCAAgC,EAChC,EAAE,EACF,yCAAyC,CAC5C;QACD,OAAO,EAAE,KAAK,CACV,gCAAgC,EAChC,0BAA0B,EAC1B,EAAE,EACF,yCAAyC,EACzC,0CAA0C,EAC1C,EAAE,EACF,yCAAyC,EACzC,sDAAsD,EACtD,yBAAyB,EACzB,KAAK,CACR;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,oCAAoC,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAChF;IACD;QACI,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,qCAAqC;QAC5C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,KAAK,CACP,2CAA2C,EAC3C,EAAE,EACF,2CAA2C,EAC3C,gEAAgE,EAChE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,uCAAuC,EACvC,EAAE,EACF,iDAAiD,EACjD,wDAAwD,EACxD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,8BAA8B;QACrC,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,iEAAiE,EACjE,EAAE,EACF,iFAAiF,EACjF,4BAA4B,EAC5B,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,iEAAiE,EACjE,EAAE,EACF,iFAAiF,EACjF,qCAAqC,EACrC,mCAAmC,EACnC,wBAAwB,EACxB,OAAO,EACP,mDAAmD,EACnD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,6BAA6B,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE;KAChF;IACD;QACI,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,8CAA8C;QACrD,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,qBAAqB,EACrB,gCAAgC,CACnC;QACD,OAAO,EAAE,KAAK,CACV,qBAAqB,EACrB,gCAAgC,EAChC,EAAE,EACF,4BAA4B,EAC5B,iCAAiC,EACjC,iBAAiB,CACpB;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,mDAAmD;QAC1D,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,gCAAgC,EAChC,+DAA+D,EAC/D,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,gCAAgC,EAChC,+DAA+D,EAC/D,GAAG,EACH,EAAE,EACF,kCAAkC,EAClC,wCAAwC,EACxC,gDAAgD,EAChD,mFAAmF,EACnF,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC3D;IACD;QACI,EAAE,EAAE,oBAAoB;QACxB,KAAK,EAAE,iDAAiD;QACxD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,kCAAkC,EAClC,eAAe,EACf,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,kCAAkC,EAClC,kBAAkB,EAClB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,OAAO,EACP,iBAAiB,EACjB,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClD;IACD;QACI,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,qBAAqB,EACrB,kBAAkB,CACrB;QACD,OAAO,EAAE,KAAK,CACV,qBAAqB,EACrB,yCAAyC,EACzC,gDAAgD,CACnD;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KACzD;IACD,+EAA+E;IAC/E;QACI,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,mCAAmC;QAC1C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,mCAAmC,EACnC,oBAAoB,EACpB,8CAA8C,EAC9C,iDAAiD,EACjD,OAAO,EACP,mBAAmB,EACnB,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,mCAAmC,EACnC,6EAA6E,EAC7E,GAAG,CACN;KACJ;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,oCAAoC;QAC3C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,EACH,EAAE,EACF,oDAAoD,EACpD,oGAAoG,EACpG,GAAG,CACN;KACJ;IACD;QACI,EAAE,EAAE,aAAa;QACjB,KAAK,EAAE,2BAA2B;QAClC,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,6CAA6C,EAC7C,EAAE,EACF,6CAA6C,EAC7C,mCAAmC,EACnC,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,6CAA6C,EAC7C,EAAE,EACF,yDAAyD,EACzD,uCAAuC,EACvC,mCAAmC,EACnC,GAAG,CACN;KACJ;CACJ,CAAC"}

package/dist/bench/score.d.ts

@@ -0,0 +1,46 @@
+import type { Severity } from '../types/index.js';
+import type { BenchCase } from './cases.js';
+/** A reported finding, reduced to what scoring needs. */
+export interface BenchFinding {
+    filePath: string;
+    line: number;
+    type: string;
+    severity: Severity;
+    content: string;
+}
+export interface CaseResult {
+    id: string;
+    title: string;
+    kind: 'bug' | 'control';
+    /** Bug cases: the planted defect was reported */
+    caught: boolean;
+    /** Findings that matched the planted defect */
+    matched: number;
+    /** Unmatched findings at medium or above (false alarms on controls, noise on bug cases) */
+    noise: number;
+    findings: BenchFinding[];
+    error?: string;
+    durationMs: number;
+}
+export interface BenchSummary {
+    bugCases: number;
+    caught: number;
+    /** caught / bugCases */
+    recall: number;
+    /** matched findings / (matched + noise) over all cases; 1 when nothing was reported */
+    precision: number;
+    controlCases: number;
+    /** Control cases with at least one false alarm */
+    controlsFlagged: number;
+    errors: number;
+}
+/** How far (in lines) a finding may sit from the planted line and still count. */
+export declare const LINE_TOLERANCE = 3;
+/** Noise threshold: low/info nits never count against a reviewer. */
+export declare const NOISE_SEVERITY: Severity;
+/** 1-based line of the planted defect in the changed file. */
+export declare function plantedLine(c: BenchCase): number;
+export declare function matchesPlanted(c: BenchCase, f: BenchFinding): boolean;
+export declare function scoreCase(c: BenchCase, findings: BenchFinding[], durationMs?: number, error?: string): CaseResult;
+export declare function summarize(results: CaseResult[]): BenchSummary;
+//# sourceMappingURL=score.d.ts.map

package/dist/bench/score.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/bench/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,GAAG,SAAS,CAAC;IACxB,iDAAiD;IACjD,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,2FAA2F;IAC3F,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,qEAAqE;AACrE,eAAO,MAAM,cAAc,EAAE,QAAmB,CAAC;AAEjD,8DAA8D;AAC9D,wBAAgB,WAAW,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAKhD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,GAAG,OAAO,CAKrE;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,UAAU,SAAI,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,CAkB5G;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,YAAY,CAe7D"}

package/dist/bench/score.js

@@ -0,0 +1,61 @@
+import { meetsSeverity } from '../utils/severity.js';
+/** How far (in lines) a finding may sit from the planted line and still count. */
+export const LINE_TOLERANCE = 3;
+/** Noise threshold: low/info nits never count against a reviewer. */
+export const NOISE_SEVERITY = 'medium';
+/** 1-based line of the planted defect in the changed file. */
+export function plantedLine(c) {
+    if (!c.expect)
+        return 0;
+    const idx = c.changed.split('\n').findIndex((l) => l.includes(c.expect.needle));
+    if (idx === -1)
+        throw new Error(`Bench case ${c.id}: needle not found in changed file`);
+    return idx + 1;
+}
+export function matchesPlanted(c, f) {
+    if (!c.expect)
+        return false;
+    if (f.filePath !== c.file)
+        return false;
+    if (Math.abs(f.line - plantedLine(c)) > LINE_TOLERANCE)
+        return false;
+    return c.expect.types.includes(f.type);
+}
+export function scoreCase(c, findings, durationMs = 0, error) {
+    let matched = 0;
+    let noise = 0;
+    for (const f of findings) {
+        if (matchesPlanted(c, f))
+            matched++;
+        else if (meetsSeverity(f.severity, NOISE_SEVERITY))
+            noise++;
+    }
+    return {
+        id: c.id,
+        title: c.title,
+        kind: c.expect ? 'bug' : 'control',
+        caught: matched > 0,
+        matched,
+        noise,
+        findings,
+        error,
+        durationMs,
+    };
+}
+export function summarize(results) {
+    const bugs = results.filter((r) => r.kind === 'bug');
+    const controls = results.filter((r) => r.kind === 'control');
+    const caught = bugs.filter((r) => r.caught).length;
+    const matched = results.reduce((n, r) => n + r.matched, 0);
+    const noise = results.reduce((n, r) => n + r.noise, 0);
+    return {
+        bugCases: bugs.length,
+        caught,
+        recall: bugs.length ? caught / bugs.length : 0,
+        precision: matched + noise ? matched / (matched + noise) : 1,
+        controlCases: controls.length,
+        controlsFlagged: controls.filter((r) => r.noise > 0).length,
+        errors: results.filter((r) => r.error).length,
+    };
+}
+//# sourceMappingURL=score.js.map

package/dist/bench/score.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/bench/score.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAChC,qEAAqE;AACrE,MAAM,CAAC,MAAM,cAAc,GAAa,QAAQ,CAAC;AAEjD,8DAA8D;AAC9D,MAAM,UAAU,WAAW,CAAC,CAAY;IACpC,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjF,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACxF,OAAO,GAAG,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAY,EAAE,CAAe;IACxD,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC5B,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc;QAAE,OAAO,KAAK,CAAC;IACrE,OAAQ,CAAC,CAAC,MAAM,CAAC,KAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,CAAY,EAAE,QAAwB,EAAE,UAAU,GAAG,CAAC,EAAE,KAAc;IAC5F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvB,IAAI,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;aAC/B,IAAI,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,cAAc,CAAC;YAAE,KAAK,EAAE,CAAC;IAChE,CAAC;IACD,OAAO;QACH,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QAClC,MAAM,EAAE,OAAO,GAAG,CAAC;QACnB,OAAO;QACP,KAAK;QACL,QAAQ;QACR,KAAK;QACL,UAAU;KACb,CAAC;AACN,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAqB;IAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACvD,OAAO;QACH,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,YAAY,EAAE,QAAQ,CAAC,MAAM;QAC7B,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;KAChD,CAAC;AACN,CAAC"}

package/dist/commands/bench.d.ts

@@ -0,0 +1,9 @@
+import { Command } from 'commander';
+import type { AIProvider } from '../ai/provider.js';
+import { type BenchCase } from '../bench/cases.js';
+import { type BenchSummary, type CaseResult } from '../bench/score.js';
+export declare const benchCommand: Command;
+/** Build a throwaway repo for one case, stage the change, and review it. Nothing is persisted. */
+export declare function runCase(c: BenchCase, provider: AIProvider): Promise<CaseResult>;
+export declare function toMarkdown(results: CaseResult[], s: BenchSummary, model: string): string;
+//# sourceMappingURL=bench.d.ts.map

package/dist/commands/bench.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bench.d.ts","sourceRoot":"","sources":["../../src/commands/bench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAe,KAAK,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAA2C,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKhH,eAAO,MAAM,YAAY,SAgEnB,CAAC;AAiBP,kGAAkG;AAClG,wBAAsB,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CA6BrF;AA4BD,wBAAgB,UAAU,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAexF"}