ownkey 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/dist/ai/factory.d.ts +3 -1
- package/dist/ai/factory.d.ts.map +1 -1
- package/dist/ai/factory.js +5 -5
- package/dist/ai/factory.js.map +1 -1
- package/dist/bench/cases.d.ts +25 -0
- package/dist/bench/cases.d.ts.map +1 -0
- package/dist/bench/cases.js +142 -0
- package/dist/bench/cases.js.map +1 -0
- package/dist/bench/score.d.ts +46 -0
- package/dist/bench/score.d.ts.map +1 -0
- package/dist/bench/score.js +61 -0
- package/dist/bench/score.js.map +1 -0
- package/dist/commands/bench.d.ts +9 -0
- package/dist/commands/bench.d.ts.map +1 -0
- package/dist/commands/bench.js +169 -0
- package/dist/commands/bench.js.map +1 -0
- package/dist/commands/review.d.ts.map +1 -1
- package/dist/commands/review.js +5 -24
- package/dist/commands/review.js.map +1 -1
- package/dist/core/review/run.d.ts +20 -0
- package/dist/core/review/run.d.ts.map +1 -0
- package/dist/core/review/run.js +32 -0
- package/dist/core/review/run.js.map +1 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -112,6 +112,7 @@ For a whole-project audit of any folder, git or not, use `ownkey suggest .`
|
|
|
112
112
|
| `ownkey scan [path]` | List the files OwnKey would analyze, honoring `.gitignore` and `.ownkeyignore`. |
|
|
113
113
|
| `ownkey apply [id]` | Apply one suggestion, or all pending ones with `--all`, optionally verified. |
|
|
114
114
|
| `ownkey undo [backup-id]` | Restore files from a backup created by `apply`. |
|
|
115
|
+
| `ownkey bench` | Measure review quality on seeded-bug cases with your model. |
|
|
115
116
|
|
|
116
117
|
<details>
|
|
117
118
|
<summary><strong>Full option reference</strong></summary>
|
|
@@ -147,6 +148,17 @@ For a whole-project audit of any folder, git or not, use `ownkey suggest .`
|
|
|
147
148
|
|
|
148
149
|
Exit codes: `0` success, `1` the `--fail-on` gate tripped, `2` the review could not run.
|
|
149
150
|
|
|
151
|
+
#### `ownkey bench`
|
|
152
|
+
|
|
153
|
+
| Option | Description |
|
|
154
|
+
|---|---|
|
|
155
|
+
| `--list` | List the cases and exit. |
|
|
156
|
+
| `--case <ids>` | Comma-separated case ids to run. Default: all. |
|
|
157
|
+
| `--model <model>` | Use this model for the run instead of the configured one. |
|
|
158
|
+
| `--format <format>` | `text` (default), `json`, or `markdown`. |
|
|
159
|
+
| `--output <file>` | Write the report to a file. |
|
|
160
|
+
| `--min-recall <ratio>` | Exit with code 1 when recall is below this ratio, from `0` to `1`. |
|
|
161
|
+
|
|
150
162
|
#### `ownkey scan [path]`
|
|
151
163
|
|
|
152
164
|
| Option | Description |
|
|
@@ -190,6 +202,21 @@ Exit codes: `0` success, `1` the `--fail-on` gate tripped, `2` the review could
|
|
|
190
202
|
|
|
191
203
|
</details>
|
|
192
204
|
|
|
205
|
+
## Measuring review quality
|
|
206
|
+
|
|
207
|
+
`ownkey bench` shows how well a model reviews before you trust it with your pull requests. It runs a fixed set of small cases, each a throwaway git repository with one staged change:
|
|
208
|
+
|
|
209
|
+
- **Bug cases** plant one real defect: an off-by-one, SQL injection, a missing `await`, a null dereference, a hardcoded secret, path traversal, command injection, an inverted auth check, a mutable default argument, a numeric sort with the string comparator, a switch fallthrough, and `is` used to compare strings. They cover JavaScript, TypeScript, and Python.
|
|
210
|
+
- **Control cases** are correct changes. Anything reported there at medium severity or above is a false alarm.
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
ownkey bench # all cases with your configured model
|
|
214
|
+
ownkey bench --model claude-opus-5-5 # compare another model on the same key
|
|
215
|
+
ownkey bench --format markdown --output bench.md
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
It reports **recall** (planted bugs reported within 3 lines of the defect with a matching type), **precision** (matching findings over all findings at medium or above), and how many controls stayed clean. Low and info findings never count against a model. Bench runs in temporary directories and stores nothing, but it does send every case to your provider, so a full run costs about 15 small review calls.
|
|
219
|
+
|
|
193
220
|
## Providers
|
|
194
221
|
|
|
195
222
|
| Provider | Setup | Default model |
|
package/dist/ai/factory.d.ts
CHANGED
|
@@ -6,7 +6,9 @@ export declare class AIProviderFactory {
|
|
|
6
6
|
/**
|
|
7
7
|
* Create an AI provider instance based on configuration
|
|
8
8
|
*/
|
|
9
|
-
static createProvider(
|
|
9
|
+
static createProvider(overrides?: {
|
|
10
|
+
model?: string;
|
|
11
|
+
}): Promise<AIProvider>;
|
|
10
12
|
/**
|
|
11
13
|
* Test provider connection
|
|
12
14
|
*/
|
package/dist/ai/factory.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C;;GAEG;AACH,qBAAa,iBAAiB;IAC1B;;OAEG;WACU,cAAc,
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAU3C;;GAEG;AACH,qBAAa,iBAAiB;IAC1B;;OAEG;WACU,cAAc,CAAC,SAAS,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,UAAU,CAAC;IA6CpF;;OAEG;WACU,YAAY,CAAC,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBjE;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAepD;;OAEG;WACU,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAmBvF"}
|
package/dist/ai/factory.js
CHANGED
|
@@ -13,7 +13,7 @@ export class AIProviderFactory {
|
|
|
13
13
|
/**
|
|
14
14
|
* Create an AI provider instance based on configuration
|
|
15
15
|
*/
|
|
16
|
-
static async createProvider() {
|
|
16
|
+
static async createProvider(overrides = {}) {
|
|
17
17
|
const config = await configManager.load();
|
|
18
18
|
if (!config.activeProvider) {
|
|
19
19
|
throw new ConfigError('No active AI provider configured. Run: ownkey config');
|
|
@@ -26,23 +26,23 @@ export class AIProviderFactory {
|
|
|
26
26
|
if (!apiKey) {
|
|
27
27
|
throw new ConfigError('Gemini API key not found. Run: ownkey config');
|
|
28
28
|
}
|
|
29
|
-
const geminiModel = config.providers.gemini?.model || GeminiProvider.getRecommendedModel();
|
|
29
|
+
const geminiModel = overrides.model || config.providers.gemini?.model || GeminiProvider.getRecommendedModel();
|
|
30
30
|
return new GeminiProvider(apiKey, geminiModel);
|
|
31
31
|
case 'openai':
|
|
32
32
|
if (!apiKey) {
|
|
33
33
|
throw new ConfigError('OpenAI API key not found. Run: ownkey config');
|
|
34
34
|
}
|
|
35
|
-
const openaiModel = config.providers.openai?.model || DEFAULT_MODELS.openai;
|
|
35
|
+
const openaiModel = overrides.model || config.providers.openai?.model || DEFAULT_MODELS.openai;
|
|
36
36
|
return new OpenAIProvider(apiKey, openaiModel, config.providers.openai?.endpoint);
|
|
37
37
|
case 'anthropic':
|
|
38
38
|
if (!apiKey) {
|
|
39
39
|
throw new ConfigError('Anthropic API key not found. Run: ownkey config');
|
|
40
40
|
}
|
|
41
|
-
const anthropicModel = config.providers.anthropic?.model || DEFAULT_MODELS.anthropic;
|
|
41
|
+
const anthropicModel = overrides.model || config.providers.anthropic?.model || DEFAULT_MODELS.anthropic;
|
|
42
42
|
return new AnthropicProvider(apiKey, anthropicModel);
|
|
43
43
|
case 'ollama':
|
|
44
44
|
// Ollama doesn't require API key - runs locally
|
|
45
|
-
const ollamaModel = config.providers.ollama?.model || DEFAULT_MODELS.ollama;
|
|
45
|
+
const ollamaModel = overrides.model || config.providers.ollama?.model || DEFAULT_MODELS.ollama;
|
|
46
46
|
const ollamaBaseUrl = config.providers.ollama?.endpoint || 'http://localhost:11434';
|
|
47
47
|
return new OllamaProvider(undefined, ollamaModel, ollamaBaseUrl);
|
|
48
48
|
default:
|
package/dist/ai/factory.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAC1B;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc;
|
|
1
|
+
{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../src/ai/factory.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C;;GAEG;AACH,MAAM,OAAO,iBAAiB;IAC1B;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,YAAgC,EAAE;QAC1D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAE1C,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACzB,MAAM,IAAI,WAAW,CAAC,sDAAsD,CAAC,CAAC;QAClF,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,CAAC,cAAc,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAEvD,MAAM,CAAC,OAAO,CAAC,YAAY,QAAQ,cAAc,CAAC,CAAC;QAEnD,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,mBAAmB,EAAE,CAAC;gBAC9G,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;YAEnD,KAAK,QAAQ;gBACT,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,8CAA8C,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC/F,OAAO,IAAI,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YAEtF,KAAK,WAAW;gBACZ,IAAI,CAAC,MAAM,EAAE,CAAC;oBACV,MAAM,IAAI,WAAW,CAAC,iDAAiD,CAAC,CAAC;gBAC7E,CAAC;gBACD,MAAM,cAAc,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,SAAS,EAAE,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC;gBACxG,OAAO,IAAI,iBAAiB,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;YAEzD,KAAK,QAAQ;gBACT,gDAAgD;gBAChD,MAAM,WAAW,GAAG,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,cAAc,CAAC,MAAM,CAAC;gBAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,EAAE,QAAQ,IAAI,wBAAwB,CAAC;gBACpF,OAAO,IAAI,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;YAErE;gBACI,MAAM,IAAI,WAAW,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,QAAoB;QAC1C,IAAI,CAAC;YACD,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACT,MAAM,CAAC,OAAO,CAAC,gCAAgC,CAAC,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACJ,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC/C,CAAC;YACD,OAAO,MAAM,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAc,CAAC,CAAC;YACpD,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,mBAAmB,CAAC,QAAgB;QACvC,QAAQ,QAAQ,EAAE,CAAC;YACf,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,mBAAmB,EAAE,CAAC;YAChD,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC,KAAK,WAAW;gBACZ,OAAO,cAAc,CAAC,SAAS,CAAC;YACpC,KAAK,QAAQ;gBACT,OAAO,cAAc,CAAC,MAAM,CAAC;YACjC;gBACI,OAAO,EAAE,CAAC;QAClB,CAAC;IACL,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAgB,EAAE,MAAc;QAC5D,IAAI,CAAC;YACD,QAAQ,QAAQ,EAAE,CAAC;gBACf,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC3D,KAAK,WAAW;oBACZ,OAAO,MAAM,iBAAiB,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;gBAC9D,KAAK,QAAQ;oBACT,OAAO,MAAM,cAAc,CAAC,kBAAkB,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC;gBACxE;oBACI,OAAO,EAAE,CAAC;YAClB,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,GAAG,EAAE,KAAc,CAAC,CAAC;YACxE,OAAO,CAAC,iBAAiB,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;CACJ"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { SuggestionType } from '../types/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Seeded-bug benchmark cases.
|
|
4
|
+
*
|
|
5
|
+
* Each case is a tiny repository: `base` is committed, then `changed` is staged and reviewed.
|
|
6
|
+
* A bug case plants exactly one defect in the change; `expect.needle` is a substring of the
|
|
7
|
+
* line that carries it. A control case is a correct change: anything the reviewer reports
|
|
8
|
+
* there at medium severity or above counts as a false alarm.
|
|
9
|
+
*/
|
|
10
|
+
export interface BenchCase {
|
|
11
|
+
id: string;
|
|
12
|
+
title: string;
|
|
13
|
+
language: string;
|
|
14
|
+
file: string;
|
|
15
|
+
base: string;
|
|
16
|
+
changed: string;
|
|
17
|
+
/** Absent for control cases */
|
|
18
|
+
expect?: {
|
|
19
|
+
needle: string;
|
|
20
|
+
/** Finding types that count as a catch */
|
|
21
|
+
types: SuggestionType[];
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export declare const BENCH_CASES: BenchCase[];
|
|
25
|
+
//# sourceMappingURL=cases.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cases.d.ts","sourceRoot":"","sources":["../../src/bench/cases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,MAAM,CAAC,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,0CAA0C;QAC1C,KAAK,EAAE,cAAc,EAAE,CAAC;KAC3B,CAAC;CACL;AAOD,eAAO,MAAM,WAAW,EAAE,SAAS,EA4VlC,CAAC"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
const lines = (...l) => l.join('\n') + '\n';
|
|
2
|
+
/** Fake, non-working payment key for the hardcoded-secret case. */
|
|
3
|
+
const FAKE_LIVE_KEY = ['sk', 'live', '51Hx9aQ2eZvKYlo2C8b7fW3mN0pR4tUv'].join('_');
|
|
4
|
+
export const BENCH_CASES = [
|
|
5
|
+
{
|
|
6
|
+
id: 'off-by-one',
|
|
7
|
+
title: 'Loop reads one past the end of the array',
|
|
8
|
+
language: 'javascript',
|
|
9
|
+
file: 'src/cart.js',
|
|
10
|
+
base: lines('export function subtotal(items) {', ' return 0;', '}'),
|
|
11
|
+
changed: lines('export function subtotal(items) {', ' let total = 0;', ' for (let i = 0; i <= items.length; i++) {', ' total += items[i].price * items[i].qty;', ' }', ' return total;', '}'),
|
|
12
|
+
expect: { needle: 'i <= items.length', types: ['bug'] },
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
id: 'sql-injection',
|
|
16
|
+
title: 'User input concatenated into a SQL query',
|
|
17
|
+
language: 'typescript',
|
|
18
|
+
file: 'src/users.ts',
|
|
19
|
+
base: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', " return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}'),
|
|
20
|
+
changed: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', " return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}', '', 'export async function searchUsers(name: string) {', " return db.query(`SELECT id, email FROM users WHERE name LIKE '%${name}%'`);", '}'),
|
|
21
|
+
expect: { needle: "LIKE '%${name}%'", types: ['security'] },
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
id: 'missing-await',
|
|
25
|
+
title: 'Async call not awaited inside try/catch',
|
|
26
|
+
language: 'typescript',
|
|
27
|
+
file: 'src/orders.ts',
|
|
28
|
+
base: lines("import { saveOrder } from './store';", '', 'export async function placeOrder(order: { id: string }) {', ' await saveOrder(order);', ' return { ok: true };', '}'),
|
|
29
|
+
changed: lines("import { saveOrder } from './store';", "import { log } from './log';", '', 'export async function placeOrder(order: { id: string }) {', ' try {', ' saveOrder(order);', ' return { ok: true };', ' } catch (err) {', " log.error('order failed', err);", ' return { ok: false };', ' }', '}'),
|
|
30
|
+
expect: { needle: ' saveOrder(order);', types: ['bug'] },
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: 'null-deref',
|
|
34
|
+
title: 'Optional lookup result used without a check',
|
|
35
|
+
language: 'typescript',
|
|
36
|
+
file: 'src/profile.ts',
|
|
37
|
+
base: lines('interface User { id: string; name: string; address?: { city: string } }', '', 'export function displayName(user: User): string {', ' return user.name;', '}'),
|
|
38
|
+
changed: lines('interface User { id: string; name: string; address?: { city: string } }', '', 'export function displayName(user: User): string {', ' return user.name;', '}', '', 'export function cityLabel(users: User[], id: string): string {', ' const user = users.find((u) => u.id === id);', ' return `${user.name} (${user.address.city})`;', '}'),
|
|
39
|
+
expect: { needle: '${user.name} (${user.address.city})', types: ['bug'] },
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
id: 'hardcoded-secret',
|
|
43
|
+
title: 'Live API key committed in source',
|
|
44
|
+
language: 'python',
|
|
45
|
+
file: 'app/payments.py',
|
|
46
|
+
base: lines('import os', 'import requests', '', 'def charge(amount_cents, token):', " key = os.environ['PAYMENTS_KEY']", " return requests.post('https://api.payments.example/charge', auth=(key, ''), data={'amount': amount_cents, 'source': token})"),
|
|
47
|
+
changed: lines('import requests', '',
|
|
48
|
+
// Assembled at runtime so this fake key never appears whole in OwnKey's own source
|
|
49
|
+
// (secret scanners would flag it); the case file on disk gets the full value.
|
|
50
|
+
`PAYMENTS_KEY = '${FAKE_LIVE_KEY}'`, '', 'def charge(amount_cents, token):', " return requests.post('https://api.payments.example/charge', auth=(PAYMENTS_KEY, ''), data={'amount': amount_cents, 'source': token})"),
|
|
51
|
+
expect: { needle: "PAYMENTS_KEY = '", types: ['security'] },
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
id: 'path-traversal',
|
|
55
|
+
title: 'Request parameter joined into a file path',
|
|
56
|
+
language: 'javascript',
|
|
57
|
+
file: 'src/files.js',
|
|
58
|
+
base: lines("import express from 'express';", '', 'export const router = express.Router();'),
|
|
59
|
+
changed: lines("import express from 'express';", "import path from 'path';", '', 'export const router = express.Router();', "const UPLOADS = path.resolve('uploads');", '', "router.get('/download', (req, res) => {", ' const file = path.join(UPLOADS, req.query.name);', ' res.sendFile(file);', '});'),
|
|
60
|
+
expect: { needle: 'path.join(UPLOADS, req.query.name)', types: ['security'] },
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: 'command-injection',
|
|
64
|
+
title: 'Shell command built from user input',
|
|
65
|
+
language: 'javascript',
|
|
66
|
+
file: 'src/thumbs.js',
|
|
67
|
+
base: lines("import { execFile } from 'child_process';", '', 'export function thumbnail(src, out, cb) {', " execFile('convert', [src, '-resize', '200x200', out], cb);", '}'),
|
|
68
|
+
changed: lines("import { exec } from 'child_process';", '', 'export function thumbnail(src, out, size, cb) {', ' exec(`convert ${src} -resize ${size} ${out}`, cb);', '}'),
|
|
69
|
+
expect: { needle: 'exec(`convert ${src}', types: ['security'] },
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: 'inverted-auth',
|
|
73
|
+
title: 'Authorization check inverted',
|
|
74
|
+
language: 'typescript',
|
|
75
|
+
file: 'src/admin.ts',
|
|
76
|
+
base: lines("import type { Request, Response, NextFunction } from 'express';", '', 'export function requireAdmin(req: Request, res: Response, next: NextFunction) {', " res.status(403).end();", '}'),
|
|
77
|
+
changed: lines("import type { Request, Response, NextFunction } from 'express';", '', 'export function requireAdmin(req: Request, res: Response, next: NextFunction) {', ' const user = (req as any).user;', ' if (!user || !user.isAdmin) {', ' return next();', ' }', " res.status(403).json({ error: 'forbidden' });", '}'),
|
|
78
|
+
expect: { needle: 'if (!user || !user.isAdmin)', types: ['security', 'bug'] },
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
id: 'mutable-default',
|
|
82
|
+
title: 'Mutable default argument shared across calls',
|
|
83
|
+
language: 'python',
|
|
84
|
+
file: 'app/tags.py',
|
|
85
|
+
base: lines('def normalize(tag):', ' return tag.strip().lower()'),
|
|
86
|
+
changed: lines('def normalize(tag):', ' return tag.strip().lower()', '', 'def add_tag(tag, tags=[]):', ' tags.append(normalize(tag))', ' return tags'),
|
|
87
|
+
expect: { needle: 'def add_tag(tag, tags=[])', types: ['bug'] },
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
id: 'numeric-sort',
|
|
91
|
+
title: 'Numbers sorted with the default string comparator',
|
|
92
|
+
language: 'javascript',
|
|
93
|
+
file: 'src/stats.js',
|
|
94
|
+
base: lines('export function mean(values) {', ' return values.reduce((a, b) => a + b, 0) / values.length;', '}'),
|
|
95
|
+
changed: lines('export function mean(values) {', ' return values.reduce((a, b) => a + b, 0) / values.length;', '}', '', 'export function median(values) {', ' const sorted = [...values].sort();', ' const mid = Math.floor(sorted.length / 2);', ' return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;', '}'),
|
|
96
|
+
expect: { needle: '[...values].sort()', types: ['bug'] },
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
id: 'switch-fallthrough',
|
|
100
|
+
title: 'Missing break makes one case fall into the next',
|
|
101
|
+
language: 'javascript',
|
|
102
|
+
file: 'src/pricing.js',
|
|
103
|
+
base: lines('export function discount(tier) {', ' return 0;', '}'),
|
|
104
|
+
changed: lines('export function discount(tier) {', ' let pct = 0;', ' switch (tier) {', " case 'gold':", ' pct = 20;', " case 'silver':", ' pct = 10;', ' break;', ' default:', ' pct = 0;', ' }', ' return pct;', '}'),
|
|
105
|
+
expect: { needle: 'pct = 20;', types: ['bug'] },
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
id: 'string-identity',
|
|
109
|
+
title: 'String compared with `is` instead of `==`',
|
|
110
|
+
language: 'python',
|
|
111
|
+
file: 'app/roles.py',
|
|
112
|
+
base: lines('def can_edit(user):', ' return False'),
|
|
113
|
+
changed: lines('def can_edit(user):', " role = user.get('role', '').lower()", " return role is 'editor' or role is 'admin'"),
|
|
114
|
+
expect: { needle: "role is 'editor'", types: ['bug'] },
|
|
115
|
+
},
|
|
116
|
+
// Controls: correct changes. Reports here at medium or above are false alarms.
|
|
117
|
+
{
|
|
118
|
+
id: 'clean-refactor',
|
|
119
|
+
title: 'Control: loop rewritten as reduce',
|
|
120
|
+
language: 'javascript',
|
|
121
|
+
file: 'src/cart.js',
|
|
122
|
+
base: lines('export function subtotal(items) {', ' let total = 0;', ' for (let i = 0; i < items.length; i++) {', ' total += items[i].price * items[i].qty;', ' }', ' return total;', '}'),
|
|
123
|
+
changed: lines('export function subtotal(items) {', ' return items.reduce((total, item) => total + item.price * item.qty, 0);', '}'),
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
id: 'clean-feature',
|
|
127
|
+
title: 'Control: parameterized query added',
|
|
128
|
+
language: 'typescript',
|
|
129
|
+
file: 'src/users.ts',
|
|
130
|
+
base: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', " return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}'),
|
|
131
|
+
changed: lines("import { db } from './db';", '', 'export async function findUser(id: number) {', " return db.query('SELECT id, email FROM users WHERE id = $1', [id]);", '}', '', 'export async function findByEmail(email: string) {', " return db.query('SELECT id, email FROM users WHERE email = $1', [email.trim().toLowerCase()]);", '}'),
|
|
132
|
+
},
|
|
133
|
+
{
|
|
134
|
+
id: 'clean-guard',
|
|
135
|
+
title: 'Control: null check added',
|
|
136
|
+
language: 'typescript',
|
|
137
|
+
file: 'src/profile.ts',
|
|
138
|
+
base: lines('interface User { id: string; name: string }', '', 'export function greet(user: User): string {', ' return `Hello, ${user.name}`;', '}'),
|
|
139
|
+
changed: lines('interface User { id: string; name: string }', '', 'export function greet(user: User | undefined): string {', " if (!user) return 'Hello, guest';", ' return `Hello, ${user.name}`;', '}'),
|
|
140
|
+
},
|
|
141
|
+
];
|
|
142
|
+
//# sourceMappingURL=cases.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cases.js","sourceRoot":"","sources":["../../src/bench/cases.ts"],"names":[],"mappings":"AAyBA,MAAM,KAAK,GAAG,CAAC,GAAG,CAAW,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAEtD,mEAAmE;AACnE,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,kCAAkC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEnF,MAAM,CAAC,MAAM,WAAW,GAAgB;IACpC;QACI,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,mCAAmC,EACnC,eAAe,EACf,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,mCAAmC,EACnC,oBAAoB,EACpB,+CAA+C,EAC/C,iDAAiD,EACjD,OAAO,EACP,mBAAmB,EACnB,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC1D;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,0CAA0C;QACjD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,EACH,EAAE,EACF,mDAAmD,EACnD,iFAAiF,EACjF,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAC9D;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,yCAAyC;QAChD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,KAAK,CACP,sCAAsC,EACtC,EAAE,EACF,2DAA2D,EAC3D,6BAA6B,EAC7B,0BAA0B,EAC1B,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,sCAAsC,EACtC,8BAA8B,EAC9B,EAAE,EACF,2DAA2D,EAC3D,WAAW,EACX,2BAA2B,EAC3B,8BAA8B,EAC9B,qBAAqB,EACrB,yCAAyC,EACzC,+BAA+B,EAC/B,OAAO,EACP,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,YAAY;QAChB,KAAK,EAAE,6CAA6C;QACpD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,yEAAyE,EACzE,EAAE,EACF,mDAAmD,EACnD,uBAAuB,EACvB,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,yEAAyE,EACzE,EAAE,EACF,mDAAmD,EACnD,uBAAuB,EACvB,GAAG,EACH,EAAE,EACF,gEAAgE,EAChE,kDAAkD,EAClD,mDAAmD,EACnD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,qCAAqC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC5E;IACD;QACI,EAAE,EAAE,kBAAkB;QACtB,KAAK,EAAE,kCAAkC;QACzC,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,iBAAiB;QACvB,IAAI,EAAE,KAAK,CACP,WAAW,EACX,iBAAiB,EACjB,EAAE,EACF,kCAAkC,EAClC,sCAAsC,EACtC,iIAAiI,CACpI;QACD,OAAO,EAAE,KAAK,CACV,iBAAiB,EACjB,EAAE;QACF,mFAAmF;QACnF,8EAA8E;QAC9E,mBAAmB,aAAa,GAAG,EACnC,EAAE,EACF,kCAAkC,EAClC,0IAA0I,CAC7I;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAC9D;IACD;QACI,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,gCAAgC,EAChC,EAAE,EACF,yCAAyC,CAC5C;QACD,OAAO,EAAE,KAAK,CACV,gCAAgC,EAChC,0BAA0B,EAC1B,EAAE,EACF,yCAAyC,EACzC,0CAA0C,EAC1C,EAAE,EACF,yCAAyC,EACzC,sDAAsD,EACtD,yBAAyB,EACzB,KAAK,CACR;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,oCAAoC,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAChF;IACD;QACI,EAAE,EAAE,mBAAmB;QACvB,KAAK,EAAE,qCAAqC;QAC5C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,eAAe;QACrB,IAAI,EAAE,KAAK,CACP,2CAA2C,EAC3C,EAAE,EACF,2CAA2C,EAC3C,gEAAgE,EAChE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,uCAAuC,EACvC,EAAE,EACF,iDAAiD,EACjD,wDAAwD,EACxD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,CAAC,UAAU,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,8BAA8B;QACrC,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,iEAAiE,EACjE,EAAE,EACF,iFAAiF,EACjF,4BAA4B,EAC5B,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,iEAAiE,EACjE,EAAE,EACF,iFAAiF,EACjF,qCAAqC,EACrC,mCAAmC,EACnC,wBAAwB,EACxB,OAAO,EACP,mDAAmD,EACnD,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,6BAA6B,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,EAAE;KAChF;IACD;QACI,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,8CAA8C;QACrD,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,qBAAqB,EACrB,gCAAgC,CACnC;QACD,OAAO,EAAE,KAAK,CACV,qBAAqB,EACrB,gCAAgC,EAChC,EAAE,EACF,4BAA4B,EAC5B,iCAAiC,EACjC,iBAAiB,CACpB;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClE;IACD;QACI,EAAE,EAAE,cAAc;QAClB,KAAK,EAAE,mDAAmD;QAC1D,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,gCAAgC,EAChC,+DAA+D,EAC/D,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,gCAAgC,EAChC,+DAA+D,EAC/D,GAAG,EACH,EAAE,EACF,kCAAkC,EAClC,wCAAwC,EACxC,gDAAgD,EAChD,mFAAmF,EACnF,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,oBAAoB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAC3D;IACD;QACI,EAAE,EAAE,oBAAoB;QACxB,KAAK,EAAE,iDAAiD;QACxD,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,kCAAkC,EAClC,eAAe,EACf,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,kCAAkC,EAClC,kBAAkB,EAClB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,wBAAwB,EACxB,uBAAuB,EACvB,oBAAoB,EACpB,kBAAkB,EAClB,sBAAsB,EACtB,OAAO,EACP,iBAAiB,EACjB,GAAG,CACN;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KAClD;IACD;QACI,EAAE,EAAE,iBAAiB;QACrB,KAAK,EAAE,2CAA2C;QAClD,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,qBAAqB,EACrB,kBAAkB,CACrB;QACD,OAAO,EAAE,KAAK,CACV,qBAAqB,EACrB,yCAAyC,EACzC,gDAAgD,CACnD;QACD,MAAM,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;KACzD;IACD,+EAA+E;IAC/E;QACI,EAAE,EAAE,gBAAgB;QACpB,KAAK,EAAE,mCAAmC;QAC1C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,aAAa;QACnB,IAAI,EAAE,KAAK,CACP,mCAAmC,EACnC,oBAAoB,EACpB,8CAA8C,EAC9C,iDAAiD,EACjD,OAAO,EACP,mBAAmB,EACnB,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,mCAAmC,EACnC,6EAA6E,EAC7E,GAAG,CACN;KACJ;IACD;QACI,EAAE,EAAE,eAAe;QACnB,KAAK,EAAE,oCAAoC;QAC3C,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,KAAK,CACP,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,4BAA4B,EAC5B,EAAE,EACF,8CAA8C,EAC9C,yEAAyE,EACzE,GAAG,EACH,EAAE,EACF,oDAAoD,EACpD,oGAAoG,EACpG,GAAG,CACN;KACJ;IACD;QACI,EAAE,EAAE,aAAa;QACjB,KAAK,EAAE,2BAA2B;QAClC,QAAQ,EAAE,YAAY;QACtB,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,KAAK,CACP,6CAA6C,EAC7C,EAAE,EACF,6CAA6C,EAC7C,mCAAmC,EACnC,GAAG,CACN;QACD,OAAO,EAAE,KAAK,CACV,6CAA6C,EAC7C,EAAE,EACF,yDAAyD,EACzD,uCAAuC,EACvC,mCAAmC,EACnC,GAAG,CACN;KACJ;CACJ,CAAC"}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { Severity } from '../types/index.js';
|
|
2
|
+
import type { BenchCase } from './cases.js';
|
|
3
|
+
/** A reported finding, reduced to what scoring needs. */
|
|
4
|
+
export interface BenchFinding {
|
|
5
|
+
filePath: string;
|
|
6
|
+
line: number;
|
|
7
|
+
type: string;
|
|
8
|
+
severity: Severity;
|
|
9
|
+
content: string;
|
|
10
|
+
}
|
|
11
|
+
export interface CaseResult {
|
|
12
|
+
id: string;
|
|
13
|
+
title: string;
|
|
14
|
+
kind: 'bug' | 'control';
|
|
15
|
+
/** Bug cases: the planted defect was reported */
|
|
16
|
+
caught: boolean;
|
|
17
|
+
/** Findings that matched the planted defect */
|
|
18
|
+
matched: number;
|
|
19
|
+
/** Unmatched findings at medium or above (false alarms on controls, noise on bug cases) */
|
|
20
|
+
noise: number;
|
|
21
|
+
findings: BenchFinding[];
|
|
22
|
+
error?: string;
|
|
23
|
+
durationMs: number;
|
|
24
|
+
}
|
|
25
|
+
export interface BenchSummary {
|
|
26
|
+
bugCases: number;
|
|
27
|
+
caught: number;
|
|
28
|
+
/** caught / bugCases */
|
|
29
|
+
recall: number;
|
|
30
|
+
/** matched findings / (matched + noise) over all cases; 1 when nothing was reported */
|
|
31
|
+
precision: number;
|
|
32
|
+
controlCases: number;
|
|
33
|
+
/** Control cases with at least one false alarm */
|
|
34
|
+
controlsFlagged: number;
|
|
35
|
+
errors: number;
|
|
36
|
+
}
|
|
37
|
+
/** How far (in lines) a finding may sit from the planted line and still count. */
|
|
38
|
+
export declare const LINE_TOLERANCE = 3;
|
|
39
|
+
/** Noise threshold: low/info nits never count against a reviewer. */
|
|
40
|
+
export declare const NOISE_SEVERITY: Severity;
|
|
41
|
+
/** 1-based line of the planted defect in the changed file. */
|
|
42
|
+
export declare function plantedLine(c: BenchCase): number;
|
|
43
|
+
export declare function matchesPlanted(c: BenchCase, f: BenchFinding): boolean;
|
|
44
|
+
export declare function scoreCase(c: BenchCase, findings: BenchFinding[], durationMs?: number, error?: string): CaseResult;
|
|
45
|
+
export declare function summarize(results: CaseResult[]): BenchSummary;
|
|
46
|
+
//# sourceMappingURL=score.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/bench/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,yDAAyD;AACzD,MAAM,WAAW,YAAY;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,QAAQ,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,UAAU;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,KAAK,GAAG,SAAS,CAAC;IACxB,iDAAiD;IACjD,MAAM,EAAE,OAAO,CAAC;IAChB,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;IAChB,2FAA2F;IAC3F,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,YAAY,EAAE,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,uFAAuF;IACvF,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,kDAAkD;IAClD,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,kFAAkF;AAClF,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,qEAAqE;AACrE,eAAO,MAAM,cAAc,EAAE,QAAmB,CAAC;AAEjD,8DAA8D;AAC9D,wBAAgB,WAAW,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAKhD;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,YAAY,GAAG,OAAO,CAKrE;AAED,wBAAgB,SAAS,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,EAAE,UAAU,SAAI,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,UAAU,CAkB5G;AAED,wBAAgB,SAAS,CAAC,OAAO,EAAE,UAAU,EAAE,GAAG,YAAY,CAe7D"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { meetsSeverity } from '../utils/severity.js';
|
|
2
|
+
/** How far (in lines) a finding may sit from the planted line and still count. */
|
|
3
|
+
export const LINE_TOLERANCE = 3;
|
|
4
|
+
/** Noise threshold: low/info nits never count against a reviewer. */
|
|
5
|
+
export const NOISE_SEVERITY = 'medium';
|
|
6
|
+
/** 1-based line of the planted defect in the changed file. */
|
|
7
|
+
export function plantedLine(c) {
|
|
8
|
+
if (!c.expect)
|
|
9
|
+
return 0;
|
|
10
|
+
const idx = c.changed.split('\n').findIndex((l) => l.includes(c.expect.needle));
|
|
11
|
+
if (idx === -1)
|
|
12
|
+
throw new Error(`Bench case ${c.id}: needle not found in changed file`);
|
|
13
|
+
return idx + 1;
|
|
14
|
+
}
|
|
15
|
+
export function matchesPlanted(c, f) {
|
|
16
|
+
if (!c.expect)
|
|
17
|
+
return false;
|
|
18
|
+
if (f.filePath !== c.file)
|
|
19
|
+
return false;
|
|
20
|
+
if (Math.abs(f.line - plantedLine(c)) > LINE_TOLERANCE)
|
|
21
|
+
return false;
|
|
22
|
+
return c.expect.types.includes(f.type);
|
|
23
|
+
}
|
|
24
|
+
export function scoreCase(c, findings, durationMs = 0, error) {
|
|
25
|
+
let matched = 0;
|
|
26
|
+
let noise = 0;
|
|
27
|
+
for (const f of findings) {
|
|
28
|
+
if (matchesPlanted(c, f))
|
|
29
|
+
matched++;
|
|
30
|
+
else if (meetsSeverity(f.severity, NOISE_SEVERITY))
|
|
31
|
+
noise++;
|
|
32
|
+
}
|
|
33
|
+
return {
|
|
34
|
+
id: c.id,
|
|
35
|
+
title: c.title,
|
|
36
|
+
kind: c.expect ? 'bug' : 'control',
|
|
37
|
+
caught: matched > 0,
|
|
38
|
+
matched,
|
|
39
|
+
noise,
|
|
40
|
+
findings,
|
|
41
|
+
error,
|
|
42
|
+
durationMs,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
export function summarize(results) {
|
|
46
|
+
const bugs = results.filter((r) => r.kind === 'bug');
|
|
47
|
+
const controls = results.filter((r) => r.kind === 'control');
|
|
48
|
+
const caught = bugs.filter((r) => r.caught).length;
|
|
49
|
+
const matched = results.reduce((n, r) => n + r.matched, 0);
|
|
50
|
+
const noise = results.reduce((n, r) => n + r.noise, 0);
|
|
51
|
+
return {
|
|
52
|
+
bugCases: bugs.length,
|
|
53
|
+
caught,
|
|
54
|
+
recall: bugs.length ? caught / bugs.length : 0,
|
|
55
|
+
precision: matched + noise ? matched / (matched + noise) : 1,
|
|
56
|
+
controlCases: controls.length,
|
|
57
|
+
controlsFlagged: controls.filter((r) => r.noise > 0).length,
|
|
58
|
+
errors: results.filter((r) => r.error).length,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/bench/score.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAwCrD,kFAAkF;AAClF,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAChC,qEAAqE;AACrE,MAAM,CAAC,MAAM,cAAc,GAAa,QAAQ,CAAC;AAEjD,8DAA8D;AAC9D,MAAM,UAAU,WAAW,CAAC,CAAY;IACpC,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACxB,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACjF,IAAI,GAAG,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACxF,OAAO,GAAG,GAAG,CAAC,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAY,EAAE,CAAe;IACxD,IAAI,CAAC,CAAC,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC5B,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,cAAc;QAAE,OAAO,KAAK,CAAC;IACrE,OAAQ,CAAC,CAAC,MAAM,CAAC,KAAkB,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,CAAY,EAAE,QAAwB,EAAE,UAAU,GAAG,CAAC,EAAE,KAAc;IAC5F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvB,IAAI,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC;YAAE,OAAO,EAAE,CAAC;aAC/B,IAAI,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,cAAc,CAAC;YAAE,KAAK,EAAE,CAAC;IAChE,CAAC;IACD,OAAO;QACH,EAAE,EAAE,CAAC,CAAC,EAAE;QACR,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;QAClC,MAAM,EAAE,OAAO,GAAG,CAAC;QACnB,OAAO;QACP,KAAK;QACL,QAAQ;QACR,KAAK;QACL,UAAU;KACb,CAAC;AACN,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,OAAqB;IAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACvD,OAAO;QACH,QAAQ,EAAE,IAAI,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9C,SAAS,EAAE,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,YAAY,EAAE,QAAQ,CAAC,MAAM;QAC7B,eAAe,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM;KAChD,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import type { AIProvider } from '../ai/provider.js';
|
|
3
|
+
import { type BenchCase } from '../bench/cases.js';
|
|
4
|
+
import { type BenchSummary, type CaseResult } from '../bench/score.js';
|
|
5
|
+
export declare const benchCommand: Command;
|
|
6
|
+
/** Build a throwaway repo for one case, stage the change, and review it. Nothing is persisted. */
|
|
7
|
+
export declare function runCase(c: BenchCase, provider: AIProvider): Promise<CaseResult>;
|
|
8
|
+
export declare function toMarkdown(results: CaseResult[], s: BenchSummary, model: string): string;
|
|
9
|
+
//# sourceMappingURL=bench.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bench.d.ts","sourceRoot":"","sources":["../../src/commands/bench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAUpC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAe,KAAK,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAA2C,KAAK,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAKhH,eAAO,MAAM,YAAY,SAgEnB,CAAC;AAiBP,kGAAkG;AAClG,wBAAsB,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CA6BrF;AA4BD,wBAAgB,UAAU,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAexF"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import { Command } from 'commander';
|
|
2
|
+
import { execFileSync } from 'child_process';
|
|
3
|
+
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'fs';
|
|
4
|
+
import { writeFile } from 'fs/promises';
|
|
5
|
+
import { tmpdir } from 'os';
|
|
6
|
+
import { dirname, join } from 'path';
|
|
7
|
+
import chalk from 'chalk';
|
|
8
|
+
import { logger } from '../utils/logger.js';
|
|
9
|
+
import { configManager } from '../config/manager.js';
|
|
10
|
+
import { AIProviderFactory } from '../ai/factory.js';
|
|
11
|
+
import { analyzeChanges, prepareReview } from '../core/review/run.js';
|
|
12
|
+
import { anchorLine } from '../core/review/report.js';
|
|
13
|
+
import { BENCH_CASES } from '../bench/cases.js';
|
|
14
|
+
import { scoreCase, summarize } from '../bench/score.js';
|
|
15
|
+
const FORMATS = ['text', 'json', 'markdown'];
|
|
16
|
+
export const benchCommand = new Command('bench')
|
|
17
|
+
.description('Measure review quality on seeded-bug cases with your configured model')
|
|
18
|
+
.option('--case <ids>', 'Comma-separated case ids to run (default: all)')
|
|
19
|
+
.option('--list', 'List the cases and exit')
|
|
20
|
+
.option('--model <model>', 'Model to use for this run instead of the configured one')
|
|
21
|
+
.option('--format <format>', 'Output format: text, json, markdown', 'text')
|
|
22
|
+
.option('--output <file>', 'Write the report to a file')
|
|
23
|
+
.option('--min-recall <ratio>', 'Exit with code 1 if recall is below this ratio (0 to 1)')
|
|
24
|
+
.action(async (options) => {
|
|
25
|
+
try {
|
|
26
|
+
if (options.list) {
|
|
27
|
+
for (const c of BENCH_CASES) {
|
|
28
|
+
console.log(`${c.id.padEnd(20)} ${(c.expect ? 'bug' : 'control').padEnd(8)} ${c.language.padEnd(11)} ${c.title}`);
|
|
29
|
+
}
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const format = String(options.format).toLowerCase();
|
|
33
|
+
if (!FORMATS.includes(format))
|
|
34
|
+
throw new Error(`Unknown format "${options.format}". Use one of: ${FORMATS.join(', ')}`);
|
|
35
|
+
const minRecall = options.minRecall === undefined ? undefined : Number(options.minRecall);
|
|
36
|
+
if (minRecall !== undefined && !(minRecall >= 0 && minRecall <= 1))
|
|
37
|
+
throw new Error('--min-recall must be a number from 0 to 1');
|
|
38
|
+
const cases = selectCases(options.case);
|
|
39
|
+
const config = await configManager.load();
|
|
40
|
+
if (!(await configManager.exists()) || !config.activeProvider) {
|
|
41
|
+
throw new Error('No AI provider configured. Run: ownkey config --provider <name> (see ownkey config --help)');
|
|
42
|
+
}
|
|
43
|
+
const provider = await AIProviderFactory.createProvider({ model: options.model });
|
|
44
|
+
const modelLabel = `${config.activeProvider}/${provider.getModel()}`;
|
|
45
|
+
const machine = format !== 'text' && !options.output;
|
|
46
|
+
if (machine)
|
|
47
|
+
logger.useStderr();
|
|
48
|
+
const say = (msg) => (machine ? process.stderr.write(msg + '\n') : console.log(msg));
|
|
49
|
+
say(chalk.bold(`\nOwnKey bench: ${cases.length} case(s) with ${modelLabel}\n`));
|
|
50
|
+
const results = [];
|
|
51
|
+
for (const c of cases) {
|
|
52
|
+
const r = await runCase(c, provider);
|
|
53
|
+
results.push(r);
|
|
54
|
+
say(caseLine(r));
|
|
55
|
+
}
|
|
56
|
+
const summary = summarize(results);
|
|
57
|
+
let body;
|
|
58
|
+
if (format === 'json')
|
|
59
|
+
body = JSON.stringify({ model: modelLabel, summary, results }, null, 2) + '\n';
|
|
60
|
+
else if (format === 'markdown')
|
|
61
|
+
body = toMarkdown(results, summary, modelLabel);
|
|
62
|
+
else
|
|
63
|
+
body = summaryText(summary);
|
|
64
|
+
if (options.output) {
|
|
65
|
+
await writeFile(options.output, body, 'utf-8');
|
|
66
|
+
console.log(summaryText(summary));
|
|
67
|
+
console.log(chalk.green(` Wrote ${format} report to ${options.output}`));
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
process.stdout.write(body);
|
|
71
|
+
}
|
|
72
|
+
if (minRecall !== undefined && summary.recall < minRecall) {
|
|
73
|
+
say(chalk.red(`\n Recall ${pct(summary.recall)} is below --min-recall ${pct(minRecall)}.\n`));
|
|
74
|
+
process.exitCode = 1;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
logger.error(`Bench failed: ${error.message}`);
|
|
79
|
+
process.exitCode = 2;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
82
|
+
function selectCases(ids) {
|
|
83
|
+
if (!ids)
|
|
84
|
+
return BENCH_CASES;
|
|
85
|
+
const wanted = String(ids).split(',').map((s) => s.trim()).filter(Boolean);
|
|
86
|
+
const unknown = wanted.filter((id) => !BENCH_CASES.some((c) => c.id === id));
|
|
87
|
+
if (unknown.length)
|
|
88
|
+
throw new Error(`Unknown case(s): ${unknown.join(', ')}. See ownkey bench --list`);
|
|
89
|
+
return BENCH_CASES.filter((c) => wanted.includes(c.id));
|
|
90
|
+
}
|
|
91
|
+
function git(cwd, ...args) {
|
|
92
|
+
execFileSync('git', ['-c', 'user.name=ownkey-bench', '-c', 'user.email=bench@ownkey.local', '-c', 'commit.gpgsign=false', ...args], {
|
|
93
|
+
cwd,
|
|
94
|
+
stdio: 'ignore',
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
/** Build a throwaway repo for one case, stage the change, and review it. Nothing is persisted. */
|
|
98
|
+
export async function runCase(c, provider) {
|
|
99
|
+
const started = Date.now();
|
|
100
|
+
const repo = mkdtempSync(join(tmpdir(), `ownkey-bench-${c.id}-`));
|
|
101
|
+
try {
|
|
102
|
+
const file = join(repo, c.file);
|
|
103
|
+
mkdirSync(dirname(file), { recursive: true });
|
|
104
|
+
writeFileSync(file, c.base);
|
|
105
|
+
git(repo, 'init', '-q');
|
|
106
|
+
git(repo, 'add', '.');
|
|
107
|
+
git(repo, 'commit', '-qm', 'base');
|
|
108
|
+
writeFileSync(file, c.changed);
|
|
109
|
+
git(repo, 'add', '.');
|
|
110
|
+
const scope = { kind: 'staged' };
|
|
111
|
+
const target = await prepareReview(repo, scope);
|
|
112
|
+
const suggestions = await analyzeChanges(repo, scope, target, provider);
|
|
113
|
+
const findings = suggestions.map((s) => ({
|
|
114
|
+
filePath: s.filePath,
|
|
115
|
+
line: anchorLine(s, target.changes.changedLines.get(s.filePath)),
|
|
116
|
+
type: s.type,
|
|
117
|
+
severity: s.severity,
|
|
118
|
+
content: s.content,
|
|
119
|
+
}));
|
|
120
|
+
return scoreCase(c, findings, Date.now() - started);
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
return scoreCase(c, [], Date.now() - started, error.message);
|
|
124
|
+
}
|
|
125
|
+
finally {
|
|
126
|
+
rmSync(repo, { recursive: true, force: true });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
const pct = (x) => `${Math.round(x * 100)}%`;
|
|
130
|
+
function caseLine(r) {
|
|
131
|
+
const secs = chalk.gray(`${(r.durationMs / 1000).toFixed(1)}s`);
|
|
132
|
+
if (r.error)
|
|
133
|
+
return ` ${chalk.red('ERROR ')} ${r.id.padEnd(20)} ${chalk.red(r.error)} ${secs}`;
|
|
134
|
+
const noise = r.noise ? chalk.yellow(` +${r.noise} noise`) : '';
|
|
135
|
+
if (r.kind === 'control') {
|
|
136
|
+
return ` ${r.noise ? chalk.yellow('FLAG ') : chalk.green('CLEAN ')} ${r.id.padEnd(20)} ${chalk.gray('control')}${noise} ${secs}`;
|
|
137
|
+
}
|
|
138
|
+
return ` ${r.caught ? chalk.green('CAUGHT') : chalk.red('MISSED')} ${r.id.padEnd(20)} ${chalk.gray(r.title)}${noise} ${secs}`;
|
|
139
|
+
}
|
|
140
|
+
function summaryText(s) {
|
|
141
|
+
return [
|
|
142
|
+
'',
|
|
143
|
+
chalk.bold(` Recall ${pct(s.recall)} (${s.caught}/${s.bugCases} planted bugs caught)`),
|
|
144
|
+
chalk.bold(` Precision ${pct(s.precision)} (matched findings vs. medium+ findings that matched nothing)`),
|
|
145
|
+
chalk.bold(` Controls ${s.controlCases - s.controlsFlagged}/${s.controlCases} clean changes passed without a false alarm`),
|
|
146
|
+
s.errors ? chalk.red(` Errors ${s.errors} case(s) could not run`) : '',
|
|
147
|
+
'',
|
|
148
|
+
]
|
|
149
|
+
.filter((l) => l !== '')
|
|
150
|
+
.join('\n')
|
|
151
|
+
.concat('\n\n');
|
|
152
|
+
}
|
|
153
|
+
export function toMarkdown(results, s, model) {
|
|
154
|
+
const out = [
|
|
155
|
+
`## OwnKey bench: ${model}`,
|
|
156
|
+
'',
|
|
157
|
+
`**Recall ${pct(s.recall)}** (${s.caught}/${s.bugCases} planted bugs caught) · **Precision ${pct(s.precision)}** · **Controls ${s.controlCases - s.controlsFlagged}/${s.controlCases} clean**`,
|
|
158
|
+
'',
|
|
159
|
+
'| Case | Kind | Result | Noise |',
|
|
160
|
+
'| --- | --- | --- | --- |',
|
|
161
|
+
];
|
|
162
|
+
for (const r of results) {
|
|
163
|
+
const result = r.error ? `error: ${r.error}` : r.kind === 'control' ? (r.noise ? 'false alarm' : 'clean') : r.caught ? 'caught' : 'missed';
|
|
164
|
+
out.push(`| ${r.id} | ${r.kind} | ${result} | ${r.noise} |`);
|
|
165
|
+
}
|
|
166
|
+
out.push('', '_Recall: planted bugs reported on the right line (within 3) with a matching type. Precision: matching findings over all findings at medium or above. Low and info findings never count as noise._', '');
|
|
167
|
+
return out.join('\n');
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=bench.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bench.js","sourceRoot":"","sources":["../../src/commands/bench.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAErD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,WAAW,EAAkB,MAAM,mBAAmB,CAAC;AAChE,OAAO,EAAE,SAAS,EAAE,SAAS,EAAyD,MAAM,mBAAmB,CAAC;AAEhH,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,CAAU,CAAC;AAGtD,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC;KAC3C,WAAW,CAAC,uEAAuE,CAAC;KACpF,MAAM,CAAC,cAAc,EAAE,gDAAgD,CAAC;KACxE,MAAM,CAAC,QAAQ,EAAE,yBAAyB,CAAC;KAC3C,MAAM,CAAC,iBAAiB,EAAE,yDAAyD,CAAC;KACpF,MAAM,CAAC,mBAAmB,EAAE,qCAAqC,EAAE,MAAM,CAAC;KAC1E,MAAM,CAAC,iBAAiB,EAAE,4BAA4B,CAAC;KACvD,MAAM,CAAC,sBAAsB,EAAE,yDAAyD,CAAC;KACzF,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACtB,IAAI,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;gBAC1B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;YACtH,CAAC;YACD,OAAO;QACX,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,EAAY,CAAC;QAC9D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,MAAM,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxH,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAC1F,IAAI,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAEjI,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAC;QAClH,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,cAAc,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QAClF,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,cAAc,IAAI,QAAQ,CAAC,QAAQ,EAAE,EAAE,CAAC;QAErE,MAAM,OAAO,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD,IAAI,OAAO;YAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAE7F,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,KAAK,CAAC,MAAM,iBAAiB,UAAU,IAAI,CAAC,CAAC,CAAC;QAChF,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACpB,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QACD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;QAEnC,IAAI,IAAY,CAAC;QACjB,IAAI,MAAM,KAAK,MAAM;YAAE,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;aACjG,IAAI,MAAM,KAAK,UAAU;YAAE,IAAI,GAAG,UAAU,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC;;YAC3E,IAAI,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEjC,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,MAAM,cAAc,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9E,CAAC;aAAM,CAAC;YACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,SAAS,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YACxD,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,0BAA0B,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/F,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,iBAAkB,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC,CAAC,CAAC;AAEP,SAAS,WAAW,CAAC,GAAY;IAC7B,IAAI,CAAC,GAAG;QAAE,OAAO,WAAW,CAAC;IAC7B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC7E,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;IACvG,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,GAAG,CAAC,GAAW,EAAE,GAAG,IAAc;IACvC,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,+BAA+B,EAAE,IAAI,EAAE,sBAAsB,EAAE,GAAG,IAAI,CAAC,EAAE;QAChI,GAAG;QACH,KAAK,EAAE,QAAQ;KAClB,CAAC,CAAC;AACP,CAAC;AAED,kGAAkG;AAClG,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,CAAY,EAAE,QAAoB;IAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,gBAAgB,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAChC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC5B,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACxB,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACtB,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACnC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QAC/B,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAEtB,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAW,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChD,MAAM,WAAW,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxE,MAAM,QAAQ,GAAmB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrD,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,IAAI,EAAE,UAAU,CAAC,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAChE,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;SACrB,CAAC,CAAC,CAAC;QACJ,OAAO,SAAS,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAG,KAAe,CAAC,OAAO,CAAC,CAAC;IAC5E,CAAC;YAAS,CAAC;QACP,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnD,CAAC;AACL,CAAC;AAED,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC;AAErD,SAAS,QAAQ,CAAC,CAAa;IAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAChE,IAAI,CAAC,CAAC,KAAK;QAAE,OAAO,KAAK,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;IAChG,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;IACvI,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,IAAI,EAAE,CAAC;AACnI,CAAC;AAED,SAAS,WAAW,CAAC,CAAe;IAChC,OAAO;QACH,EAAE;QACF,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,uBAAuB,CAAC;QAC5F,KAAK,CAAC,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,gEAAgE,CAAC;QAC5G,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,6CAA6C,CAAC;QAC7H,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,MAAM,wBAAwB,CAAC,CAAC,CAAC,CAAC,EAAE;QAC3E,EAAE;KACL;SACI,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC;SACV,MAAM,CAAC,MAAM,CAAC,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAqB,EAAE,CAAe,EAAE,KAAa;IAC5E,MAAM,GAAG,GAAG;QACR,oBAAoB,KAAK,EAAE;QAC3B,EAAE;QACF,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,uCAAuC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,UAAU;QAC9L,EAAE;QACF,kCAAkC;QAClC,2BAA2B;KAC9B,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC3I,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,IAAI,MAAM,MAAM,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACjE,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,mMAAmM,EAAE,EAAE,CAAC,CAAC;IACtN,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"review.d.ts","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"review.d.ts","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAiBpC,eAAO,MAAM,aAAa,SAgFpB,CAAC"}
|
package/dist/commands/review.js
CHANGED
|
@@ -4,12 +4,12 @@ import { writeFile } from 'fs/promises';
|
|
|
4
4
|
import { createRequire } from 'module';
|
|
5
5
|
import chalk from 'chalk';
|
|
6
6
|
import { logger } from '../utils/logger.js';
|
|
7
|
-
import { FileScanner } from '../core/scanner.js';
|
|
8
7
|
import { AIProviderFactory } from '../ai/factory.js';
|
|
9
8
|
import { configManager } from '../config/manager.js';
|
|
10
9
|
import { meetsSeverity, parseSeverity } from '../utils/severity.js';
|
|
11
10
|
import { persistSuggestions } from '../core/persist.js';
|
|
12
|
-
import {
|
|
11
|
+
import { describeScope, gitRoot, resolveScope } from '../core/review/git.js';
|
|
12
|
+
import { analyzeChanges, prepareReview } from '../core/review/run.js';
|
|
13
13
|
import { anchorLine, sortFindings, toMarkdown, toSarif } from '../core/review/report.js';
|
|
14
14
|
const FORMATS = ['text', 'markdown', 'json', 'sarif'];
|
|
15
15
|
export const reviewCommand = new Command('review')
|
|
@@ -49,42 +49,23 @@ export const reviewCommand = new Command('review')
|
|
|
49
49
|
}
|
|
50
50
|
const root = gitRoot(resolve(path));
|
|
51
51
|
const scope = resolveScope(root, { staged: options.staged, base: options.base });
|
|
52
|
-
const changes = collectChanges(root, scope);
|
|
53
52
|
const scopeLabel = describeScope(scope);
|
|
54
|
-
// Honour .gitignore/.ownkeyignore, size and binary rules.
|
|
55
|
-
const scanner = new FileScanner(root);
|
|
56
|
-
const scanned = await scanner.scan();
|
|
57
|
-
const reviewable = new Set(scanned.files.map((f) => f.relativePath.replace(/\\/g, '/')));
|
|
58
|
-
const files = changes.files.filter((f) => reviewable.has(f));
|
|
59
53
|
const maxFiles = parseInt(options.maxFiles, 10) || 25;
|
|
54
|
+
const target = await prepareReview(root, scope, maxFiles);
|
|
55
|
+
const { changes, reviewable: files, selected } = target;
|
|
60
56
|
if (files.length === 0) {
|
|
61
57
|
say(chalk.green(`\n Nothing to review in ${scopeLabel}.\n`));
|
|
62
58
|
await emit([], format, options.output, scopeLabel, config.activeProvider);
|
|
63
59
|
await writeExtras([], options, scopeLabel, config.activeProvider);
|
|
64
60
|
return;
|
|
65
61
|
}
|
|
66
|
-
const selected = files.slice(0, maxFiles);
|
|
67
62
|
say(chalk.bold(`\nReviewing ${selected.length} file(s) in ${scopeLabel}`) + (files.length > selected.length ? chalk.yellow(` (${files.length - selected.length} skipped by --max-files)`) : ''));
|
|
68
63
|
for (const f of selected)
|
|
69
64
|
say(chalk.gray(` ${f}`));
|
|
70
|
-
const byPath = new Map(scanned.files.map((f) => [f.relativePath.replace(/\\/g, '/'), f]));
|
|
71
|
-
const contents = selected.map((f) => ({ ...byPath.get(f), content: readForReview(root, scope, f) }));
|
|
72
|
-
const patch = changes.patch;
|
|
73
|
-
const context = {
|
|
74
|
-
files: contents,
|
|
75
|
-
projectInfo: { name: basename(root), rootPath: root, fileCount: contents.length },
|
|
76
|
-
reviewPatch: patch,
|
|
77
|
-
};
|
|
78
65
|
const provider = await AIProviderFactory.createProvider();
|
|
79
66
|
const model = provider.getModel();
|
|
80
67
|
say(chalk.gray(`\n Model: ${config.activeProvider} / ${model}\n`));
|
|
81
|
-
const
|
|
82
|
-
for await (const s of provider.analyze(context))
|
|
83
|
-
raw.push(s);
|
|
84
|
-
const inScope = new Set(selected);
|
|
85
|
-
const kept = raw
|
|
86
|
-
.map((s) => ({ ...s, filePath: s.filePath.replace(/\\/g, '/').replace(/^\.\//, '') }))
|
|
87
|
-
.filter((s) => inScope.has(s.filePath))
|
|
68
|
+
const kept = (await analyzeChanges(root, scope, target, provider))
|
|
88
69
|
.filter((s) => !typeFilter || s.type === typeFilter)
|
|
89
70
|
.filter((s) => meetsSeverity(s.severity, minSeverity));
|
|
90
71
|
const stored = await persistSuggestions(root, basename(root), kept, `review ${scopeLabel}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"review.js","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"review.js","sourceRoot":"","sources":["../../src/commands/review.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtE,OAAO,EAAW,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,0BAA0B,CAAC;AAElG,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAU,CAAC;AAG/D,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;KAC7C,WAAW,CAAC,4EAA4E,CAAC;KACzF,QAAQ,CAAC,QAAQ,EAAE,gCAAgC,EAAE,GAAG,CAAC;KACzD,MAAM,CAAC,UAAU,EAAE,4BAA4B,CAAC;KAChD,MAAM,CAAC,cAAc,EAAE,yEAAyE,CAAC;KACjG,MAAM,CAAC,oBAAoB,EAAE,gEAAgE,CAAC;KAC9F,MAAM,CAAC,eAAe,EAAE,yDAAyD,CAAC;KAClF,MAAM,CAAC,mBAAmB,EAAE,4CAA4C,EAAE,MAAM,CAAC;KACjF,MAAM,CAAC,iBAAiB,EAAE,8CAA8C,CAAC;KACzE,MAAM,CAAC,uBAAuB,EAAE,+DAA+D,CAAC;KAChG,MAAM,CAAC,0BAA0B,EAAE,qEAAqE,CAAC;KACzG,MAAM,CAAC,mBAAmB,EAAE,uEAAuE,CAAC;KACpG,MAAM,CAAC,sBAAsB,EAAE,iCAAiC,EAAE,IAAI,CAAC;KACvE,MAAM,CAAC,KAAK,EAAE,IAAY,EAAE,OAAO,EAAE,EAAE;IACpC,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,EAAY,CAAC;QAC9D,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,MAAM,kBAAkB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,MAAM,WAAW,GAAG,aAAa,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAuB,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC;QACnE,MAAM,OAAO,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACrD,IAAI,OAAO;YAAE,MAAM,CAAC,SAAS,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE;YACxB,IAAI,OAAO;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;;gBACzC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,4FAA4F,CAAC,CAAC;QAClH,CAAC;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACpC,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QACjF,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC1D,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;QAExD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,4BAA4B,UAAU,KAAK,CAAC,CAAC,CAAC;YAC9D,MAAM,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;YAC1E,MAAM,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;YAClE,OAAO;QACX,CAAC;QAED,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,MAAM,eAAe,UAAU,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,0BAA0B,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACjM,KAAK,MAAM,CAAC,IAAI,QAAQ;YAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAEpD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAClC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,cAAc,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC;QAEpE,MAAM,IAAI,GAAG,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;aAC7D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;aACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAE3D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,UAAU,EAAE,CAAC,CAAC;QAC5F,MAAM,QAAQ,GAAG,YAAY,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YAChB,GAAG,CAAC;YACJ,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACjB,IAAI,EAAE,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;SAC5D,CAAC,CAAC,CACN,CAAC;QAEF,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,cAAc,IAAI,KAAK,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QACrE,MAAM,WAAW,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAE7D,IAAI,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,0CAA0C,MAAM,MAAM,CAAC,CAAC,CAAC;YACvE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,KAAK,CAAC,kBAAmB,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzB,CAAC;AACL,CAAC,CAAC,CAAC;AAEP,KAAK,UAAU,WAAW,CAAC,QAAmB,EAAE,OAA0D,EAAE,UAAkB,EAAE,KAAa;IACzI,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACtB,MAAM,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IACvH,CAAC;IACD,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;QACzB,MAAM,SAAS,CAAC,OAAO,CAAC,cAAc,EAAE,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9F,CAAC;AACL,CAAC;AAED,SAAS,cAAc;IACnB,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAC,CAAC,OAAO,CAAC;AACxE,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,QAAmB,EAAE,MAAc,EAAE,MAA0B,EAAE,UAAkB,EAAE,KAAa;IAClH,IAAI,IAAY,CAAC;IACjB,QAAQ,MAAM,EAAE,CAAC;QACb,KAAK,OAAO,CAAC,CAAC,CAAC;YACX,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;YAC3E,MAAM;QACV,CAAC;QACD,KAAK,MAAM;YACP,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;YAC9E,MAAM;QACV,KAAK,UAAU;YACX,IAAI,GAAG,UAAU,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM;QACV;YACI,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,MAAM,EAAE,CAAC;QACT,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,MAAM,cAAc,MAAM,EAAE,CAAC,CAAC,CAAC;QAClE,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7D,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CAAC,QAAmB;IACnC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACjI,MAAM,GAAG,GAAa,CAAC,EAAE,CAAC,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACvB,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QAC/I,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE;YAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAClF,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,MAAM,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClK,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { type ChangeSet, type ReviewScope } from './git.js';
|
|
2
|
+
import type { AIProvider } from '../../ai/provider.js';
|
|
3
|
+
import type { FileMetadata, Suggestion } from '../../types/index.js';
|
|
4
|
+
export interface ReviewTarget {
|
|
5
|
+
changes: ChangeSet;
|
|
6
|
+
/** Changed files that pass ignore/size/binary rules */
|
|
7
|
+
reviewable: string[];
|
|
8
|
+
/** Files actually sent to the model (reviewable, capped by maxFiles) */
|
|
9
|
+
selected: string[];
|
|
10
|
+
/** Scanner metadata by repo-relative path */
|
|
11
|
+
metadata: Map<string, FileMetadata>;
|
|
12
|
+
}
|
|
13
|
+
/** Work out which changed files to review, honouring .gitignore/.ownkeyignore, size and binary rules. */
|
|
14
|
+
export declare function prepareReview(root: string, scope: ReviewScope, maxFiles?: number): Promise<ReviewTarget>;
|
|
15
|
+
/**
|
|
16
|
+
* Send the selected files (in full, for context) and the patch (as focus) to the model.
|
|
17
|
+
* Returns suggestions on the selected files only, with normalized paths.
|
|
18
|
+
*/
|
|
19
|
+
export declare function analyzeChanges(root: string, scope: ReviewScope, target: ReviewTarget, provider: AIProvider): Promise<Suggestion[]>;
|
|
20
|
+
//# sourceMappingURL=run.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/core/review/run.ts"],"names":[],"mappings":"AAEA,OAAO,EAAiC,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,UAAU,CAAC;AAC3F,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EAA4B,YAAY,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAE/F,MAAM,WAAW,YAAY;IACzB,OAAO,EAAE,SAAS,CAAC;IACnB,uDAAuD;IACvD,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,yGAAyG;AACzG,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,SAAK,GAAG,OAAO,CAAC,YAAY,CAAC,CAO1G;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAgBxI"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { basename } from 'path';
|
|
2
|
+
import { FileScanner } from '../scanner.js';
|
|
3
|
+
import { collectChanges, readForReview } from './git.js';
|
|
4
|
+
/** Work out which changed files to review, honouring .gitignore/.ownkeyignore, size and binary rules. */
|
|
5
|
+
export async function prepareReview(root, scope, maxFiles = 25) {
|
|
6
|
+
const changes = collectChanges(root, scope);
|
|
7
|
+
const scanner = new FileScanner(root);
|
|
8
|
+
const scanned = await scanner.scan();
|
|
9
|
+
const metadata = new Map(scanned.files.map((f) => [f.relativePath.replace(/\\/g, '/'), f]));
|
|
10
|
+
const reviewable = changes.files.filter((f) => metadata.has(f));
|
|
11
|
+
return { changes, reviewable, selected: reviewable.slice(0, maxFiles), metadata };
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Send the selected files (in full, for context) and the patch (as focus) to the model.
|
|
15
|
+
* Returns suggestions on the selected files only, with normalized paths.
|
|
16
|
+
*/
|
|
17
|
+
export async function analyzeChanges(root, scope, target, provider) {
|
|
18
|
+
const contents = target.selected.map((f) => ({ ...target.metadata.get(f), content: readForReview(root, scope, f) }));
|
|
19
|
+
const context = {
|
|
20
|
+
files: contents,
|
|
21
|
+
projectInfo: { name: basename(root), rootPath: root, fileCount: contents.length },
|
|
22
|
+
reviewPatch: target.changes.patch,
|
|
23
|
+
};
|
|
24
|
+
const raw = [];
|
|
25
|
+
for await (const s of provider.analyze(context))
|
|
26
|
+
raw.push(s);
|
|
27
|
+
const inScope = new Set(target.selected);
|
|
28
|
+
return raw
|
|
29
|
+
.map((s) => ({ ...s, filePath: s.filePath.replace(/\\/g, '/').replace(/^\.\//, '') }))
|
|
30
|
+
.filter((s) => inScope.has(s.filePath));
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../../src/core/review/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,aAAa,EAAoC,MAAM,UAAU,CAAC;AAc3F,yGAAyG;AACzG,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,KAAkB,EAAE,QAAQ,GAAG,EAAE;IAC/E,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAU,CAAC,CAAC,CAAC;IACrG,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;AACtF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,KAAkB,EAAE,MAAoB,EAAE,QAAoB;IAC7G,MAAM,QAAQ,GAAkB,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,EAAE,OAAO,EAAE,aAAa,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAErI,MAAM,OAAO,GAAgB;QACzB,KAAK,EAAE,QAAQ;QACf,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE;QACjF,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK;KACpC,CAAC;IAEF,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE7D,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACzC,OAAO,GAAG;SACL,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;SACrF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAChD,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -10,6 +10,7 @@ import { suggestCommand } from './commands/suggest.js';
|
|
|
10
10
|
import { applyCommand } from './commands/apply.js';
|
|
11
11
|
import { undoCommand } from './commands/undo.js';
|
|
12
12
|
import { reviewCommand } from './commands/review.js';
|
|
13
|
+
import { benchCommand } from './commands/bench.js';
|
|
13
14
|
const program = new Command();
|
|
14
15
|
program
|
|
15
16
|
.name('ownkey')
|
|
@@ -27,6 +28,7 @@ program.addCommand(suggestCommand);
|
|
|
27
28
|
program.addCommand(reviewCommand);
|
|
28
29
|
program.addCommand(applyCommand);
|
|
29
30
|
program.addCommand(undoCommand);
|
|
31
|
+
program.addCommand(benchCommand);
|
|
30
32
|
// Pre-action hook: set up logging and storage.
|
|
31
33
|
// Note: the hook is registered on the root program, so `thisCommand` is always the
|
|
32
34
|
// root. The command actually being run is the second argument (`actionCommand`).
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC,kBAAkB;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AAEvC,kBAAkB;AAClB,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACF,IAAI,CAAC,QAAQ,CAAC;KACd,WAAW,CAAC,yDAAyD,CAAC;KACtE,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC;AAExE,iBAAiB;AACjB,OAAO,CAAC,MAAM,CAAC,cAAc,EAAE,kCAAkC,CAAC,CAAC;AACnE,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;AACtD,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,sBAAsB,CAAC,CAAC;AAClD,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,+BAA+B,CAAC,CAAC;AAE3D,WAAW;AACX,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChC,OAAO,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;AACnC,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;AAClC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;AACjC,OAAO,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;AAChC,OAAO,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;AAEjC,+CAA+C;AAC/C,mFAAmF;AACnF,iFAAiF;AACjF,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,EAAE;IAC3D,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IAEpB,wCAAwC;IACxC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IAEhC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;SAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;SAAM,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,yFAAyF;IACzF,IAAI,aAAa,CAAC,IAAI,EAAE,KAAK,QAAQ,EAAE,CAAC;QACpC,OAAO;IACX,CAAC;IAED,2EAA2E;IAC3E,MAAM,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;AACpD,CAAC,CAAC,CAAC;AAEH,+BAA+B;AAC/B,OAAO,CAAC,IAAI,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE;IAClC,MAAM,QAAQ,CAAC,UAAU,EAAE,CAAC;AAChC,CAAC,CAAC,CAAC;AAEH,kBAAkB;AAClB,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ownkey",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Local-first AI code reviewer CLI: review git changes, gate PRs with SARIF, and apply verified fixes using your own OpenAI, Anthropic, Gemini, or Ollama keys",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|