@open-cr-agent/cli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/cloud.d.ts +32 -0
- package/dist/cloud.js +262 -0
- package/dist/memory.js +2 -2
- package/dist/metrics.d.ts +47 -0
- package/dist/metrics.js +227 -0
- package/dist/review/args.d.ts +9 -2
- package/dist/review/args.js +46 -2
- package/dist/review/cloud-providers.d.ts +12 -0
- package/dist/review/cloud-providers.js +59 -0
- package/dist/review/cloud-upload.d.ts +40 -0
- package/dist/review/cloud-upload.js +135 -0
- package/dist/review/command.d.ts +4 -0
- package/dist/review/command.js +54 -14
- package/dist/review/config.d.ts +66 -4
- package/dist/review/config.js +101 -10
- package/dist/review/plan-render.d.ts +1 -1
- package/dist/review/plan-render.js +1 -1
- package/dist/review/progress.js +1 -1
- package/dist/review/provenance.d.ts +6 -0
- package/dist/review/provenance.js +41 -0
- package/dist/review/remote-config.d.ts +7 -0
- package/dist/review/remote-config.js +12 -8
- package/dist/review/render.d.ts +2 -1
- package/dist/review/render.js +3 -3
- package/dist/review/runtimes.d.ts +5 -0
- package/dist/review/runtimes.js +29 -0
- package/dist/review/sarif-input.d.ts +4 -0
- package/dist/review/sarif-input.js +34 -0
- package/dist/review/sarif.js +9 -3
- package/dist/review/target.d.ts +3 -3
- package/dist/review/target.js +21 -13
- package/dist/review/terminal.js +1 -1
- package/dist/run.js +34 -5
- package/package.json +13 -8
package/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
`ocra` is the command line of [Open-CR-Agent](https://github.com/jma49/Open-CR-Agent), an open-source multi-agent code reviewer for local changes and GitHub pull requests. Deterministic code selects and bundles the files, matches rules and anchors comments; LLM reviewers make the judgment calls, and a verification pass checks what they report.
|
|
4
4
|
|
|
5
|
-
This is an early 0.x release: options and output may change between minor versions. What the reviews find and miss is measured on the [quality page](https://
|
|
5
|
+
This is an early 0.x release: options and output may change between minor versions. What the reviews find and miss is measured on the [quality page](https://ocracloud.com/en/docs/quality).
|
|
6
6
|
|
|
7
7
|
## Install
|
|
8
8
|
|
|
@@ -34,8 +34,8 @@ Pull requests can also be reviewed by the GitHub Action: inline comments, one su
|
|
|
34
34
|
|
|
35
35
|
## Documentation
|
|
36
36
|
|
|
37
|
-
- [Quickstart](https://
|
|
38
|
-
- [GitHub pull requests](https://
|
|
37
|
+
- [Quickstart](https://ocracloud.com/en/docs/quickstart) and the manual, in [English](https://ocracloud.com/en/docs) and [中文](https://ocracloud.com/zh/docs)
|
|
38
|
+
- [GitHub pull requests](https://ocracloud.com/en/docs/github)
|
|
39
39
|
- [Changelog](https://github.com/jma49/Open-CR-Agent/blob/main/CHANGELOG.md) and [issues](https://github.com/jma49/Open-CR-Agent/issues)
|
|
40
40
|
|
|
41
41
|
## License
|
package/dist/cloud.d.ts
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Output } from "./review/progress.js";
|
|
2
|
+
/** ocra Cloud's address; OCRA_CLOUD_URL names another server. */
|
|
3
|
+
export declare const DEFAULT_CLOUD_URL = "https://app.ocracloud.com";
|
|
4
|
+
export declare const LOGIN_USAGE: string;
|
|
5
|
+
export type Credentials = {
|
|
6
|
+
server: string;
|
|
7
|
+
login: string;
|
|
8
|
+
access_token: string;
|
|
9
|
+
refresh_token: string;
|
|
10
|
+
/** Epoch milliseconds when the access token expires. */
|
|
11
|
+
expires_at: number;
|
|
12
|
+
};
|
|
13
|
+
export type CloudDeps = {
|
|
14
|
+
env: Readonly<Record<string, string | undefined>>;
|
|
15
|
+
fetch: typeof fetch;
|
|
16
|
+
now: () => number;
|
|
17
|
+
sleep: (ms: number) => Promise<void>;
|
|
18
|
+
openBrowser: (url: string) => void;
|
|
19
|
+
credentialsPath: string;
|
|
20
|
+
clientName: string;
|
|
21
|
+
};
|
|
22
|
+
export declare function defaultCloudDeps(env?: CloudDeps["env"]): CloudDeps;
|
|
23
|
+
export declare function credentialsPath(env: CloudDeps["env"]): string;
|
|
24
|
+
export declare function readCredentials(path: string): Promise<Credentials | undefined>;
|
|
25
|
+
/**
|
|
26
|
+
* The saved session with a live access token, refreshed (and saved) when
|
|
27
|
+
* it has expired or is about to; undefined when not signed in or the
|
|
28
|
+
* session was revoked.
|
|
29
|
+
*/
|
|
30
|
+
export declare function cloudSession(deps: CloudDeps, minValidityMs?: number): Promise<Credentials | undefined>;
|
|
31
|
+
export declare function cloudCommand(command: "login" | "logout" | "whoami", argv: string[], out: Output, err: Output, deps: CloudDeps): Promise<number>;
|
|
32
|
+
//# sourceMappingURL=cloud.d.ts.map
|
package/dist/cloud.js
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { homedir, hostname } from "node:os";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { parseArgs } from "node:util";
|
|
6
|
+
import { errorMessage } from "@open-cr-agent/core/internal";
|
|
7
|
+
import { UsageError } from "./review/args.js";
|
|
8
|
+
import { forTerminal } from "./review/terminal.js";
|
|
9
|
+
import { VERSION } from "./version.js";
|
|
10
|
+
// ocra Cloud sign-in (ADR-0024): `ocra login` runs the OAuth device flow
|
|
11
|
+
// (RFC 8628) against ocra Cloud, `ocra whoami` shows the account, and
|
|
12
|
+
// `ocra logout` ends the session. Nothing here runs unless one of these
|
|
13
|
+
// commands is used: a self-hosted user never sees a login prompt.
|
|
14
|
+
/** ocra Cloud's address; OCRA_CLOUD_URL names another server. */
|
|
15
|
+
export const DEFAULT_CLOUD_URL = "https://app.ocracloud.com";
|
|
16
|
+
export const LOGIN_USAGE = `Usage: ocra login [--no-browser]
|
|
17
|
+
ocra logout
|
|
18
|
+
ocra whoami
|
|
19
|
+
|
|
20
|
+
Sign in to ocra Cloud with GitHub, so reviews can use the model keys stored
|
|
21
|
+
there. ocra login shows a code, opens the browser to confirm it, and saves the
|
|
22
|
+
session to ${credentialsHint()} (readable only by you).
|
|
23
|
+
|
|
24
|
+
Environment:
|
|
25
|
+
OCRA_CLOUD=off Disable ocra Cloud: these commands refuse to run
|
|
26
|
+
OCRA_CLOUD_URL Another ocra Cloud server (default ${DEFAULT_CLOUD_URL})
|
|
27
|
+
`;
|
|
28
|
+
export function defaultCloudDeps(env = process.env) {
|
|
29
|
+
return {
|
|
30
|
+
env,
|
|
31
|
+
fetch: globalThis.fetch,
|
|
32
|
+
now: Date.now,
|
|
33
|
+
sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
34
|
+
openBrowser,
|
|
35
|
+
credentialsPath: credentialsPath(env),
|
|
36
|
+
clientName: `${hostname()} (ocra ${VERSION})`,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function credentialsHint() {
|
|
40
|
+
return process.platform === "win32"
|
|
41
|
+
? "%APPDATA%\\ocra\\credentials.json"
|
|
42
|
+
: "~/.config/ocra/credentials.json";
|
|
43
|
+
}
|
|
44
|
+
export function credentialsPath(env) {
|
|
45
|
+
const base = process.platform === "win32"
|
|
46
|
+
? (env.APPDATA ?? join(homedir(), "AppData", "Roaming"))
|
|
47
|
+
: (env.XDG_CONFIG_HOME ?? join(homedir(), ".config"));
|
|
48
|
+
return join(base, "ocra", "credentials.json");
|
|
49
|
+
}
|
|
50
|
+
function cloudUrl(env) {
|
|
51
|
+
const named = env.OCRA_CLOUD_URL || DEFAULT_CLOUD_URL;
|
|
52
|
+
let url;
|
|
53
|
+
try {
|
|
54
|
+
url = new URL(named);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
throw new UsageError(`OCRA_CLOUD_URL is not a URL: ${named}`);
|
|
58
|
+
}
|
|
59
|
+
if (url.protocol !== "https:" && url.hostname !== "localhost" && url.hostname !== "127.0.0.1") {
|
|
60
|
+
throw new UsageError(`OCRA_CLOUD_URL must use https: ${url}`);
|
|
61
|
+
}
|
|
62
|
+
return url.origin;
|
|
63
|
+
}
|
|
64
|
+
function openBrowser(url) {
|
|
65
|
+
const [cmd, args] = process.platform === "darwin"
|
|
66
|
+
? ["open", [url]]
|
|
67
|
+
: process.platform === "win32"
|
|
68
|
+
? ["cmd", ["/c", "start", "", url]]
|
|
69
|
+
: ["xdg-open", [url]];
|
|
70
|
+
try {
|
|
71
|
+
const child = spawn(cmd, args, { stdio: "ignore", detached: true });
|
|
72
|
+
child.on("error", () => { });
|
|
73
|
+
child.unref();
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
// The URL is printed as well; a missing opener is not an error.
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
export async function readCredentials(path) {
|
|
80
|
+
let text;
|
|
81
|
+
try {
|
|
82
|
+
text = await readFile(path, "utf8");
|
|
83
|
+
}
|
|
84
|
+
catch {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
const c = JSON.parse(text);
|
|
88
|
+
if (!c.server || !c.access_token || !c.refresh_token || typeof c.expires_at !== "number") {
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
91
|
+
return c;
|
|
92
|
+
}
|
|
93
|
+
async function writeCredentials(path, c) {
|
|
94
|
+
await mkdir(dirname(path), { recursive: true, mode: 0o700 });
|
|
95
|
+
await writeFile(path, `${JSON.stringify(c, null, 2)}\n`, { mode: 0o600 });
|
|
96
|
+
// writeFile keeps the mode of a file that already exists.
|
|
97
|
+
await chmod(path, 0o600);
|
|
98
|
+
}
|
|
99
|
+
async function postJson(deps, url, body, token) {
|
|
100
|
+
const res = await deps.fetch(url, {
|
|
101
|
+
method: "POST",
|
|
102
|
+
headers: {
|
|
103
|
+
"content-type": "application/json",
|
|
104
|
+
"user-agent": `ocra/${VERSION}`,
|
|
105
|
+
...(token ? { authorization: `Bearer ${token}` } : {}),
|
|
106
|
+
},
|
|
107
|
+
body: JSON.stringify(body),
|
|
108
|
+
signal: AbortSignal.timeout(30_000),
|
|
109
|
+
});
|
|
110
|
+
return {
|
|
111
|
+
status: res.status,
|
|
112
|
+
body: (await res.json().catch(() => ({}))),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
async function fetchMe(deps, server, token) {
|
|
116
|
+
const res = await deps.fetch(`${server}/api/me`, {
|
|
117
|
+
headers: { authorization: `Bearer ${token}`, "user-agent": `ocra/${VERSION}` },
|
|
118
|
+
signal: AbortSignal.timeout(30_000),
|
|
119
|
+
});
|
|
120
|
+
if (res.status === 401)
|
|
121
|
+
return undefined;
|
|
122
|
+
if (!res.ok)
|
|
123
|
+
throw new Error(`ocra Cloud answered ${res.status}`);
|
|
124
|
+
return (await res.json());
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* The saved session with a live access token, refreshed (and saved) when
|
|
128
|
+
* it has expired or is about to; undefined when not signed in or the
|
|
129
|
+
* session was revoked.
|
|
130
|
+
*/
|
|
131
|
+
export async function cloudSession(deps, minValidityMs = 60_000) {
|
|
132
|
+
const saved = await readCredentials(deps.credentialsPath);
|
|
133
|
+
if (!saved)
|
|
134
|
+
return undefined;
|
|
135
|
+
if (saved.expires_at - deps.now() > minValidityMs)
|
|
136
|
+
return saved;
|
|
137
|
+
const { status, body } = await postJson(deps, `${saved.server}/api/device/refresh`, {
|
|
138
|
+
refresh_token: saved.refresh_token,
|
|
139
|
+
});
|
|
140
|
+
const t = body;
|
|
141
|
+
if (status !== 200 || !t.access_token || !t.refresh_token || !t.expires_in)
|
|
142
|
+
return undefined;
|
|
143
|
+
const next = {
|
|
144
|
+
...saved,
|
|
145
|
+
access_token: t.access_token,
|
|
146
|
+
refresh_token: t.refresh_token,
|
|
147
|
+
expires_at: deps.now() + t.expires_in * 1000,
|
|
148
|
+
};
|
|
149
|
+
await writeCredentials(deps.credentialsPath, next);
|
|
150
|
+
return next;
|
|
151
|
+
}
|
|
152
|
+
export async function cloudCommand(command, argv, out, err, deps) {
|
|
153
|
+
const { values } = parseArgs({
|
|
154
|
+
args: argv,
|
|
155
|
+
strict: true,
|
|
156
|
+
options: { "no-browser": { type: "boolean" }, help: { type: "boolean", short: "h" } },
|
|
157
|
+
});
|
|
158
|
+
if (values.help) {
|
|
159
|
+
out.write(LOGIN_USAGE);
|
|
160
|
+
return 0;
|
|
161
|
+
}
|
|
162
|
+
if (deps.env.OCRA_CLOUD === "off") {
|
|
163
|
+
err.write("ocra Cloud is off (OCRA_CLOUD=off).\n");
|
|
164
|
+
return 2;
|
|
165
|
+
}
|
|
166
|
+
if (command === "whoami")
|
|
167
|
+
return whoami(out, err, deps);
|
|
168
|
+
if (command === "logout")
|
|
169
|
+
return logout(out, deps);
|
|
170
|
+
return login(out, err, deps, !values["no-browser"]);
|
|
171
|
+
}
|
|
172
|
+
async function whoami(out, err, deps) {
|
|
173
|
+
const session = await cloudSession(deps);
|
|
174
|
+
if (!session) {
|
|
175
|
+
err.write("Not signed in. Run ocra login.\n");
|
|
176
|
+
return 1;
|
|
177
|
+
}
|
|
178
|
+
const me = await fetchMe(deps, session.server, session.access_token);
|
|
179
|
+
if (!me) {
|
|
180
|
+
err.write("The saved session was revoked. Run ocra login.\n");
|
|
181
|
+
return 1;
|
|
182
|
+
}
|
|
183
|
+
out.write(forTerminal(`${me.login} on ${session.server}\n`));
|
|
184
|
+
return 0;
|
|
185
|
+
}
|
|
186
|
+
async function logout(out, deps) {
|
|
187
|
+
const saved = await readCredentials(deps.credentialsPath);
|
|
188
|
+
if (saved) {
|
|
189
|
+
// Ends the session on the server too; the local file goes either way.
|
|
190
|
+
const session = await cloudSession(deps).catch(() => undefined);
|
|
191
|
+
if (session) {
|
|
192
|
+
await postJson(deps, `${session.server}/api/auth/logout`, {}, session.access_token).catch(() => { });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
await rm(deps.credentialsPath, { force: true });
|
|
196
|
+
out.write(saved ? "Signed out.\n" : "Not signed in.\n");
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
async function login(out, err, deps, browser) {
|
|
200
|
+
const server = cloudUrl(deps.env);
|
|
201
|
+
const start = await postJson(deps, `${server}/api/device/code`, { client_name: deps.clientName });
|
|
202
|
+
const code = start.body;
|
|
203
|
+
if (start.status !== 200 ||
|
|
204
|
+
!code.device_code ||
|
|
205
|
+
!code.user_code ||
|
|
206
|
+
!code.verification_uri_complete) {
|
|
207
|
+
err.write(`ocra Cloud did not start a login (HTTP ${start.status}). Try again in a minute.\n`);
|
|
208
|
+
return 2;
|
|
209
|
+
}
|
|
210
|
+
out.write(forTerminal(`Confirm this code in your browser: ${code.user_code}\n` +
|
|
211
|
+
` ${code.verification_uri_complete}\n` +
|
|
212
|
+
`(or open ${code.verification_uri} and type it)\n\nWaiting for approval...\n`));
|
|
213
|
+
if (browser)
|
|
214
|
+
deps.openBrowser(code.verification_uri_complete);
|
|
215
|
+
let interval = (code.interval ?? 5) * 1000;
|
|
216
|
+
const deadline = deps.now() + (code.expires_in ?? 600) * 1000;
|
|
217
|
+
while (deps.now() < deadline) {
|
|
218
|
+
await deps.sleep(interval);
|
|
219
|
+
let answer;
|
|
220
|
+
try {
|
|
221
|
+
answer = await postJson(deps, `${server}/api/device/token`, {
|
|
222
|
+
device_code: code.device_code,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
// A dropped connection is retried on the next tick.
|
|
227
|
+
err.write(forTerminal(`(${errorMessage(error)}; retrying)\n`));
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
const t = answer.body;
|
|
231
|
+
if (answer.status === 200 && t.access_token && t.refresh_token && t.expires_in) {
|
|
232
|
+
const me = await fetchMe(deps, server, t.access_token);
|
|
233
|
+
const credentials = {
|
|
234
|
+
server,
|
|
235
|
+
login: me?.login ?? "",
|
|
236
|
+
access_token: t.access_token,
|
|
237
|
+
refresh_token: t.refresh_token,
|
|
238
|
+
expires_at: deps.now() + t.expires_in * 1000,
|
|
239
|
+
};
|
|
240
|
+
await writeCredentials(deps.credentialsPath, credentials);
|
|
241
|
+
out.write(forTerminal(`Signed in as ${credentials.login}.\n`));
|
|
242
|
+
return 0;
|
|
243
|
+
}
|
|
244
|
+
if (t.error === "authorization_pending")
|
|
245
|
+
continue;
|
|
246
|
+
if (t.error === "slow_down") {
|
|
247
|
+
interval = typeof t.interval === "number" ? t.interval * 1000 : interval + 5000;
|
|
248
|
+
continue;
|
|
249
|
+
}
|
|
250
|
+
if (t.error === "access_denied") {
|
|
251
|
+
err.write("Login denied in the browser.\n");
|
|
252
|
+
return 1;
|
|
253
|
+
}
|
|
254
|
+
if (t.error === "expired_token")
|
|
255
|
+
break;
|
|
256
|
+
err.write(forTerminal(`Login failed: ${t.error ?? `HTTP ${answer.status}`}\n`));
|
|
257
|
+
return 2;
|
|
258
|
+
}
|
|
259
|
+
err.write("The code expired before it was confirmed. Run ocra login again.\n");
|
|
260
|
+
return 1;
|
|
261
|
+
}
|
|
262
|
+
//# sourceMappingURL=cloud.js.map
|
package/dist/memory.js
CHANGED
|
@@ -2,8 +2,8 @@ import { constants, existsSync } from "node:fs";
|
|
|
2
2
|
import { lstat, mkdir, open, readdir, readFile } from "node:fs/promises";
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
import { parseArgs } from "node:util";
|
|
5
|
-
import { MEMORY_PATH, parseMemory, serializeMemory
|
|
6
|
-
import { findRepositoryRoot } from "@open-cr-agent/vcs-local";
|
|
5
|
+
import { MEMORY_PATH, parseMemory, serializeMemory } from "@open-cr-agent/core/internal";
|
|
6
|
+
import { findRepositoryRoot } from "@open-cr-agent/vcs-local/internal";
|
|
7
7
|
import { UsageError } from "./review/args.js";
|
|
8
8
|
import { SESSIONS_DIR } from "./review/command.js";
|
|
9
9
|
import { forTerminal } from "./review/terminal.js";
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { type Severity, type Verification } from "@open-cr-agent/core";
|
|
2
|
+
import type { Output } from "./review/progress.js";
|
|
3
|
+
export declare const METRICS_VERSION = 1;
|
|
4
|
+
export declare const METRICS_USAGE = "Usage: ocra metrics [options]\n\nCounts over the finished reviews in .ocra/sessions: runs, cost, findings,\nwhat reviewers later fixed or dismissed, per reviewer. Reads each session's\nreport.json (report version 1); nothing calls a model.\n\nOptions:\n --sessions <dir> Sessions directory (default: .ocra/sessions of this repository)\n --since <date> Only runs started at or after this ISO 8601 date\n --format <text|json> Output format (default: text); the JSON carries \"version\": 1\n -h, --help Show help\n";
|
|
5
|
+
export interface ReviewerMetrics {
|
|
6
|
+
tasks: number;
|
|
7
|
+
failedTasks: number;
|
|
8
|
+
findings: number;
|
|
9
|
+
costUsd: number;
|
|
10
|
+
fixed: number;
|
|
11
|
+
dismissed: number;
|
|
12
|
+
acceptanceRate: number | null;
|
|
13
|
+
}
|
|
14
|
+
export interface Metrics {
|
|
15
|
+
version: typeof METRICS_VERSION;
|
|
16
|
+
sessionsDir: string;
|
|
17
|
+
since?: string;
|
|
18
|
+
runs: {
|
|
19
|
+
total: number;
|
|
20
|
+
byVerdict: Record<string, number>;
|
|
21
|
+
incomplete: number;
|
|
22
|
+
unreadable: number;
|
|
23
|
+
};
|
|
24
|
+
cost: {
|
|
25
|
+
usd: number;
|
|
26
|
+
inputTokens: number;
|
|
27
|
+
outputTokens: number;
|
|
28
|
+
usdPerRun: number | null;
|
|
29
|
+
};
|
|
30
|
+
findings: {
|
|
31
|
+
reported: number;
|
|
32
|
+
unique: number;
|
|
33
|
+
bySeverity: Record<Severity, number>;
|
|
34
|
+
byVerification: Record<Verification, number>;
|
|
35
|
+
};
|
|
36
|
+
outcomes: {
|
|
37
|
+
fixed: number;
|
|
38
|
+
dismissed: number;
|
|
39
|
+
acceptanceRate: number | null;
|
|
40
|
+
};
|
|
41
|
+
reviewers: Record<string, ReviewerMetrics>;
|
|
42
|
+
}
|
|
43
|
+
export declare function metricsCommand(argv: string[], out: Output, cwd: string): Promise<number>;
|
|
44
|
+
export declare function sessionStart(id: string): Date | undefined;
|
|
45
|
+
export declare function collectMetrics(sessionsDir: string, since?: Date): Promise<Metrics>;
|
|
46
|
+
export declare function renderMetrics(m: Metrics): string;
|
|
47
|
+
//# sourceMappingURL=metrics.d.ts.map
|
package/dist/metrics.js
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
import { parseArgs } from "node:util";
|
|
4
|
+
import { reportOutputSchema, } from "@open-cr-agent/core";
|
|
5
|
+
import { findRepositoryRoot } from "@open-cr-agent/vcs-local/internal";
|
|
6
|
+
import { UsageError } from "./review/args.js";
|
|
7
|
+
import { SESSIONS_DIR } from "./review/command.js";
|
|
8
|
+
import { forTerminal } from "./review/terminal.js";
|
|
9
|
+
export const METRICS_VERSION = 1;
|
|
10
|
+
export const METRICS_USAGE = `Usage: ocra metrics [options]
|
|
11
|
+
|
|
12
|
+
Counts over the finished reviews in ${SESSIONS_DIR}: runs, cost, findings,
|
|
13
|
+
what reviewers later fixed or dismissed, per reviewer. Reads each session's
|
|
14
|
+
report.json (report version 1); nothing calls a model.
|
|
15
|
+
|
|
16
|
+
Options:
|
|
17
|
+
--sessions <dir> Sessions directory (default: ${SESSIONS_DIR} of this repository)
|
|
18
|
+
--since <date> Only runs started at or after this ISO 8601 date
|
|
19
|
+
--format <text|json> Output format (default: text); the JSON carries "version": ${METRICS_VERSION}
|
|
20
|
+
-h, --help Show help
|
|
21
|
+
`;
|
|
22
|
+
export async function metricsCommand(argv, out, cwd) {
|
|
23
|
+
const { values } = parseArgs({
|
|
24
|
+
args: argv,
|
|
25
|
+
allowPositionals: false,
|
|
26
|
+
strict: true,
|
|
27
|
+
options: {
|
|
28
|
+
sessions: { type: "string" },
|
|
29
|
+
since: { type: "string" },
|
|
30
|
+
format: { type: "string" },
|
|
31
|
+
help: { type: "boolean", short: "h" },
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
if (values.help) {
|
|
35
|
+
out.write(METRICS_USAGE);
|
|
36
|
+
return 0;
|
|
37
|
+
}
|
|
38
|
+
const format = values.format ?? "text";
|
|
39
|
+
if (format !== "text" && format !== "json")
|
|
40
|
+
throw new UsageError("--format must be text or json");
|
|
41
|
+
const since = values.since === undefined ? undefined : parseSince(values.since);
|
|
42
|
+
const sessionsDir = values.sessions
|
|
43
|
+
? resolve(cwd, values.sessions)
|
|
44
|
+
: join(await findRepositoryRoot(cwd), SESSIONS_DIR);
|
|
45
|
+
const metrics = await collectMetrics(sessionsDir, since);
|
|
46
|
+
out.write(format === "json" ? `${JSON.stringify(metrics, null, 2)}\n` : renderMetrics(metrics));
|
|
47
|
+
return 0;
|
|
48
|
+
}
|
|
49
|
+
function parseSince(text) {
|
|
50
|
+
const date = new Date(text);
|
|
51
|
+
if (Number.isNaN(date.getTime()))
|
|
52
|
+
throw new UsageError(`--since needs an ISO 8601 date, not "${text}"`);
|
|
53
|
+
return date;
|
|
54
|
+
}
|
|
55
|
+
// A session directory is named by its run id, whose first part is the start
|
|
56
|
+
// time (newRunId in core); that is what --since compares against.
|
|
57
|
+
export function sessionStart(id) {
|
|
58
|
+
const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z/.exec(id);
|
|
59
|
+
if (!m)
|
|
60
|
+
return undefined;
|
|
61
|
+
const [, y, mo, d, h, mi, s] = m;
|
|
62
|
+
return new Date(`${y}-${mo}-${d}T${h}:${mi}:${s}Z`);
|
|
63
|
+
}
|
|
64
|
+
export async function collectMetrics(sessionsDir, since) {
|
|
65
|
+
const names = (await readdir(sessionsDir).catch(() => []))
|
|
66
|
+
.filter((n) => !n.startsWith("."))
|
|
67
|
+
.filter((n) => {
|
|
68
|
+
const start = sessionStart(n);
|
|
69
|
+
return since === undefined || (start !== undefined && start >= since);
|
|
70
|
+
})
|
|
71
|
+
.sort();
|
|
72
|
+
const reports = [];
|
|
73
|
+
let unreadable = 0;
|
|
74
|
+
for (const name of names) {
|
|
75
|
+
const report = await readReport(join(sessionsDir, name, "report.json"));
|
|
76
|
+
if (report)
|
|
77
|
+
reports.push(report);
|
|
78
|
+
else
|
|
79
|
+
unreadable += 1;
|
|
80
|
+
}
|
|
81
|
+
return aggregate(reports, {
|
|
82
|
+
sessionsDir,
|
|
83
|
+
unreadable,
|
|
84
|
+
...(since ? { since: since.toISOString() } : {}),
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
async function readReport(path) {
|
|
88
|
+
let data;
|
|
89
|
+
try {
|
|
90
|
+
data = JSON.parse(await readFile(path, "utf8"));
|
|
91
|
+
}
|
|
92
|
+
catch {
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
const parsed = reportOutputSchema.safeParse(data);
|
|
96
|
+
return parsed.success ? parsed.data : undefined;
|
|
97
|
+
}
|
|
98
|
+
function aggregate(reports, meta) {
|
|
99
|
+
const byVerdict = {};
|
|
100
|
+
const bySeverity = { critical: 0, warning: 0, suggestion: 0 };
|
|
101
|
+
const byVerification = { confirmed: 0, uncertain: 0, unchecked: 0 };
|
|
102
|
+
const reviewers = new Map();
|
|
103
|
+
const reviewerOf = new Map();
|
|
104
|
+
const unique = new Set();
|
|
105
|
+
const fixed = new Set();
|
|
106
|
+
const dismissed = new Set();
|
|
107
|
+
let incomplete = 0;
|
|
108
|
+
let reported = 0;
|
|
109
|
+
let usd = 0;
|
|
110
|
+
let inputTokens = 0;
|
|
111
|
+
let outputTokens = 0;
|
|
112
|
+
const forReviewer = (id) => {
|
|
113
|
+
let r = reviewers.get(id);
|
|
114
|
+
if (!r) {
|
|
115
|
+
r = {
|
|
116
|
+
tasks: 0,
|
|
117
|
+
failedTasks: 0,
|
|
118
|
+
findings: 0,
|
|
119
|
+
costUsd: 0,
|
|
120
|
+
fixed: 0,
|
|
121
|
+
dismissed: 0,
|
|
122
|
+
acceptanceRate: null,
|
|
123
|
+
};
|
|
124
|
+
reviewers.set(id, r);
|
|
125
|
+
}
|
|
126
|
+
return r;
|
|
127
|
+
};
|
|
128
|
+
for (const report of reports) {
|
|
129
|
+
byVerdict[report.verdict] = (byVerdict[report.verdict] ?? 0) + 1;
|
|
130
|
+
if (report.coverage.some((c) => c.status === "failed" || c.status === "unreviewed"))
|
|
131
|
+
incomplete += 1;
|
|
132
|
+
usd += report.usage.costUsd;
|
|
133
|
+
inputTokens += report.usage.inputTokens;
|
|
134
|
+
outputTokens += report.usage.outputTokens;
|
|
135
|
+
for (const task of report.tasks) {
|
|
136
|
+
const r = forReviewer(task.reviewer);
|
|
137
|
+
r.tasks += 1;
|
|
138
|
+
if (task.status === "failed" || task.status === "timed_out")
|
|
139
|
+
r.failedTasks += 1;
|
|
140
|
+
r.costUsd += task.usage.costUsd;
|
|
141
|
+
}
|
|
142
|
+
for (const finding of report.findings) {
|
|
143
|
+
reported += 1;
|
|
144
|
+
unique.add(finding.fingerprint);
|
|
145
|
+
bySeverity[finding.severity] += 1;
|
|
146
|
+
byVerification[finding.verification] += 1;
|
|
147
|
+
forReviewer(finding.reviewer).findings += 1;
|
|
148
|
+
if (!reviewerOf.has(finding.fingerprint))
|
|
149
|
+
reviewerOf.set(finding.fingerprint, finding.reviewer);
|
|
150
|
+
}
|
|
151
|
+
for (const f of report.rereview?.fixed ?? [])
|
|
152
|
+
fixed.add(f.fingerprint);
|
|
153
|
+
for (const f of report.rereview?.dismissed ?? [])
|
|
154
|
+
dismissed.add(f.fingerprint);
|
|
155
|
+
}
|
|
156
|
+
// A fingerprint can be dismissed in one review and gone in a later one:
|
|
157
|
+
// the reviewer's decision stands.
|
|
158
|
+
for (const fp of dismissed)
|
|
159
|
+
fixed.delete(fp);
|
|
160
|
+
for (const fp of fixed) {
|
|
161
|
+
const reviewer = reviewerOf.get(fp);
|
|
162
|
+
if (reviewer)
|
|
163
|
+
forReviewer(reviewer).fixed += 1;
|
|
164
|
+
}
|
|
165
|
+
for (const fp of dismissed) {
|
|
166
|
+
const reviewer = reviewerOf.get(fp);
|
|
167
|
+
if (reviewer)
|
|
168
|
+
forReviewer(reviewer).dismissed += 1;
|
|
169
|
+
}
|
|
170
|
+
for (const r of reviewers.values())
|
|
171
|
+
r.acceptanceRate = rate(r.fixed, r.dismissed);
|
|
172
|
+
return {
|
|
173
|
+
version: METRICS_VERSION,
|
|
174
|
+
sessionsDir: meta.sessionsDir,
|
|
175
|
+
...(meta.since ? { since: meta.since } : {}),
|
|
176
|
+
runs: { total: reports.length, byVerdict, incomplete, unreadable: meta.unreadable },
|
|
177
|
+
cost: {
|
|
178
|
+
usd,
|
|
179
|
+
inputTokens,
|
|
180
|
+
outputTokens,
|
|
181
|
+
usdPerRun: reports.length > 0 ? usd / reports.length : null,
|
|
182
|
+
},
|
|
183
|
+
findings: { reported, unique: unique.size, bySeverity, byVerification },
|
|
184
|
+
outcomes: {
|
|
185
|
+
fixed: fixed.size,
|
|
186
|
+
dismissed: dismissed.size,
|
|
187
|
+
acceptanceRate: rate(fixed.size, dismissed.size),
|
|
188
|
+
},
|
|
189
|
+
reviewers: Object.fromEntries([...reviewers].sort(([a], [b]) => a.localeCompare(b))),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
function rate(fixed, dismissed) {
|
|
193
|
+
return fixed + dismissed > 0 ? fixed / (fixed + dismissed) : null;
|
|
194
|
+
}
|
|
195
|
+
const percent = (value) => (value === null ? "n/a" : `${Math.round(value * 100)}%`);
|
|
196
|
+
const dollars = (value) => (value === null ? "n/a" : `$${value.toFixed(2)}`);
|
|
197
|
+
export function renderMetrics(m) {
|
|
198
|
+
const lines = [
|
|
199
|
+
`Runs: ${m.runs.total}${m.since ? ` since ${m.since}` : ""} (${m.runs.incomplete} incomplete, ${m.runs.unreadable} unreadable)`,
|
|
200
|
+
` by verdict: ${entries(m.runs.byVerdict)}`,
|
|
201
|
+
`Cost: ${dollars(m.cost.usd)} total, ${dollars(m.cost.usdPerRun)} per run · ${m.cost.inputTokens} in / ${m.cost.outputTokens} out tokens`,
|
|
202
|
+
`Findings: ${m.findings.reported} reported, ${m.findings.unique} unique`,
|
|
203
|
+
` by severity: ${entries(m.findings.bySeverity)}`,
|
|
204
|
+
` by verification: ${entries(m.findings.byVerification)}`,
|
|
205
|
+
`Outcomes: ${m.outcomes.fixed} fixed, ${m.outcomes.dismissed} dismissed, acceptance ${percent(m.outcomes.acceptanceRate)}`,
|
|
206
|
+
"",
|
|
207
|
+
"Reviewer Tasks Failed Findings Cost Fixed Dismissed Acceptance",
|
|
208
|
+
];
|
|
209
|
+
for (const [id, r] of Object.entries(m.reviewers)) {
|
|
210
|
+
lines.push([
|
|
211
|
+
id.padEnd(18),
|
|
212
|
+
String(r.tasks).padStart(5),
|
|
213
|
+
String(r.failedTasks).padStart(7),
|
|
214
|
+
String(r.findings).padStart(9),
|
|
215
|
+
dollars(r.costUsd).padStart(9),
|
|
216
|
+
String(r.fixed).padStart(6),
|
|
217
|
+
String(r.dismissed).padStart(10),
|
|
218
|
+
percent(r.acceptanceRate).padStart(11),
|
|
219
|
+
].join(" "));
|
|
220
|
+
}
|
|
221
|
+
return `${forTerminal(lines.join("\n"))}\n`;
|
|
222
|
+
}
|
|
223
|
+
function entries(record) {
|
|
224
|
+
const parts = Object.entries(record).filter(([, n]) => n > 0);
|
|
225
|
+
return parts.length === 0 ? "none" : parts.map(([k, n]) => `${k} ${n}`).join(", ");
|
|
226
|
+
}
|
|
227
|
+
//# sourceMappingURL=metrics.js.map
|
package/dist/review/args.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { OcraError } from "@open-cr-agent/core";
|
|
1
2
|
import type { LocalTarget } from "@open-cr-agent/vcs-local";
|
|
2
3
|
export type OutputFormat = "text" | "json" | "sarif";
|
|
3
4
|
export interface PullRequestTarget {
|
|
@@ -22,9 +23,15 @@ export interface ReviewArgs {
|
|
|
22
23
|
ultra?: true;
|
|
23
24
|
full?: true;
|
|
24
25
|
plan?: true;
|
|
26
|
+
importSarif?: string[];
|
|
27
|
+
configFile?: string;
|
|
28
|
+
noUpload?: true;
|
|
29
|
+
temperature?: number;
|
|
30
|
+
seed?: number;
|
|
25
31
|
}
|
|
26
|
-
export declare class UsageError extends
|
|
32
|
+
export declare class UsageError extends OcraError {
|
|
33
|
+
constructor(message: string);
|
|
27
34
|
}
|
|
28
|
-
export declare const REVIEW_USAGE = "Usage: ocra review [options]\n\nReviews uncommitted changes by default.\n\nOptions:\n --from <ref> Review changes on --to since it diverged from <ref>\n --to <ref> End of the range (default: HEAD)\n --commit <sha> Review a single commit\n --pr <number> Review a GitHub pull request (needs GITHUB_TOKEN)\n --repo <owner/name> Repository of --pr (default: GITHUB_REPOSITORY or origin)\n --mr <iid> Review a GitLab merge request (needs GITLAB_TOKEN)\n --project <id|path> Project of --mr (default: CI_PROJECT_ID or origin)\n --publish With --pr or --mr: post the review to it\n --full With --pr or --mr: review every file, not only what\n changed since the previous review\n --format <format> text (default), json, or sarif (SARIF 2.1.0; not with --plan)\n --output <file> Write the result to a file instead of stdout\n --reviewers <ids> Run only these reviewers (comma-separated)\n --max-cost-usd <n> Spend limit: reviews use 80%, verification and judging the rest\n --plan Show files, bundles, review tasks and prompt sizes; call no model\n --ultra Favor recall: all reviewers at every tier, two samples each (about 2x cost)\n --no-repo-config Ignore .ocra/config.json and its plugins (for untrusted\n code); models come from OCRA_MODEL_* variables\n -h, --help Show help\n";
|
|
35
|
+
export declare const REVIEW_USAGE = "Usage: ocra review [options]\n\nReviews uncommitted changes by default.\n\nOptions:\n --from <ref> Review changes on --to since it diverged from <ref>\n --to <ref> End of the range (default: HEAD)\n --commit <sha> Review a single commit\n --pr <number> Review a GitHub pull request (needs GITHUB_TOKEN)\n --repo <owner/name> Repository of --pr (default: GITHUB_REPOSITORY or origin)\n --mr <iid> Review a GitLab merge request (needs GITLAB_TOKEN)\n --project <id|path> Project of --mr (default: CI_PROJECT_ID or origin)\n --publish With --pr or --mr: post the review to it\n --full With --pr or --mr: review every file, not only what\n changed since the previous review\n --format <format> text (default), json, or sarif (SARIF 2.1.0; not with --plan)\n --output <file> Write the result to a file instead of stdout\n --reviewers <ids> Run only these reviewers (comma-separated)\n --max-cost-usd <n> Spend limit: reviews use 80%, verification and judging the rest\n --import-sarif <file> Add the results of a SARIF 2.1.0 log (Semgrep, CodeQL, \u2026)\n that fall on the change; repeatable; not with --plan\n --plan Show files, bundles, review tasks and prompt sizes; call no model\n --ultra Favor recall: all reviewers at every tier, two samples each (about 2x cost)\n --temperature <n> Sampling temperature, 0 to 2 (default: the provider's, or sampling in config)\n --seed <n> Sampling seed, where the runtime and provider support one\n --config <file> Read this configuration file instead of the repository's\n .ocra/config.json; it applies with --no-repo-config too\n --no-repo-config Ignore .ocra/config.json and its plugins (for untrusted\n code); models come from OCRA_MODEL_* variables or --config\n --no-upload When signed in to ocra Cloud, do not send this review's\n counts (verdict, numbers of findings, files, tokens)\n -h, --help Show help\n";
|
|
29
36
|
export declare function parseReviewArgs(argv: string[]): ReviewArgs | "help";
|
|
30
37
|
//# sourceMappingURL=args.d.ts.map
|